Merge branch 'fix/r3-02-stale-good-family' — R3-02: one quality contract at four seams — comms loss marks raw+UNS value variables BadNoCommunication (WriteStatuses sink member, recovery audit), Calculation + VT hold on Bad / worst-of sub-code / (value,status) dedup, MQTT initial push + ConnectionLost stale, Sparkplug birth replay (archreview 03/S15, 05/STAB-19, 05/STAB-18, 02/S14)
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
|
||||
/// <summary>
|
||||
@@ -18,6 +20,34 @@ public interface IVirtualTagEvaluator
|
||||
/// <param name="dependencies">Read-only dictionary of variable names to values for expression evaluation.</param>
|
||||
/// <returns>Result containing success flag, evaluated value, and optional failure reason.</returns>
|
||||
VirtualTagEvalResult Evaluate(string virtualTagId, string expression, IReadOnlyDictionary<string, object?> dependencies);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluate <paramref name="expression"/> against dependency <b>snapshots</b> — value, status
|
||||
/// code and source timestamp per input — so a script's <c>ctx.GetTag(path).StatusCode</c> reports
|
||||
/// what the source actually said (archreview 02/S14).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Default-implemented on purpose.</b> It projects to values and calls the older overload, so
|
||||
/// every existing implementation — the Roslyn evaluator aside, that is the null evaluator and the
|
||||
/// test fakes — compiles and behaves unchanged; an implementation that can do better (the Roslyn
|
||||
/// one) overrides it. Making it abstract would have been a breaking change for a fix whose whole
|
||||
/// point is that it changes nothing on the Good path.
|
||||
/// <para>The default necessarily discards the status codes it was given, which is exactly the
|
||||
/// laundering this overload exists to end — so a consumer that cares must call an implementation
|
||||
/// that overrides it.</para>
|
||||
/// </remarks>
|
||||
/// <param name="virtualTagId">The unique identifier of the virtual tag being evaluated.</param>
|
||||
/// <param name="expression">The expression string to evaluate.</param>
|
||||
/// <param name="dependencies">Read-only dictionary of dependency snapshots, keyed by tag path.</param>
|
||||
/// <returns>Result containing success flag, evaluated value, and optional failure reason.</returns>
|
||||
VirtualTagEvalResult Evaluate(
|
||||
string virtualTagId, string expression, IReadOnlyDictionary<string, DataValueSnapshot> dependencies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dependencies);
|
||||
var values = new Dictionary<string, object?>(dependencies.Count, StringComparer.Ordinal);
|
||||
foreach (var kv in dependencies) values[kv.Key] = kv.Value?.Value;
|
||||
return Evaluate(virtualTagId, expression, values);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Result of one virtual-tag expression eval. Stash a Reason on every Failure so
|
||||
|
||||
@@ -104,6 +104,15 @@ public static class OtOpcUaTelemetry
|
||||
Meter.CreateCounter<long>("otopcua.redundancy.service_level_change", unit: "{change}",
|
||||
description: "OPC UA Server.ServiceLevel transitions emitted by the redundancy state.");
|
||||
|
||||
/// <summary>Archreview 03/S15 — tag refs a driver left <c>BadNoCommunication</c> after it reconnected:
|
||||
/// the host marked them Bad on the outage, and the driver never re-published them. Non-zero means a
|
||||
/// driver that reconnects without a full re-publish (a push driver missing its initial push, or a
|
||||
/// value-only dedup swallowing an unchanged value), which leaves those nodes Bad indefinitely.
|
||||
/// Tagged driver + driver_type.</summary>
|
||||
public static readonly Counter<long> DriverCommsLatchedRefs =
|
||||
Meter.CreateCounter<long>("otopcua.driver.comms_latched_refs", unit: "{ref}",
|
||||
description: "Refs still BadNoCommunication after their driver reconnected (driver, driver_type).");
|
||||
|
||||
/// <summary>Inbound operations denied by the Primary data-plane gate (archreview 03/S4) —
|
||||
/// site=write|ack|alarm-emit, reason=secondary|detached|role-unknown.</summary>
|
||||
public static readonly Counter<long> PrimaryGateDenied =
|
||||
|
||||
@@ -32,6 +32,14 @@ public sealed class DeferredAddressSpaceSink : IOpcUaAddressSpaceSink, ISurgical
|
||||
// (actors inject THIS wrapper, not the inner SdkAddressSpaceSink) — same trap as WireAlarmNotifiers above.
|
||||
public void WriteValues(IReadOnlyList<ValueWrite> values) => _inner.WriteValues(values);
|
||||
|
||||
/// <inheritdoc />
|
||||
// Forward the status-only batch (03/S15). Unforwarded, the interface's DEFAULT runs here instead —
|
||||
// and that default writes a NULL value, so a comms-loss annotation would erase the last known value
|
||||
// of every node on the driver rather than retaining it, on every driver-role host (actors inject
|
||||
// THIS wrapper, not the inner SdkAddressSpaceSink). Same trap as WriteValues / WireAlarmNotifiers
|
||||
// above, except this one degrades to wrong data rather than to slow.
|
||||
public void WriteStatuses(IReadOnlyList<StatusWrite> statuses) => _inner.WriteStatuses(statuses);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime sourceTimestampUtc, AddressSpaceRealm realm)
|
||||
=> _inner.WriteAlarmCondition(alarmNodeId, state, sourceTimestampUtc, realm);
|
||||
|
||||
@@ -42,6 +42,31 @@ public interface IOpcUaAddressSpaceSink
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Batched <b>status-only</b> write: each node keeps its CURRENT value and only its
|
||||
/// StatusCode + SourceTimestamp change. This is the comms-loss annotation for value variables
|
||||
/// (archreview 03/S15) — a driver that has stopped communicating leaves the node holding its last
|
||||
/// known value (the Part 8 convention SCADA clients expect) under a
|
||||
/// <see cref="OpcUaStatus.BadNoCommunication"/> status, rather than nulling it.
|
||||
/// <para>A sink that HOLDS the node's value must retain it — the SDK-backed sink does, because the
|
||||
/// node itself is where the value lives. This DEFAULT implementation cannot: the host holds no value
|
||||
/// cache (v3's single-source design), so it degrades to <see cref="WriteValue"/> with a null value,
|
||||
/// which is the honest statement for a value-less sink and keeps every fake compiling unchanged.</para>
|
||||
/// <para>Same per-item isolation contract as <see cref="WriteValues"/>. An unknown or
|
||||
/// unmaterialised NodeId is a <b>no-op, never a create</b>: a status carries nothing to create a node
|
||||
/// from, and the caller's index can legitimately name a node a surgical remove has just dropped.</para></summary>
|
||||
/// <param name="statuses">The status writes to apply, in order.</param>
|
||||
/// <exception cref="ValueWriteBatchException">A batching implementation throws this when some — but not
|
||||
/// necessarily all — items failed; every other item has been applied.</exception>
|
||||
void WriteStatuses(IReadOnlyList<StatusWrite> statuses)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(statuses);
|
||||
for (var i = 0; i < statuses.Count; i++)
|
||||
{
|
||||
var s = statuses[i];
|
||||
WriteValue(s.NodeId, null, s.StatusCode, s.SourceTimestampUtc, s.Realm);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Write an alarm-condition's full Part 9 state. When a real condition node has been
|
||||
/// materialised for <paramref name="alarmNodeId"/> via <see cref="MaterialiseAlarmCondition"/>,
|
||||
/// this projects the whole <see cref="AlarmConditionSnapshot"/>
|
||||
@@ -190,6 +215,16 @@ public interface IOpcUaAddressSpaceSink
|
||||
public readonly record struct ValueWrite(
|
||||
string NodeId, object? Value, uint StatusCode, DateTime SourceTimestampUtc, AddressSpaceRealm Realm);
|
||||
|
||||
/// <summary>One status-only write within a <see cref="IOpcUaAddressSpaceSink.WriteStatuses"/> batch: the
|
||||
/// node's current VALUE is retained and only <paramref name="StatusCode"/> +
|
||||
/// <paramref name="SourceTimestampUtc"/> change.</summary>
|
||||
/// <param name="NodeId">The OPC UA node id of the variable, within <paramref name="Realm"/>.</param>
|
||||
/// <param name="StatusCode">The full status code, sub-code included, passed to the SDK verbatim (#519).</param>
|
||||
/// <param name="SourceTimestampUtc">The source timestamp in UTC.</param>
|
||||
/// <param name="Realm">Which of the two v3 namespaces <paramref name="NodeId"/> lives in.</param>
|
||||
public readonly record struct StatusWrite(
|
||||
string NodeId, uint StatusCode, DateTime SourceTimestampUtc, AddressSpaceRealm Realm);
|
||||
|
||||
/// <summary>Aggregate outcome of a partially-failed <see cref="IOpcUaAddressSpaceSink.WriteValues"/> batch:
|
||||
/// every non-failing item HAS been applied; this carries what the caller needs to log and meter honestly.</summary>
|
||||
/// <remarks>A batched sink must NOT abandon the tail of a batch when one item throws: the poll engine
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// The one worst-of-input rule every derived-value seam uses — scripted alarms, virtual tags and
|
||||
/// the Calculation driver.
|
||||
///
|
||||
/// <para><b>The contract it implements</b> (archreview R3-02, "the stale-Good family"): a value
|
||||
/// consumer never reports a quality better than its source's. A node derived from driver values
|
||||
/// carries, at worst, the worst status among its inputs — a Bad input makes the derived node Bad
|
||||
/// with that input's sub-code, an Uncertain input makes it Uncertain, and a source that has
|
||||
/// stopped communicating is a Bad input for everything downstream of it.</para>
|
||||
///
|
||||
/// <para><b>Why it lives here.</b> Core.Abstractions is the one assembly all three seams already
|
||||
/// reference (Commons, Runtime and the driver assemblies all sit above it), and
|
||||
/// <see cref="DataValueSnapshot"/> — the shape the rule reads — is declared beside it. Stating
|
||||
/// the rule three times is how the three seams drifted apart in the first place.</para>
|
||||
///
|
||||
/// <para><b>Severity, not equality.</b> Comparison is on the OPC UA Part 4 severity bits (the top
|
||||
/// two: <c>00</c> Good, <c>01</c> Uncertain, <c>10</c>/<c>11</c> Bad); the sub-code below them is
|
||||
/// carried through verbatim because the sub-code is the diagnostic
|
||||
/// (<c>BadNoCommunication</c> = go look at the device, <c>BadWaitingForInitialData</c> = it never
|
||||
/// arrived, <c>BadInternalError</c> = the script).</para>
|
||||
/// </summary>
|
||||
public static class InputQuality
|
||||
{
|
||||
/// <summary>
|
||||
/// No value has ever been received for this input — the status a fresh node is materialised
|
||||
/// with, and what <see cref="WorstOf"/> reports for a dependency slot that holds no snapshot
|
||||
/// at all. Spelled here rather than taken from <c>Commons.OpcUa.OpcUaStatus</c> because
|
||||
/// Core.Abstractions sits <em>below</em> Commons and deliberately references nothing.
|
||||
/// </summary>
|
||||
public const uint BadWaitingForInitialData = 0x80320000u;
|
||||
|
||||
/// <summary>
|
||||
/// The worst status among <paramref name="inputs"/>: the highest Part 4 severity, carrying
|
||||
/// that input's own sub-code. A <see langword="null"/> entry is a dependency that has never
|
||||
/// been pushed and reports <see cref="BadWaitingForInitialData"/> — reporting Good for it is
|
||||
/// the laundering this helper exists to stop. An empty set is Good: there is nothing to be
|
||||
/// worse than.
|
||||
/// </summary>
|
||||
/// <param name="inputs">The dependency snapshots, in a stable order.</param>
|
||||
/// <returns>The worst status code observed.</returns>
|
||||
/// <remarks>
|
||||
/// Among <em>equal</em> severity the first entry seen wins, so a re-evaluation over the same
|
||||
/// inputs reports the same sub-code instead of oscillating between two equally-bad codes.
|
||||
/// </remarks>
|
||||
public static uint WorstOf(IEnumerable<DataValueSnapshot?> inputs)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inputs);
|
||||
|
||||
var worst = 0u;
|
||||
var worstSeverity = 0u;
|
||||
foreach (var input in inputs)
|
||||
{
|
||||
var status = input?.StatusCode ?? BadWaitingForInitialData;
|
||||
var severity = status >> 30;
|
||||
if (severity <= worstSeverity) continue;
|
||||
worstSeverity = severity;
|
||||
worst = status;
|
||||
}
|
||||
|
||||
return worst;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when a consumer may evaluate against a value carrying <paramref name="statusCode"/>.
|
||||
/// Good and Uncertain are usable — an Uncertain value is present and a script can make a
|
||||
/// qualified decision from it; Bad is not, and a consumer holds instead (the alarm engine's
|
||||
/// long-standing policy, generalised).
|
||||
/// </summary>
|
||||
/// <param name="statusCode">The status code to classify.</param>
|
||||
/// <returns><see langword="true"/> for Good or Uncertain severity.</returns>
|
||||
public static bool IsUsable(uint statusCode) => (statusCode >> 30) < 2;
|
||||
}
|
||||
@@ -919,26 +919,18 @@ public sealed class ScriptedAlarmEngine : IDisposable
|
||||
/// the case the skip was hiding was the case that matters most. The upstream source now stamps a
|
||||
/// never-pushed path <see cref="StatusNeverPushed"/> and nothing else does, so the two are tellable
|
||||
/// apart. A null snapshot is still skipped: that is a cache the refill did not populate, not an
|
||||
/// observation.</para></summary>
|
||||
/// observation.</para>
|
||||
/// <para><b>The comparison itself lives in <see cref="InputQuality.WorstOf"/></b> (archreview R3-02)
|
||||
/// — the same rule the virtual-tag and Calculation seams now use, so "worst input wins, carrying its
|
||||
/// sub-code, first-seen on a severity tie" is stated once. What stays here is the alarm-specific
|
||||
/// <em>filter</em>: this engine's two skips are deliberately NOT part of the shared rule, because the
|
||||
/// other two seams treat a never-pushed input as Bad rather than as absent.</para></summary>
|
||||
private static uint WorstInputStatus(IReadOnlyDictionary<string, DataValueSnapshot> cache)
|
||||
{
|
||||
uint worst = 0u;
|
||||
var worstSeverity = 0u;
|
||||
foreach (var kv in cache)
|
||||
{
|
||||
// No entry at all — nothing was observed, so there is nothing to report on.
|
||||
if (kv.Value is null) continue;
|
||||
// Observed as "no data has ever arrived" — the cold-start placeholder, not a fault.
|
||||
if (kv.Value.StatusCode == StatusNeverPushed) continue;
|
||||
var status = kv.Value.StatusCode;
|
||||
var severity = status >> 30;
|
||||
if (severity > worstSeverity)
|
||||
{
|
||||
worstSeverity = severity;
|
||||
worst = status;
|
||||
}
|
||||
}
|
||||
return worst;
|
||||
// No entry at all — nothing was observed, so there is nothing to report on.
|
||||
// Observed as "no data has ever arrived" — the cold-start placeholder, not a fault.
|
||||
var observed = cache.Values.Where(v => v is not null && v.StatusCode != StatusNeverPushed);
|
||||
return InputQuality.WorstOf(observed);
|
||||
}
|
||||
|
||||
/// <summary>Update the tracked worst-quality bucket for an alarm; return true iff the 3-state bucket
|
||||
|
||||
@@ -82,7 +82,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
// SHORT-HELD state lock — it is taken for dictionary work only and is NEVER held across a script run
|
||||
// (Driver.Calculation-005).
|
||||
private readonly object _lock = new();
|
||||
private readonly Dictionary<string, object?> _depValues = new(StringComparer.Ordinal);
|
||||
// Dependency SNAPSHOTS, not bare values (archreview 05/STAB-19): the status code is what tells a
|
||||
// comms-lost input apart from a genuine Good null, and storing values only is what let this driver
|
||||
// compute a Good output from a dead source.
|
||||
private readonly Dictionary<string, DataValueSnapshot> _depValues = new(StringComparer.Ordinal);
|
||||
private readonly HashSet<string> _arrivedDeps = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, TagState> _stateByRawPath = new(StringComparer.Ordinal);
|
||||
|
||||
@@ -332,7 +335,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
if (!_running) return;
|
||||
lock (_lock)
|
||||
{
|
||||
_depValues[rawPath] = value;
|
||||
_depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc);
|
||||
_arrivedDeps.Add(rawPath);
|
||||
if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return;
|
||||
|
||||
@@ -391,9 +394,16 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
var now = DateTime.UtcNow;
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_stateByRawPath.TryGetValue(reference, out var st) || !st.HasValue)
|
||||
// Gated on HAS PUBLISHED, not on "has a value": a tag whose only publish was an inputs-Bad
|
||||
// report has no value of its own, and answering BadWaitingForInitialData for it would tell a
|
||||
// polled reader "never observed" while the subscriber on the same node had been told
|
||||
// BadNoCommunication. Something HAS been observed about such a tag — that its input is dead.
|
||||
if (!_stateByRawPath.TryGetValue(reference, out var st) || !st.HasPublished)
|
||||
return new DataValueSnapshot(null, StatusBadWaitingForInitialData, null, now);
|
||||
var status = st.LastPublishedBad ? StatusBadInternalError : StatusGood;
|
||||
// Exactly what the subscriber was last told — including a propagated input sub-code. Deriving
|
||||
// it from a two-state flag here would make a read disagree with the subscription over the
|
||||
// same node (05/STAB-19).
|
||||
var status = st.LastPublishedStatus;
|
||||
return new DataValueSnapshot(st.LastValue, status, st.LastTimestampUtc, st.LastTimestampUtc);
|
||||
}
|
||||
}
|
||||
@@ -404,12 +414,16 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
/// trigger fired, and that trigger's timestamp. Built under <see cref="_lock"/> by
|
||||
/// <see cref="PrepareEvaluation"/> and executed outside it, so the script itself needs no shared state.</summary>
|
||||
/// <param name="Definition">The calc tag being evaluated.</param>
|
||||
/// <param name="Dependencies">Its declared reads' values, as of the trigger.</param>
|
||||
/// <param name="Dependencies">Its declared reads' snapshots — value, status and timestamps — as of the trigger.</param>
|
||||
/// <param name="TimestampUtc">The trigger's timestamp, stamped on whatever is published.</param>
|
||||
/// <param name="WorstInputStatus">The worst status among <paramref name="Dependencies"/>
|
||||
/// (<see cref="InputQuality.WorstOf"/>). Bad ⇒ the script is HELD and this status is published;
|
||||
/// Good or Uncertain ⇒ the script runs and its result is stamped with it.</param>
|
||||
private sealed record PendingEvaluation(
|
||||
CalculationTagDefinition Definition,
|
||||
IReadOnlyDictionary<string, object?> Dependencies,
|
||||
DateTime TimestampUtc);
|
||||
IReadOnlyDictionary<string, DataValueSnapshot> Dependencies,
|
||||
DateTime TimestampUtc,
|
||||
uint WorstInputStatus);
|
||||
|
||||
/// <summary>
|
||||
/// Prepare one calc tag's evaluation: resolve the definition, apply the change-gate, and snapshot
|
||||
@@ -441,10 +455,14 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
if (string.IsNullOrWhiteSpace(def.ScriptSource)) return null; // unresolved script — never computes
|
||||
if (!def.DependencyRefs.All(_arrivedDeps.Contains)) return null; // still warming up — do not evaluate
|
||||
|
||||
var deps = new Dictionary<string, object?>(StringComparer.Ordinal);
|
||||
var deps = new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
||||
foreach (var dep in def.DependencyRefs)
|
||||
if (_depValues.TryGetValue(dep, out var v)) deps[dep] = v;
|
||||
return new PendingEvaluation(def, deps, timestampUtc);
|
||||
|
||||
// Worst-of-input, computed from the SAME snapshot the evaluation will use, so the status
|
||||
// published can never disagree with the values it was derived from (05/STAB-19).
|
||||
var worst = InputQuality.WorstOf(def.DependencyRefs.Select(r => deps.GetValueOrDefault(r)));
|
||||
return new PendingEvaluation(def, deps, timestampUtc, worst);
|
||||
}
|
||||
|
||||
/// <summary>Start the evaluation pump if it is not already running. MUST be called under
|
||||
@@ -513,6 +531,28 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
private void RunEvaluation(PendingEvaluation pending)
|
||||
{
|
||||
var def = pending.Definition;
|
||||
|
||||
// HOLD ON BAD (05/STAB-19, matching ScriptedAlarmEngine's AreInputsReady): a Bad input is not
|
||||
// usable, so the script does NOT run — which removes the "(double)ctx.GetTag(x).Value threw"
|
||||
// storm at its source rather than catching it downstream — and the tag reports the input's own
|
||||
// sub-code. Uncertain IS usable: the script runs and the output inherits the Uncertain code.
|
||||
if (!InputQuality.IsUsable(pending.WorstInputStatus))
|
||||
{
|
||||
(object? Value, uint StatusCode)? badPublish;
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return;
|
||||
badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc);
|
||||
}
|
||||
|
||||
// No script log: an input dying is not a script fault, and the script log is the per-failure
|
||||
// SCRIPT diagnostic. Here the status code is the diagnostic.
|
||||
if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc);
|
||||
return;
|
||||
}
|
||||
|
||||
// Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are
|
||||
// then the input's real ones rather than a fabricated Good stamped at evaluation time (05/STAB-19).
|
||||
var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies);
|
||||
|
||||
string? failureReason = null;
|
||||
@@ -530,7 +570,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
}
|
||||
else
|
||||
{
|
||||
toPublish = MarkEvaluationSucceeded(st, result.Value, pending.TimestampUtc);
|
||||
toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -539,23 +579,51 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
}
|
||||
|
||||
/// <summary>Record a successful evaluation. MUST be called under <see cref="_lock"/>; returns what to
|
||||
/// publish, or null when the value is deduped. The dedup is bypassed while recovering from Bad so a
|
||||
/// recovery whose value equals the pre-failure value still republishes Good (otherwise the node would
|
||||
/// stay Bad forever).</summary>
|
||||
/// publish, or null when the result is deduped.
|
||||
/// <para><b>The dedup key is (value, status), not value alone</b> — 05/STAB-19 generalising the
|
||||
/// Bad-recovery bypass 02/S13 introduced. Recovery is a publish, never an assumption: the node is
|
||||
/// currently carrying the previous status, so a result whose VALUE is unchanged but whose STATUS
|
||||
/// differs — the shape of every recovery, since a device that comes back usually comes back holding
|
||||
/// the same reading — must still land, or the node stays Bad forever.</para></summary>
|
||||
/// <param name="st">The tag's published state.</param>
|
||||
/// <param name="value">The computed value.</param>
|
||||
/// <param name="statusCode">The status to stamp: the worst input status (Good, or an Uncertain sub-code).</param>
|
||||
/// <param name="timestampUtc">The trigger's timestamp.</param>
|
||||
/// <returns>The snapshot to publish, or null when deduped.</returns>
|
||||
private (object? Value, uint StatusCode)? MarkEvaluationSucceeded(TagState st, object? value, DateTime timestampUtc)
|
||||
private (object? Value, uint StatusCode)? MarkEvaluationSucceeded(
|
||||
TagState st, object? value, uint statusCode, DateTime timestampUtc)
|
||||
{
|
||||
if (!st.LastPublishedBad && st.HasValue && Equals(st.LastValue, value)) return null;
|
||||
if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && Equals(st.LastValue, value))
|
||||
return null;
|
||||
|
||||
st.HasValue = true;
|
||||
st.LastValue = value;
|
||||
st.LastPublishedBad = false;
|
||||
st.HasPublished = true;
|
||||
st.LastPublishedStatus = statusCode;
|
||||
st.LastTimestampUtc = timestampUtc;
|
||||
_lastComputeUtc = DateTime.UtcNow;
|
||||
return (value, StatusGood);
|
||||
return (value, statusCode);
|
||||
}
|
||||
|
||||
/// <summary>Record that this tag's INPUTS are Bad, so no evaluation ran (05/STAB-19). Publishes the
|
||||
/// worst input's status — carrying the last known value — <b>once per change of that status</b>: a
|
||||
/// repeat of the same Bad is silent, while <c>BadNoCommunication</c> arriving over an earlier
|
||||
/// <c>BadInternalError</c> (or the reverse) is not masked, because the sub-code is what names which
|
||||
/// failure the operator is looking at. MUST be called under <see cref="_lock"/>.</summary>
|
||||
/// <param name="st">The tag's published state.</param>
|
||||
/// <param name="worstInputStatus">The worst status among the tag's declared inputs.</param>
|
||||
/// <param name="timestampUtc">The trigger's timestamp.</param>
|
||||
/// <returns>The snapshot to publish, or null when that status was already published.</returns>
|
||||
private static (object? Value, uint StatusCode)? MarkInputsBad(
|
||||
TagState st, uint worstInputStatus, DateTime timestampUtc)
|
||||
{
|
||||
if (st.HasPublished && st.LastPublishedStatus == worstInputStatus) return null;
|
||||
|
||||
st.HasPublished = true;
|
||||
st.LastPublishedStatus = worstInputStatus;
|
||||
st.LastTimestampUtc = timestampUtc;
|
||||
// Carry the last-known value (null if none) with the input's Bad status.
|
||||
return (st.HasValue ? st.LastValue : null, worstInputStatus);
|
||||
}
|
||||
|
||||
/// <summary>Failure tail (mirrors <c>VirtualTagActor</c> 02/S13): the caller always emits the
|
||||
@@ -568,9 +636,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
/// <returns>The Bad snapshot to publish on a fresh Good→Bad transition, else null.</returns>
|
||||
private static (object? Value, uint StatusCode)? MarkEvaluationFailed(TagState st, DateTime timestampUtc)
|
||||
{
|
||||
if (st.LastPublishedBad) return null;
|
||||
if (st.HasPublished && st.LastPublishedStatus == StatusBadInternalError) return null;
|
||||
|
||||
st.LastPublishedBad = true;
|
||||
st.HasPublished = true;
|
||||
st.LastPublishedStatus = StatusBadInternalError;
|
||||
st.LastTimestampUtc = timestampUtc;
|
||||
// Carry the last-known value (null if none) with Bad quality.
|
||||
return (st.HasValue ? st.LastValue : null, StatusBadInternalError);
|
||||
@@ -643,7 +712,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
{
|
||||
public bool HasValue;
|
||||
public object? LastValue;
|
||||
public bool LastPublishedBad;
|
||||
|
||||
/// <summary>Whether anything has been published for this tag yet. Distinct from
|
||||
/// <see cref="HasValue"/>: an inputs-Bad publish carries no value of its own.</summary>
|
||||
public bool HasPublished;
|
||||
|
||||
/// <summary>The status code last published for this tag. Replaces the old two-state
|
||||
/// <c>LastPublishedBad</c> flag (05/STAB-19): the dedup key is (value, status), and a Bad status
|
||||
/// now carries the failing INPUT's sub-code rather than a fixed <c>BadInternalError</c>.</summary>
|
||||
public uint LastPublishedStatus;
|
||||
public DateTime LastTimestampUtc;
|
||||
}
|
||||
|
||||
|
||||
@@ -75,7 +75,44 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
|
||||
/// <param name="calcTagId">Identity of the calc tag (bound to the script log; in the live path equals the script id).</param>
|
||||
/// <param name="scriptSource">The C# script source producing the tag's value.</param>
|
||||
/// <param name="dependencies">Read-only snapshot of dependency values keyed by RawPath.</param>
|
||||
/// <remarks>
|
||||
/// <b>Value-only overload.</b> Every dependency reaches the script stamped <c>Good</c> with an
|
||||
/// evaluation-time timestamp, because bare values carry nothing better. Prefer
|
||||
/// <see cref="Evaluate(string, string, IReadOnlyDictionary{string, DataValueSnapshot})"/>, which the
|
||||
/// driver uses: it carries each input's real status and source timestamp, so
|
||||
/// <c>ctx.GetTag(path).StatusCode</c> is honest (archreview 05/STAB-19). This overload remains for
|
||||
/// callers that genuinely hold only values — chiefly tests.
|
||||
/// </remarks>
|
||||
public VirtualTagEvalResult Evaluate(string calcTagId, string scriptSource, IReadOnlyDictionary<string, object?> dependencies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dependencies);
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var snapshots = new Dictionary<string, DataValueSnapshot>(dependencies.Count, StringComparer.Ordinal);
|
||||
foreach (var kv in dependencies)
|
||||
snapshots[kv.Key] = new DataValueSnapshot(kv.Value, StatusCode: 0u, nowUtc, nowUtc);
|
||||
|
||||
return Evaluate(calcTagId, scriptSource, snapshots);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluate <paramref name="scriptSource"/> for the calc tag <paramref name="calcTagId"/> against
|
||||
/// the dependency <b>snapshots</b> in <paramref name="dependencies"/> — value, status code and
|
||||
/// source timestamp per input. Never throws; script faults surface as
|
||||
/// <see cref="VirtualTagEvalResult.Failure"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The snapshots pass <b>through</b> to the script's read cache rather than being re-wrapped as
|
||||
/// Good (archreview 05/STAB-19). The old wrapping meant a script that inspected
|
||||
/// <c>ctx.GetTag(path).StatusCode</c> was told Good for every input, including one whose driver had
|
||||
/// stopped communicating — so a quality-aware script could not be written at all. The driver holds
|
||||
/// evaluation entirely while any input is Bad, so what a script sees here is Good or Uncertain.
|
||||
/// </remarks>
|
||||
/// <param name="calcTagId">Identity of the calc tag (bound to the script log).</param>
|
||||
/// <param name="scriptSource">The C# script source producing the tag's value.</param>
|
||||
/// <param name="dependencies">Read-only snapshot of dependency values keyed by RawPath.</param>
|
||||
/// <returns>The evaluation outcome.</returns>
|
||||
public VirtualTagEvalResult Evaluate(
|
||||
string calcTagId, string scriptSource, IReadOnlyDictionary<string, DataValueSnapshot> dependencies)
|
||||
{
|
||||
if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed");
|
||||
if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression");
|
||||
@@ -86,12 +123,15 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
|
||||
// returns null — Ok(null) here too. Near-misses fall through to the compiled path.
|
||||
if (PassthroughScript.TryMatch(scriptSource, out var passthroughRef))
|
||||
{
|
||||
return dependencies.TryGetValue(passthroughRef, out var ptValue)
|
||||
? VirtualTagEvalResult.Ok(ptValue)
|
||||
// The VALUE only — the caller stamps the output with the worst input status, which for a
|
||||
// single-dependency passthrough IS this input's status. Returning it here as well would put
|
||||
// the same decision in two places.
|
||||
return dependencies.TryGetValue(passthroughRef, out var ptSnapshot)
|
||||
? VirtualTagEvalResult.Ok(ptSnapshot.Value)
|
||||
: VirtualTagEvalResult.Ok(null);
|
||||
}
|
||||
|
||||
var readCache = BuildReadCache(dependencies);
|
||||
var readCache = dependencies;
|
||||
// Per-evaluation script logger: bind both ScriptId and CalcTagId (via the VirtualTagId slot, which
|
||||
// the Script-log page attributes on) from the calc-tag id so each line is attributable.
|
||||
var scriptLog = _scriptRoot
|
||||
@@ -173,20 +213,6 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable
|
||||
/// <inheritdoc />
|
||||
public void ClearCompiledScripts() => _cache.Clear();
|
||||
|
||||
private static IReadOnlyDictionary<string, DataValueSnapshot> BuildReadCache(
|
||||
IReadOnlyDictionary<string, object?> deps)
|
||||
{
|
||||
// VirtualTagContext.GetTag returns a DataValueSnapshot — wrap each raw dep value as Good-quality
|
||||
// so the script's `(int)ctx.GetTag("a").Value` pattern works. Null values stay null.
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var cache = new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
||||
foreach (var kv in deps)
|
||||
{
|
||||
cache[kv.Key] = new DataValueSnapshot(kv.Value, StatusCode: 0u, SourceTimestampUtc: nowUtc, ServerTimestampUtc: nowUtc);
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
/// <summary>Disposes the evaluator, draining + disposing every cached compile's collectible ALC.</summary>
|
||||
public void Dispose()
|
||||
{
|
||||
|
||||
@@ -349,6 +349,31 @@ public sealed class MqttConnection : IAsyncDisposable, IMqttSubscribeTransport,
|
||||
/// </remarks>
|
||||
public event Func<CancellationToken, Task>? Reconnected;
|
||||
|
||||
/// <summary>
|
||||
/// Raised on the <c>Connected → Reconnecting</c> edge — the session this connection had is
|
||||
/// gone. Ingest paths stale their cached values to <c>BadNoCommunication</c> here (archreview
|
||||
/// 05/STAB-18): without it, MQTT reports Good over a dead broker until the host's 90 s
|
||||
/// comms-loss watchdog notices, and MQTTnet's own supervisor usually reconnects well inside
|
||||
/// that window so the outage is never surfaced at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Runs on MQTTnet's dispatcher thread</b> (this is raised from the library's
|
||||
/// <c>DisconnectedAsync</c> callback), so a handler must be non-blocking, must not do I/O
|
||||
/// and must not await — the same constraint <see cref="MessageReceived"/> carries. A
|
||||
/// throwing handler is caught and logged rather than escaping into the pump.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Exactly once per edge, and only for a real outage.</b> It does not fire for a failed
|
||||
/// <i>initial</i> connect (there was no session to lose — the values are still
|
||||
/// <c>BadWaitingForInitialData</c>) nor during <see cref="DisposeAsync"/> (an intentional
|
||||
/// teardown, which every redeploy performs). It is deliberately NOT the mirror of
|
||||
/// <see cref="Reconnected"/>: recovery is a value the broker republishes, never an
|
||||
/// assumption this connection makes on the caller's behalf.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public event Action? ConnectionLost;
|
||||
|
||||
/// <summary>
|
||||
/// Raised for every inbound application message, on MQTTnet's own dispatcher thread. This
|
||||
/// connection does no routing, parsing or typing of its own — that is the subscription
|
||||
@@ -939,6 +964,19 @@ public sealed class MqttConnection : IAsyncDisposable, IMqttSubscribeTransport,
|
||||
if (State == MqttConnectionState.Connected)
|
||||
{
|
||||
SetState(MqttConnectionState.Reconnecting);
|
||||
|
||||
// The quality edge (05/STAB-18): exactly here, once per outage, and never for an initial
|
||||
// connect failure or a dispose — both of which are excluded by the state guard above.
|
||||
// Contained, because this runs on the library's dispatcher: a throwing ingest handler must
|
||||
// not stop the reconnect wake below.
|
||||
try
|
||||
{
|
||||
ConnectionLost?.Invoke();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger?.LogError(ex, "MQTT driver '{DriverId}': a connection-lost handler threw.", _driverId);
|
||||
}
|
||||
}
|
||||
|
||||
// Released unconditionally rather than only when the client had been connected: a missed
|
||||
|
||||
@@ -89,6 +89,16 @@ public sealed class MqttSubscriptionManager
|
||||
/// <summary>The broker refused, or never answered, the SUBSCRIBE for this reference's topic.</summary>
|
||||
private const uint StatusBadCommunicationError = 0x80050000u;
|
||||
|
||||
/// <summary>
|
||||
/// No value has ever been observed for this reference — <see cref="LastValueCache.Read"/>'s
|
||||
/// answer for a RawPath nothing has published under yet. The initial push skips it: pushing
|
||||
/// this would claim an observation the driver has not made.
|
||||
/// </summary>
|
||||
private const uint StatusBadWaitingForInitialData = 0x80320000u;
|
||||
|
||||
/// <summary>The broker session this driver was observing is gone — see <see cref="OnConnectionLost"/>.</summary>
|
||||
private const uint StatusBadNoCommunication = 0x80310000u;
|
||||
|
||||
private readonly int _defaultQos;
|
||||
private readonly string _driverId;
|
||||
private readonly ILogger? _logger;
|
||||
@@ -230,6 +240,45 @@ public sealed class MqttSubscriptionManager
|
||||
_transport = connection;
|
||||
connection.MessageReceived += HandleMessage;
|
||||
connection.Reconnected += OnReconnectedAsync;
|
||||
connection.ConnectionLost += OnConnectionLost;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks every already-observed reference <c>BadNoCommunication</c>, <b>keeping its last
|
||||
/// value</b>, and publishes the change — the driver's answer to a lost broker session
|
||||
/// (archreview 05/STAB-18 shape a). Wired to <see cref="MqttConnection.ConnectionLost"/> by
|
||||
/// <see cref="AttachTo"/>; public so tests (and Sparkplug's ingestor) can drive the edge
|
||||
/// directly.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>This is what makes the reconnect recover.</b> The steady-state suppression in
|
||||
/// <see cref="Deliver"/> compares value AND status against the driver-owned cache, which
|
||||
/// survives reconnects and redeploys — so the retained replay of an unchanged payload was
|
||||
/// suppressed and every node latched Bad by the host stayed Bad until the plant value
|
||||
/// happened to change. Staling the status first makes that identical replay a change.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A never-observed reference is left alone (it is already
|
||||
/// <see cref="StatusBadWaitingForInitialData"/>, which says something more precise), and an
|
||||
/// already-Bad one is not re-stamped. Runs on MQTTnet's dispatcher thread: dictionary work
|
||||
/// and synchronous <see cref="OnDataChange"/> invokes only — exactly what
|
||||
/// <see cref="Deliver"/> already does there — and never awaits.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public void OnConnectionLost()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
foreach (var rawPath in _authored.ByRawPath.Keys)
|
||||
{
|
||||
var prior = Values.Read(rawPath);
|
||||
if (prior.StatusCode == StatusBadWaitingForInitialData) continue; // never observed
|
||||
if ((prior.StatusCode >> 30) >= 2) continue; // already Bad
|
||||
|
||||
var staled = prior with { StatusCode = StatusBadNoCommunication, ServerTimestampUtc = now };
|
||||
Values.Update(rawPath, staled);
|
||||
Raise(rawPath, staled);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -356,6 +405,13 @@ public sealed class MqttSubscriptionManager
|
||||
_handleByRawPath[def.Name] = handle;
|
||||
}
|
||||
|
||||
// Initial push (05/STAB-18 shape b): every ref this driver has already observed is republished
|
||||
// onto the new handle before any broker traffic. A poll driver gets this free — PollGroupEngine
|
||||
// runs the first poll of a new subscription with forceRaise: true — and CalculationDriver does
|
||||
// it explicitly; MQTT did not, so a node reseeded BadWaitingForInitialData by an address-space
|
||||
// rebuild stayed there until the plant published again, which for a quiet topic is never.
|
||||
PushObservedSnapshots(defs);
|
||||
|
||||
// One SUBSCRIBE per distinct topic, and only for topics not already live: re-SUBSCRIBEing an
|
||||
// established filter is legal but pointless, and it re-triggers the retained seed.
|
||||
var pending = defs.Where(d => !_subscribedTopics.ContainsKey(d.Topic)).ToList();
|
||||
@@ -756,6 +812,12 @@ public sealed class MqttSubscriptionManager
|
||||
// timestamp — and only the notification is skipped. "Cached but not raised" is already a valid
|
||||
// state on this path (see Raise: an authored-but-unsubscribed tag does exactly that).
|
||||
//
|
||||
// What makes this suppression SAFE is the pair of edge behaviours around it (05/STAB-18):
|
||||
// PushObservedSnapshots re-delivers the cached snapshot on every (re)subscribe, and
|
||||
// OnConnectionLost stales every observed ref to BadNoCommunication — so the reconnect's
|
||||
// retained replay of an unchanged payload differs in STATUS and is not suppressed. Without
|
||||
// those, this comparison is what pinned a comms-lost tag Bad until the plant value changed.
|
||||
//
|
||||
// Sparkplug bypasses this entirely. SparkplugIngestor.cs updates Values and raises directly
|
||||
// (its own Values.Update()/Raise() calls, not through this Deliver method), so Sparkplug B
|
||||
// ingestion gets no unchanged-value suppression — deliberately out of scope here.
|
||||
@@ -788,6 +850,29 @@ public sealed class MqttSubscriptionManager
|
||||
return !Equals(previous, current);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Republishes the cached snapshot of every already-observed reference in
|
||||
/// <paramref name="defs"/> — the push-model equivalent of <c>PollGroupEngine</c>'s
|
||||
/// <c>forceRaise: true</c> on the first poll of a new subscription (05/STAB-18 shape b).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A never-observed reference is skipped: its cached status is
|
||||
/// <see cref="StatusBadWaitingForInitialData"/>, and republishing that would assert an
|
||||
/// observation this driver has not made. The cache is deliberately NOT re-<c>Update</c>d here —
|
||||
/// nothing new was observed, only re-delivered, so the stored timestamps stay honest and the
|
||||
/// steady-state suppression still compares against the real last observation.
|
||||
/// </remarks>
|
||||
/// <param name="defs">The definitions the new handle covers.</param>
|
||||
private void PushObservedSnapshots(IReadOnlyList<MqttTagDefinition> defs)
|
||||
{
|
||||
for (var i = 0; i < defs.Count; i++)
|
||||
{
|
||||
var snapshot = Values.Read(defs[i].Name);
|
||||
if (snapshot.StatusCode == StatusBadWaitingForInitialData) continue;
|
||||
Raise(defs[i].Name, snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Raises <see cref="OnDataChange"/> for a reference, if a live subscription covers it.
|
||||
/// A throwing subscriber is contained so it cannot starve the rest of the fan-out — or, worse,
|
||||
|
||||
@@ -101,9 +101,28 @@ public sealed class SparkplugIngestor
|
||||
/// OpcUaClient and this driver's own plain-mode path all emit it, while no driver emits
|
||||
/// BadNoCommunication at all (it appears once, in a CLI formatter's label map). Following the
|
||||
/// precedent keeps one code meaning "not communicating" across the whole fleet.
|
||||
/// <para><b>Superseded in part (archreview R3-02):</b> <see cref="StatusBadNoCommunication"/> now
|
||||
/// has a distinct, fleet-wide meaning — "the link to the source is down" — written by the host's
|
||||
/// comms-loss fan-out and by <see cref="OnConnectionLost"/>. This code keeps the narrower one it
|
||||
/// always had here: the <i>source itself</i> announced it is offline, or a birth voided the
|
||||
/// scope.</para>
|
||||
/// </summary>
|
||||
private const uint StatusBadCommunicationError = 0x80050000u;
|
||||
|
||||
/// <summary>
|
||||
/// The broker session this driver was observing is gone — see <see cref="OnConnectionLost"/>.
|
||||
/// Distinct from <see cref="StatusBadCommunicationError"/>, which means the source announced its
|
||||
/// own death.
|
||||
/// </summary>
|
||||
private const uint StatusBadNoCommunication = 0x80310000u;
|
||||
|
||||
/// <summary>
|
||||
/// No value has ever been observed for this reference — <c>LastValueCache.Read</c>'s answer for
|
||||
/// a RawPath nothing has published under yet. Neither the initial push nor the connection-loss
|
||||
/// stale touches such a reference: both would claim an observation that never happened.
|
||||
/// </summary>
|
||||
private const uint StatusBadWaitingForInitialData = 0x80320000u;
|
||||
|
||||
/// <summary>The metric's value does not fit the tag's effective data type — including an explicit Sparkplug null.</summary>
|
||||
private const uint StatusBadTypeMismatch = 0x80740000u;
|
||||
|
||||
@@ -317,6 +336,47 @@ public sealed class SparkplugIngestor
|
||||
_publishTransport = connection;
|
||||
connection.MessageReceived += HandleMessage;
|
||||
connection.Reconnected += OnReconnectedAsync;
|
||||
connection.ConnectionLost += OnConnectionLost;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks every already-observed metric <c>BadNoCommunication</c>, <b>keeping its last value</b>,
|
||||
/// and publishes the change — this driver's answer to a lost broker session (archreview
|
||||
/// 05/STAB-18 shape a). Wired to <see cref="MqttConnection.ConnectionLost"/> by
|
||||
/// <see cref="AttachTo"/>; public so tests can drive the edge directly.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>A death is a different event and keeps its own code.</b> NDEATH/DDEATH is the source
|
||||
/// announcing it is offline and stales with <see cref="StatusBadCommunicationError"/> as it
|
||||
/// always has; this is <i>our</i> transport losing the broker, and it reports
|
||||
/// <see cref="StatusBadNoCommunication"/> — the same code the host's comms-loss fan-out
|
||||
/// writes onto the value nodes, so an operator sees one story from both ends.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>It is what makes the reconnect recover.</b> A reconnect replays NBIRTH/DBIRTH carrying
|
||||
/// values that are routinely unchanged; the birth path publishes them, but every node the
|
||||
/// host latched Bad needs a change to un-latch. Staling first guarantees the replay differs.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Runs on MQTTnet's dispatcher thread: dictionary reads and synchronous
|
||||
/// <see cref="OnDataChange"/> invokes only, never an await. A never-observed metric and an
|
||||
/// already-Bad one are both left alone.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public void OnConnectionLost()
|
||||
{
|
||||
var now = _utcNow();
|
||||
foreach (var rawPath in _authored.ByRawPath.Keys)
|
||||
{
|
||||
var prior = Values.Read(rawPath);
|
||||
if (prior.StatusCode == StatusBadWaitingForInitialData) continue; // never observed
|
||||
if ((prior.StatusCode >> 30) >= 2) continue; // already Bad
|
||||
|
||||
var staled = prior with { StatusCode = StatusBadNoCommunication, ServerTimestampUtc = now };
|
||||
Values.Update(rawPath, staled);
|
||||
Raise(rawPath, staled);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -502,6 +562,13 @@ public sealed class SparkplugIngestor
|
||||
_handleByRawPath[def.Name] = handle;
|
||||
}
|
||||
|
||||
// Initial push (05/STAB-18 shape b): every metric this driver has already observed is
|
||||
// republished onto the new handle. A poll driver gets this free from PollGroupEngine's
|
||||
// forceRaise on the first poll of a new subscription; Sparkplug pushed nothing, so an
|
||||
// address-space rebuild — which reseeds every node BadWaitingForInitialData and triggers no
|
||||
// rebirth — left a quiet metric waiting for the plant indefinitely.
|
||||
PushObservedSnapshots(defs);
|
||||
|
||||
return Task.FromResult<ISubscriptionHandle>(handle);
|
||||
}
|
||||
|
||||
@@ -580,10 +647,16 @@ public sealed class SparkplugIngestor
|
||||
/// already empty, so calling it three times during one connect costs three no-ops.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values are deliberately <b>not</b> staled here. A session change is a driver-side event,
|
||||
/// not a statement about the plant; last observed values remain the best available answer
|
||||
/// until a birth lands, and the connection's own health surface already reports the outage.
|
||||
/// Only a death — the source saying it is offline — stales a tag.
|
||||
/// <b>Values are not staled here — but they ARE staled on a connection loss</b> (archreview
|
||||
/// R3-02; this remark used to claim the opposite and that was the stale-Good bug). What is
|
||||
/// still true is the narrow part: a session reset for a <i>config rebuild</i> on a live
|
||||
/// connection says nothing about the plant, so last observed values remain the best answer
|
||||
/// until a birth lands. What was wrong was the reason given for the general case — "the
|
||||
/// connection's own health surface already reports the outage". Health is not quality: an
|
||||
/// OPC UA client subscribed to the metric's node sees neither the health page nor the
|
||||
/// outage, and reported Good over a dead broker. The transport loss is now handled by
|
||||
/// <see cref="OnConnectionLost"/> (<see cref="StatusBadNoCommunication"/>, value retained);
|
||||
/// a death still stales the way it always did (<see cref="StatusBadCommunicationError"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="reason">Why the reset is happening, for the diagnostic log.</param>
|
||||
@@ -1376,6 +1449,28 @@ public sealed class SparkplugIngestor
|
||||
return new DataValueSnapshot(null, StatusBadTypeMismatch, stamp, serverUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Republishes the cached snapshot of every already-observed reference in
|
||||
/// <paramref name="defs"/> — the push-model equivalent of <c>PollGroupEngine</c>'s
|
||||
/// <c>forceRaise: true</c> on the first poll of a new subscription (05/STAB-18 shape b).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A never-observed reference is skipped: its cached status is
|
||||
/// <see cref="StatusBadWaitingForInitialData"/>, which is a more precise statement than any
|
||||
/// value this could republish. The cache is not re-<c>Update</c>d — nothing new was observed,
|
||||
/// only re-delivered — so the stored timestamps stay honest.
|
||||
/// </remarks>
|
||||
/// <param name="defs">The definitions the new handle covers.</param>
|
||||
private void PushObservedSnapshots(IReadOnlyList<MqttTagDefinition> defs)
|
||||
{
|
||||
for (var i = 0; i < defs.Count; i++)
|
||||
{
|
||||
var snapshot = Values.Read(defs[i].Name);
|
||||
if (snapshot.StatusCode == StatusBadWaitingForInitialData) continue;
|
||||
Raise(defs[i].Name, snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Marks every authored tag in <paramref name="scope"/> as no longer communicating.</summary>
|
||||
private void FanStale(SparkplugScope scope, DateTime? sourceUtc, string reason)
|
||||
{
|
||||
|
||||
@@ -59,7 +59,34 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <b>Value-only overload.</b> Every dependency reaches the script stamped <c>Good</c> with an
|
||||
/// evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
|
||||
/// overload instead (archreview 02/S14); this one remains for callers that genuinely hold only
|
||||
/// values.
|
||||
/// </remarks>
|
||||
public VirtualTagEvalResult Evaluate(string virtualTagId, string expression, IReadOnlyDictionary<string, object?> dependencies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dependencies);
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var snapshots = new Dictionary<string, DataValueSnapshot>(dependencies.Count, StringComparer.Ordinal);
|
||||
foreach (var kv in dependencies)
|
||||
snapshots[kv.Key] = new DataValueSnapshot(kv.Value, StatusCode: 0u, nowUtc, nowUtc);
|
||||
|
||||
return Evaluate(virtualTagId, expression, snapshots);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Overrides the interface's default (which projects to values and loses the status codes): the
|
||||
/// snapshots pass <b>through</b> as the script's read cache, so
|
||||
/// <c>ctx.GetTag(path).StatusCode</c> and <c>.SourceTimestampUtc</c> are the input's own rather
|
||||
/// than a fabricated <c>Good</c> stamped at evaluation time (archreview 02/S14). The virtual-tag
|
||||
/// actor holds evaluation entirely while any input is Bad, so what a script observes here is Good
|
||||
/// or Uncertain.
|
||||
/// </remarks>
|
||||
public VirtualTagEvalResult Evaluate(
|
||||
string virtualTagId, string expression, IReadOnlyDictionary<string, DataValueSnapshot> dependencies)
|
||||
{
|
||||
if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed");
|
||||
if (string.IsNullOrWhiteSpace(expression)) return VirtualTagEvalResult.Failure("empty expression");
|
||||
@@ -71,12 +98,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
// so the script returns null — Ok(null) here too.
|
||||
if (PassthroughScript.TryMatch(expression, out var passthroughRef))
|
||||
{
|
||||
return dependencies.TryGetValue(passthroughRef, out var ptValue)
|
||||
? VirtualTagEvalResult.Ok(ptValue)
|
||||
// The VALUE only — the caller stamps the result with the worst input status, which for a
|
||||
// single-dependency passthrough IS this input's status.
|
||||
return dependencies.TryGetValue(passthroughRef, out var ptSnapshot)
|
||||
? VirtualTagEvalResult.Ok(ptSnapshot?.Value)
|
||||
: VirtualTagEvalResult.Ok(null);
|
||||
}
|
||||
|
||||
var readCache = BuildReadCache(dependencies);
|
||||
var readCache = dependencies;
|
||||
// Per-evaluation script logger: bind both ScriptId and VirtualTagId from the virtual-tag id
|
||||
// (in the live path the script id equals the virtual-tag id) so the Script-log page can
|
||||
// attribute each line. EquipmentId stays unbound for now.
|
||||
@@ -162,21 +191,6 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
/// <inheritdoc />
|
||||
public void ClearCompiledScripts() => _cache.Clear();
|
||||
|
||||
private static IReadOnlyDictionary<string, DataValueSnapshot> BuildReadCache(
|
||||
IReadOnlyDictionary<string, object?> deps)
|
||||
{
|
||||
// VirtualTagContext.GetTag returns a DataValueSnapshot — we wrap each raw dep value
|
||||
// as Good-quality so the script's `(int)ctx.GetTag("a").Value` pattern works. Null
|
||||
// values stay null; the script can null-check via GetTag(path).Value.
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var cache = new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
||||
foreach (var kv in deps)
|
||||
{
|
||||
cache[kv.Key] = new DataValueSnapshot(kv.Value, StatusCode: 0u, SourceTimestampUtc: nowUtc, ServerTimestampUtc: nowUtc);
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
/// <summary>Disposes the evaluator and cleans up all cached script evaluators.</summary>
|
||||
public void Dispose()
|
||||
{
|
||||
|
||||
@@ -649,6 +649,109 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Apply a batch of <b>status-only</b> writes: each node keeps its CURRENT value and only its
|
||||
/// StatusCode + Timestamp change (archreview 03/S15 — the comms-loss annotation for value variables).
|
||||
/// Same shape as <see cref="WriteValues"/>: an outside-the-lock validate/resolve pre-pass, then
|
||||
/// <see cref="MaxWritesPerLockAcquisition"/>-sized chunks under one <c>Lock</c> each, per-item isolation,
|
||||
/// and a <see cref="ValueWriteBatchException"/> aggregate.
|
||||
/// <para><b>It never creates a node.</b> A status carries nothing to create one from, and the caller's
|
||||
/// per-driver ref index can legitimately name a node a surgical remove has just dropped — so an unknown
|
||||
/// id is skipped, not materialised, and is not counted as a failure either (it is an expected race, not
|
||||
/// a fault).</para></summary>
|
||||
/// <param name="statuses">The status writes to apply, in order.</param>
|
||||
/// <exception cref="ValueWriteBatchException">When one or more items failed validation or threw. Every
|
||||
/// other item HAS been applied.</exception>
|
||||
public void WriteStatuses(IReadOnlyList<StatusWrite> statuses)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(statuses);
|
||||
EnsureAddressSpaceCreated();
|
||||
if (statuses.Count == 0) return;
|
||||
|
||||
var resolved = new ResolvedWrite[statuses.Count];
|
||||
string? firstFailedNodeId = null;
|
||||
Exception? firstFailure = null;
|
||||
var failed = 0;
|
||||
|
||||
for (var i = 0; i < statuses.Count; i++)
|
||||
{
|
||||
var nodeId = statuses[i].NodeId;
|
||||
if (string.IsNullOrEmpty(nodeId))
|
||||
{
|
||||
failed++;
|
||||
firstFailedNodeId ??= nodeId ?? string.Empty;
|
||||
firstFailure ??= new ArgumentException(
|
||||
$"WriteStatuses item {i} has a null or empty NodeId.", nameof(statuses));
|
||||
continue;
|
||||
}
|
||||
|
||||
resolved[i] = new ResolvedWrite(
|
||||
nodeId, NamespaceIndexForRealm(statuses[i].Realm), MapKey(statuses[i].Realm, nodeId), IsValid: true);
|
||||
}
|
||||
|
||||
var succeeded = 0;
|
||||
var skipped = 0;
|
||||
for (var start = 0; start < statuses.Count; start += MaxWritesPerLockAcquisition)
|
||||
{
|
||||
var end = Math.Min(start + MaxWritesPerLockAcquisition, statuses.Count);
|
||||
lock (Lock)
|
||||
{
|
||||
for (var i = start; i < end; i++)
|
||||
{
|
||||
if (!resolved[i].IsValid) continue;
|
||||
var s = statuses[i];
|
||||
try
|
||||
{
|
||||
if (WriteStatusLocked(resolved[i].Key, s.StatusCode, s.SourceTimestampUtc)) succeeded++;
|
||||
else skipped++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failed++;
|
||||
firstFailedNodeId ??= s.NodeId;
|
||||
firstFailure ??= ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (skipped > 0)
|
||||
{
|
||||
// Debug, not Warning: the caller's index racing a surgical remove is expected, and a driver
|
||||
// whose nodes were all just removed would otherwise log once per ref. This manager carries no
|
||||
// ILogger, so log via the SDK's static trace (see ServeNodeAsync).
|
||||
#pragma warning disable CS0618 // Type or member is obsolete
|
||||
Utils.LogDebug(
|
||||
"OtOpcUaNodeManager: WriteStatuses skipped {0} of {1} item(s) — node not materialised.",
|
||||
skipped,
|
||||
statuses.Count);
|
||||
#pragma warning restore CS0618
|
||||
}
|
||||
|
||||
if (failed > 0)
|
||||
{
|
||||
throw new ValueWriteBatchException(
|
||||
firstFailedNodeId ?? string.Empty, failed, succeeded, firstFailure!);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The body of a single status-only write. <b>Caller MUST hold <c>Lock</c></b>.</summary>
|
||||
/// <param name="key">The realm-qualified map key.</param>
|
||||
/// <param name="statusCode">The full OPC UA status code, sub-code included.</param>
|
||||
/// <param name="sourceTimestampUtc">The timestamp of the annotation in UTC.</param>
|
||||
/// <returns><see langword="true"/> when the node existed and was stamped; <see langword="false"/> when
|
||||
/// no such node is materialised (skipped — deliberately not a create; see <see cref="WriteStatuses"/>).</returns>
|
||||
private bool WriteStatusLocked(RealmKey key, uint statusCode, DateTime sourceTimestampUtc)
|
||||
{
|
||||
if (!_variables.TryGetValue(key, out var variable)) return false;
|
||||
|
||||
// Value deliberately untouched — that IS the feature: the node keeps its last known value under a
|
||||
// Bad status (OPC UA Part 8's last-known-value convention).
|
||||
variable.StatusCode = new StatusCode(statusCode);
|
||||
variable.Timestamp = sourceTimestampUtc;
|
||||
variable.ClearChangeMasks(SystemContext, includeChildren: false);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>One <see cref="WriteValues"/> item after the outside-the-lock pre-pass: its namespace index and
|
||||
/// map key already resolved, or <paramref name="IsValid"/> false when the item failed validation.</summary>
|
||||
/// <param name="NodeId">The item's bare <c>s=</c> identifier.</param>
|
||||
|
||||
@@ -31,6 +31,13 @@ public sealed class SdkAddressSpaceSink : IOpcUaAddressSpaceSink, ISurgicalAddre
|
||||
public void WriteValues(IReadOnlyList<ValueWrite> values)
|
||||
=> _nodeManager.WriteValues(values);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Overrides the interface's default on purpose, and here the default is not merely slower but
|
||||
/// <b>wrong</b>: it writes a null value (a value-less sink has nothing else to write), which would erase
|
||||
/// the last known value this annotation exists to preserve. The manager's own entry point retains it.</remarks>
|
||||
public void WriteStatuses(IReadOnlyList<StatusWrite> statuses)
|
||||
=> _nodeManager.WriteStatuses(statuses);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime sourceTimestampUtc, AddressSpaceRealm realm)
|
||||
=> _nodeManager.WriteAlarmCondition(alarmNodeId, state, sourceTimestampUtc, realm);
|
||||
|
||||
@@ -91,6 +91,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// connect work, so it should not run more often than an ordinary reconnect attempt.</summary>
|
||||
public static readonly TimeSpan DefaultReprovisionBackoff = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs still
|
||||
/// latched <see cref="OpcUaStatus.BadNoCommunication"/>. Sized so an ordinary poll driver has had
|
||||
/// several poll cycles — and a slow one at least one — to re-publish before anything is said; the
|
||||
/// audit's job is to name a driver that will NEVER re-publish, not to race a slow first poll.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan DefaultCommsRecoveryAuditDelay = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
|
||||
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
|
||||
@@ -426,6 +434,27 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <see cref="DefaultApplyWedgeTimeout"/>).</summary>
|
||||
private readonly TimeSpan _applyWedgeTimeout;
|
||||
|
||||
/// <summary>How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs
|
||||
/// still latched <see cref="OpcUaStatus.BadNoCommunication"/> (see <see cref="CommsRecoveryAudit"/>).</summary>
|
||||
private readonly TimeSpan _commsRecoveryAuditDelay;
|
||||
|
||||
/// <summary>
|
||||
/// Archreview <b>03/S15</b> — RawPaths this host has stamped <c>BadNoCommunication</c> on the value
|
||||
/// nodes of, per driver, and has not yet seen a value for. Set by
|
||||
/// <see cref="FanCommsLossToValueNodes"/>, pruned by <see cref="ForwardBatchToMux"/> as each ref
|
||||
/// re-publishes, and read once by <see cref="CommsRecoveryAudit"/> after the driver reconnects.
|
||||
/// <para><b>It is a tripwire, not a mechanism.</b> Nothing depends on it being correct for the data
|
||||
/// plane to behave: the host publishes nothing Good on reconnect either way. What it buys is that a
|
||||
/// driver which reconnects and then fails to re-publish — the pre-STAB-18 MQTT shape, and what any
|
||||
/// future push driver without an initial push will do — produces a Warning and a metered count
|
||||
/// instead of a subscription that silently reports Bad forever.</para>
|
||||
/// <para>Keyed by RawPath, so a raw node and its UNS mirrors count once: the operator is being told
|
||||
/// how many <i>tags</i> did not come back, not how many address-space nodes they project onto. The
|
||||
/// entry is removed as soon as it empties, so the steady-state cost in
|
||||
/// <see cref="ForwardBatchToMux"/> is one dictionary miss per burst.</para>
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, HashSet<string>> _commsLatched = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Per-driver rate limit for the supervision-reprovision hook (see
|
||||
/// <see cref="DefaultReprovisionBackoff"/>).</summary>
|
||||
private readonly TimeSpan _reprovisionBackoff;
|
||||
@@ -483,6 +512,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <param name="OperatorUser">The authenticated principal performing the acknowledge.</param>
|
||||
public sealed record RouteNativeAlarmAck(string ConditionNodeId, string? Comment, string OperatorUser);
|
||||
|
||||
/// <summary>Self-sent by the 03/S15 audit timer, <see cref="_commsRecoveryAuditDelay"/> after a driver
|
||||
/// reconnected: report any of its refs still latched <c>BadNoCommunication</c>. See
|
||||
/// <see cref="HandleCommsRecoveryAudit"/>.</summary>
|
||||
/// <param name="DriverInstanceId">The driver whose recovery is being audited.</param>
|
||||
public sealed record CommsRecoveryAudit(string DriverInstanceId);
|
||||
|
||||
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
|
||||
{
|
||||
// Convenience accessors for sites that don't need the full spec.
|
||||
@@ -548,6 +583,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// VirtualTag + ScriptedAlarm hosts. Defaults to null (admin-only nodes / tests skip the emit).</param>
|
||||
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
|
||||
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>.</param>
|
||||
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
|
||||
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
|
||||
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; tests shorten it.</param>
|
||||
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
|
||||
public static Props Props(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
|
||||
@@ -577,7 +615,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
IHistorizationRoleView? historizationRoleView = null,
|
||||
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
|
||||
TimeSpan? applyWedgeTimeout = null,
|
||||
TimeSpan? reprovisionBackoff = null) =>
|
||||
TimeSpan? reprovisionBackoff = null,
|
||||
TimeSpan? commsRecoveryAuditDelay = null) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -589,7 +628,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
|
||||
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff));
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -648,6 +687,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
|
||||
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>; a test shortens
|
||||
/// it so the watchdog is observable in a test's lifetime.</param>
|
||||
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
|
||||
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
|
||||
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; a test shortens it for the same reason.</param>
|
||||
public DriverHostActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
|
||||
CommonsNodeId localNode,
|
||||
@@ -676,10 +718,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
IHistorizationRoleView? historizationRoleView = null,
|
||||
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
|
||||
TimeSpan? applyWedgeTimeout = null,
|
||||
TimeSpan? reprovisionBackoff = null)
|
||||
TimeSpan? reprovisionBackoff = null,
|
||||
TimeSpan? commsRecoveryAuditDelay = null)
|
||||
{
|
||||
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
|
||||
_reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
|
||||
_commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
|
||||
_historizationRoleView = historizationRoleView;
|
||||
_sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal;
|
||||
_telemetryHub = telemetryHub;
|
||||
@@ -1110,6 +1154,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
|
||||
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
|
||||
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
|
||||
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
|
||||
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
|
||||
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
|
||||
Receive<RestartDriver>(HandleRestartDriver);
|
||||
@@ -1167,6 +1212,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
|
||||
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
|
||||
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
|
||||
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
|
||||
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
|
||||
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
|
||||
Receive<RestartDriver>(HandleRestartDriver);
|
||||
@@ -1209,6 +1255,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// One envelope per burst; the mux filters per item (it registered a batch receive in Task 9).
|
||||
_dependencyMux?.Tell(msg);
|
||||
|
||||
// 03/S15 recovery audit: clear whatever this burst re-published from the driver's Bad latch. Guarded
|
||||
// by ONE dictionary probe per burst — with no outage outstanding (the steady state) this is a single
|
||||
// miss, not per-item work.
|
||||
if (_commsLatched.ContainsKey(msg.DriverInstanceId))
|
||||
{
|
||||
for (var i = 0; i < msg.Items.Count; i++)
|
||||
TrackCommsLatchPrune(msg.DriverInstanceId, msg.Items[i].FullReference);
|
||||
}
|
||||
|
||||
if (_opcUaPublishActor is null) return;
|
||||
|
||||
// Indexed for-loop, not foreach: IReadOnlyList<AttributeValueItem> enumerates through an interface
|
||||
@@ -1285,13 +1340,34 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// the source had died. <see cref="FanCommsLossToDependencyMux"/> closes that by pushing a
|
||||
/// synthetic Bad for every value ref the driver owns.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Archreview 03/S15 — and the EXTERNAL data plane, which the two layers above still left
|
||||
/// out.</b> Conditions and internal consumers learned of the outage; the driver's raw value
|
||||
/// variables and their UNS mirrors — what an OPC UA client actually subscribes to — did not, so
|
||||
/// they served the last value at <c>Good</c> for the whole outage.
|
||||
/// <see cref="FanCommsLossToValueNodes"/> closes that with one status-only batch that keeps each
|
||||
/// node's value and changes only its status. All three fan-outs run on the disconnect edge only;
|
||||
/// the reconnect edge writes nothing to values (recovery is the driver's own re-publish) and
|
||||
/// re-annotates conditions Good, which is a statement about the LINK, not about any tag's value.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void OnDriverConnectivityChanged(DriverInstanceActor.ConnectivityChanged msg)
|
||||
{
|
||||
// #481: the mux fan-out is independent of the OPC UA sink — a node with scripted alarms but no
|
||||
// publish actor (the mux-only test/dev path) must still see comms loss. Done before the early
|
||||
// return for exactly that reason.
|
||||
if (!msg.Connected) FanCommsLossToDependencyMux(msg.DriverInstanceId);
|
||||
if (!msg.Connected)
|
||||
{
|
||||
FanCommsLossToDependencyMux(msg.DriverInstanceId);
|
||||
// 03/S15: and the EXTERNAL data plane, which #481 stopped one seam short of.
|
||||
FanCommsLossToValueNodes(msg.DriverInstanceId);
|
||||
}
|
||||
else
|
||||
{
|
||||
// 03/S15: nothing Good is published here — instead, check LATER whether the driver republished
|
||||
// what this host marked Bad. See HandleCommsRecoveryAudit.
|
||||
TrackCommsLatchArmAudit(msg.DriverInstanceId);
|
||||
}
|
||||
|
||||
if (_opcUaPublishActor is null) return;
|
||||
|
||||
@@ -1323,6 +1399,160 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
_localNode, msg.DriverInstanceId, msg.Connected ? "connected" : "disconnected", annotated, statusCode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Archreview <b>03/S15</b> — the value-node half of the comms-loss fan-out: stamp
|
||||
/// <see cref="OpcUaStatus.BadNoCommunication"/> on every raw NodeId <b>and</b> every referencing UNS
|
||||
/// NodeId the just-disconnected driver owns, as ONE status batch through the same publish actor the
|
||||
/// value fan-out uses.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why it was missing.</b> #481 built the per-driver ref index and wired comms loss to the
|
||||
/// internal consumers (the mux) and to Part 9 conditions, and stopped one seam short of the
|
||||
/// external OPC UA data plane. A client subscribed to a value node therefore saw the last value
|
||||
/// at <b>Good</b> quality for the whole outage, for every driver whose failure shape is "the
|
||||
/// reader stops producing" rather than "publish a Bad snapshot per tag". (Modbus does the
|
||||
/// latter, quickly, which is why the gap looked smaller than it was — a coalesced-read throw, a
|
||||
/// dropped session on a push driver, or simply a driver whose last act was a Good publish all
|
||||
/// held Good indefinitely.)
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status-only, so the VALUE is retained.</b> This actor holds no value cache — v3's
|
||||
/// single-source design deliberately removed the independent buffer — so only the node can keep
|
||||
/// the last known value, which is what
|
||||
/// <see cref="OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate"/> exists for. A Bad node still
|
||||
/// carrying its last reading is OPC UA Part 8's convention and is what a SCADA consumer expects.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Nothing is written on reconnect.</b> Publishing Good would assert a quality this actor has
|
||||
/// not observed; recovery is the driver's own re-publish (see
|
||||
/// <see cref="FanCommsLossToDependencyMux"/>'s remark on the detach that makes the poll engine
|
||||
/// force-raise, and MQTT's initial push).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Ungated by redundancy role, like the condition annotation: a Secondary keeps its address
|
||||
/// space honest for failover. Re-stamping an already-Bad node is harmless — the SDK's
|
||||
/// <c>StatusValue</c> trigger dedups an unchanged status at the monitored item.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="driverInstanceId">The driver child that just reported comms loss.</param>
|
||||
private void FanCommsLossToValueNodes(string driverInstanceId)
|
||||
{
|
||||
if (_opcUaPublishActor is null) return;
|
||||
if (!_valueRefsByDriver.TryGetValue(driverInstanceId, out var entries) || entries.Count == 0) return;
|
||||
|
||||
var ts = DateTime.UtcNow;
|
||||
var writes = new List<StatusWrite>(entries.Count);
|
||||
foreach (var (_, nodes) in entries)
|
||||
{
|
||||
foreach (var n in nodes)
|
||||
{
|
||||
writes.Add(new StatusWrite(n.NodeId, OpcUaStatus.BadNoCommunication, ts, n.Realm));
|
||||
}
|
||||
}
|
||||
|
||||
if (writes.Count == 0) return;
|
||||
|
||||
_opcUaPublishActor.Tell(new OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate(writes));
|
||||
TrackCommsLatchSet(driverInstanceId, entries);
|
||||
|
||||
_log.Info(
|
||||
"DriverHost {Node}: driver {Driver} disconnected — {Count} value node(s) marked BadNoCommunication (value retained)",
|
||||
_localNode, driverInstanceId, writes.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records every RawPath just latched <c>BadNoCommunication</c> for <paramref name="driverInstanceId"/>
|
||||
/// (03/S15's recovery audit — see <see cref="_commsLatched"/>). Replaces any previous latch wholesale:
|
||||
/// a second outage re-arms from the current ref set rather than accumulating refs a redeploy removed.
|
||||
/// </summary>
|
||||
/// <param name="driverInstanceId">The driver that just went comms-lost.</param>
|
||||
/// <param name="entries">Its value refs, from the per-driver index.</param>
|
||||
private void TrackCommsLatchSet(
|
||||
string driverInstanceId,
|
||||
List<(string RawPath, HashSet<NodeRealmRef> Nodes)> entries)
|
||||
{
|
||||
var latched = new HashSet<string>(entries.Count, StringComparer.Ordinal);
|
||||
foreach (var (rawPath, _) in entries) latched.Add(rawPath);
|
||||
_commsLatched[driverInstanceId] = latched;
|
||||
|
||||
// A previous reconnect's audit is now meaningless — this driver is down again.
|
||||
Timers.Cancel(CommsAuditTimerKey(driverInstanceId));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clears <paramref name="fullReference"/> from <paramref name="driverInstanceId"/>'s latch, and drops
|
||||
/// the driver's entry once it empties. Called per routed value item, so the steady-state cost with no
|
||||
/// latch outstanding is one dictionary miss per burst — not per item.
|
||||
/// </summary>
|
||||
/// <param name="driverInstanceId">The publishing driver.</param>
|
||||
/// <param name="fullReference">The RawPath that just published a value.</param>
|
||||
private void TrackCommsLatchPrune(string driverInstanceId, string fullReference)
|
||||
{
|
||||
if (!_commsLatched.TryGetValue(driverInstanceId, out var latched)) return;
|
||||
if (!latched.Remove(fullReference)) return;
|
||||
if (latched.Count == 0) _commsLatched.Remove(driverInstanceId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Arms the recovery audit for a driver that has just reconnected: after
|
||||
/// <see cref="_commsRecoveryAuditDelay"/>, <see cref="HandleCommsRecoveryAudit"/> reports whatever is
|
||||
/// still latched. Nothing is armed when the driver has no latch — it never went Bad here.
|
||||
/// </summary>
|
||||
/// <param name="driverInstanceId">The driver that just reconnected.</param>
|
||||
private void TrackCommsLatchArmAudit(string driverInstanceId)
|
||||
{
|
||||
if (!_commsLatched.ContainsKey(driverInstanceId)) return;
|
||||
Timers.StartSingleTimer(
|
||||
CommsAuditTimerKey(driverInstanceId),
|
||||
new CommsRecoveryAudit(driverInstanceId),
|
||||
_commsRecoveryAuditDelay);
|
||||
}
|
||||
|
||||
private static string CommsAuditTimerKey(string driverInstanceId) => $"comms-audit:{driverInstanceId}";
|
||||
|
||||
/// <summary>
|
||||
/// The 03/S15 tripwire firing: refs this host latched <c>BadNoCommunication</c> that the driver has
|
||||
/// still not re-published <see cref="_commsRecoveryAuditDelay"/> after it reconnected.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>It reports, it does not repair.</b> Writing Good here would assert a quality nothing has
|
||||
/// observed — the very thing this whole path refuses to do. What the operator gets instead is a
|
||||
/// named driver and a count, plus the <c>otopcua.driver.comms_latched_refs</c> counter, which is
|
||||
/// the operator-visible signature of a driver that reconnects without re-publishing (the
|
||||
/// pre-STAB-18 MQTT shape, or any future push driver that forgets its initial push).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The latch is cleared afterwards either way: it has been reported, and leaving it would make
|
||||
/// the next outage's audit report refs from this one.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="msg">Names the driver whose audit is due.</param>
|
||||
private void HandleCommsRecoveryAudit(CommsRecoveryAudit msg)
|
||||
{
|
||||
if (!_commsLatched.TryGetValue(msg.DriverInstanceId, out var latched) || latched.Count == 0)
|
||||
{
|
||||
_commsLatched.Remove(msg.DriverInstanceId);
|
||||
return;
|
||||
}
|
||||
|
||||
var driverType = _children.TryGetValue(msg.DriverInstanceId, out var child) ? child.DriverType : "unknown";
|
||||
OtOpcUaTelemetry.DriverCommsLatchedRefs.Add(
|
||||
latched.Count,
|
||||
new KeyValuePair<string, object?>("driver", msg.DriverInstanceId),
|
||||
new KeyValuePair<string, object?>("driver_type", driverType));
|
||||
|
||||
_log.Warning(
|
||||
"DriverHost {Node}: driver {Driver} ({Type}) — {Count} ref(s) still BadNoCommunication {Delay} after reconnect; "
|
||||
+ "the driver did not re-publish them (missing initial push, or a value-only dedup swallowed an unchanged value). "
|
||||
+ "First: {First}",
|
||||
_localNode, msg.DriverInstanceId, driverType, latched.Count, _commsRecoveryAuditDelay,
|
||||
latched.OrderBy(r => r, StringComparer.Ordinal).First());
|
||||
|
||||
_commsLatched.Remove(msg.DriverInstanceId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #481 (Layer 4) — push a synthetic <see cref="OpcUaStatus.BadNoCommunication"/> into the dependency
|
||||
/// mux for every value ref the just-disconnected driver owns, so scripted alarms and virtual tags see
|
||||
@@ -1778,6 +2008,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
|
||||
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
|
||||
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
|
||||
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
|
||||
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
|
||||
Receive<CommsRecoveryAudit>(_ => { });
|
||||
// A late DeltaApplied (an apply completed just before the DB went Stale) — re-register the driver's
|
||||
// mux adapter anyway; it simply re-reads the driver's current refs (harmless, no DB access).
|
||||
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
|
||||
|
||||
@@ -1423,6 +1423,21 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
// Archreview 03/S15 — announce comms loss on the way out. Every OTHER route into Reconnecting
|
||||
// (watchdog trip, failed initial connect, Force Reconnect) announces it from the Reconnecting
|
||||
// entry; stopping did not, so an operator's Restart and a supervision respawn tore this child down
|
||||
// silently and its value nodes went on serving their last value at Good across the gap. The fresh
|
||||
// child's re-subscribe + initial push (03/S16) is what clears them again; on a deploy that REMOVES
|
||||
// the driver the host has already rebuilt its ref index without it, so the Tell finds nothing.
|
||||
//
|
||||
// Gated on having actually reached Connected: a child that never connected leaves its nodes at
|
||||
// BadWaitingForInitialData, which says something more precise than "communication lost", and
|
||||
// overwriting it would claim an outage that never happened.
|
||||
if (_connectedSinceUtc != default)
|
||||
{
|
||||
Context.Parent.Tell(new ConnectivityChanged(_driverInstanceId, Connected: false));
|
||||
}
|
||||
|
||||
DetachSubscription();
|
||||
// MUST happen: the IDriver instance can outlive this actor (the host respawns a child around the
|
||||
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
|
||||
|
||||
@@ -56,6 +56,15 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// their realm each (<see cref="ValueWrite"/>); status codes ride verbatim (#519).</summary>
|
||||
public sealed record AttributeValueBatchUpdate(IReadOnlyList<ValueWrite> Items);
|
||||
|
||||
/// <summary>Status-only batch: every named node keeps its CURRENT value and only its status +
|
||||
/// timestamp change (archreview 03/S15). The comms-loss fan-out sends one of these covering every
|
||||
/// NodeId a disconnected driver owns — the raw node and each referencing UNS node — so a client sees
|
||||
/// <see cref="OpcUaStatus.BadNoCommunication"/> over the last known value instead of a stale Good.
|
||||
/// <para>Deliberately a separate message from <see cref="AttributeValueBatchUpdate"/> rather than a
|
||||
/// flag on it: the host holds no value cache (v3's single-source design), so it has nothing to put in
|
||||
/// a <see cref="ValueWrite"/>, and only the node knows what value to retain.</para></summary>
|
||||
public sealed record AttributeStatusBatchUpdate(IReadOnlyList<StatusWrite> Items);
|
||||
|
||||
/// <summary>Carries the full Part 9 condition state for a scripted alarm to the sink. The
|
||||
/// <paramref name="State"/> snapshot is the Commons projection the Runtime host maps from the engine's
|
||||
/// Core <c>AlarmConditionState</c> + severity/message — the actor stays decoupled from
|
||||
@@ -292,6 +301,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
|
||||
Receive<AttributeValueUpdate>(HandleAttributeUpdate);
|
||||
Receive<AttributeValueBatchUpdate>(HandleAttributeBatchUpdate);
|
||||
Receive<AttributeStatusBatchUpdate>(HandleAttributeStatusBatchUpdate);
|
||||
Receive<AlarmStateUpdate>(HandleAlarmUpdate);
|
||||
Receive<AlarmQualityUpdate>(HandleAlarmQualityUpdate);
|
||||
Receive<RebuildAddressSpace>(HandleRebuild);
|
||||
@@ -368,6 +378,36 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Routes a status-only batch to <see cref="IOpcUaAddressSpaceSink.WriteStatuses"/> — the
|
||||
/// comms-loss annotation (03/S15). Same shape as <see cref="HandleAttributeBatchUpdate"/>: metered per
|
||||
/// ITEM (batching is a transport change, so dashboards must not drop by the batch factor), a
|
||||
/// <see cref="ValueWriteBatchException"/> is a partial failure that still counts what landed and names
|
||||
/// the offender, and any other throw is swallowed so a sink fault cannot restart the publish actor and
|
||||
/// drop live traffic. Metered under <c>kind:"status"</c> so a status annotation is separable from a
|
||||
/// value write on the same counter.</summary>
|
||||
private void HandleAttributeStatusBatchUpdate(AttributeStatusBatchUpdate msg)
|
||||
{
|
||||
try
|
||||
{
|
||||
_sink.WriteStatuses(msg.Items);
|
||||
Interlocked.Add(ref _writes, msg.Items.Count);
|
||||
OtOpcUaTelemetry.OpcUaSinkWrite.Add(msg.Items.Count, new KeyValuePair<string, object?>("kind", "status"));
|
||||
}
|
||||
catch (ValueWriteBatchException ex)
|
||||
{
|
||||
_log.Warning(ex,
|
||||
"OpcUaPublish: sink.WriteStatuses partially failed for a {Count}-item batch — {Failed} failed " +
|
||||
"(first: {Node}), {Succeeded} applied",
|
||||
msg.Items.Count, ex.FailedCount, ex.FirstFailedNodeId, ex.SucceededCount);
|
||||
Interlocked.Add(ref _writes, ex.SucceededCount);
|
||||
OtOpcUaTelemetry.OpcUaSinkWrite.Add(ex.SucceededCount, new KeyValuePair<string, object?>("kind", "status"));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "OpcUaPublish: sink.WriteStatuses threw for a {Count}-item batch", msg.Items.Count);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleAlarmUpdate(AlarmStateUpdate msg)
|
||||
{
|
||||
try
|
||||
|
||||
@@ -66,10 +66,23 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor
|
||||
_log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Forwards one dependency value to the consumer, <b>status code included</b> (archreview
|
||||
/// 05/STAB-19).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This used to pass a hard-coded <c>0u</c> behind a comment saying quality was stripped at the
|
||||
/// mux boundary. That was true when the adapter was written and stopped being true at #478/#519 —
|
||||
/// <see cref="VirtualTagActor.DependencyValueChanged"/> has carried the source driver's full status
|
||||
/// code, sub-code and all, ever since. The stale comment outliving the fact is what kept a
|
||||
/// comms-lost input arriving at the Calculation driver as <i>arrived, Good, null</i>:
|
||||
/// indistinguishable from a genuine Good null, so the warm-up gate was satisfied by a value that
|
||||
/// never arrived and the calc output stayed Good over a dead source.
|
||||
/// </remarks>
|
||||
/// <param name="msg">The dependency value, as the mux delivered it.</param>
|
||||
private void OnDependencyValueChanged(VirtualTagActor.DependencyValueChanged msg)
|
||||
{
|
||||
// Quality is stripped at the mux boundary (value + timestamp only), so forward Good (0).
|
||||
_consumer.OnDependencyValue(msg.TagId, msg.Value, statusCode: 0u, msg.TimestampUtc);
|
||||
_consumer.OnDependencyValue(msg.TagId, msg.Value, msg.StatusCode, msg.TimestampUtc);
|
||||
}
|
||||
|
||||
/// <summary>The mux's batched delivery: one message carrying this adapter's whole matched slice of a
|
||||
|
||||
@@ -6,6 +6,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
@@ -31,9 +32,13 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
public const string ScriptLogsTopic = "script-logs";
|
||||
|
||||
/// <summary>A dependency's value changed. <paramref name="StatusCode"/> (#478) carries the source
|
||||
/// driver's OPC UA status code so the scripted-alarm host can derive a condition's worst-of-input
|
||||
/// quality; it defaults to <see cref="OpcUaStatus.Good"/>, and the virtual-tag engine ignores it.
|
||||
/// Since #519 it carries the driver's sub-code, so a worst-of-input winner keeps the reason it won.</summary>
|
||||
/// driver's OPC UA status code — since #519 including its sub-code, so a worst-of-input winner keeps
|
||||
/// the reason it won — and it defaults to <see cref="OpcUaStatus.Good"/>.
|
||||
/// <para><b>The virtual-tag engine consumes it</b> (archreview 02/S14). This doc used to say the
|
||||
/// engine ignored it, which was true and became the defect: a Bad input was stored as a bare value,
|
||||
/// the script ran against it, and the tag published a fresh-Good result over a dead source. The actor
|
||||
/// now holds evaluation while any declared input is Bad and inherits the worst input's status
|
||||
/// otherwise. The Calculation driver reads it through the same mux (05/STAB-19).</para></summary>
|
||||
public sealed record DependencyValueChanged(
|
||||
string TagId, object? Value, DateTime TimestampUtc, uint StatusCode = OpcUaStatus.Good);
|
||||
|
||||
@@ -72,13 +77,22 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
private readonly IActorRef? _mux;
|
||||
private readonly ITelemetryLocalHub? _telemetryHub;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
private readonly Dictionary<string, object?> _dependencies = new(StringComparer.Ordinal);
|
||||
// Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a
|
||||
// comms-lost input from a genuine Good null, and it is what the script reads through
|
||||
// ctx.GetTag(path).StatusCode.
|
||||
private readonly Dictionary<string, DataValueSnapshot> _dependencies = new(StringComparer.Ordinal);
|
||||
|
||||
private bool _hasLastValue;
|
||||
private object? _lastValue;
|
||||
// 02/S13: true once a Bad EvaluationResult has been published and not yet cleared by a Good
|
||||
// recovery. Gates Bad publishes to once-per-transition and force-publishes the recovery value.
|
||||
private bool _lastPublishedBad;
|
||||
|
||||
/// <summary>Whether anything has been published for this tag yet — distinct from
|
||||
/// <see cref="_hasLastValue"/>, because an inputs-Bad publish carries no value of its own.</summary>
|
||||
private bool _hasPublished;
|
||||
|
||||
/// <summary>The status last published. Replaces 02/S13's two-state <c>_lastPublishedBad</c> flag
|
||||
/// (02/S14): the dedup key is (value, status), a Bad may now carry the failing INPUT's sub-code rather
|
||||
/// than a fixed BadInternalError, and a reassert re-emits this rather than guessing Good.</summary>
|
||||
private uint _lastPublishedStatus = OpcUaStatus.Good;
|
||||
|
||||
/// <summary>Factory method to create Props for a VirtualTagActor.</summary>
|
||||
/// <param name="virtualTagId">Unique identifier for the virtual tag.</param>
|
||||
@@ -178,15 +192,17 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
/// </para></summary>
|
||||
private void OnReassertValue()
|
||||
{
|
||||
if (!_hasLastValue && !_lastPublishedBad) return;
|
||||
var statusCode = _lastPublishedBad ? OpcUaStatus.BadInternalError : OpcUaStatus.Good;
|
||||
if (!_hasLastValue && !_hasPublished) return;
|
||||
// The status LAST PUBLISHED, not a guess (02/S14). A VT whose input is comms-lost re-asserts
|
||||
// BadNoCommunication onto the freshly-materialised node; re-asserting Good here would put the
|
||||
// stale-Good lie back at every deploy.
|
||||
Context.Parent.Tell(new EvaluationResult(
|
||||
_virtualTagId, _lastValue, DateTime.UtcNow, CorrelationId.NewId(), statusCode, IsReassert: true));
|
||||
_virtualTagId, _lastValue, DateTime.UtcNow, CorrelationId.NewId(), _lastPublishedStatus, IsReassert: true));
|
||||
}
|
||||
|
||||
private void OnDependencyChanged(DependencyValueChanged msg)
|
||||
{
|
||||
_dependencies[msg.TagId] = msg.Value;
|
||||
_dependencies[msg.TagId] = new DataValueSnapshot(msg.Value, msg.StatusCode, msg.TimestampUtc, msg.TimestampUtc);
|
||||
Evaluate(msg.TimestampUtc);
|
||||
}
|
||||
|
||||
@@ -210,7 +226,8 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
_dependencies[item.TagId] = item.Value;
|
||||
_dependencies[item.TagId] =
|
||||
new DataValueSnapshot(item.Value, item.StatusCode, item.TimestampUtc, item.TimestampUtc);
|
||||
}
|
||||
|
||||
Evaluate(items[count - 1].TimestampUtc);
|
||||
@@ -222,6 +239,22 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
/// <param name="timestampUtc">The triggering dependency change's timestamp, stamped on the result.</param>
|
||||
private void Evaluate(DateTime timestampUtc)
|
||||
{
|
||||
// Worst-of-input FIRST (02/S14), before the evaluator is even considered.
|
||||
//
|
||||
// Over the deps that have ARRIVED, deliberately — a never-pushed dep is cold start, not a fault,
|
||||
// and it is already handled by the inputs-ready gate in the two publish tails (which suppress the
|
||||
// warm-up flash). Counting it as Bad here would instead HOLD evaluation during warm-up, which is
|
||||
// a behaviour change this finding does not ask for: a partially-warmed-up VT evaluates today and
|
||||
// must keep doing so, or every Good-path assertion in the existing suites changes meaning. Same
|
||||
// skip ScriptedAlarmEngine.WorstInputStatus has always made.
|
||||
var worst = InputQuality.WorstOf(
|
||||
_dependencyRefs.Select(r => _dependencies.GetValueOrDefault(r)).Where(s => s is not null));
|
||||
if (!InputQuality.IsUsable(worst))
|
||||
{
|
||||
OnInputsBad(worst, timestampUtc);
|
||||
return;
|
||||
}
|
||||
|
||||
VirtualTagEvalResult result;
|
||||
try
|
||||
{
|
||||
@@ -249,9 +282,10 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
return;
|
||||
}
|
||||
|
||||
// Value dedup — but bypass it while recovering from Bad, so a recovery whose value equals the
|
||||
// pre-failure value still republishes Good (otherwise the node would stay Bad forever).
|
||||
if (!_lastPublishedBad && _hasLastValue && Equals(_lastValue, result.Value))
|
||||
// Dedup on (value, STATUS) — 02/S14 generalising 02/S13's recover-from-Bad bypass. Keying on
|
||||
// value alone swallows every recovery whose value is unchanged, which is what a device coming
|
||||
// back usually produces, and the node then stays Bad forever.
|
||||
if (_hasPublished && _lastPublishedStatus == worst && _hasLastValue && Equals(_lastValue, result.Value))
|
||||
{
|
||||
OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "skip"));
|
||||
return;
|
||||
@@ -259,12 +293,55 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
|
||||
_hasLastValue = true;
|
||||
_lastValue = result.Value;
|
||||
_lastPublishedBad = false;
|
||||
_hasPublished = true;
|
||||
// The output inherits the worst input's quality: Good, or an Uncertain sub-code (Bad never
|
||||
// reaches here — it was held above).
|
||||
_lastPublishedStatus = worst;
|
||||
OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "ok"));
|
||||
var evalResult = new EvaluationResult(_virtualTagId, result.Value, timestampUtc, CorrelationId.NewId());
|
||||
var evalResult = new EvaluationResult(_virtualTagId, result.Value, timestampUtc, CorrelationId.NewId(), worst);
|
||||
Context.Parent.Tell(evalResult);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 02/S14 — this tag's INPUTS are Bad, so the script is not run and the tag reports the worst
|
||||
/// input's status, carrying its last known value.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Holding rather than evaluating-and-degrading</b> is what removes the
|
||||
/// <c>(double)ctx.GetTag(x).Value</c> NRE storm at source: a comms-lost input used to arrive as
|
||||
/// Good/null and the script ran against it on every trigger. It is also the alarm engine's
|
||||
/// long-standing policy (<c>AreInputsReady</c>), so all three script seams now agree.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>No script-log entry</b>, deliberately: an input dying is not a script fault, and the
|
||||
/// script log is 02/S13's per-failure SCRIPT diagnostic. Here the status code is the
|
||||
/// diagnostic, and it names which failure the operator is looking at.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Published <b>once per change of the published status</b>, and only once every declared
|
||||
/// dependency has arrived — R2-03's inputs-ready gate, kept as-is, so a cold start or an actor
|
||||
/// respawn does not flash Bad while the mux is still filling the map.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="worstInputStatus">The worst status among this tag's declared dependencies.</param>
|
||||
/// <param name="timestampUtc">The triggering change's timestamp, stamped on the result.</param>
|
||||
private void OnInputsBad(uint worstInputStatus, DateTime timestampUtc)
|
||||
{
|
||||
if (!AllDependenciesArrived()) return;
|
||||
if (_hasPublished && _lastPublishedStatus == worstInputStatus) return;
|
||||
|
||||
// Metered BELOW both guards, so the counter measures tags that actually REPORT Bad inputs. Above
|
||||
// them it also counted every warm-up trigger — a dependency that has simply not arrived yet — which
|
||||
// is ordinary cold start, and burying the real signal in it is how a counter stops being trusted.
|
||||
OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "inputs-bad"));
|
||||
|
||||
_hasPublished = true;
|
||||
_lastPublishedStatus = worstInputStatus;
|
||||
Context.Parent.Tell(new EvaluationResult(
|
||||
_virtualTagId, _hasLastValue ? _lastValue : null, timestampUtc, CorrelationId.NewId(), worstInputStatus));
|
||||
}
|
||||
|
||||
/// <summary>02/S13 failure tail shared by the evaluator-threw catch and the <c>!Success</c> branch.
|
||||
/// Always meters + publishes the per-failure script-log entry (the pre-existing operator
|
||||
/// diagnostic); additionally publishes a Bad <see cref="EvaluationResult"/> — carrying the
|
||||
@@ -278,12 +355,13 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "fail"));
|
||||
PublishLog(level, reason);
|
||||
|
||||
if (_lastPublishedBad || !AllDependenciesArrived())
|
||||
if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastPublishedBad = true;
|
||||
_hasPublished = true;
|
||||
_lastPublishedStatus = OpcUaStatus.BadInternalError;
|
||||
Context.Parent.Tell(new EvaluationResult(
|
||||
_virtualTagId, _hasLastValue ? _lastValue : null, timestampUtc, CorrelationId.NewId(), OpcUaStatus.BadInternalError));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user