dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'fix/r3-02-stale-good-family' — R3-02: one quality contract at four seams — comms loss marks raw+UNS value variables BadNoCommunication (WriteStatuses sink member, recovery audit), Calculation + VT hold on Bad / worst-of sub-code / (value,status) dedup, MQTT initial push + ConnectionLost stale, Sparkplug birth replay (archreview 03/S15, 05/STAB-19, 05/STAB-18, 02/S14)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-20 11:49:06 -04:00
parent f1b976f5e2 886082fec7
commit 157d01c2c2
46 changed files with 4340 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+237 -4
View File
@@ -91,6 +91,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// connect work, so it should not run more often than an ordinary reconnect attempt.</summary>
public static readonly TimeSpan DefaultReprovisionBackoff = TimeSpan.FromSeconds(10);
/// <summary>
/// How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs still
/// latched <see cref="OpcUaStatus.BadNoCommunication"/>. Sized so an ordinary poll driver has had
/// several poll cycles — and a slow one at least one — to re-publish before anything is said; the
/// audit's job is to name a driver that will NEVER re-publish, not to race a slow first poll.
/// </summary>
public static readonly TimeSpan DefaultCommsRecoveryAuditDelay = TimeSpan.FromSeconds(30);
/// <summary>
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
@@ -426,6 +434,27 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <see cref="DefaultApplyWedgeTimeout"/>).</summary>
private readonly TimeSpan _applyWedgeTimeout;
/// <summary>How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs
/// still latched <see cref="OpcUaStatus.BadNoCommunication"/> (see <see cref="CommsRecoveryAudit"/>).</summary>
private readonly TimeSpan _commsRecoveryAuditDelay;
/// <summary>
/// Archreview <b>03/S15</b> — RawPaths this host has stamped <c>BadNoCommunication</c> on the value
/// nodes of, per driver, and has not yet seen a value for. Set by
/// <see cref="FanCommsLossToValueNodes"/>, pruned by <see cref="ForwardBatchToMux"/> as each ref
/// re-publishes, and read once by <see cref="CommsRecoveryAudit"/> after the driver reconnects.
/// <para><b>It is a tripwire, not a mechanism.</b> Nothing depends on it being correct for the data
/// plane to behave: the host publishes nothing Good on reconnect either way. What it buys is that a
/// driver which reconnects and then fails to re-publish — the pre-STAB-18 MQTT shape, and what any
/// future push driver without an initial push will do — produces a Warning and a metered count
/// instead of a subscription that silently reports Bad forever.</para>
/// <para>Keyed by RawPath, so a raw node and its UNS mirrors count once: the operator is being told
/// how many <i>tags</i> did not come back, not how many address-space nodes they project onto. The
/// entry is removed as soon as it empties, so the steady-state cost in
/// <see cref="ForwardBatchToMux"/> is one dictionary miss per burst.</para>
/// </summary>
private readonly Dictionary<string, HashSet<string>> _commsLatched = new(StringComparer.Ordinal);
/// <summary>Per-driver rate limit for the supervision-reprovision hook (see
/// <see cref="DefaultReprovisionBackoff"/>).</summary>
private readonly TimeSpan _reprovisionBackoff;
@@ -483,6 +512,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <param name="OperatorUser">The authenticated principal performing the acknowledge.</param>
public sealed record RouteNativeAlarmAck(string ConditionNodeId, string? Comment, string OperatorUser);
/// <summary>Self-sent by the 03/S15 audit timer, <see cref="_commsRecoveryAuditDelay"/> after a driver
/// reconnected: report any of its refs still latched <c>BadNoCommunication</c>. See
/// <see cref="HandleCommsRecoveryAudit"/>.</summary>
/// <param name="DriverInstanceId">The driver whose recovery is being audited.</param>
public sealed record CommsRecoveryAudit(string DriverInstanceId);
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
{
// Convenience accessors for sites that don't need the full spec.
@@ -548,6 +583,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// VirtualTag + ScriptedAlarm hosts. Defaults to null (admin-only nodes / tests skip the emit).</param>
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>.</param>
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; tests shorten it.</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
@@ -577,7 +615,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null) =>
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -589,7 +628,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff));
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -648,6 +687,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>; a test shortens
/// it so the watchdog is observable in a test's lifetime.</param>
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; a test shortens it for the same reason.</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
@@ -676,10 +718,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null)
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null)
{
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
_reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
_commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
_historizationRoleView = historizationRoleView;
_sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal;
_telemetryHub = telemetryHub;
@@ -1110,6 +1154,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
@@ -1167,6 +1212,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
@@ -1209,6 +1255,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// One envelope per burst; the mux filters per item (it registered a batch receive in Task 9).
_dependencyMux?.Tell(msg);
// 03/S15 recovery audit: clear whatever this burst re-published from the driver's Bad latch. Guarded
// by ONE dictionary probe per burst — with no outage outstanding (the steady state) this is a single
// miss, not per-item work.
if (_commsLatched.ContainsKey(msg.DriverInstanceId))
{
for (var i = 0; i < msg.Items.Count; i++)
TrackCommsLatchPrune(msg.DriverInstanceId, msg.Items[i].FullReference);
}
if (_opcUaPublishActor is null) return;
// Indexed for-loop, not foreach: IReadOnlyList<AttributeValueItem> enumerates through an interface
@@ -1285,13 +1340,34 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// the source had died. <see cref="FanCommsLossToDependencyMux"/> closes that by pushing a
/// synthetic Bad for every value ref the driver owns.
/// </para>
/// <para>
/// <b>Archreview 03/S15 — and the EXTERNAL data plane, which the two layers above still left
/// out.</b> Conditions and internal consumers learned of the outage; the driver's raw value
/// variables and their UNS mirrors — what an OPC UA client actually subscribes to — did not, so
/// they served the last value at <c>Good</c> for the whole outage.
/// <see cref="FanCommsLossToValueNodes"/> closes that with one status-only batch that keeps each
/// node's value and changes only its status. All three fan-outs run on the disconnect edge only;
/// the reconnect edge writes nothing to values (recovery is the driver's own re-publish) and
/// re-annotates conditions Good, which is a statement about the LINK, not about any tag's value.
/// </para>
/// </summary>
private void OnDriverConnectivityChanged(DriverInstanceActor.ConnectivityChanged msg)
{
// #481: the mux fan-out is independent of the OPC UA sink — a node with scripted alarms but no
// publish actor (the mux-only test/dev path) must still see comms loss. Done before the early
// return for exactly that reason.
if (!msg.Connected) FanCommsLossToDependencyMux(msg.DriverInstanceId);
if (!msg.Connected)
{
FanCommsLossToDependencyMux(msg.DriverInstanceId);
// 03/S15: and the EXTERNAL data plane, which #481 stopped one seam short of.
FanCommsLossToValueNodes(msg.DriverInstanceId);
}
else
{
// 03/S15: nothing Good is published here — instead, check LATER whether the driver republished
// what this host marked Bad. See HandleCommsRecoveryAudit.
TrackCommsLatchArmAudit(msg.DriverInstanceId);
}
if (_opcUaPublishActor is null) return;
@@ -1323,6 +1399,160 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
_localNode, msg.DriverInstanceId, msg.Connected ? "connected" : "disconnected", annotated, statusCode);
}
/// <summary>
/// Archreview <b>03/S15</b> — the value-node half of the comms-loss fan-out: stamp
/// <see cref="OpcUaStatus.BadNoCommunication"/> on every raw NodeId <b>and</b> every referencing UNS
/// NodeId the just-disconnected driver owns, as ONE status batch through the same publish actor the
/// value fan-out uses.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why it was missing.</b> #481 built the per-driver ref index and wired comms loss to the
/// internal consumers (the mux) and to Part 9 conditions, and stopped one seam short of the
/// external OPC UA data plane. A client subscribed to a value node therefore saw the last value
/// at <b>Good</b> quality for the whole outage, for every driver whose failure shape is "the
/// reader stops producing" rather than "publish a Bad snapshot per tag". (Modbus does the
/// latter, quickly, which is why the gap looked smaller than it was — a coalesced-read throw, a
/// dropped session on a push driver, or simply a driver whose last act was a Good publish all
/// held Good indefinitely.)
/// </para>
/// <para>
/// <b>Status-only, so the VALUE is retained.</b> This actor holds no value cache — v3's
/// single-source design deliberately removed the independent buffer — so only the node can keep
/// the last known value, which is what
/// <see cref="OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate"/> exists for. A Bad node still
/// carrying its last reading is OPC UA Part 8's convention and is what a SCADA consumer expects.
/// </para>
/// <para>
/// <b>Nothing is written on reconnect.</b> Publishing Good would assert a quality this actor has
/// not observed; recovery is the driver's own re-publish (see
/// <see cref="FanCommsLossToDependencyMux"/>'s remark on the detach that makes the poll engine
/// force-raise, and MQTT's initial push).
/// </para>
/// <para>
/// Ungated by redundancy role, like the condition annotation: a Secondary keeps its address
/// space honest for failover. Re-stamping an already-Bad node is harmless — the SDK's
/// <c>StatusValue</c> trigger dedups an unchanged status at the monitored item.
/// </para>
/// </remarks>
/// <param name="driverInstanceId">The driver child that just reported comms loss.</param>
private void FanCommsLossToValueNodes(string driverInstanceId)
{
if (_opcUaPublishActor is null) return;
if (!_valueRefsByDriver.TryGetValue(driverInstanceId, out var entries) || entries.Count == 0) return;
var ts = DateTime.UtcNow;
var writes = new List<StatusWrite>(entries.Count);
foreach (var (_, nodes) in entries)
{
foreach (var n in nodes)
{
writes.Add(new StatusWrite(n.NodeId, OpcUaStatus.BadNoCommunication, ts, n.Realm));
}
}
if (writes.Count == 0) return;
_opcUaPublishActor.Tell(new OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate(writes));
TrackCommsLatchSet(driverInstanceId, entries);
_log.Info(
"DriverHost {Node}: driver {Driver} disconnected — {Count} value node(s) marked BadNoCommunication (value retained)",
_localNode, driverInstanceId, writes.Count);
}
/// <summary>
/// Records every RawPath just latched <c>BadNoCommunication</c> for <paramref name="driverInstanceId"/>
/// (03/S15's recovery audit — see <see cref="_commsLatched"/>). Replaces any previous latch wholesale:
/// a second outage re-arms from the current ref set rather than accumulating refs a redeploy removed.
/// </summary>
/// <param name="driverInstanceId">The driver that just went comms-lost.</param>
/// <param name="entries">Its value refs, from the per-driver index.</param>
private void TrackCommsLatchSet(
string driverInstanceId,
List<(string RawPath, HashSet<NodeRealmRef> Nodes)> entries)
{
var latched = new HashSet<string>(entries.Count, StringComparer.Ordinal);
foreach (var (rawPath, _) in entries) latched.Add(rawPath);
_commsLatched[driverInstanceId] = latched;
// A previous reconnect's audit is now meaningless — this driver is down again.
Timers.Cancel(CommsAuditTimerKey(driverInstanceId));
}
/// <summary>
/// Clears <paramref name="fullReference"/> from <paramref name="driverInstanceId"/>'s latch, and drops
/// the driver's entry once it empties. Called per routed value item, so the steady-state cost with no
/// latch outstanding is one dictionary miss per burst — not per item.
/// </summary>
/// <param name="driverInstanceId">The publishing driver.</param>
/// <param name="fullReference">The RawPath that just published a value.</param>
private void TrackCommsLatchPrune(string driverInstanceId, string fullReference)
{
if (!_commsLatched.TryGetValue(driverInstanceId, out var latched)) return;
if (!latched.Remove(fullReference)) return;
if (latched.Count == 0) _commsLatched.Remove(driverInstanceId);
}
/// <summary>
/// Arms the recovery audit for a driver that has just reconnected: after
/// <see cref="_commsRecoveryAuditDelay"/>, <see cref="HandleCommsRecoveryAudit"/> reports whatever is
/// still latched. Nothing is armed when the driver has no latch — it never went Bad here.
/// </summary>
/// <param name="driverInstanceId">The driver that just reconnected.</param>
private void TrackCommsLatchArmAudit(string driverInstanceId)
{
if (!_commsLatched.ContainsKey(driverInstanceId)) return;
Timers.StartSingleTimer(
CommsAuditTimerKey(driverInstanceId),
new CommsRecoveryAudit(driverInstanceId),
_commsRecoveryAuditDelay);
}
private static string CommsAuditTimerKey(string driverInstanceId) => $"comms-audit:{driverInstanceId}";
/// <summary>
/// The 03/S15 tripwire firing: refs this host latched <c>BadNoCommunication</c> that the driver has
/// still not re-published <see cref="_commsRecoveryAuditDelay"/> after it reconnected.
/// </summary>
/// <remarks>
/// <para>
/// <b>It reports, it does not repair.</b> Writing Good here would assert a quality nothing has
/// observed — the very thing this whole path refuses to do. What the operator gets instead is a
/// named driver and a count, plus the <c>otopcua.driver.comms_latched_refs</c> counter, which is
/// the operator-visible signature of a driver that reconnects without re-publishing (the
/// pre-STAB-18 MQTT shape, or any future push driver that forgets its initial push).
/// </para>
/// <para>
/// The latch is cleared afterwards either way: it has been reported, and leaving it would make
/// the next outage's audit report refs from this one.
/// </para>
/// </remarks>
/// <param name="msg">Names the driver whose audit is due.</param>
private void HandleCommsRecoveryAudit(CommsRecoveryAudit msg)
{
if (!_commsLatched.TryGetValue(msg.DriverInstanceId, out var latched) || latched.Count == 0)
{
_commsLatched.Remove(msg.DriverInstanceId);
return;
}
var driverType = _children.TryGetValue(msg.DriverInstanceId, out var child) ? child.DriverType : "unknown";
OtOpcUaTelemetry.DriverCommsLatchedRefs.Add(
latched.Count,
new KeyValuePair<string, object?>("driver", msg.DriverInstanceId),
new KeyValuePair<string, object?>("driver_type", driverType));
_log.Warning(
"DriverHost {Node}: driver {Driver} ({Type}) — {Count} ref(s) still BadNoCommunication {Delay} after reconnect; "
+ "the driver did not re-publish them (missing initial push, or a value-only dedup swallowed an unchanged value). "
+ "First: {First}",
_localNode, msg.DriverInstanceId, driverType, latched.Count, _commsRecoveryAuditDelay,
latched.OrderBy(r => r, StringComparer.Ordinal).First());
_commsLatched.Remove(msg.DriverInstanceId);
}
/// <summary>
/// #481 (Layer 4) — push a synthetic <see cref="OpcUaStatus.BadNoCommunication"/> into the dependency
/// mux for every value ref the just-disconnected driver owns, so scripted alarms and virtual tags see
@@ -1778,6 +2008,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
Receive<CommsRecoveryAudit>(_ => { });
// A late DeltaApplied (an apply completed just before the DB went Stale) — re-register the driver's
// mux adapter anyway; it simply re-reads the driver's current refs (harmless, no DB access).
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);