dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(runtime): audit refs still BadNoCommunication after a reconnect (dedup-trap tripwire)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-20 10:16:01 -04:00
parent 97f4ea0c9e
commit 7d98ef9332
6 changed files with 368 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+160 -3
View File
@@ -82,6 +82,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// </remarks>
public static readonly TimeSpan DefaultApplyWedgeTimeout = TimeSpan.FromMinutes(5);
/// <summary>
/// How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs still
/// latched <see cref="OpcUaStatus.BadNoCommunication"/>. Sized so an ordinary poll driver has had
/// several poll cycles — and a slow one at least one — to re-publish before anything is said; the
/// audit's job is to name a driver that will NEVER re-publish, not to race a slow first poll.
/// </summary>
public static readonly TimeSpan DefaultCommsRecoveryAuditDelay = TimeSpan.FromSeconds(30);
/// <summary>
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
@@ -401,6 +409,27 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <see cref="DefaultApplyWedgeTimeout"/>).</summary>
private readonly TimeSpan _applyWedgeTimeout;
/// <summary>How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs
/// still latched <see cref="OpcUaStatus.BadNoCommunication"/> (see <see cref="CommsRecoveryAudit"/>).</summary>
private readonly TimeSpan _commsRecoveryAuditDelay;
/// <summary>
/// Archreview <b>03/S15</b> — RawPaths this host has stamped <c>BadNoCommunication</c> on the value
/// nodes of, per driver, and has not yet seen a value for. Set by
/// <see cref="FanCommsLossToValueNodes"/>, pruned by <see cref="ForwardBatchToMux"/> as each ref
/// re-publishes, and read once by <see cref="CommsRecoveryAudit"/> after the driver reconnects.
/// <para><b>It is a tripwire, not a mechanism.</b> Nothing depends on it being correct for the data
/// plane to behave: the host publishes nothing Good on reconnect either way. What it buys is that a
/// driver which reconnects and then fails to re-publish — the pre-STAB-18 MQTT shape, and what any
/// future push driver without an initial push will do — produces a Warning and a metered count
/// instead of a subscription that silently reports Bad forever.</para>
/// <para>Keyed by RawPath, so a raw node and its UNS mirrors count once: the operator is being told
/// how many <i>tags</i> did not come back, not how many address-space nodes they project onto. The
/// entry is removed as soon as it empties, so the steady-state cost in
/// <see cref="ForwardBatchToMux"/> is one dictionary miss per burst.</para>
/// </summary>
private readonly Dictionary<string, HashSet<string>> _commsLatched = new(StringComparer.Ordinal);
/// <summary>
/// Host of this node's LocalDb replication partner — the only node that holds a copy of this
/// node's alarm queue, and therefore the only node it may ever stand down in favour of.
@@ -449,6 +478,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <param name="OperatorUser">The authenticated principal performing the acknowledge.</param>
public sealed record RouteNativeAlarmAck(string ConditionNodeId, string? Comment, string OperatorUser);
/// <summary>Self-sent by the 03/S15 audit timer, <see cref="_commsRecoveryAuditDelay"/> after a driver
/// reconnected: report any of its refs still latched <c>BadNoCommunication</c>. See
/// <see cref="HandleCommsRecoveryAudit"/>.</summary>
/// <param name="DriverInstanceId">The driver whose recovery is being audited.</param>
public sealed record CommsRecoveryAudit(string DriverInstanceId);
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
{
// Convenience accessors for sites that don't need the full spec.
@@ -514,6 +549,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// VirtualTag + ScriptedAlarm hosts. Defaults to null (admin-only nodes / tests skip the emit).</param>
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>.</param>
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; tests shorten it.</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
@@ -542,7 +580,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
IDriverPrimaryRoleSource? primaryRoleSource = null,
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null) =>
TimeSpan? applyWedgeTimeout = null,
TimeSpan? commsRecoveryAuditDelay = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -554,7 +593,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout));
sharesHistorizationScopeWithLocal, applyWedgeTimeout, commsRecoveryAuditDelay));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -613,6 +652,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <param name="applyWedgeTimeout">How long the host may sit in <see cref="Applying"/> before the
/// wedge watchdog logs a Warning. Defaults to <see cref="DefaultApplyWedgeTimeout"/>; a test shortens
/// it so the watchdog is observable in a test's lifetime.</param>
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; a test shortens it for the same reason.</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
@@ -640,9 +682,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
IDriverPrimaryRoleSource? primaryRoleSource = null,
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null)
TimeSpan? applyWedgeTimeout = null,
TimeSpan? commsRecoveryAuditDelay = null)
{
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
_commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
_historizationRoleView = historizationRoleView;
_sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal;
_telemetryHub = telemetryHub;
@@ -1073,6 +1117,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
@@ -1128,6 +1173,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
@@ -1168,6 +1214,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// One envelope per burst; the mux filters per item (it registered a batch receive in Task 9).
_dependencyMux?.Tell(msg);
// 03/S15 recovery audit: clear whatever this burst re-published from the driver's Bad latch. Guarded
// by ONE dictionary probe per burst — with no outage outstanding (the steady state) this is a single
// miss, not per-item work.
if (_commsLatched.ContainsKey(msg.DriverInstanceId))
{
for (var i = 0; i < msg.Items.Count; i++)
TrackCommsLatchPrune(msg.DriverInstanceId, msg.Items[i].FullReference);
}
if (_opcUaPublishActor is null) return;
// Indexed for-loop, not foreach: IReadOnlyList<AttributeValueItem> enumerates through an interface
@@ -1266,6 +1321,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// 03/S15: and the EXTERNAL data plane, which #481 stopped one seam short of.
FanCommsLossToValueNodes(msg.DriverInstanceId);
}
else
{
// 03/S15: nothing Good is published here — instead, check LATER whether the driver republished
// what this host marked Bad. See HandleCommsRecoveryAudit.
TrackCommsLatchArmAudit(msg.DriverInstanceId);
}
if (_opcUaPublishActor is null) return;
@@ -1352,12 +1413,105 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
if (writes.Count == 0) return;
_opcUaPublishActor.Tell(new OpcUa.OpcUaPublishActor.AttributeStatusBatchUpdate(writes));
TrackCommsLatchSet(driverInstanceId, entries);
_log.Info(
"DriverHost {Node}: driver {Driver} disconnected — {Count} value node(s) marked BadNoCommunication (value retained)",
_localNode, driverInstanceId, writes.Count);
}
/// <summary>
/// Records every RawPath just latched <c>BadNoCommunication</c> for <paramref name="driverInstanceId"/>
/// (03/S15's recovery audit — see <see cref="_commsLatched"/>). Replaces any previous latch wholesale:
/// a second outage re-arms from the current ref set rather than accumulating refs a redeploy removed.
/// </summary>
/// <param name="driverInstanceId">The driver that just went comms-lost.</param>
/// <param name="entries">Its value refs, from the per-driver index.</param>
private void TrackCommsLatchSet(
string driverInstanceId,
List<(string RawPath, HashSet<NodeRealmRef> Nodes)> entries)
{
var latched = new HashSet<string>(entries.Count, StringComparer.Ordinal);
foreach (var (rawPath, _) in entries) latched.Add(rawPath);
_commsLatched[driverInstanceId] = latched;
// A previous reconnect's audit is now meaningless — this driver is down again.
Timers.Cancel(CommsAuditTimerKey(driverInstanceId));
}
/// <summary>
/// Clears <paramref name="fullReference"/> from <paramref name="driverInstanceId"/>'s latch, and drops
/// the driver's entry once it empties. Called per routed value item, so the steady-state cost with no
/// latch outstanding is one dictionary miss per burst — not per item.
/// </summary>
/// <param name="driverInstanceId">The publishing driver.</param>
/// <param name="fullReference">The RawPath that just published a value.</param>
private void TrackCommsLatchPrune(string driverInstanceId, string fullReference)
{
if (!_commsLatched.TryGetValue(driverInstanceId, out var latched)) return;
if (!latched.Remove(fullReference)) return;
if (latched.Count == 0) _commsLatched.Remove(driverInstanceId);
}
/// <summary>
/// Arms the recovery audit for a driver that has just reconnected: after
/// <see cref="_commsRecoveryAuditDelay"/>, <see cref="HandleCommsRecoveryAudit"/> reports whatever is
/// still latched. Nothing is armed when the driver has no latch — it never went Bad here.
/// </summary>
/// <param name="driverInstanceId">The driver that just reconnected.</param>
private void TrackCommsLatchArmAudit(string driverInstanceId)
{
if (!_commsLatched.ContainsKey(driverInstanceId)) return;
Timers.StartSingleTimer(
CommsAuditTimerKey(driverInstanceId),
new CommsRecoveryAudit(driverInstanceId),
_commsRecoveryAuditDelay);
}
private static string CommsAuditTimerKey(string driverInstanceId) => $"comms-audit:{driverInstanceId}";
/// <summary>
/// The 03/S15 tripwire firing: refs this host latched <c>BadNoCommunication</c> that the driver has
/// still not re-published <see cref="_commsRecoveryAuditDelay"/> after it reconnected.
/// </summary>
/// <remarks>
/// <para>
/// <b>It reports, it does not repair.</b> Writing Good here would assert a quality nothing has
/// observed — the very thing this whole path refuses to do. What the operator gets instead is a
/// named driver and a count, plus the <c>otopcua.driver.comms_latched_refs</c> counter, which is
/// the operator-visible signature of a driver that reconnects without re-publishing (the
/// pre-STAB-18 MQTT shape, or any future push driver that forgets its initial push).
/// </para>
/// <para>
/// The latch is cleared afterwards either way: it has been reported, and leaving it would make
/// the next outage's audit report refs from this one.
/// </para>
/// </remarks>
/// <param name="msg">Names the driver whose audit is due.</param>
private void HandleCommsRecoveryAudit(CommsRecoveryAudit msg)
{
if (!_commsLatched.TryGetValue(msg.DriverInstanceId, out var latched) || latched.Count == 0)
{
_commsLatched.Remove(msg.DriverInstanceId);
return;
}
var driverType = _children.TryGetValue(msg.DriverInstanceId, out var child) ? child.DriverType : "unknown";
OtOpcUaTelemetry.DriverCommsLatchedRefs.Add(
latched.Count,
new KeyValuePair<string, object?>("driver", msg.DriverInstanceId),
new KeyValuePair<string, object?>("driver_type", driverType));
_log.Warning(
"DriverHost {Node}: driver {Driver} ({Type}) — {Count} ref(s) still BadNoCommunication {Delay} after reconnect; "
+ "the driver did not re-publish them (missing initial push, or a value-only dedup swallowed an unchanged value). "
+ "First: {First}",
_localNode, msg.DriverInstanceId, driverType, latched.Count, _commsRecoveryAuditDelay,
latched.OrderBy(r => r, StringComparer.Ordinal).First());
_commsLatched.Remove(msg.DriverInstanceId);
}
/// <summary>
/// #481 (Layer 4) — push a synthetic <see cref="OpcUaStatus.BadNoCommunication"/> into the dependency
/// mux for every value ref the just-disconnected driver owns, so scripted alarms and virtual tags see
@@ -1807,6 +1961,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
Receive<CommsRecoveryAudit>(_ => { });
// A late DeltaApplied (an apply completed just before the DB went Stale) — re-register the driver's
// mux adapter anyway; it simply re-reads the driver's current refs (harmless, no DB access).
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);