dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(runtime): audit refs still BadNoCommunication after a reconnect (dedup-trap tripwire)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-20 10:16:01 -04:00
parent 97f4ea0c9e
commit 7d98ef9332
6 changed files with 368 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorCommsLossValueNodesTests.cs
+172
View File
@@ -1,6 +1,8 @@
using System.Diagnostics.Metrics;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
@@ -32,6 +34,17 @@ public sealed class DriverHostActorCommsLossValueNodesTests : RuntimeActorTestBa
{
private static readonly NodeId TestNode = NodeId.Parse("driver-commsloss-values");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('c', 64));
private static readonly DateTime TsA = new(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
/// <summary>Audit delay used by the T9 cases — long enough that a same-turn prune wins the race,
/// short enough to observe.</summary>
private static readonly TimeSpan AuditDelay = TimeSpan.FromMilliseconds(400);
// Distinct driver ids per metered test: the meter tag is what the recorder filters on, and a
// MeterListener sees every measurement in the process — including a parallel test's (#503).
private const string SilentDriver = "drv-audit-silent";
private const string RepublishingDriver = "drv-audit-republish";
private const string PartialDriver = "drv-audit-partial";
/// <summary>
/// A disconnect produces ONE status batch naming exactly the disconnected driver's raw nodes plus
@@ -102,6 +115,122 @@ public sealed class DriverHostActorCommsLossValueNodesTests : RuntimeActorTestBa
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
// ---------------------------------------------------------------------------------
// T9 — the recovery audit (the dedup-trap tripwire)
// ---------------------------------------------------------------------------------
/// <summary>
/// The host publishes nothing Good on reconnect, so the only way a stuck-Bad node becomes visible
/// is this audit: refs still latched some time after the driver reconnected are logged and metered.
/// Here the driver re-publishes nothing at all — the pre-STAB-18 MQTT shape, and the exact failure
/// any future push driver without an initial push would produce.
/// </summary>
[Fact]
public void Refs_still_latched_after_a_reconnect_are_logged_and_metered()
{
var db = NewInMemoryDbFactory();
var deploymentId = SeedV3Deployment(db, RevA,
rawTags: new[]
{
(Driver: SilentDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
(Driver: SilentDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
},
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, AuditDelay);
using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs");
actor.Tell(new DriverInstanceActor.ConnectivityChanged(SilentDriver, Connected: false));
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(TimeSpan.FromSeconds(10), () =>
actor.Tell(new DriverInstanceActor.ConnectivityChanged(SilentDriver, Connected: true)));
// Two REFS, not three nodes: the latch is keyed by RawPath, and the UNS mirror shares its
// backing tag's ref — the operator is being told how many tags did not come back, not how many
// address-space nodes they project onto.
AwaitCondition(() => meter.WithTag("driver", SilentDriver) > 0, TimeSpan.FromSeconds(5));
meter.SumWithTag("driver", SilentDriver).ShouldBe(2);
}
/// <summary>
/// <b>The control.</b> A driver that DOES re-publish every latched ref on reconnect trips nothing —
/// without this, the test above would pass against an audit that fires unconditionally, which would
/// make the Warning noise rather than signal.
/// </summary>
[Fact]
public void A_driver_that_republishes_every_ref_trips_no_audit()
{
var db = NewInMemoryDbFactory();
var deploymentId = SeedV3Deployment(db, RevA,
rawTags: new[]
{
(Driver: RepublishingDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
(Driver: RepublishingDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
},
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, AuditDelay);
using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs");
actor.Tell(new DriverInstanceActor.ConnectivityChanged(RepublishingDriver, Connected: false));
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
EventFilter.Warning(contains: "still BadNoCommunication").Expect(0, TimeSpan.FromSeconds(5), () =>
{
actor.Tell(new DriverInstanceActor.ConnectivityChanged(RepublishingDriver, Connected: true));
// The driver's own re-publish — identical values, which is precisely the case a value-only
// dedup would swallow; here it is the driver that force-raised, so both refs are pruned.
actor.Tell(new DriverInstanceActor.AttributeValueBatchPublished(RepublishingDriver, new[]
{
new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/speed", 42.0, OpcUaStatus.Good, TsA),
new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/temp", 7.5, OpcUaStatus.Good, TsA),
}));
publish.ExpectMsg<OpcUaPublishActor.AttributeValueBatchUpdate>(PresenceBudget);
});
meter.WithTag("driver", RepublishingDriver).ShouldBe(0);
}
/// <summary>
/// A partial recovery is still reported, and reports only what is missing: one ref comes back, the
/// other does not, and the audit counts exactly one. An audit that reported the whole latch (or
/// nothing at all once anything arrived) would be useless on precisely the mixed case an operator
/// needs it for.
/// </summary>
[Fact]
public void A_partial_recovery_reports_only_the_refs_that_did_not_come_back()
{
var db = NewInMemoryDbFactory();
var deploymentId = SeedV3Deployment(db, RevA,
rawTags: new[]
{
(Driver: PartialDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
(Driver: PartialDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
},
unsRefs: Array.Empty<(string, string, string, string, string?)>());
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, AuditDelay);
using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs");
actor.Tell(new DriverInstanceActor.ConnectivityChanged(PartialDriver, Connected: false));
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(TimeSpan.FromSeconds(10), () =>
{
actor.Tell(new DriverInstanceActor.ConnectivityChanged(PartialDriver, Connected: true));
actor.Tell(new DriverInstanceActor.AttributeValueBatchPublished(PartialDriver, new[]
{
new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/speed", 42.0, OpcUaStatus.Good, TsA),
}));
publish.ExpectMsg<OpcUaPublishActor.AttributeValueBatchUpdate>(PresenceBudget);
});
AwaitCondition(() => meter.WithTag("driver", PartialDriver) > 0, TimeSpan.FromSeconds(5));
meter.SumWithTag("driver", PartialDriver).ShouldBe(1, "only the ref that never came back is reported");
}
/// <summary>
/// A driver that owns no value refs produces no message at all — an empty batch would cost a hop
/// and a sink call for nothing on every flap of an alarm-only or freshly-added driver.
@@ -123,4 +252,47 @@ public sealed class DriverHostActorCommsLossValueNodesTests : RuntimeActorTestBa
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
/// <summary>
/// Sums one counter's measurements, <b>filtered by tag</b>. A <see cref="MeterListener"/> observes
/// every measurement in the process, so filtering on instrument name alone would fold in a parallel
/// test's (#503) — each test here therefore uses its own driver id and asserts on that tag.
/// </summary>
private sealed class MeterRecorder : IDisposable
{
private readonly MeterListener _listener;
private readonly List<(long Value, KeyValuePair<string, object?>[] Tags)> _measurements = new();
private readonly object _gate = new();
public MeterRecorder(string instrumentName)
{
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == instrumentName)
listener.EnableMeasurementEvents(instrument);
},
};
_listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
{
lock (_gate) _measurements.Add((value, tags.ToArray()));
});
_listener.Start();
}
public int WithTag(string key, string value)
{
lock (_gate) return _measurements.Count(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value)));
}
public long SumWithTag(string key, string value)
{
lock (_gate)
return _measurements.Where(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value)))
.Sum(m => m.Value);
}
public void Dispose() => _listener.Dispose();
}
}
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs
+9 -4
View File
@@ -114,13 +114,17 @@ public static class V3ArtifactSeeder
/// <param name="localNode">The host's local node id.</param>
/// <param name="deploymentId">The deployment to dispatch.</param>
/// <param name="rev">The revision hash the deployment was sealed under.</param>
/// <param name="commsRecoveryAuditDelay">Optional 03/S15 audit delay — how long after a reconnect the
/// host checks for refs still latched <c>BadNoCommunication</c>. Null keeps the production default,
/// which is far longer than any test's lifetime.</param>
/// <returns>The host actor plus the publish and mux probes wired into it.</returns>
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux) SpawnHostAndApply(
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
NodeId localNode, DeploymentId deploymentId, RevisionHash rev)
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
TimeSpan? commsRecoveryAuditDelay = null)
{
var publish = kit.CreateTestProbe();
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref);
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, commsRecoveryAuditDelay);
Dispatch(actor, coordinator, deploymentId, rev);
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
@@ -147,7 +151,7 @@ public static class V3ArtifactSeeder
private static (IActorRef Actor, TestProbe Mux, TestProbe Coordinator) Spawn(
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db, NodeId localNode,
IActorRef? opcUaPublishActor)
IActorRef? opcUaPublishActor, TimeSpan? commsRecoveryAuditDelay = null)
{
var coordinator = kit.CreateTestProbe();
var mux = kit.CreateTestProbe();
@@ -159,7 +163,8 @@ public static class V3ArtifactSeeder
dependencyMux: mux.Ref,
opcUaPublishActor: opcUaPublishActor,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: vtHost.Ref));
virtualTagHostOverride: vtHost.Ref,
commsRecoveryAuditDelay: commsRecoveryAuditDelay));
return (actor, mux, coordinator);
}