using System.Diagnostics.Metrics; using Akka.Actor; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; using static ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness.V3ArtifactSeeder; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// /// Archreview 03/S15 — the value-node half of the comms-loss fan-out. Until this, a driver /// declared comms-lost annotated its Part 9 conditions and pushed a synthetic Bad into the dependency /// mux, but its raw and UNS value variables — the product's external surface — got nothing: an /// OPC UA client subscribed to a value node saw the last value at Good quality for the whole /// outage, for every driver whose failure shape is "reader stops producing" rather than "per-tag Bad /// snapshot". /// /// Status-only, value retained. The write goes through /// so the node keeps its last known value /// under BadNoCommunication (OPC UA Part 8's convention). The host holds no value cache — v3's /// single-source design — so only the node itself can retain it. /// /// Nothing is published on reconnect, deliberately: publishing Good would assert a /// quality this actor has not observed. Recovery is the driver's own re-publish (the poll engine's /// forceRaise on a fresh subscription, or the push-model initial push). That contract is /// end-to-end tested in . /// public sealed class DriverHostActorCommsLossValueNodesTests : RuntimeActorTestBase { private static readonly NodeId TestNode = NodeId.Parse("driver-commsloss-values"); private static readonly RevisionHash RevA = RevisionHash.Parse(new string('c', 64)); private static readonly DateTime TsA = new(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc); /// Audit delay used by the T9 cases — long enough that a same-turn prune wins the race, /// short enough to observe. private static readonly TimeSpan AuditDelay = TimeSpan.FromMilliseconds(400); // Distinct driver ids per metered test: the meter tag is what the recorder filters on, and a // MeterListener sees every measurement in the process — including a parallel test's (#503). private const string SilentDriver = "drv-audit-silent"; private const string RepublishingDriver = "drv-audit-republish"; private const string PartialDriver = "drv-audit-partial"; /// /// A disconnect produces ONE status batch naming exactly the disconnected driver's raw nodes plus /// every referencing UNS mirror, all BadNoCommunication. The second driver's ref is the /// control: a fan-out that walked every value ref on the node instead of the per-driver index would /// mark a perfectly healthy driver's nodes Bad, and no assertion about the first driver's nodes /// would notice. /// [Fact] public void Disconnect_marks_the_drivers_raw_and_uns_value_nodes_BadNoCommunication() { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false), (Driver: "drv-2", DriverName: "S7", Device: "dev2", Tag: "other", DataType: "Double", Writable: false), }, unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) }); var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA); actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-1", Connected: false)); var batch = publish.ExpectMsg(PresenceBudget); batch.Items.Select(i => i.NodeId).ShouldBe(new[] { "Plant/Modbus/dev1/speed", "filling/line1/station1/speed", "Plant/Modbus/dev1/temp", }, ignoreOrder: true); batch.Items.ShouldAllBe(i => i.StatusCode == OpcUaStatus.BadNoCommunication); batch.Items.Single(i => i.NodeId == "Plant/Modbus/dev1/speed").Realm.ShouldBe(AddressSpaceRealm.Raw); batch.Items.Single(i => i.NodeId == "filling/line1/station1/speed").Realm.ShouldBe(AddressSpaceRealm.Uns); // The control: the OTHER driver's node is untouched by drv-1's outage. batch.Items.ShouldNotContain(i => i.NodeId == "Plant/S7/dev2/other"); // ONE batch, not one message per node. publish.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); } /// /// A reconnect publishes nothing to the value nodes. Writing Good here would claim a quality /// the host has not observed — the device may be answering the transport and still returning /// garbage — and it would race the driver's own first real value. /// [Fact] public void Reconnect_publishes_no_value_or_status_message() { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), }, unsRefs: Array.Empty<(string, string, string, string, string?)>()); var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA); actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-1", Connected: false)); publish.ExpectMsg(PresenceBudget); actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-1", Connected: true)); publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); } // --------------------------------------------------------------------------------- // T9 — the recovery audit (the dedup-trap tripwire) // --------------------------------------------------------------------------------- /// /// The host publishes nothing Good on reconnect, so the only way a stuck-Bad node becomes visible /// is this audit: refs still latched some time after the driver reconnected are logged and metered. /// Here the driver re-publishes nothing at all — the pre-STAB-18 MQTT shape, and the exact failure /// any future push driver without an initial push would produce. /// [Fact] public void Refs_still_latched_after_a_reconnect_are_logged_and_metered() { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: SilentDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), (Driver: SilentDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false), }, unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) }); var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay); using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs"); actor.Tell(new DriverInstanceActor.ConnectivityChanged(SilentDriver, Connected: false)); publish.ExpectMsg(PresenceBudget); EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(TimeSpan.FromSeconds(10), () => actor.Tell(new DriverInstanceActor.ConnectivityChanged(SilentDriver, Connected: true))); // Two REFS, not three nodes: the latch is keyed by RawPath, and the UNS mirror shares its // backing tag's ref — the operator is being told how many tags did not come back, not how many // address-space nodes they project onto. AwaitCondition(() => meter.WithTag("driver_instance", SilentDriver) > 0, TimeSpan.FromSeconds(5)); meter.SumWithTag("driver_instance", SilentDriver).ShouldBe(2); } /// /// The control. A driver that DOES re-publish every latched ref on reconnect trips nothing — /// without this, the test above would pass against an audit that fires unconditionally, which would /// make the Warning noise rather than signal. /// [Fact] public void A_driver_that_republishes_every_ref_trips_no_audit() { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: RepublishingDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), (Driver: RepublishingDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false), }, unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) }); var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay); using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs"); actor.Tell(new DriverInstanceActor.ConnectivityChanged(RepublishingDriver, Connected: false)); publish.ExpectMsg(PresenceBudget); EventFilter.Warning(contains: "still BadNoCommunication").Expect(0, TimeSpan.FromSeconds(5), () => { actor.Tell(new DriverInstanceActor.ConnectivityChanged(RepublishingDriver, Connected: true)); // The driver's own re-publish — identical values, which is precisely the case a value-only // dedup would swallow; here it is the driver that force-raised, so both refs are pruned. actor.Tell(new DriverInstanceActor.AttributeValueBatchPublished(RepublishingDriver, new[] { new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/speed", 42.0, OpcUaStatus.Good, TsA), new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/temp", 7.5, OpcUaStatus.Good, TsA), })); publish.ExpectMsg(PresenceBudget); }); meter.WithTag("driver_instance", RepublishingDriver).ShouldBe(0); } /// /// A partial recovery is still reported, and reports only what is missing: one ref comes back, the /// other does not, and the audit counts exactly one. An audit that reported the whole latch (or /// nothing at all once anything arrived) would be useless on precisely the mixed case an operator /// needs it for. /// [Fact] public void A_partial_recovery_reports_only_the_refs_that_did_not_come_back() { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: PartialDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), (Driver: PartialDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false), }, unsRefs: Array.Empty<(string, string, string, string, string?)>()); var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay); using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs"); actor.Tell(new DriverInstanceActor.ConnectivityChanged(PartialDriver, Connected: false)); publish.ExpectMsg(PresenceBudget); EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(TimeSpan.FromSeconds(10), () => { actor.Tell(new DriverInstanceActor.ConnectivityChanged(PartialDriver, Connected: true)); actor.Tell(new DriverInstanceActor.AttributeValueBatchPublished(PartialDriver, new[] { new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/speed", 42.0, OpcUaStatus.Good, TsA), })); publish.ExpectMsg(PresenceBudget); }); AwaitCondition(() => meter.WithTag("driver_instance", PartialDriver) > 0, TimeSpan.FromSeconds(5)); meter.SumWithTag("driver_instance", PartialDriver).ShouldBe(1, "only the ref that never came back is reported"); } /// /// A driver that owns no value refs produces no message at all — an empty batch would cost a hop /// and a sink call for nothing on every flap of an alarm-only or freshly-added driver. /// [Fact] public void A_driver_with_no_value_refs_publishes_nothing() { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), }, unsRefs: Array.Empty<(string, string, string, string, string?)>()); var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA); actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-unknown", Connected: false)); publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); } /// /// Sums one counter's measurements, filtered by tag. A observes /// every measurement in the process, so filtering on instrument name alone would fold in a parallel /// test's (#503) — each test here therefore uses its own driver id and asserts on that tag. /// private sealed class MeterRecorder : IDisposable { private readonly MeterListener _listener; private readonly List<(long Value, KeyValuePair[] Tags)> _measurements = new(); private readonly object _gate = new(); public MeterRecorder(string instrumentName) { _listener = new MeterListener { InstrumentPublished = (instrument, listener) => { if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == instrumentName) listener.EnableMeasurementEvents(instrument); }, }; _listener.SetMeasurementEventCallback((_, value, tags, _) => { lock (_gate) _measurements.Add((value, tags.ToArray())); }); _listener.Start(); } public int WithTag(string key, string value) { lock (_gate) return _measurements.Count(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value))); } public long SumWithTag(string key, string value) { lock (_gate) return _measurements.Where(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value))) .Sum(m => m.Value); } public void Dispose() => _listener.Dispose(); } }