using System.Diagnostics.Metrics; using Akka.Actor; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; using static ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness.V3ArtifactSeeder; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// /// The 03/S15 contract test: comms loss → value nodes Bad → the driver's own re-publish clears /// them. A real is spawned by a real /// around a scripted driver, so the whole seam runs: the child's /// Reconnecting entry announces comms loss, the host stamps the value nodes /// BadNoCommunication, the child reconnects and re-subscribes, and the driver's initial push is /// what makes the nodes Good again. /// /// ⚠️ Vacuity rule — the pushed value is IDENTICAL to the pre-outage one. The host writes /// nothing Good on reconnect by design, so recovery depends entirely on a re-publish landing; and every /// dedup on that path (a driver's last-value cache, a consumer's last-published memory) keys on value. /// A recovery test whose driver pushes a different value therefore passes against a build with /// no force-republish anywhere — the value change alone defeats every dedup — and is not this /// contract's guard. The same-value push is the whole point. /// /// The falsification control is /// : a driver whose /// SubscribeAsync pushes nothing (the pre-STAB-18 MQTT shape) leaves the nodes Bad and is caught /// only by the audit. It proves the test above can fail. /// public sealed class DriverHostCommsLossRecoveryTests : RuntimeActorTestBase { private static readonly NodeId TestNode = NodeId.Parse("driver-commsloss-recovery"); private static readonly RevisionHash RevA = RevisionHash.Parse(new string('d', 64)); /// Audit window measured from the RECONNECT, not from the outage — the initial push lands /// within milliseconds of Connected, so this is generous. private static readonly TimeSpan AuditDelay = TimeSpan.FromSeconds(2); /// The child's reconnect retry runs on DriverInstanceActor.DefaultReconnectInterval /// (10 s) and the host exposes no seam to shorten it, so every budget here clears that. private static readonly TimeSpan ReconnectBudget = TimeSpan.FromSeconds(25); private const string SpeedRef = "Plant/Modbus/dev1/speed"; private const string UnsSpeedNode = "filling/line1/station1/speed"; [Fact] public void An_identical_value_pushed_on_resubscribe_recovers_the_nodes() { var driver = new ScriptedSubscribeDriver { InstanceId = "drv-recover", InitialPush = (SpeedRef, 5.0, OpcUaStatus.Good) }; var (host, publish) = Arrange(driver, "drv-recover"); using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs"); // Steady state: the driver's FIRST subscribe already pushed 5 Good, and the host fanned it to // raw + UNS. (The push on subscribe is the behaviour under test, so it is also how the pre-outage // value gets there — no separate seeding call, which would leave an extra batch in the probe.) var first = publish.ExpectMsg(PresenceBudget); first.Items.Select(i => i.NodeId).ShouldBe(new[] { SpeedRef, UnsSpeedNode }, ignoreOrder: true); first.Items.ShouldAllBe(i => Equals(i.Value, 5.0)); // The outage: the child re-enters Reconnecting and announces comms loss to the host. host.Tell(new ReconnectDriver("c1", "drv-recover", "test-operator", Guid.NewGuid())); var staled = publish.ExpectMsg(ReconnectBudget); staled.Items.Select(i => i.NodeId).ShouldBe(new[] { SpeedRef, UnsSpeedNode }, ignoreOrder: true); staled.Items.ShouldAllBe(i => i.StatusCode == OpcUaStatus.BadNoCommunication); // Recovery: the reconnect re-subscribes, and the driver pushes the SAME value it held before. var recovered = publish.ExpectMsg(ReconnectBudget); recovered.Items.Select(i => i.NodeId).ShouldBe(new[] { SpeedRef, UnsSpeedNode }, ignoreOrder: true); recovered.Items.ShouldAllBe(i => i.StatusCode == OpcUaStatus.Good); recovered.Items.ShouldAllBe(i => Equals(i.Value, 5.0)); // …and the audit finds nothing outstanding. EventFilter.Warning(contains: "still BadNoCommunication") .Expect(0, AuditDelay + TimeSpan.FromSeconds(3), () => { }); meter.SumWithTag("driver_instance", "drv-recover").ShouldBe(0); } /// /// The falsification control. Same scenario, but the driver's SubscribeAsync pushes /// nothing — exactly what MQTT did before 05/STAB-18, and what any push driver without an initial /// push does. The nodes stay Bad (the host writes no Good of its own), and the ONLY thing that says /// so is the recovery audit. /// [Fact] public void A_silent_resubscribe_leaves_the_nodes_Bad_and_trips_the_audit() { var driver = new ScriptedSubscribeDriver { InstanceId = "drv-silent", InitialPush = null }; var (host, publish) = Arrange(driver, "drv-silent"); using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs"); driver.FireDataChange(SpeedRef, 5.0, OpcUaStatus.Good); publish.ExpectMsg(PresenceBudget); EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(ReconnectBudget, () => { host.Tell(new ReconnectDriver("c1", "drv-silent", "test-operator", Guid.NewGuid())); publish.ExpectMsg(ReconnectBudget); // Nothing Good ever arrives — assert it rather than merely not looking. publish.ExpectNoMsg(TimeSpan.FromSeconds(1)); }); // One ref (the raw tag); its UNS mirror shares it. AwaitCondition(() => meter.SumWithTag("driver_instance", "drv-silent") == 1, TimeSpan.FromSeconds(5)); driver.SubscribeCount.ShouldBeGreaterThanOrEqualTo(2, "the child must actually have re-subscribed — otherwise this proves nothing about the push"); } private (IActorRef Host, Akka.TestKit.TestProbe Publish) Arrange(ScriptedSubscribeDriver driver, string driverId) { var db = NewInMemoryDbFactory(); var deploymentId = SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: driverId, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false), }, unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) }, enabled: true); var (host, publish, _) = SpawnHostAndApply( this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay, driverFactory: new SingleDriverFactory(driver)); // The child is spawned + initialised by the apply; wait for its first subscribe so the scripted // pushes below are attributed to a live subscription. AwaitCondition(() => driver.SubscribeCount >= 1, PresenceBudget); return (host, publish); } /// Hands the host one scripted driver, whatever type/instance it asks for. private sealed class SingleDriverFactory(IDriver driver) : IDriverFactory { public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) => driver; public IReadOnlyCollection SupportedTypes { get; } = ["Modbus"]; } /// /// A subscribable driver whose optionally performs an initial /// push — the poll engine's forceRaise: true and MQTT's post-STAB-18 behaviour, modelled /// at the seam the host actually observes. With null it is the silent /// shape the control test needs. /// private sealed class ScriptedSubscribeDriver : IDriver, ISubscribable { private readonly StubHandle _handle = new(); private int _subscribeCount; public event EventHandler? OnDataChange; /// The value re-published on every (re)subscribe, or null for a silent driver. public (string Ref, object? Value, uint StatusCode)? InitialPush { get; init; } /// The instance id the child actor adopts — DriverInstanceActor takes it from the /// DRIVER, not from the spec, and the host's ref map is keyed by the spec's id. They must agree or /// every published value is dropped as unmapped. public required string InstanceId { get; init; } public int SubscribeCount => Volatile.Read(ref _subscribeCount); public string DriverInstanceId => InstanceId; public string DriverType => "Modbus"; public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask; // Stable health: the #527 watchdog must not fire on its own here — the outage in these tests is // driven explicitly, and a health-driven trip would make the timing non-deterministic. public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null); public Task SubscribeAsync( IReadOnlyList fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) { Interlocked.Increment(ref _subscribeCount); if (InitialPush is { } push && fullReferences.Contains(push.Ref)) FireDataChange(push.Ref, push.Value, push.StatusCode); return Task.FromResult(_handle); } public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask; public void FireDataChange(string fullRef, object? value, uint statusCode) { var snapshot = new DataValueSnapshot(value, statusCode, DateTime.UtcNow, DateTime.UtcNow); OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, snapshot)); } } /// Tag-filtered counter recorder — see the note in /// DriverHostActorCommsLossValueNodesTests on why the instrument name alone is not enough. private sealed class MeterRecorder : IDisposable { private readonly MeterListener _listener; private readonly List<(long Value, KeyValuePair[] Tags)> _measurements = new(); private readonly object _gate = new(); public MeterRecorder(string instrumentName) { _listener = new MeterListener { InstrumentPublished = (instrument, listener) => { if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == instrumentName) listener.EnableMeasurementEvents(instrument); }, }; _listener.SetMeasurementEventCallback((_, value, tags, _) => { lock (_gate) _measurements.Add((value, tags.ToArray())); }); _listener.Start(); } public long SumWithTag(string key, string value) { lock (_gate) return _measurements.Where(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value))) .Sum(m => m.Value); } public void Dispose() => _listener.Dispose(); } }