using Akka.Actor; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// /// Gitea #527 — the comms-loss watchdog that gives /// its first production sender. /// /// /// /// The defect. Nothing in src/ ever sent DisconnectObserved. A driver that /// connected and then lost its device stayed Connected indefinitely — publishing Bad for /// every tag, never re-initialising, and not recovering when the device returned. Live-confirmed /// against a stopped Modbus fixture, which needed a node restart. /// /// /// What these tests are really pinning is the trip CONDITION, not the plumbing. The obvious /// design — "non-Healthy for N consecutive polls" — is wrong, and /// is the test that /// says so. means connected but some data is bad, and /// several drivers write health per-tag inside their read loop, so one unreadable address can leave /// a perfectly healthy device's driver Degraded forever. Tripping on state alone would reconnect /// that driver every 90 s in production — worse than the bug being fixed. The conjunction with a /// frozen is what makes the signal honest, and /// pins the other half. /// /// /// Every absence assertion carries a positive control — a proven number of health-poll ticks /// actually processed () — so "the watchdog /// did not fire" can never pass merely because nothing ran. /// /// public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTestBase { private static readonly TimeSpan FastPoll = TimeSpan.FromMilliseconds(50); private static readonly TimeSpan FastTimeout = TimeSpan.FromMilliseconds(250); /// Drives the actor to Connected and returns once the connect has been announced. private (IActorRef Actor, Akka.TestKit.TestProbe Parent) ConnectDriver( HealthControlledStubDriver driver, TimeSpan? commsLossTimeout = null, TimeSpan? reconnectInterval = null) { var parent = CreateTestProbe(); var actor = parent.ChildActorOf(DriverInstanceActor.Props( driver, reconnectInterval: reconnectInterval ?? TimeSpan.FromSeconds(30), healthPollInterval: FastPoll, commsLossTimeout: commsLossTimeout ?? FastTimeout)); actor.Tell(new DriverInstanceActor.InitializeRequested("{}")); // Connected entry announces Connected=true; consuming it here leaves the probe holding only // whatever the watchdog does (or does not) send next. parent.ExpectMsg(TimeSpan.FromSeconds(3)) .Connected.ShouldBeTrue(); return (actor, parent); } /// /// The core of #527: a Connected driver whose device has gone away — non-Healthy, and no successful /// read since well before the threshold — is declared disconnected and announces comms loss to the /// host, which is the signal #477 and #481 both hang off. /// [Fact] public void A_connected_driver_that_stops_reading_successfully_is_declared_disconnected() { var driver = new HealthControlledStubDriver { // The live shape: reads fail, the driver reports Degraded, and LastSuccessfulRead is frozen // at the instant the device died. Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"), }; var (_, parent) = ConnectDriver(driver); parent.ExpectMsg(TimeSpan.FromSeconds(3)) .Connected.ShouldBeFalse(); } /// /// The falsifying control for the trip condition, and the reason it is not "non-Healthy for N polls". /// This driver is permanently Degraded — modelling one unreadable address among many, which /// ModbusDriver.ReadAsync leaves latched whenever the bad tag is read last — but it is still /// reading successfully, so LastSuccessfulRead keeps advancing. It must be left alone. /// A state-only watchdog would tear this driver down on every threshold, forever. /// [Fact] public void A_degraded_driver_that_is_still_reading_successfully_is_left_alone() { var driver = new HealthControlledStubDriver { // Recomputed per call: Degraded, but LastSuccessfulRead is always fresh. HealthFactory = () => new DriverHealth(DriverState.Degraded, DateTime.UtcNow, "one bad tag"), }; var (_, parent) = ConnectDriver(driver); // Positive control: prove the watchdog genuinely ran, many times, over a span far longer than the // trip threshold. Without this the absence assertion below would pass against a watchdog that // never executed at all. AwaitCondition( () => driver.HealthReadCount >= 12, TimeSpan.FromSeconds(5), "health-poll ticks did not run — the absence assertion below would be vacuous"); driver.ObservedSpan.ShouldBeGreaterThan(FastTimeout, "the observation window never exceeded the trip threshold"); parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } /// /// The other conjunct. A Healthy driver with nothing subscribed legitimately never reads, so its /// LastSuccessfulRead is arbitrarily stale. Staleness alone must not declare it dead. /// [Fact] public void An_idle_healthy_driver_with_a_stale_last_read_is_left_alone() { var driver = new HealthControlledStubDriver { Health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow.AddHours(-2), null), }; var (_, parent) = ConnectDriver(driver); AwaitCondition( () => driver.HealthReadCount >= 12, TimeSpan.FromSeconds(5), "health-poll ticks did not run — the absence assertion below would be vacuous"); parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } /// /// A driver that connects but never manages a single read reports LastSuccessfulRead = null, /// so staleness has no natural baseline. The watchdog measures from the Connected entry instead — /// otherwise a never-read driver would be the one case that could never be declared dead. /// [Fact] public void A_driver_that_never_reads_after_connecting_is_declared_disconnected() { var driver = new HealthControlledStubDriver { Health = new DriverHealth(DriverState.Degraded, LastSuccessfulRead: null, "never read"), }; var (_, parent) = ConnectDriver(driver); parent.ExpectMsg(TimeSpan.FromSeconds(3)) .Connected.ShouldBeFalse(); } /// /// A that throws is a failed observation, not evidence that the /// device is gone. Treating it as a trip would let a buggy health accessor cycle a working driver. /// [Fact] public void A_throwing_GetHealth_is_not_evidence_of_comms_loss() { var driver = new HealthControlledStubDriver { GetHealthShouldThrow = true }; var (_, parent) = ConnectDriver(driver); AwaitCondition( () => driver.HealthReadCount >= 12, TimeSpan.FromSeconds(5), "health-poll ticks did not run — the absence assertion below would be vacuous"); parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } /// /// #527's second acceptance criterion, end to end: the driver recovers by itself once the device /// returns, with no node restart. Trip → Reconnecting → retry re-initialises → Connected, and the /// host is told connectivity is back. /// [Fact] public void The_driver_recovers_on_its_own_once_the_device_returns() { var driver = new HealthControlledStubDriver { Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"), }; var (_, parent) = ConnectDriver(driver, reconnectInterval: TimeSpan.FromMilliseconds(100)); parent.ExpectMsg(TimeSpan.FromSeconds(3)) .Connected.ShouldBeFalse(); // The device comes back: the next initialise attempt succeeds and reads resume. driver.Health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); parent.ExpectMsg(TimeSpan.FromSeconds(5)) .Connected.ShouldBeTrue(); driver.InitializeCount.ShouldBeGreaterThan(1, "the driver was never re-initialised"); } /// /// The leak guard. Several drivers overwrite an transport field /// inside InitializeAsync without disposing the previous one (ModbusDriver._transport, /// MqttDriver._connection), so re-initialising without a teardown leaks one socket per /// attempt. Latent while only an operator's Force Reconnect re-entered this path; the watchdog /// makes it automatic and unbounded, so the retry must shut down first. /// [Fact] public void A_reconnect_attempt_tears_the_dead_transport_down_before_re_initialising() { var driver = new HealthControlledStubDriver { Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"), }; var (_, parent) = ConnectDriver(driver, reconnectInterval: TimeSpan.FromMilliseconds(100)); parent.ExpectMsg(TimeSpan.FromSeconds(3)) .Connected.ShouldBeFalse(); AwaitCondition( () => driver.InitializeCount >= 2, TimeSpan.FromSeconds(5), "the reconnect retry never re-initialised the driver"); // Ordering, not just counts: EVERY re-initialise must be immediately preceded by a shutdown, since // it is the initialise that overwrites the transport field. Asserting ShutdownCount > 0 alone would // also pass on a teardown that happened afterwards, which frees nothing before the overwrite. // // The sequence is deliberately not pinned to a fixed length: this driver reports stale health even // after a successful re-init, so the watchdog trips again and the cycle repeats — the real // behaviour for a device that accepts a connection but serves no data. Every additional cycle is // another chance for the invariant to be violated, so a longer run is a stronger test, not a // flakier one. var lifecycle = driver.Lifecycle; lifecycle[0].ShouldBe("init", "the first connect must not tear down a driver that was never up"); for (var i = 1; i < lifecycle.Count; i++) { if (lifecycle[i] != "init") continue; lifecycle[i - 1].ShouldBe( "shutdown", $"re-initialise at index {i} was not preceded by a teardown — the previous transport leaks. " + $"Full sequence: [{string.Join(", ", lifecycle)}]"); } } /// /// The escape hatch is real: a zero threshold disables the watchdog entirely, leaving the /// pre-#527 behaviour. Pinned because tests elsewhere rely on a Connected driver staying Connected /// while reporting non-Healthy. /// [Fact] public void A_zero_threshold_disables_the_watchdog() { var driver = new HealthControlledStubDriver { Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"), }; var (_, parent) = ConnectDriver(driver, commsLossTimeout: TimeSpan.Zero); AwaitCondition( () => driver.HealthReadCount >= 12, TimeSpan.FromSeconds(5), "health-poll ticks did not run — the absence assertion below would be vacuous"); parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } } /// /// A whose health is fully driven by the test, counting observations so an /// absence assertion can prove the watchdog actually ran, and recording lifecycle call ORDER so the /// teardown-before-reinitialise guard can assert sequence rather than mere counts. /// internal sealed class HealthControlledStubDriver : StubDriver { private readonly Lock _gate = new(); private int _healthReadCount; private DateTime _firstReadUtc; private DateTime _lastReadUtc; /// The health returned by every call; ignored when /// is set. public DriverHealth Health { get; set; } = new(DriverState.Healthy, DateTime.UtcNow, null); /// Per-call health, for modelling a driver whose LastSuccessfulRead keeps advancing. public Func? HealthFactory { get; set; } /// When true, throws — a failed observation. public bool GetHealthShouldThrow { get; set; } /// Number of calls; the positive control for absence assertions. public int HealthReadCount => Volatile.Read(ref _healthReadCount); /// Wall-clock span between the first and last observation — proves an absence window /// genuinely exceeded the trip threshold rather than merely counting fast ticks. public TimeSpan ObservedSpan { get { lock (_gate) return _lastReadUtc - _firstReadUtc; } } /// Ordered lifecycle log: "init" / "shutdown", in call order. public IReadOnlyList Lifecycle { get { lock (_gate) return _lifecycle.ToArray(); } } private readonly List _lifecycle = new(); /// public override async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { lock (_gate) _lifecycle.Add("init"); await base.InitializeAsync(driverConfigJson, cancellationToken); } /// public override Task ShutdownAsync(CancellationToken cancellationToken) { lock (_gate) _lifecycle.Add("shutdown"); return base.ShutdownAsync(cancellationToken); } /// public override DriverHealth GetHealth() { Interlocked.Increment(ref _healthReadCount); lock (_gate) { var now = DateTime.UtcNow; if (_firstReadUtc == default) _firstReadUtc = now; _lastReadUtc = now; } if (GetHealthShouldThrow) throw new InvalidOperationException("stub-health-fail"); return HealthFactory is { } factory ? factory() : Health; } }