using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
///
/// Gitea #527 — the comms-loss watchdog that gives
/// its first production sender.
///
///
///
/// The defect. Nothing in src/ ever sent DisconnectObserved. A driver that
/// connected and then lost its device stayed Connected indefinitely — publishing Bad for
/// every tag, never re-initialising, and not recovering when the device returned. Live-confirmed
/// against a stopped Modbus fixture, which needed a node restart.
///
///
/// What these tests are really pinning is the trip CONDITION, not the plumbing. The obvious
/// design — "non-Healthy for N consecutive polls" — is wrong, and
/// is the test that
/// says so. means connected but some data is bad, and
/// several drivers write health per-tag inside their read loop, so one unreadable address can leave
/// a perfectly healthy device's driver Degraded forever. Tripping on state alone would reconnect
/// that driver every 90 s in production — worse than the bug being fixed. The conjunction with a
/// frozen is what makes the signal honest, and
/// pins the other half.
///
///
/// Every absence assertion carries a positive control — a proven number of health-poll ticks
/// actually processed () — so "the watchdog
/// did not fire" can never pass merely because nothing ran.
///
///
public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTestBase
{
private static readonly TimeSpan FastPoll = TimeSpan.FromMilliseconds(50);
private static readonly TimeSpan FastTimeout = TimeSpan.FromMilliseconds(250);
/// The threshold used by the per-connection grace pair (03/S17). Deliberately an order of
/// magnitude above so the absence window below is a decision, not a race.
private static readonly TimeSpan Grace = TimeSpan.FromMilliseconds(1500);
/// Drives the actor to Connected and returns once the connect has been announced.
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) ConnectDriver(
HealthControlledStubDriver driver,
TimeSpan? commsLossTimeout = null,
TimeSpan? reconnectInterval = null)
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver,
reconnectInterval: reconnectInterval ?? TimeSpan.FromSeconds(30),
healthPollInterval: FastPoll,
commsLossTimeout: commsLossTimeout ?? FastTimeout));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
// Connected entry announces Connected=true; consuming it here leaves the probe holding only
// whatever the watchdog does (or does not) send next.
parent.ExpectMsg(TimeSpan.FromSeconds(3))
.Connected.ShouldBeTrue();
return (actor, parent);
}
///
/// The core of #527: a Connected driver whose device has gone away — non-Healthy, and no successful
/// read since well before the threshold — is declared disconnected and announces comms loss to the
/// host, which is the signal #477 and #481 both hang off.
///
[Fact]
public void A_connected_driver_that_stops_reading_successfully_is_declared_disconnected()
{
var driver = new HealthControlledStubDriver
{
// The live shape: reads fail, the driver reports Degraded, and LastSuccessfulRead is frozen
// at the instant the device died.
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"),
};
var (_, parent) = ConnectDriver(driver);
parent.ExpectMsg(TimeSpan.FromSeconds(3))
.Connected.ShouldBeFalse();
}
///
/// The falsifying control for the trip condition, and the reason it is not "non-Healthy for N polls".
/// This driver is permanently Degraded — modelling one unreadable address among many, which
/// ModbusDriver.ReadAsync leaves latched whenever the bad tag is read last — but it is still
/// reading successfully, so LastSuccessfulRead keeps advancing. It must be left alone.
/// A state-only watchdog would tear this driver down on every threshold, forever.
///
/// Falsification partner (03/S17):
/// .
/// The silence baseline is max(LastSuccessfulRead, _connectedSinceUtc), and each test kills
/// one term of that max — this one goes red if the LastSuccessfulRead term is dropped (its
/// connect ages far past the threshold while its reads keep succeeding), that one goes red if the
/// _connectedSinceUtc term is dropped (i.e. if the original ?? is restored).
///
///
[Fact]
public void A_degraded_driver_that_is_still_reading_successfully_is_left_alone()
{
var driver = new HealthControlledStubDriver
{
// Recomputed per call: Degraded, but LastSuccessfulRead is always fresh.
HealthFactory = () => new DriverHealth(DriverState.Degraded, DateTime.UtcNow, "one bad tag"),
};
var (_, parent) = ConnectDriver(driver);
// Positive control: prove the watchdog genuinely ran, many times, over a span far longer than the
// trip threshold. Without this the absence assertion below would pass against a watchdog that
// never executed at all.
AwaitCondition(
() => driver.HealthReadCount >= 12,
TimeSpan.FromSeconds(5),
"health-poll ticks did not run — the absence assertion below would be vacuous");
driver.ObservedSpan.ShouldBeGreaterThan(FastTimeout, "the observation window never exceeded the trip threshold");
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
///
/// The other conjunct. A Healthy driver with nothing subscribed legitimately never reads, so its
/// LastSuccessfulRead is arbitrarily stale. Staleness alone must not declare it dead.
///
[Fact]
public void An_idle_healthy_driver_with_a_stale_last_read_is_left_alone()
{
var driver = new HealthControlledStubDriver
{
Health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow.AddHours(-2), null),
};
var (_, parent) = ConnectDriver(driver);
AwaitCondition(
() => driver.HealthReadCount >= 12,
TimeSpan.FromSeconds(5),
"health-poll ticks did not run — the absence assertion below would be vacuous");
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
///
/// A driver that connects but never manages a single read reports LastSuccessfulRead = null,
/// so staleness has no natural baseline. The watchdog measures from the Connected entry instead —
/// otherwise a never-read driver would be the one case that could never be declared dead.
///
[Fact]
public void A_driver_that_never_reads_after_connecting_is_declared_disconnected()
{
var driver = new HealthControlledStubDriver
{
Health = new DriverHealth(DriverState.Degraded, LastSuccessfulRead: null, "never read"),
};
var (_, parent) = ConnectDriver(driver);
parent.ExpectMsg(TimeSpan.FromSeconds(3))
.Connected.ShouldBeFalse();
}
///
/// A that throws is a failed observation, not evidence that the
/// device is gone. Treating it as a trip would let a buggy health accessor cycle a working driver.
///
[Fact]
public void A_throwing_GetHealth_is_not_evidence_of_comms_loss()
{
var driver = new HealthControlledStubDriver { GetHealthShouldThrow = true };
var (_, parent) = ConnectDriver(driver);
AwaitCondition(
() => driver.HealthReadCount >= 12,
TimeSpan.FromSeconds(5),
"health-poll ticks did not run — the absence assertion below would be vacuous");
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
///
/// #527's second acceptance criterion, end to end: the driver recovers by itself once the device
/// returns, with no node restart. Trip → Reconnecting → retry re-initialises → Connected, and the
/// host is told connectivity is back.
///
[Fact]
public void The_driver_recovers_on_its_own_once_the_device_returns()
{
var driver = new HealthControlledStubDriver
{
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"),
};
var (_, parent) = ConnectDriver(driver, reconnectInterval: TimeSpan.FromMilliseconds(100));
parent.ExpectMsg(TimeSpan.FromSeconds(3))
.Connected.ShouldBeFalse();
// The device comes back: the next initialise attempt succeeds and reads resume.
driver.Health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
parent.ExpectMsg(TimeSpan.FromSeconds(5))
.Connected.ShouldBeTrue();
driver.InitializeCount.ShouldBeGreaterThan(1, "the driver was never re-initialised");
}
///
/// The leak guard. Several drivers overwrite an transport field
/// inside InitializeAsync without disposing the previous one (ModbusDriver._transport,
/// MqttDriver._connection), so re-initialising without a teardown leaks one socket per
/// attempt. Latent while only an operator's Force Reconnect re-entered this path; the watchdog
/// makes it automatic and unbounded, so the retry must shut down first.
///
[Fact]
public void A_reconnect_attempt_tears_the_dead_transport_down_before_re_initialising()
{
var driver = new HealthControlledStubDriver
{
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"),
};
var (_, parent) = ConnectDriver(driver, reconnectInterval: TimeSpan.FromMilliseconds(100));
parent.ExpectMsg(TimeSpan.FromSeconds(3))
.Connected.ShouldBeFalse();
AwaitCondition(
() => driver.InitializeCount >= 2,
TimeSpan.FromSeconds(5),
"the reconnect retry never re-initialised the driver");
// Ordering, not just counts: EVERY re-initialise must be immediately preceded by a shutdown, since
// it is the initialise that overwrites the transport field. Asserting ShutdownCount > 0 alone would
// also pass on a teardown that happened afterwards, which frees nothing before the overwrite.
//
// The sequence is deliberately not pinned to a fixed length: this driver reports stale health even
// after a successful re-init, so the watchdog trips again and the cycle repeats — the real
// behaviour for a device that accepts a connection but serves no data. Every additional cycle is
// another chance for the invariant to be violated, so a longer run is a stronger test, not a
// flakier one.
var lifecycle = driver.Lifecycle;
lifecycle[0].ShouldBe("init", "the first connect must not tear down a driver that was never up");
for (var i = 1; i < lifecycle.Count; i++)
{
if (lifecycle[i] != "init") continue;
lifecycle[i - 1].ShouldBe(
"shutdown",
$"re-initialise at index {i} was not preceded by a teardown — the previous transport leaks. " +
$"Full sequence: [{string.Join(", ", lifecycle)}]");
}
}
///
/// The escape hatch is real: a zero threshold disables the watchdog entirely, leaving the
/// pre-#527 behaviour. Pinned because tests elsewhere rely on a Connected driver staying Connected
/// while reporting non-Healthy.
///
[Fact]
public void A_zero_threshold_disables_the_watchdog()
{
var driver = new HealthControlledStubDriver
{
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "socket timeout"),
};
var (_, parent) = ConnectDriver(driver, commsLossTimeout: TimeSpan.Zero);
AwaitCondition(
() => driver.HealthReadCount >= 12,
TimeSpan.FromSeconds(5),
"health-poll ticks did not run — the absence assertion below would be vacuous");
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
///
/// 03/S17 — the grace is per connection. Drivers deliberately preserve
/// across Degraded/Faulted health writes (that
/// preservation is the #527 design), so for a device that accepts a TCP connection but never serves
/// data the stamp stays frozen from before the outage — already older than the threshold. Measuring
/// silence from that stamp alone (??) makes the first health tick of every new
/// connection re-trip, so the reconnect cycle runs at health-tick cadence (~30 s in production)
/// instead of the designed grace. Measuring from max(LastSuccessfulRead, _connectedSinceUtc)
/// gives each connection its own full window, which is exactly what the _connectedSinceUtc
/// re-stamp on every Connected entry already promises in its comment.
///
/// Falsification partner:
/// — see its
/// remarks for how the pair covers both terms of the max.
///
///
/// The absence window carries the house positive control: a proven number of health-poll ticks
/// processed after the reconnect, so "the watchdog did not re-trip" cannot pass because
/// the actor stopped ticking. It is also deliberately far shorter than , so
/// it asserts a delay, not a disablement —
/// is the anti-vacuity half.
///
///
[Fact]
public void A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again()
{
var driver = new HealthControlledStubDriver
{
// The half-broken device: the socket connects, every read fails, and the last successful read is
// frozen well before the outage — preserved across the Degraded writes, as every driver does.
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"),
};
var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100));
// The first trip is correct and expected — the device really has been silent for five minutes.
parent.ExpectMsg(PresenceBudget)
.Connected.ShouldBeFalse();
// The retry re-initialises successfully (the socket still accepts), so a NEW connection starts here.
parent.ExpectMsg(PresenceBudget)
.Connected.ShouldBeTrue();
var ticksAtReconnect = driver.HealthReadCount;
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(400));
(driver.HealthReadCount - ticksAtReconnect).ShouldBeGreaterThanOrEqualTo(
4,
"health-poll ticks did not run after the reconnect — the absence assertion above would be vacuous");
}
///
/// The anti-vacuity half of the 03/S17 pair: the per-connection grace delays the watchdog, it
/// does not disable it. The same half-broken device is still declared lost once the grace elapses —
/// without this, a "fix" that simply stopped evaluating the watchdog after the first connection would
/// pass
/// and look correct.
///
[Fact]
public void A_half_broken_device_is_still_declared_lost_once_the_grace_elapses()
{
var driver = new HealthControlledStubDriver
{
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"),
};
var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100));
parent.ExpectMsg(PresenceBudget).Connected.ShouldBeFalse();
parent.ExpectMsg(PresenceBudget).Connected.ShouldBeTrue();
// The second connection serves no data either, so once ITS grace elapses the watchdog trips again.
parent.ExpectMsg(PresenceBudget).Connected.ShouldBeFalse();
}
}
///
/// A whose health is fully driven by the test, counting observations so an
/// absence assertion can prove the watchdog actually ran, and recording lifecycle call ORDER so the
/// teardown-before-reinitialise guard can assert sequence rather than mere counts.
///
internal sealed class HealthControlledStubDriver : StubDriver
{
private readonly Lock _gate = new();
private int _healthReadCount;
private DateTime _firstReadUtc;
private DateTime _lastReadUtc;
/// The health returned by every call; ignored when
/// is set.
public DriverHealth Health { get; set; } = new(DriverState.Healthy, DateTime.UtcNow, null);
/// Per-call health, for modelling a driver whose LastSuccessfulRead keeps advancing.
public Func? HealthFactory { get; set; }
/// When true, throws — a failed observation.
public bool GetHealthShouldThrow { get; set; }
/// Number of calls; the positive control for absence assertions.
public int HealthReadCount => Volatile.Read(ref _healthReadCount);
/// Wall-clock span between the first and last observation — proves an absence window
/// genuinely exceeded the trip threshold rather than merely counting fast ticks.
public TimeSpan ObservedSpan
{
get { lock (_gate) return _lastReadUtc - _firstReadUtc; }
}
/// Ordered lifecycle log: "init" / "shutdown", in call order.
public IReadOnlyList Lifecycle
{
get { lock (_gate) return _lifecycle.ToArray(); }
}
private readonly List _lifecycle = new();
///
public override async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
{
lock (_gate) _lifecycle.Add("init");
await base.InitializeAsync(driverConfigJson, cancellationToken);
}
///
public override Task ShutdownAsync(CancellationToken cancellationToken)
{
lock (_gate) _lifecycle.Add("shutdown");
return base.ShutdownAsync(cancellationToken);
}
///
public override DriverHealth GetHealth()
{
Interlocked.Increment(ref _healthReadCount);
lock (_gate)
{
var now = DateTime.UtcNow;
if (_firstReadUtc == default) _firstReadUtc = now;
_lastReadUtc = now;
}
if (GetHealthShouldThrow) throw new InvalidOperationException("stub-health-fail");
return HealthFactory is { } factory ? factory() : Health;
}
}