fix(runtime): comms-loss grace is per connection — max(LastSuccessfulRead, connectedSince) (archreview 03/S17)

2026-08-20 09:32:24 -04:00
parent 5fd65d190e
commit a801e2aefb
4 changed files with 141 additions and 3 deletions
@@ -40,6 +40,10 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest
    private static readonly TimeSpan FastPoll = TimeSpan.FromMilliseconds(50);
    private static readonly TimeSpan FastTimeout = TimeSpan.FromMilliseconds(250);

+    /// <summary>The threshold used by the per-connection grace pair (03/S17). Deliberately an order of
+    /// magnitude above <see cref="FastPoll"/> so the absence window below is a decision, not a race.</summary>
+    private static readonly TimeSpan Grace = TimeSpan.FromMilliseconds(1500);
+
    /// <summary>Drives the actor to Connected and returns once the connect has been announced.</summary>
    private (IActorRef Actor, Akka.TestKit.TestProbe Parent) ConnectDriver(
        HealthControlledStubDriver driver,
@@ -87,6 +91,14 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest
    ///     <c>ModbusDriver.ReadAsync</c> leaves latched whenever the bad tag is read last — but it is still
    ///     reading successfully, so <c>LastSuccessfulRead</c> keeps advancing. It must be left alone.
    ///     A state-only watchdog would tear this driver down on every threshold, forever.
+    ///     <para>
+    ///         <b>Falsification partner (03/S17):</b>
+    ///         <see cref="A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again"/>.
+    ///         The silence baseline is <c>max(LastSuccessfulRead, _connectedSinceUtc)</c>, and each test kills
+    ///         one term of that max — this one goes red if the <c>LastSuccessfulRead</c> term is dropped (its
+    ///         connect ages far past the threshold while its reads keep succeeding), that one goes red if the
+    ///         <c>_connectedSinceUtc</c> term is dropped (i.e. if the original <c>??</c> is restored).
+    ///     </para>
    /// </summary>
    [Fact]
    public void A_degraded_driver_that_is_still_reading_successfully_is_left_alone()
@@ -258,6 +270,77 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest

        parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
    }
+
+    /// <summary>
+    ///     03/S17 — <b>the grace is per connection.</b> Drivers deliberately <em>preserve</em>
+    ///     <see cref="DriverHealth.LastSuccessfulRead"/> across Degraded/Faulted health writes (that
+    ///     preservation is the #527 design), so for a device that accepts a TCP connection but never serves
+    ///     data the stamp stays frozen from before the outage — already older than the threshold. Measuring
+    ///     silence from that stamp alone (<c>??</c>) makes the <b>first</b> health tick of every new
+    ///     connection re-trip, so the reconnect cycle runs at health-tick cadence (~30 s in production)
+    ///     instead of the designed grace. Measuring from <c>max(LastSuccessfulRead, _connectedSinceUtc)</c>
+    ///     gives each connection its own full window, which is exactly what the <c>_connectedSinceUtc</c>
+    ///     re-stamp on every Connected entry already promises in its comment.
+    ///     <para>
+    ///         <b>Falsification partner:</b>
+    ///         <see cref="A_degraded_driver_that_is_still_reading_successfully_is_left_alone"/> — see its
+    ///         remarks for how the pair covers both terms of the max.
+    ///     </para>
+    ///     <para>
+    ///         The absence window carries the house positive control: a proven number of health-poll ticks
+    ///         processed <em>after</em> the reconnect, so "the watchdog did not re-trip" cannot pass because
+    ///         the actor stopped ticking. It is also deliberately far shorter than <see cref="Grace"/>, so
+    ///         it asserts a delay, not a disablement — <see cref="A_half_broken_device_is_still_declared_lost_once_the_grace_elapses"/>
+    ///         is the anti-vacuity half.
+    ///     </para>
+    /// </summary>
+    [Fact]
+    public void A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again()
+    {
+        var driver = new HealthControlledStubDriver
+        {
+            // The half-broken device: the socket connects, every read fails, and the last successful read is
+            // frozen well before the outage — preserved across the Degraded writes, as every driver does.
+            Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"),
+        };
+        var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100));
+
+        // The first trip is correct and expected — the device really has been silent for five minutes.
+        parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget)
+            .Connected.ShouldBeFalse();
+        // The retry re-initialises successfully (the socket still accepts), so a NEW connection starts here.
+        parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget)
+            .Connected.ShouldBeTrue();
+
+        var ticksAtReconnect = driver.HealthReadCount;
+        parent.ExpectNoMsg(TimeSpan.FromMilliseconds(400));
+        (driver.HealthReadCount - ticksAtReconnect).ShouldBeGreaterThanOrEqualTo(
+            4,
+            "health-poll ticks did not run after the reconnect — the absence assertion above would be vacuous");
+    }
+
+    /// <summary>
+    ///     The anti-vacuity half of the 03/S17 pair: the per-connection grace <b>delays</b> the watchdog, it
+    ///     does not disable it. The same half-broken device is still declared lost once the grace elapses —
+    ///     without this, a "fix" that simply stopped evaluating the watchdog after the first connection would
+    ///     pass <see cref="A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again"/>
+    ///     and look correct.
+    /// </summary>
+    [Fact]
+    public void A_half_broken_device_is_still_declared_lost_once_the_grace_elapses()
+    {
+        var driver = new HealthControlledStubDriver
+        {
+            Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"),
+        };
+        var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100));
+
+        parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget).Connected.ShouldBeFalse();
+        parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget).Connected.ShouldBeTrue();
+
+        // The second connection serves no data either, so once ITS grace elapses the watchdog trips again.
+        parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget).Connected.ShouldBeFalse();
+    }
 }

 /// <summary>