fix(runtime): single-flight reconnect attempts, bounded pre-reconnect teardown (A2)

The Reconnecting retry timer started a new ShutdownAsync + InitializeAsync pair on every tick whether or not the previous attempt had finished; the generation token only discarded the older RESULT, so with an Initialize bound longer than the cadence the teardown ran underneath an attempt still inside the driver (the dispose-before-overwrite race). An _initInFlight gate, cleared by the piped InitializeSucceeded/Failed, skips a tick while an attempt is running; the pre-reconnect teardown is bounded by the shutdown timeout; PostStop cancels an in-flight attempt before shutting the driver down. Review ref: 02/H2. Refs #731 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 06:26:45 -04:00
parent eb23053700
commit bfd5b7b015
2 changed files with 195 additions and 5 deletions
@@ -0,0 +1,126 @@
+using Akka.Actor;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
+
+/// <summary>
+///     Review 02 H2: the <c>Reconnecting</c> retry timer fired every <c>reconnectInterval</c> whether or not the
+///     previous attempt was still running — <c>_initGeneration</c> only discarded the older attempt's RESULT. With an
+///     Initialize bound longer than the cadence (Tier C's 15 s vs. the 10 s timer, or the test harness's unbounded
+///     invoker) attempt N+1's <c>ShutdownAsync</c> ran while attempt N's <c>InitializeAsync</c> was still inside the
+///     driver: the dispose-before-overwrite race the drivers' rule file forbids, once per overlap, for as long as
+///     the device stayed down. The retry is now single-flight.
+/// </summary>
+public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActorTestBase
+{
+    [Fact]
+    public void Shutdown_and_Initialize_never_overlap_while_the_init_outlasts_the_retry_cadence()
+    {
+        // A 400 ms failing init against a 50 ms retry cadence: without the gate the timer starts a new
+        // Shutdown+Initialize pair every 50 ms on top of the one still running (eight-deep overlap).
+        var driver = new InterleavingDriver(initDuration: TimeSpan.FromMilliseconds(400), failUntilAttempt: 4);
+        var parent = CreateTestProbe();
+        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
+            driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero));
+
+        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
+
+        // Positive control: the gate skips ticks, it does not stop retrying — the 4th attempt succeeds and the
+        // actor connects.
+        parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
+
+        driver.Attempts.ShouldBeGreaterThanOrEqualTo(4);
+        driver.Overlaps.ShouldBe(0,
+            $"ShutdownAsync entered while an InitializeAsync was still running. Interleaving: {driver.Interleaving}");
+    }
+
+    [Fact]
+    public void A_pre_reconnect_teardown_that_hangs_is_abandoned_at_the_bound_and_the_init_still_runs()
+    {
+        var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 2)
+        {
+            // Only the SECOND shutdown (the reconnect teardown) hangs; the first attempt has none.
+            ShutdownBehavior = _ => never.Task,
+        };
+        var parent = CreateTestProbe();
+        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
+            driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
+            shutdownTimeout: TimeSpan.FromMilliseconds(200)));
+
+        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
+
+        parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
+        never.Task.IsCompleted.ShouldBeFalse("the teardown is still hung — the attempt did not wait for it");
+        driver.Attempts.ShouldBe(2);
+    }
+
+    /// <summary>Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.</summary>
+    private sealed class InterleavingDriver : IDriver
+    {
+        private readonly TimeSpan _initDuration;
+        private readonly int _failUntilAttempt;
+        private readonly Lock _gate = new();
+        private readonly List<string> _events = new();
+        private int _attempts;
+        private int _initsInProgress;
+        private int _overlaps;
+
+        public InterleavingDriver(TimeSpan initDuration, int failUntilAttempt)
+        {
+            _initDuration = initDuration;
+            _failUntilAttempt = failUntilAttempt;
+        }
+
+        public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
+
+        public int Attempts => Volatile.Read(ref _attempts);
+        public int Overlaps => Volatile.Read(ref _overlaps);
+        public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }
+
+        public string DriverInstanceId => "interleaving-1";
+        public string DriverType => "Interleaving";
+
+        public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
+        {
+            var attempt = Interlocked.Increment(ref _attempts);
+            Interlocked.Increment(ref _initsInProgress);
+            Record($"init{attempt}>");
+            try
+            {
+                if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
+                if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
+            }
+            finally
+            {
+                Record($"<init{attempt}");
+                Interlocked.Decrement(ref _initsInProgress);
+            }
+        }
+
+        public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
+
+        public async Task ShutdownAsync(CancellationToken cancellationToken)
+        {
+            if (Volatile.Read(ref _initsInProgress) > 0) Interlocked.Increment(ref _overlaps);
+            Record("shutdown>");
+            try
+            {
+                if (ShutdownBehavior is { } behavior) await behavior(cancellationToken);
+            }
+            finally
+            {
+                Record("<shutdown");
+            }
+        }
+
+        public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
+
+        private void Record(string e) { lock (_gate) _events.Add(e); }
+    }
+}