fix(abcip): single-flight cold-connect per device — backoff survives the parallel read window

2026-08-16 04:24:59 -04:00
parent fe8d132c64
commit d5f573b019
3 changed files with 213 additions and 31 deletions
@@ -168,6 +168,103 @@ public sealed class AbCipParallelReadTests
        snapshots.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.Good);
    }

+    /// <summary>
+    ///     Counts + overlaps fake <c>InitializeAsync</c> (the Forward-Open stand-in) calls. Holds
+    ///     every in-flight connect until either <paramref name="target"/> overlap (proving the
+    ///     connects were NOT single-flight) or a short grace period elapses, so a correctly
+    ///     serialised run still completes instead of deadlocking.
+    /// </summary>
+    private sealed class ConnectProbe(int target)
+    {
+        private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private int _inFlight;
+        private int _attempts;
+
+        /// <summary>Gets the highest number of connect attempts observed in flight simultaneously.</summary>
+        public int MaxConcurrent { get; private set; }
+
+        /// <summary>Gets the total number of connect attempts that reached the wire.</summary>
+        public int Attempts => Volatile.Read(ref _attempts);
+
+        /// <summary>Records one in-flight connect and holds it open until the probe releases.</summary>
+        /// <param name="cancellationToken">The connect's cancellation token.</param>
+        public async Task HoldAsync(CancellationToken cancellationToken)
+        {
+            Interlocked.Increment(ref _attempts);
+            var n = Interlocked.Increment(ref _inFlight);
+            lock (this)
+            {
+                if (n > MaxConcurrent) MaxConcurrent = n;
+            }
+            try
+            {
+                if (n >= target) _release.TrySetResult();
+                try
+                {
+                    await _release.Task.WaitAsync(TimeSpan.FromMilliseconds(150), cancellationToken)
+                        .ConfigureAwait(false);
+                }
+                catch (TimeoutException)
+                {
+                    // No second connect overlapped — the single-flight case.
+                }
+            }
+            finally
+            {
+                Interlocked.Decrement(ref _inFlight);
+            }
+        }
+    }
+
+    /// <summary>A fake standing in for a device that is down — every connect attempt fails.</summary>
+    private sealed class DownDeviceFake(AbCipTagCreateParams p, ConnectProbe probe) : FakeAbCipTag(p)
+    {
+        /// <inheritdoc />
+        public override async Task InitializeAsync(CancellationToken cancellationToken)
+        {
+            await probe.HoldAsync(cancellationToken).ConfigureAwait(false);
+            throw new InvalidOperationException("fake Forward-Open failure");
+        }
+    }
+
+    /// <summary>
+    ///     Review fix — cold (never-connected) tags on ONE down device must connect single-flight
+    ///     under the parallel read window. The window let up to <c>MaxConcurrentReads</c> work items
+    ///     run the unsynchronised <c>ShouldAttempt</c> → connect → <c>RecordFailure</c> sequence on
+    ///     the shared per-device <c>ConnectionBackoff</c> concurrently: every cold tag passed
+    ///     <c>ShouldAttempt</c> before any failure was recorded (N simultaneous Forward-Opens — a
+    ///     thundering herd at the device), and the unsynchronised <c>_consecutiveFailures++</c> lost
+    ///     increments, weakening the 05/STAB-8 backoff into a reconnect storm.
+    /// </summary>
+    [Fact]
+    public async Task Cold_tags_on_one_down_device_connect_single_flight()
+    {
+        var probe = new ConnectProbe(target: 2);
+        var factory = new FakeAbCipTagFactory { Customise = p => new DownDeviceFake(p, probe) };
+        var drv = NewDriver(factory, maxConcurrentReads: 8,
+            Tag("A"), Tag("B"), Tag("C"), Tag("D"), Tag("E"), Tag("F"));
+        await drv.InitializeAsync("{}", CancellationToken.None);
+
+        var snapshots = await drv.ReadAsync(["A", "B", "C", "D", "E", "F"], CancellationToken.None);
+
+        probe.MaxConcurrent.ShouldBe(1,
+            "cold connects against ONE device must be single-flight — the backoff check + connect + " +
+            "record sequence is not internally synchronized");
+        probe.Attempts.ShouldBe(1,
+            "the one recorded failure must be visible to every sibling still inside the read window, " +
+            "so only ONE Forward-Open reaches a down device per batch");
+        snapshots.Count.ShouldBe(6);
+        snapshots.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.BadCommunicationError);
+
+        // The failure increment landed and is durable: the 05/STAB-8 window opened by that single
+        // recorded failure (base 1 s) is still open, so a second batch makes NO further attempt.
+        var again = await drv.ReadAsync(["A", "B", "C", "D", "E", "F"], CancellationToken.None);
+
+        probe.Attempts.ShouldBe(1,
+            "no lost increment — the recorded failure must still hold the backoff window shut");
+        again.ShouldAllBe(s => s.StatusCode == AbCipStatusMapper.BadCommunicationError);
+    }
+
    /// <summary>
    ///     Health is written once after the batch: the worst per-item outcome wins, and
    ///     <c>LastSuccessfulRead</c> still advances because at least one tag read succeeded — the
@@ -129,8 +129,15 @@ internal sealed class ArrayFakeAbCipTag : FakeAbCipTag
 /// <summary>Test factory that produces <see cref="FakeAbCipTag"/>s and indexes them for assertion.</summary>
 internal sealed class FakeAbCipTagFactory : IAbCipTagFactory
 {
-    /// <summary>Gets a dictionary of created tags indexed by tag name for assertion.</summary>
-    public Dictionary<string, FakeAbCipTag> Tags { get; } = new(StringComparer.OrdinalIgnoreCase);
+    /// <summary>
+    ///     Gets a dictionary of created tags indexed by tag name for assertion. Concurrent because
+    ///     <see cref="AbCipDriver.ReadAsync"/> creates cold runtimes from inside its bounded-parallel
+    ///     read window — two work items for different tags can call <see cref="Create"/> at the same
+    ///     time, which a plain <see cref="Dictionary{TKey,TValue}"/> would answer with a corrupt
+    ///     bucket chain or a lost entry rather than an honest failure.
+    /// </summary>
+    public System.Collections.Concurrent.ConcurrentDictionary<string, FakeAbCipTag> Tags { get; } =
+        new(StringComparer.OrdinalIgnoreCase);
    /// <summary>Gets or sets an optional customization function to override the tag creation.</summary>
    public Func<AbCipTagCreateParams, FakeAbCipTag>? Customise { get; set; }