feat(core): PollGroupEngine v2 — failure backoff + caller-token-filtered OCE exit (CONV-1/STAB-8; STAB-14-class engine guard)

2026-07-13 11:51:50 -04:00
parent b8cf3cce0b
commit ca6530c7f5
3 changed files with 177 additions and 9 deletions
@@ -461,6 +461,143 @@ public sealed class PollGroupEngineTests
        events.Count.ShouldBeGreaterThanOrEqualTo(1);
    }

+    // ---- PollGroupEngine v2: failure backoff + caller-token-filtered OCE (CONV-1 / STAB-8 / STAB-14) ----
+
+    /// <summary>
+    ///     Sustained reader failures must stretch the poll cadence (capped exponential backoff)
+    ///     instead of hammering a dead device at the base interval every tick.
+    /// </summary>
+    [Fact]
+    public async Task Backoff_SustainedReaderFailures_StretchesCadenceToCap()
+    {
+        var readCount = 0;
+        Task<IReadOnlyList<DataValueSnapshot>> Reader(IReadOnlyList<string> refs, CancellationToken ct)
+        {
+            Interlocked.Increment(ref readCount);
+            throw new InvalidOperationException("dead device");
+        }
+
+        await using var engine = new PollGroupEngine(
+            Reader,
+            (_, _, _) => { },
+            minInterval: TimeSpan.FromMilliseconds(20),
+            onError: _ => { },
+            backoffCap: TimeSpan.FromMilliseconds(120));
+
+        var handle = engine.Subscribe(["X"], TimeSpan.FromMilliseconds(20));
+        await Task.Delay(700, TestContext.Current.CancellationToken);
+        engine.Unsubscribe(handle);
+
+        // Backoff schedule from a 20 ms base: 20,40,80,120,120,... → ≈8-9 reads in 700 ms.
+        // A no-backoff 20 ms loop would fire ≈35. Assert the cadence is clearly stretched.
+        readCount.ShouldBeGreaterThanOrEqualTo(2);
+        readCount.ShouldBeLessThan(18);
+    }
+
+    /// <summary>A successful poll resets the failure count so the cadence snaps back to the base interval.</summary>
+    [Fact]
+    public async Task Backoff_SuccessResetsToInterval()
+    {
+        var readCount = 0;
+        var generation = 0;
+        Task<IReadOnlyList<DataValueSnapshot>> Reader(IReadOnlyList<string> refs, CancellationToken ct)
+        {
+            // First few reads fail (winding the backoff up toward the cap); thereafter every read
+            // returns a fresh changing value so each successful poll raises a change event.
+            if (Interlocked.Increment(ref readCount) <= 3)
+                throw new InvalidOperationException("warming up failures");
+            var gen = Interlocked.Increment(ref generation);
+            var now = DateTime.UtcNow;
+            return Task.FromResult<IReadOnlyList<DataValueSnapshot>>(
+                refs.Select(_ => new DataValueSnapshot(gen, 0u, now, now)).ToList());
+        }
+
+        var events = new ConcurrentQueue<string>();
+        await using var engine = new PollGroupEngine(
+            Reader,
+            (_, r, _) => events.Enqueue(r),
+            minInterval: TimeSpan.FromMilliseconds(20),
+            onError: _ => { },
+            backoffCap: TimeSpan.FromMilliseconds(300));
+
+        var handle = engine.Subscribe(["X"], TimeSpan.FromMilliseconds(20));
+        // After recovery the cadence must be the fast 20 ms base, not the wound-up cap — so many
+        // change events accumulate quickly. A non-resetting impl stays near the 300 ms cap (≈3).
+        await WaitForAsync(() => events.Count >= 8, TimeSpan.FromSeconds(3));
+        engine.Unsubscribe(handle);
+
+        events.Count.ShouldBeGreaterThanOrEqualTo(8);
+    }
+
+    /// <summary>
+    ///     STAB-14-class engine guard: a reader that throws <see cref="OperationCanceledException"/>
+    ///     WITHOUT the caller/loop token being cancelled (a driver-internal timeout CTS) must be
+    ///     treated as a normal failure — reported + retried — NOT as a loop teardown. Before the
+    ///     fix the bare <c>catch (OCE) { return; }</c> killed the whole subscription on the first
+    ///     such OCE.
+    /// </summary>
+    [Fact]
+    public async Task ReaderOperationCanceled_WithoutCallerCancellation_TreatedAsFailureNotTeardown()
+    {
+        var observed = new ConcurrentQueue<Exception>();
+        var events = new ConcurrentQueue<string>();
+        var readCount = 0;
+        Task<IReadOnlyList<DataValueSnapshot>> Reader(IReadOnlyList<string> refs, CancellationToken ct)
+        {
+            // A driver-internal timeout OCE — the loop token (ct) is NOT cancelled.
+            if (Interlocked.Increment(ref readCount) <= 2)
+                throw new OperationCanceledException("driver-internal read timeout");
+            var now = DateTime.UtcNow;
+            return Task.FromResult<IReadOnlyList<DataValueSnapshot>>(
+                refs.Select(_ => new DataValueSnapshot(1, 0u, now, now)).ToList());
+        }
+
+        await using var engine = new PollGroupEngine(
+            Reader,
+            (_, r, _) => events.Enqueue(r),
+            minInterval: TimeSpan.FromMilliseconds(30),
+            onError: ex => observed.Enqueue(ex),
+            backoffCap: TimeSpan.FromMilliseconds(120));
+
+        var handle = engine.Subscribe(["X"], TimeSpan.FromMilliseconds(30));
+        // The loop must SURVIVE the reader OCEs and eventually deliver a change once the reader
+        // recovers — proving the OCE was not treated as teardown.
+        await WaitForAsync(() => events.Count >= 1, TimeSpan.FromSeconds(3));
+        engine.Unsubscribe(handle);
+
+        events.Count.ShouldBeGreaterThanOrEqualTo(1);
+        observed.Count.ShouldBeGreaterThanOrEqualTo(1);
+        observed.ShouldAllBe(e => e is OperationCanceledException);
+    }
+
+    /// <summary>Caller (loop-token) cancellation still exits the loop promptly — well under the drain timeout.</summary>
+    [Fact]
+    public async Task CallerCancellation_StillExitsPromptly()
+    {
+        // Reader blocks until the loop token is cancelled (a well-behaved cancellable read).
+        Task<IReadOnlyList<DataValueSnapshot>> Reader(IReadOnlyList<string> refs, CancellationToken ct)
+            => Task.Delay(Timeout.Infinite, ct).ContinueWith(
+                _ => (IReadOnlyList<DataValueSnapshot>)new List<DataValueSnapshot>(), ct);
+
+        await using var engine = new PollGroupEngine(
+            Reader,
+            (_, _, _) => { },
+            minInterval: TimeSpan.FromMilliseconds(30),
+            onError: _ => { },
+            backoffCap: TimeSpan.FromMilliseconds(120));
+
+        var handle = engine.Subscribe(["X"], TimeSpan.FromMilliseconds(30));
+        await Task.Delay(60, TestContext.Current.CancellationToken); // let the reader begin blocking
+
+        var sw = System.Diagnostics.Stopwatch.StartNew();
+        engine.Unsubscribe(handle).ShouldBeTrue();
+        sw.Stop();
+
+        // Prompt exit: the caller-token OCE unwinds the loop immediately — nowhere near the 5 s
+        // StopState drain ceiling.
+        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(2));
+    }
+
    private sealed record DummyHandle : ISubscriptionHandle
    {
        /// <summary>Gets a diagnostic identifier for this handle.</summary>