fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

2026-08-14 19:57:08 -04:00
parent ee193cd2bb
commit 34a3f4bb69
9 changed files with 507 additions and 45 deletions
@@ -93,7 +93,8 @@ public class SiteAlarmAggregatorActorTests : TestKit
    }

    private sealed record SiteSub(
-        string CorrelationId, Action<AlarmStateChanged> OnAlarm, Action<Exception> OnError, CancellationToken Ct);
+        string CorrelationId, Action<AlarmStateChanged> OnAlarm, Action<Exception> OnError,
+        Action OnCompleted, CancellationToken Ct);

    private sealed class MockSiteAlarmStreamClient : SiteStreamGrpcClient
    {
@@ -107,9 +108,10 @@ public class SiteAlarmAggregatorActorTests : TestKit
        public MockSiteAlarmStreamClient() : base() { }

        public override Task SubscribeSiteAsync(
-            string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError, CancellationToken ct)
+            string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError,
+            Action onCompleted, CancellationToken ct)
        {
-            lock (_lock) { _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, ct)); }
+            lock (_lock) { _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, onCompleted, ct)); }
            var tcs = new TaskCompletionSource();
            ct.Register(() => tcs.TrySetResult());
            return tcs.Task; // never completes until cancelled (simulates a live stream)
@@ -445,6 +447,82 @@ public class SiteAlarmAggregatorActorTests : TestKit
        Assert.Equal(2, TotalSubs());
    }

+    // ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
+
+    [Fact]
+    public void GracefulStreamCompletion_ReopensOnReconcileTick_OnTheSameNode()
+    {
+        // The site ends every stream with OK at its 4h max lifetime. Pre-fix the client's
+        // read loop just finished, nothing was told to the actor, and the site's alarm feed
+        // stayed silently dead until the central node restarted.
+        var (_, seed, _, factory) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(300));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+
+        factory.ClientFor(GrpcNodeA).Subs[0].OnCompleted();
+
+        // Reopened by the reconcile tick, on the SAME node — a clean close is not a fault,
+        // so there is nothing to fail over from.
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(3));
+        Assert.Empty(factory.ClientFor(GrpcNodeB).Subs);
+        // The finished stream was released, so the site keeps no zombie relay actor.
+        Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).Unsubscribed);
+    }
+
+    [Fact]
+    public void GracefulStreamCompletion_NeitherSpendsNorRefunds_TheErrorRetryBudget()
+    {
+        // Reconcile is far away; reopens are driven explicitly so the budget can be observed.
+        var (actor, seed, _, factory) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+
+        // Spend the whole error budget (MaxRetries = 3), each error flipping the node.
+        factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("1"));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
+        factory.ClientFor(GrpcNodeB).Subs[0].OnError(new Exception("2"));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
+        factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
+
+        // A graceful completion reopens without flipping the node (budget not spent) …
+        factory.ClientFor(GrpcNodeB).Subs[1].OnCompleted();
+        actor.Tell(new RunReconcile());
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 3, TimeSpan.FromSeconds(5));
+        Assert.Equal(2, factory.ClientFor(GrpcNodeA).Subs.Count);
+
+        int TotalSubs() => factory.ClientFor(GrpcNodeA).Subs.Count + factory.ClientFor(GrpcNodeB).Subs.Count;
+        var before = TotalSubs();
+
+        // … and without refunding it either: the next error is the 4th, so it exceeds
+        // MaxRetries and the stream is given up rather than reconnected.
+        factory.ClientFor(GrpcNodeB).Subs[2].OnError(new Exception("4"));
+        Thread.Sleep(400);
+        Assert.Equal(before, TotalSubs());
+    }
+
+    [Fact]
+    public void LateCompletionFromAPreviousStreamGeneration_IsIgnored()
+    {
+        var (_, seed, _, factory) = CreateActor(reconcileInterval: TimeSpan.FromMinutes(10));
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+
+        var firstSub = factory.ClientFor(GrpcNodeA).Subs.Single();
+        firstSub.OnError(new Exception("real fault"));  // gen 1 dies → gen 2 opens on NodeB
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
+
+        // A completion racing out of the dead gen-1 stream must not tear down the live
+        // gen-2 stream (which would go deltaless until the next reconcile tick).
+        firstSub.OnCompleted();
+        Thread.Sleep(300);
+        Assert.DoesNotContain("corr-1", factory.ClientFor(GrpcNodeB).Unsubscribed);
+        Assert.Single(factory.ClientFor(GrpcNodeB).Subs);
+    }
+
    // ── R2 T10: live-delta publish coalescing (N6) ──

    [Fact]