fix(comms): reconnect on graceful stream completion — kills the 4h silent stream death

2026-08-14 19:57:08 -04:00
parent ee193cd2bb
commit 34a3f4bb69
9 changed files with 507 additions and 45 deletions
@@ -400,6 +400,67 @@ public class DebugStreamBridgeActorTests : TestKit
        Assert.Equal("corr-1", factory.ClientFor(GrpcNodeB).SubscribeCalls[0].CorrelationId);
    }

+    // ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
+
+    [Fact]
+    public void On_GracefulStreamCompletion_Reopens_On_The_Same_Node()
+    {
+        // The site caps every stream at GrpcMaxStreamLifetime and then ends the RPC with
+        // OK. Pre-fix the client's read loop just finished, the actor was told nothing, and
+        // the debug session went silently deaf for the rest of its life.
+        var (_, factory) = CreateBridgeWithTrackingFactory();
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
+            TimeSpan.FromSeconds(3));
+
+        factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnCompleted();
+
+        // Reopened on the SAME node — a clean close is not a fault, so there is nothing to
+        // fail over from — and the finished stream is released, not left zombie.
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 2,
+            TimeSpan.FromSeconds(5));
+        Assert.Empty(factory.ClientFor(GrpcNodeB).SubscribeCalls);
+        Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds);
+    }
+
+    [Fact]
+    public void RepeatedGracefulCompletions_DoNotConsume_TheErrorRetryBudget()
+    {
+        // Five completions — two more than MaxRetries. Had completion been routed through
+        // the error path the session would have terminated on the fourth.
+        var ctx = CreateBridgeActor();
+        ctx.CommProbe.ExpectMsg<SiteEnvelope>();
+        AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
+
+        for (var i = 1; i <= 5; i++)
+        {
+            ctx.MockGrpcClient.SubscribeCalls[i - 1].OnCompleted();
+            var expected = i + 1;
+            AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected,
+                TimeSpan.FromSeconds(5));
+        }
+
+        Assert.False(ctx.TerminatedFlag[0]);
+    }
+
+    [Fact]
+    public void LateCompletionFromAPreviousStreamGeneration_IsIgnored()
+    {
+        var (_, factory) = CreateBridgeWithTrackingFactory();
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
+            TimeSpan.FromSeconds(3));
+
+        var firstSub = factory.ClientFor(GrpcNodeA).SubscribeCalls[0];
+        firstSub.OnError(new Exception("NodeA down"));   // gen 1 dies → gen 2 opens on NodeB
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
+            TimeSpan.FromSeconds(5));
+
+        // A completion racing out of the dead gen-1 stream must not tear down the live one.
+        firstSub.OnCompleted();
+        Thread.Sleep(300);
+        Assert.DoesNotContain("corr-1", factory.ClientFor(GrpcNodeB).UnsubscribedCorrelationIds);
+        Assert.Single(factory.ClientFor(GrpcNodeB).SubscribeCalls);
+    }
+
    // ── Task 6 (arch review 02, High): teardown/failover unsubscribe is endpoint-safe ──
    // Both paths use TryGet, never GetOrCreate, so cleanup can never open a fresh
    // channel or (with (site,endpoint) keying) touch another session's channel.
@@ -894,9 +955,11 @@ internal class MockSiteStreamGrpcClient : SiteStreamGrpcClient
        string instanceUniqueName,
        Action<object> onEvent,
        Action<Exception> onError,
+        Action onCompleted,
        CancellationToken ct)
    {
-        var subscription = new MockSubscription(correlationId, instanceUniqueName, onEvent, onError, ct);
+        var subscription = new MockSubscription(
+            correlationId, instanceUniqueName, onEvent, onError, onCompleted, ct);
        lock (_lock) { _subscribeCalls.Add(subscription); }

        // Return a task that completes when cancelled (simulates long-running stream)
@@ -916,6 +979,7 @@ internal record MockSubscription(
    string InstanceUniqueName,
    Action<object> OnEvent,
    Action<Exception> OnError,
+    Action OnCompleted,
    CancellationToken CancellationToken);

 /// <summary>