fix(comm): queue failover re-seed behind in-flight fan-out; stamp stream generation on gRPC errors (plan R2-02 T11)

2026-07-13 10:18:16 -04:00
parent bb8be55382
commit c29e588905
2 changed files with 92 additions and 4 deletions
@@ -397,6 +397,50 @@ public class SiteAlarmAggregatorActorTests : TestKit
        AwaitCondition(() => TotalSubs() > before, TimeSpan.FromSeconds(3));
    }

+    // ── R2 T11: queued re-seed after reconnect + stream-generation stamp (N7.1/N7.2) ──
+
+    [Fact]
+    public void ReseedRequestedDuringInFlightFanout_IsQueued_NotDropped()
+    {
+        var (actor, seed, sink, factory) = CreateActor(publishCoalesce: TimeSpan.Zero);
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();                       // initial seed done (CallCount 1)
+        AwaitAssert(() => Assert.Equal(1, sink.Count));
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+
+        actor.Tell(new RunReconcile());            // reconcile fan-out now in flight (CallCount 2)
+        AwaitAssert(() => Assert.Equal(2, seed.CallCount));
+
+        // Stream error while the reconcile is in flight → the failover re-seed must not be
+        // silently skipped. Pre-fix: StartFanout no-ops and CallCount stays 2 until the next
+        // 60s reconcile tick.
+        factory.ClientFor(GrpcNodeA).Subs.Last().OnError(new Exception("stream fault"));
+
+        seed.CompleteNext();                       // finish the in-flight reconcile
+        AwaitAssert(() => Assert.Equal(3, seed.CallCount)); // queued re-seed ran immediately after
+    }
+
+    [Fact]
+    public void LateErrorFromAPreviousStreamGeneration_IsIgnored()
+    {
+        var (_, seed, _, factory) = CreateActor(publishCoalesce: TimeSpan.Zero);
+        AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+
+        int TotalSubs() => factory.ClientFor(GrpcNodeA).Subs.Count + factory.ClientFor(GrpcNodeB).Subs.Count;
+
+        var firstSub = factory.ClientFor(GrpcNodeA).Subs.Single();
+        firstSub.OnError(new Exception("real fault"));      // gen 1 error → flip, reconnect
+        AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5)); // gen 2 stream open
+        AwaitAssert(() => Assert.Equal(2, TotalSubs()));
+
+        firstSub.OnError(new Exception("late zombie fault")); // stale gen-1 error races in
+        // Pre-fix this burns retry budget and opens a THIRD stream; post-fix it is ignored.
+        Thread.Sleep(400);
+        Assert.Equal(2, TotalSubs());
+    }
+
    // ── R2 T10: live-delta publish coalescing (N6) ──

    [Fact]