fix(communication): streaming consumers re-resolve site gRPC addresses on every reopen; spec matches (review 2026-10-21 03/R-2, 03/D-2)

2026-10-22 06:21:33 -04:00
parent 3ed71167b3
commit 90306f0ccb
8 changed files with 438 additions and 34 deletions
@@ -420,6 +420,33 @@ public class DebugStreamBridgeActorTests : TestKit
        Assert.Equal("corr-1", factory.ClientFor(GrpcNodeB).SubscribeCalls[0].CorrelationId);
    }

+    /// <summary>Verifies that a reconnect after the site's address was edited dials the new address (review 2026-10-21 03/R-2).</summary>
+    [Fact]
+    public void Reconnect_AfterSiteAddressEdit_DialsTheNewAddress()
+    {
+        const string NewNodeB = "http://localhost:5400";
+        var commProbe = CreateTestProbe();
+        var factory = new EndpointTrackingGrpcClientFactory();
+        SiteStreamEndpointResolver resolver =
+            _ => Task.FromResult<SiteStreamEndpoints?>(new SiteStreamEndpoints(GrpcNodeA, NewNodeB));
+
+        var props = Props.Create(typeof(DebugStreamBridgeActor),
+            SiteId, InstanceName, "corr-1", commProbe.Ref,
+            (Action<object>)(_ => { }), (Action)(() => { }),
+            factory, GrpcNodeA, GrpcNodeB, resolver);
+        Sys.ActorOf(props);
+        commProbe.ExpectMsg<SiteEnvelope>();
+
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
+        factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnError(new Exception("NodeA down"));
+
+        // The failover reconnect goes to the EDITED NodeB, never the start-time one.
+        AwaitCondition(() => factory.ClientFor(NewNodeB).SubscribeCalls.Count == 1, TimeSpan.FromSeconds(5));
+        Assert.Empty(factory.ClientFor(GrpcNodeB).SubscribeCalls);
+        // The failed stream was released on the endpoint it was opened against.
+        Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds);
+    }
+
    // ── graceful (status OK) stream completion is a reconnect trigger ──

    /// <summary>Verifies that a graceful stream completion reopens the stream on the same node.</summary>
@@ -1013,4 +1013,61 @@ public class SiteAlarmAggregatorActorTests : TestKit
        seed.FaultNext();
        AwaitCondition(() => seed.CallCount == 3, TimeSpan.FromSeconds(5));
    }
+
+    // ── review 2026-10-21 03/R-2: reopen re-resolves the site's endpoints ──
+
+    /// <summary>Verifies that a reopen after the site's address was edited dials the NEW address, not the start-time one.</summary>
+    [Fact]
+    public void Reopen_AfterSiteAddressEdit_DialsTheNewAddress()
+    {
+        const string NewNodeA = "http://localhost:5300";
+        const string NewNodeB = "http://localhost:5400";
+        var edited = 0;
+        SiteStreamEndpointResolver resolver = _ => Task.FromResult<SiteStreamEndpoints?>(
+            Volatile.Read(ref edited) == 1 ? new(NewNodeA, NewNodeB) : new(GrpcNodeA, GrpcNodeB));
+
+        var seed = new SeedStub();
+        var sink = new PublishSink();
+        var factory = new MockSiteAlarmStreamClientFactory();
+        Sys.ActorOf(Props.Create(() => new SiteAlarmAggregatorActor(
+            SiteId, "corr-1", seed.Seed, sink.Publish, factory, GrpcNodeA, GrpcNodeB,
+            TimeSpan.FromMilliseconds(300), TimeSpan.Zero, TestReconnectDelay, TestStabilityWindow,
+            0.0, SiteAlarmAggregatorActor.DefaultOpenAttemptTimeout, resolver)));
+
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+
+        // The operator re-points the site; the running stream then ends gracefully.
+        Volatile.Write(ref edited, 1);
+        factory.ClientFor(GrpcNodeA).Subs[0].OnCompleted();
+
+        // The reconcile tick's reopen dials the edited NodeA, never the stale one again.
+        AwaitCondition(() => factory.ClientFor(NewNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        Assert.Single(factory.ClientFor(GrpcNodeA).Subs);
+
+        // A fault now fails over to the edited NodeB.
+        factory.ClientFor(NewNodeA).Subs[0].OnError(new Exception("down"));
+        AwaitCondition(() => factory.ClientFor(NewNodeB).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        Assert.Empty(factory.ClientFor(GrpcNodeB).Subs);
+    }
+
+    /// <summary>Verifies that a failing endpoint lookup still reopens against the known address.</summary>
+    [Fact]
+    public void Reopen_WhenTheEndpointLookupFails_UsesTheKnownAddress()
+    {
+        SiteStreamEndpointResolver resolver = _ => throw new InvalidOperationException("db down");
+        var seed = new SeedStub();
+        var sink = new PublishSink();
+        var factory = new MockSiteAlarmStreamClientFactory();
+        Sys.ActorOf(Props.Create(() => new SiteAlarmAggregatorActor(
+            SiteId, "corr-1", seed.Seed, sink.Publish, factory, GrpcNodeA, GrpcNodeB,
+            TimeSpan.FromMilliseconds(300), TimeSpan.Zero, TestReconnectDelay, TestStabilityWindow,
+            0.0, SiteAlarmAggregatorActor.DefaultOpenAttemptTimeout, resolver)));
+
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
+        seed.CompleteNext();
+        factory.ClientFor(GrpcNodeA).Subs[0].OnCompleted();
+
+        AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(3));
+    }
 }