fix(#654): bounded fast retry while the known-cluster gate is unpublished

CentralCommunicationActor: while the PSK provider is unresolved or the first ClusterNode load failed, re-arm RefreshContacts every 2 s (max 15) until the first publish; Information log on first publish; 'NEVER be published' Error now only after the retry budget is exhausted. Corrected false comments. Tests: late provider (Grpc+Dps), failed first load, retries stop, bounded budget; ControlPlaneStarterOrderTests back to the default 60 s refresh. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-08 23:32:38 -04:00
parent ca17d2cf71
commit 9dc6390d50
4 changed files with 184 additions and 26 deletions
@@ -678,6 +678,107 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
        pskCalls.ShouldBeGreaterThanOrEqualTo(2);
    }

+    /// <summary>Counts contact loads and fails the first <c>failFirst</c> of them.</summary>
+    private sealed class CountingDbFactory(
+        IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        private int _calls;
+
+        public int Calls => Volatile.Read(ref _calls);
+
+        public OtOpcUaConfigDbContext CreateDbContext()
+        {
+            if (Interlocked.Increment(ref _calls) <= failFirst)
+                throw new InvalidOperationException("ConfigDb unreachable at boot");
+            return inner.CreateDbContext();
+        }
+
+        public Task<OtOpcUaConfigDbContext> CreateDbContextAsync(CancellationToken ct = default) =>
+            Task.FromResult(CreateDbContext());
+    }
+
+    private static MeshTransportOptions ModeOptions(string mode) => new()
+    {
+        Mode = mode,
+        ContactRefreshSeconds = 60,
+        DispatchTimeoutSeconds = 10,
+    };
+
+    [Theory]
+    [InlineData(MeshTransportOptions.ModeGrpc)]
+    [InlineData(MeshTransportOptions.ModeDps)]
+    public void The_gate_publishes_within_seconds_when_the_provider_resolves_late_under_the_default_refresh(
+        string mode)
+    {
+        // #654: the first RefreshContacts can land before the registry-bound PSK graph is resolvable.
+        // The next PERIODIC tick is 60 s away; the bounded bootstrap retry must publish in seconds.
+        var factory = NewInMemoryDbFactory();
+        SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
+        var psk = new RecordingPskProvider();
+        var calls = 0;
+
+        IMeshPskProvider? PskFactory() =>
+            Interlocked.Increment(ref calls) <= 3 ? throw new InvalidOperationException("race") : psk;
+
+        EventFilter.Error().Expect(0, TimeSpan.FromSeconds(1), () =>
+        {
+            Sys.ActorOf(CentralCommunicationActor.Props(
+                factory, ModeOptions(mode), () => null, () => new RecordingDialer(), PskFactory));
+            AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(10));
+        });
+    }
+
+    [Fact]
+    public void A_failed_first_contact_load_retries_quickly_under_the_default_refresh()
+    {
+        var inner = NewInMemoryDbFactory();
+        SeedGrpcCluster(inner, "C1", ("host-a:4053", "host-a", 4056, true, false));
+        var factory = new CountingDbFactory(inner, failFirst: 1);
+        var psk = new RecordingPskProvider();
+
+        EventFilter.Warning().Expect(2, TimeSpan.FromSeconds(10), () =>
+            Sys.ActorOf(CentralCommunicationActor.Props(
+                factory, GrpcMode(), () => null, () => new RecordingDialer(), () => psk)));
+
+        AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(10));
+    }
+
+    [Fact]
+    public void Bootstrap_retries_stop_once_the_gate_is_published()
+    {
+        var inner = NewInMemoryDbFactory();
+        SeedGrpcCluster(inner, "C1", ("host-a:4053", "host-a", 4056, true, false));
+        var factory = new CountingDbFactory(inner);
+        var psk = new RecordingPskProvider();
+        Sys.ActorOf(CentralCommunicationActor.Props(
+            factory, GrpcMode(), () => null, () => new RecordingDialer(), () => psk,
+            bootstrapRetryInterval: TimeSpan.FromMilliseconds(100)));
+
+        // Positive control: the gate did publish before we assert anything about "afterwards".
+        AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(10));
+        Thread.Sleep(1000);
+
+        // Ten retry intervals elapsed with a 60 s periodic cadence: no extra loads, no republish.
+        factory.Calls.ShouldBe(1);
+        psk.KnownClusterPublications.Count.ShouldBe(1);
+    }
+
+    [Fact]
+    public void A_permanently_failing_load_retry_budget_is_bounded()
+    {
+        var inner = NewInMemoryDbFactory();
+        var factory = new CountingDbFactory(inner, failFirst: int.MaxValue);
+        Sys.ActorOf(CentralCommunicationActor.Props(
+            factory, GrpcMode(), () => null, () => new RecordingDialer(), () => new RecordingPskProvider(),
+            bootstrapRetryInterval: TimeSpan.FromMilliseconds(20)));
+
+        EventFilter.Warning().Expect(32, TimeSpan.FromSeconds(10), () => { });
+        Thread.Sleep(1500);
+        // 1 initial + 15 retries; the 60 s periodic tick has not fired again.
+        factory.Calls.ShouldBe(16);
+    }
+
    [Fact]
    public void A_permanently_unresolvable_pskProvider_is_reported_once_per_incarnation_not_once_per_refresh()
    {
@@ -702,14 +803,15 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
        }

        IActorRef actor = null!;
-        EventFilter.Error(contains: "will NEVER be published").ExpectOne(TimeSpan.FromSeconds(5), () =>
+        EventFilter.Error(contains: "will NEVER be published").ExpectOne(TimeSpan.FromSeconds(10), () =>
        {
            actor = Sys.ActorOf(CentralCommunicationActor.Props(
                factory,
                GrpcMode(),
                () => null,
                () => new RecordingDialer(),
-                pskProvider: PskFactory));
+                pskProvider: PskFactory,
+                bootstrapRetryInterval: TimeSpan.FromMilliseconds(50)));

            // pskCalls == 1 is PreStart's own attempt; reaching 2 is PROOF the first contact load
            // ran and PublishKnownClusters's property access performed the first-use retry — not
@@ -717,6 +819,13 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
            AwaitCondition(() => pskCalls >= 2, TimeSpan.FromSeconds(5));
        });

+        // The bootstrap retry is BOUNDED: with a 50 ms interval and a budget of 15 it has long
+        // finished, so the call count plateaus instead of growing with elapsed time.
+        Thread.Sleep(500);
+        var plateau = pskCalls;
+        Thread.Sleep(500);
+        pskCalls.ShouldBe(plateau);
+
        var pskCallsAfterFirstLoad = pskCalls;

        // A SECOND contact load must NOT repeat the Error. The pin only means something if that
@@ -94,8 +94,7 @@ public sealed class ControlPlaneStarterOrderTests
        // walks the same registry-bound chain the production MeshPskProvider does — which makes
        // this fact the race detector for the PreStart-vs-starter-#4 window: a lost race that
        // stayed null forever would never publish, and the TaskCompletionSource below would time
-        // out red. (Memoize-on-first-use in the actor is what heals the lost race at the first
-        // RefreshContacts message.)
+        // out red. (Memoize-on-first-use plus the bounded bootstrap retry heal a lost race.)
        //
        // The signal is a TaskCompletionSource rather than a counter, because PreStart runs on the
        // actor's own thread and the contact load is a piped DB read: there is no instant after
@@ -143,15 +142,11 @@ public sealed class ControlPlaneStarterOrderTests
            .ConfigureServices(services =>
            {
                services.AddSingleton<IOptions<AkkaClusterOptions>>(Options.Create(clusterOptions));
-                // ContactRefreshSeconds = 1 (production: 60). The memoized PSK provider retries on EVERY
-                // refresh, so a first refresh that lands before the registry-bound graph is resolvable
-                // (the lost race the resolution fact exists to heal) publishes on the NEXT tick. At 60 s
-                // that next tick was outside the fact's 20 s budget — a lost race on a loaded CI runner
-                // timed out red exactly like a never-resolved provider (the Dps case failed in 2 of 7
-                // Gitea runs, #606). A 1 s cadence
-                // still proves "resolved and used" — a provider that stayed null forever never publishes.
+                // ContactRefreshSeconds is the production default (60). A first refresh that lands before the
+                // registry-bound graph is resolvable is healed by the bounded bootstrap retry (#654), not
+                // by the next periodic tick, so the 20 s budget below proves production heals fast.
                services.AddSingleton<IOptions<MeshTransportOptions>>(Options.Create(
-                    new MeshTransportOptions { Mode = mode, DispatchTimeoutSeconds = 10, ContactRefreshSeconds = 1 }));
+                    new MeshTransportOptions { Mode = mode, DispatchTimeoutSeconds = 10 }));
                services.AddDbContextFactory<OtOpcUaConfigDbContext>(
                    o => o.UseInMemoryDatabase($"starter-order-{Guid.NewGuid():N}"));
                services.AddSingleton<IClusterRoleInfo>(new StubClusterRoleInfo());