fix(reconcile): batch startup reconcile into fixed round trips, bound central work by QueryTimeout, retry failed site passes with back-off (review 2026-10-21 03/P-1)

2026-10-22 06:25:57 -04:00
parent d6a9a79c27
commit 7c83a4aa94
9 changed files with 638 additions and 130 deletions
@@ -53,7 +53,9 @@ public class SiteReconciliationActorTests : TestKit, IDisposable
        IActorRef siteCommunicationActor,
        IDeploymentConfigFetcher fetcher,
        ILogger<SiteReconciliationActor>? logger = null,
-        TimeSpan? askTimeout = null) =>
+        TimeSpan? askTimeout = null,
+        int maxAttempts = 3,
+        TimeSpan? retryBaseDelay = null) =>
        ActorOf(Props.Create(() => new SiteReconciliationActor(
            _storage,
            fetcher,
@@ -62,7 +64,10 @@ public class SiteReconciliationActorTests : TestKit, IDisposable
            NodeId,
            logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger<SiteReconciliationActor>.Instance,
            TimeSpan.FromMilliseconds(50),                 // initial delay — fast for tests
-            askTimeout ?? TimeSpan.FromSeconds(5))));       // ask timeout
+            askTimeout ?? TimeSpan.FromSeconds(5),          // ask timeout
+            null,
+            maxAttempts,
+            retryBaseDelay)));

    /// <summary>Verifies that an instance missing at the site is fetched from central and stored.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
@@ -296,6 +301,68 @@ public class SiteReconciliationActorTests : TestKit, IDisposable
        Assert.Empty(fetcher.Calls);
    }

+    /// <summary>
+    /// review 2026-10-21 03/P-1: a pass whose Ask to central fails is retried with back-off up
+    /// to the attempt cap — not left until the next node restart — and then stops.
+    /// </summary>
+    [Fact]
+    public void CentralAskFails_IsRetried_UpToTheAttemptCap()
+    {
+        var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
+        var commProbe = CreateTestProbe();
+
+        CreateReconciliationActor(
+            commProbe, fetcher,
+            askTimeout: TimeSpan.FromMilliseconds(200),
+            maxAttempts: 3,
+            retryBaseDelay: TimeSpan.FromMilliseconds(100));
+
+        // Never replied to → three passes (1 + 2 retries), then no more.
+        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
+        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
+        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
+        commProbe.ExpectNoMsg(TimeSpan.FromSeconds(1.5));
+        Assert.Empty(fetcher.Calls);
+    }
+
+    /// <summary>
+    /// review 2026-10-21 03/P-1: a pass that completes with a failed item is retried, and the
+    /// retry heals the item.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task FailedItem_IsRetried_AndHealsOnTheNextPass()
+    {
+        const string config = "{\"instanceUniqueName\":\"Flaky\"}";
+        var calls = 0;
+        var fetcher = new FakeConfigFetcher(_ => Interlocked.Increment(ref calls) == 1
+            ? Task.FromException<string>(new DeploymentConfigFetchException("transient", isSuperseded: false))
+            : Task.FromResult(config));
+        var commProbe = CreateTestProbe();
+
+        CreateReconciliationActor(
+            commProbe, fetcher, maxAttempts: 3, retryBaseDelay: TimeSpan.FromMilliseconds(100));
+
+        var gap = new ReconcileSiteResponse(
+            [new ReconcileGapItem("Flaky", "depF", "revF", true, "tok-f")], [], "http://central:9000");
+
+        commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
+        commProbe.Reply(gap);
+
+        var retry = commProbe.ExpectMsg<ReconcileSiteRequest>(TimeSpan.FromSeconds(5));
+        Assert.False(retry.LocalNameToRevisionHash.ContainsKey("Flaky"));
+        commProbe.Reply(gap);
+
+        await AwaitAssertAsync(async () =>
+        {
+            var configs = await _storage.GetAllDeployedConfigsAsync();
+            Assert.Single(configs, c => c.InstanceUniqueName == "Flaky");
+        }, TimeSpan.FromSeconds(5));
+
+        // Healed on pass 2 → no third pass.
+        commProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(600));
+    }
+
    /// <summary>
    /// In-test fake <see cref="IDeploymentConfigFetcher"/>: runs a per-deploymentId behavior
    /// (return config JSON or throw, as a Task) and records every call's coords thread-safely