fix(deploymentmanager): bulk deploy skips Disabled + reconcile circuit-breaker + probe captures pre-flatten versions (arch-review 2026-08-19 P7-2 + P7-3 + P7-4)

2026-08-19 14:52:00 -04:00
parent b68f553f1e
commit 5cbc143b01
10 changed files with 541 additions and 32 deletions
@@ -45,7 +45,9 @@ public class DeploySiteAsyncTests : TestKit
    }

    /// <summary>Seeds <paramref name="count"/> deployable instances at the test site.</summary>
-    private List<Instance> ArrangeInstances(int count)
+    /// <param name="count">How many instances to seed.</param>
+    /// <param name="disabledIds">Ids seeded <c>Disabled</c> instead of <c>NotDeployed</c>.</param>
+    private List<Instance> ArrangeInstances(int count, params int[] disabledIds)
    {
        var instances = new List<Instance>();
        for (var i = 1; i <= count; i++)
@@ -55,7 +57,7 @@ public class DeploySiteAsyncTests : TestKit
                Id = i,
                SiteId = SiteId,
                TemplateId = 10,
-                State = InstanceState.NotDeployed
+                State = disabledIds.Contains(i) ? InstanceState.Disabled : InstanceState.NotDeployed
            };
            instances.Add(instance);

@@ -73,7 +75,11 @@ public class DeploySiteAsyncTests : TestKit
        return instances;
    }

-    private DeploymentService CreateService(IActorRef commActor, int maxParallelism, TimeSpan? perInstanceTimeout = null)
+    private DeploymentService CreateService(
+        IActorRef commActor,
+        int maxParallelism,
+        TimeSpan? perInstanceTimeout = null,
+        TimeSpan? reconcileTimeout = null)
    {
        var comms = new CommunicationService(
            Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromSeconds(30) }),
@@ -84,7 +90,8 @@ public class DeploySiteAsyncTests : TestKit
        {
            OperationLockTimeout = TimeSpan.FromSeconds(5),
            SiteDeploymentMaxParallelism = maxParallelism,
-            SiteDeploymentTimeoutPerInstance = perInstanceTimeout ?? TimeSpan.FromSeconds(30)
+            SiteDeploymentTimeoutPerInstance = perInstanceTimeout ?? TimeSpan.FromSeconds(30),
+            SiteDeploymentReconcileTimeout = reconcileTimeout ?? TimeSpan.FromSeconds(5)
        });

        return new DeploymentService(
@@ -305,6 +312,147 @@ public class DeploySiteAsyncTests : TestKit
            Arg.Any<CancellationToken>());
    }

+    /// <summary>
+    /// P7-2: a bulk site deploy must not restart deliberately-stopped equipment.
+    ///
+    /// <para>
+    /// Deploy is a legal transition out of <c>Disabled</c> and the deploy path
+    /// forces the instance to <c>Enabled</c> on success, so a site-wide push over an
+    /// unfiltered instance list silently re-enabled every instance an operator had
+    /// disabled. Bulk deploy therefore skips them outright — no flatten, no send, no
+    /// state change — and reports them so the operator can see what was left alone.
+    /// The single-instance path is unchanged: that is the explicit "yes, re-enable
+    /// this one" action.
+    /// </para>
+    /// </summary>
+    [Fact]
+    public async Task DeploySiteAsync_SkipsDisabledInstances_WithoutDeployingOrReEnablingThem()
+    {
+        var instances = ArrangeInstances(5, disabledIds: [2, 4]);
+
+        var commActor = Sys.ActorOf(Props.Create(() =>
+            new ThrottledSiteActor(new ConcurrencyTracker(), slowInstanceName: null, slowDelay: TimeSpan.Zero)));
+
+        var result = await CreateService(commActor, maxParallelism: 3).DeploySiteAsync(SiteId, "admin");
+
+        Assert.True(result.IsSuccess);
+        var summary = result.Value;
+
+        // Only the three Enabled/NotDeployed instances were attempted.
+        Assert.Equal(3, summary.InstanceResults.Count);
+        Assert.Equal(3, summary.SuccessCount);
+        Assert.Equal(0, summary.FailureCount);
+        Assert.DoesNotContain(summary.InstanceResults, r => r.UniqueName is "Inst-02" or "Inst-04");
+
+        // The Disabled pair is reported as skipped, with the reason the operator needs.
+        Assert.NotNull(summary.SkippedInstances);
+        var skipped = summary.SkippedInstances;
+        Assert.Equal(2, skipped.Count);
+        Assert.Equal(
+            new[] { "Inst-02", "Inst-04" },
+            skipped.Select(s => s.UniqueName).Order().ToArray());
+        Assert.All(skipped, s => Assert.Contains("deploy individually to re-enable", s.Reason));
+
+        // ...and left completely untouched: still Disabled, and never handed to the
+        // repository at all (no deployment record, no instance update, no snapshot).
+        foreach (var id in new[] { 2, 4 })
+        {
+            Assert.Equal(InstanceState.Disabled, instances.Single(i => i.Id == id).State);
+            await _pipeline.DidNotReceive().FlattenAndValidateAsync(
+                id, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
+        }
+
+        await _repo.DidNotReceive().UpdateInstanceAsync(
+            Arg.Is<Instance>(i => i.Id == 2 || i.Id == 4), Arg.Any<CancellationToken>());
+        await _repo.DidNotReceive().AddDeploymentRecordAsync(
+            Arg.Is<DeploymentRecord>(r => r.InstanceId == 2 || r.InstanceId == 4), Arg.Any<CancellationToken>());
+    }
+
+    /// <summary>
+    /// P7-3: phase 1 is serial, so an unreachable site used to cost the batch one
+    /// full 30-second <c>QueryTimeout</c> PER INSTANCE before a single deploy was
+    /// sent — hours for a large site, so the CLI's 5-minute bulk deadline cancelled
+    /// and the next attempt re-armed the same cost. Two bounds close it: a short
+    /// per-reconcile deadline, and a once-per-batch circuit-breaker that skips the
+    /// remaining reconciles after the first timeout.
+    ///
+    /// <para>
+    /// The site here answers the deploy but NEVER answers the reconcile query, so
+    /// the assertion that exactly ONE query was issued for six instances is the
+    /// whole finding: without the breaker there would be six.
+    /// </para>
+    /// </summary>
+    [Fact]
+    public async Task DeploySiteAsync_UnreachableSite_ReconcilesOnceThenSkipsTheRestOfTheBatch()
+    {
+        const int instanceCount = 6;
+        ArrangeInstances(instanceCount);
+
+        // Reconciliation only runs when a prior record is stuck InProgress.
+        for (var i = 1; i <= instanceCount; i++)
+        {
+            _repo.GetCurrentDeploymentStatusAsync(i, Arg.Any<CancellationToken>())
+                .Returns(new DeploymentRecord($"prior-{i}", "admin")
+                {
+                    InstanceId = i,
+                    Status = DeploymentStatus.InProgress,
+                    RevisionHash = "sha256:old"
+                });
+        }
+
+        var queryCount = 0;
+        var commActor = Sys.ActorOf(Props.Create(() =>
+            new SilentQuerySiteActor(() => Interlocked.Increment(ref queryCount))));
+
+        var reconcileTimeout = TimeSpan.FromMilliseconds(300);
+        var service = CreateService(commActor, maxParallelism: 3, reconcileTimeout: reconcileTimeout);
+
+        var sw = Stopwatch.StartNew();
+        var result = await service.DeploySiteAsync(SiteId, "admin");
+        sw.Stop();
+
+        Assert.True(result.IsSuccess);
+
+        // The circuit-breaker opened after the first unanswered reconcile.
+        Assert.Equal(1, Volatile.Read(ref queryCount));
+
+        // Prepare therefore cost ONE reconcile deadline, not six.
+        Assert.True(sw.Elapsed < TimeSpan.FromSeconds(2),
+            $"bulk prepare took {sw.ElapsedMilliseconds} ms — the per-batch reconcile bound is not holding");
+
+        // A skipped reconcile is not a failed deploy: every instance still went out.
+        Assert.Equal(instanceCount, result.Value.SuccessCount);
+        Assert.Equal(0, result.Value.FailureCount);
+    }
+
+    /// <summary>
+    /// Answers <c>RefreshDeploymentCommand</c> normally but NEVER answers a
+    /// <c>DeploymentStateQueryRequest</c>, counting how many arrived — a site that
+    /// is up for deploys yet unreachable for the reconcile query, which is the shape
+    /// a wedged or overloaded site presents.
+    /// </summary>
+    private sealed class SilentQuerySiteActor : ReceiveActor
+    {
+        public SilentQuerySiteActor(Action onQuery)
+        {
+            Receive<SiteEnvelope>(env =>
+            {
+                switch (env.Message)
+                {
+                    case DeploymentStateQueryRequest:
+                        onQuery();
+                        return;
+
+                    case RefreshDeploymentCommand cmd:
+                        Sender.Tell(new DeploymentStatusResponse(
+                            cmd.DeploymentId, cmd.InstanceUniqueName,
+                            DeploymentStatus.Success, null, DateTimeOffset.UtcNow));
+                        return;
+                }
+            });
+        }
+    }
+
    /// <summary>
    /// Answers every <c>RefreshDeploymentCommand</c> after a fixed delay, recording
    /// how old each command's staging <c>Timestamp</c> already was on arrival.