fix(deploymentmanager): bulk deploy skips Disabled + reconcile circuit-breaker + probe captures pre-flatten versions (arch-review 2026-08-19 P7-2 + P7-3 + P7-4)

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentManagerOptions.cs

@@ -41,6 +41,25 @@ public class DeploymentManagerOptions
     /// </summary>
     public TimeSpan SiteDeploymentTimeoutPerInstance { get; set; } = TimeSpan.FromSeconds(120);
 
+    /// <summary>
+    /// Deadline applied to each query-before-redeploy reconciliation round-trip
+    /// inside a bulk <c>DeploySiteAsync</c> (the single-instance path keeps the
+    /// communication layer's own <c>QueryTimeout</c>).
+    ///
+    /// <para>
+    /// Phase 1 is strictly serial, so an unreachable site otherwise costs
+    /// <c>QueryTimeout</c> (30 s) PER INSTANCE before a single deploy is sent — for
+    /// a 500-instance site roughly four hours, far past the CLI's 5-minute bulk
+    /// deploy timeout, which then cancels and re-arms the identical cost on the next
+    /// attempt. Deliberately short (seconds): a reconcile that does not answer
+    /// promptly falls through to a normal deploy anyway, which is exactly what its
+    /// own timeout produces. Works together with the once-per-batch reachability
+    /// circuit-breaker, which skips the remaining reconciles after the first
+    /// timeout.
+    /// </para>
+    /// </summary>
+    public TimeSpan SiteDeploymentReconcileTimeout { get; set; } = TimeSpan.FromSeconds(5);
+
     /// <summary>
     /// Retention window for TERMINAL deployment records (<c>Success</c>,
     /// <c>Failed</c>). Deployments are insert-only — one row per attempt, forever —

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentManagerOptionsValidator.cs

@@ -38,6 +38,11 @@ public sealed class DeploymentManagerOptionsValidator : OptionsValidatorBase<Dep
             $"(was {options.SiteDeploymentTimeoutPerInstance}); it bounds each instance's site round-trip " +
             $"inside a bulk site deployment.");
 
+        builder.RequireThat(options.SiteDeploymentReconcileTimeout > TimeSpan.Zero,
+            $"ScadaBridge:DeploymentManager:SiteDeploymentReconcileTimeout must be a positive duration " +
+            $"(was {options.SiteDeploymentReconcileTimeout}); it bounds each query-before-redeploy " +
+            $"reconciliation inside a bulk site deployment's serial prepare phase.");
+
         builder.RequireThat(options.TerminalDeploymentRecordRetention >= TimeSpan.Zero,
             $"ScadaBridge:DeploymentManager:TerminalDeploymentRecordRetention must not be negative " +
             $"(was {options.TerminalDeploymentRecordRetention}); use TimeSpan.Zero to disable purging of " +

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs

@@ -57,6 +57,13 @@ public class DeploymentService
     /// </summary>
     private const string TimeoutFailurePrefix = "Communication failure:";
 
+    /// <summary>
+    /// Reason reported on every <see cref="SkippedInstanceResult"/> a bulk site
+    /// deploy produces for a <see cref="InstanceState.Disabled"/> instance.
+    /// </summary>
+    internal const string DisabledSkipReason =
+        "Disabled — bulk deploy preserves disabled state; deploy individually to re-enable";
+
     /// <summary>Rows deleted per terminal-record purge batch.</summary>
     private const int TerminalPurgeBatchSize = 500;
 
@@ -185,8 +192,13 @@ public class DeploymentService
         // out of the same phase helpers is what keeps deployment identity,
         // idempotency, lock coverage and optimistic concurrency from drifting apart
         // between them.
+        // reconcileGate is null on this path on purpose: the batch-wide reconcile
+        // budget exists to stop a serial phase-1 loop multiplying one unreachable
+        // site's query timeout by the instance count. A single instance pays it
+        // once, and this is the explicit operator path — it keeps the full
+        // communication-layer QueryTimeout.
         var prepared = await PrepareDeploymentAsync(
-            instanceId, user, session: null, cancellationToken);
+            instanceId, user, session: null, reconcileGate: null, cancellationToken);
 
         if (prepared.EarlyResult is { } early)
             return early;
@@ -230,6 +242,11 @@ public class DeploymentService
     /// one. Sharing it across a batch is what collapses N identical template-chain
     /// walks into one.
     /// </param>
+    /// <param name="reconcileGate">
+    /// Batch-wide reconcile budget + reachability circuit-breaker for a bulk
+    /// deploy, or <see langword="null"/> on the single-instance path (which keeps
+    /// the communication layer's own query timeout).
+    /// </param>
     /// <param name="cancellationToken">Cancellation token.</param>
     /// <returns>
     /// A prepared deployment ready to send, or one carrying
@@ -240,6 +257,7 @@ public class DeploymentService
         int instanceId,
         string user,
         FlattenSession? session,
+        BulkReconcileGate? reconcileGate,
         CancellationToken cancellationToken)
     {
         // Load instance
@@ -258,7 +276,8 @@ public class DeploymentService
 
         try
         {
-            return await PrepareUnderLockAsync(instance, user, session, lockHandle, cancellationToken);
+            return await PrepareUnderLockAsync(
+                instance, user, session, reconcileGate, lockHandle, cancellationToken);
         }
         catch
         {
@@ -279,6 +298,7 @@ public class DeploymentService
         Instance instance,
         string user,
         FlattenSession? session,
+        BulkReconcileGate? reconcileGate,
         IDisposable lockHandle,
         CancellationToken cancellationToken)
     {
@@ -329,8 +349,11 @@ public class DeploymentService
         // duplicate deployment is not produced (design: "Deployment Identity &
         // Idempotency"). A clean prior Success or a fresh first-time deploy
         // skips this extra round-trip.
-        var reconciled = await TryReconcileWithSiteAsync(
-            instance, revisionHash, configJson, user, cancellationToken);
+        var reconciled = reconcileGate is null
+            ? await TryReconcileWithSiteAsync(
+                instance, revisionHash, configJson, user, cancellationToken)
+            : await ReconcileUnderBatchGateAsync(
+                instance, revisionHash, configJson, user, reconcileGate, cancellationToken);
         if (reconciled != null)
         {
             lockHandle.Dispose();
@@ -745,6 +768,36 @@ public class DeploymentService
     /// </summary>
     private readonly record struct SendOutcome(DeploymentStatusResponse? Response, Exception? Error);
 
+    /// <summary>
+    /// A bulk deploy's shared query-before-redeploy budget: the short per-reconcile
+    /// deadline every instance in the batch runs under, plus the once-per-batch
+    /// reachability circuit-breaker that skips the rest after the first timeout.
+    /// See <see cref="ReconcileUnderBatchGateAsync"/> for why both bounds exist.
+    ///
+    /// <para>
+    /// Deliberately not thread-safe: it is only ever touched from the serial
+    /// prepare phase.
+    /// </para>
+    /// </summary>
+    /// <param name="reconcileTimeout">Deadline applied to each reconcile round-trip.</param>
+    private sealed class BulkReconcileGate(TimeSpan reconcileTimeout)
+    {
+        /// <summary>Deadline applied to each reconcile round-trip in this batch.</summary>
+        public TimeSpan ReconcileTimeout { get; } = reconcileTimeout;
+
+        /// <summary>True once a reconcile in this batch hit its deadline.</summary>
+        public bool SiteUnreachable { get; private set; }
+
+        /// <summary>Number of reconciles the circuit-breaker skipped, for the batch's one summary log line.</summary>
+        public int SkippedCount { get; private set; }
+
+        /// <summary>Opens the circuit for the remainder of the batch.</summary>
+        public void MarkUnreachable() => SiteUnreachable = true;
+
+        /// <summary>Counts one reconcile skipped because the circuit is open.</summary>
+        public void CountSkipped() => SkippedCount++;
+    }
+
     /// <summary>
     /// Deploy every deployable instance at a site in one operation.
     ///
@@ -783,9 +836,18 @@ public class DeploymentService
     /// <see cref="DeployInstanceAsync"/> — its own deployment id and revision hash,
     /// its own operation lock held across the whole operation, its own
     /// optimistically-concurrent status record, and the same query-before-redeploy
-    /// idempotency check. Instances that cannot be deployed (wrong state, failed
-    /// validation, lock already held) are reported as failed rows; the batch is
-    /// NOT all-or-nothing, mirroring the artifact deployment path.
+    /// idempotency check — with two batch-level differences. Instances that cannot
+    /// be deployed (wrong state, failed validation, lock already held) are reported
+    /// as failed rows; the batch is NOT all-or-nothing, mirroring the artifact
+    /// deployment path.
+    /// </para>
+    ///
+    /// <para>
+    /// <b>Disabled instances are skipped entirely</b> and reported in
+    /// <see cref="SiteDeploymentSummary.SkippedInstances"/> — see the filter below
+    /// for why. <b>Reconciliation is bounded per batch</b> by
+    /// <see cref="ReconcileUnderBatchGateAsync"/>, so an unreachable site costs the
+    /// batch one short timeout rather than one 30-second query per instance.
     /// </para>
     /// </summary>
     /// <param name="siteId">The database ID of the site whose instances are deployed.</param>
@@ -803,12 +865,30 @@ public class DeploymentService
 
         var instances = await _siteRepository.GetInstancesBySiteIdAsync(siteId, cancellationToken);
         if (instances.Count == 0)
-            return Result<SiteDeploymentSummary>.Success(new SiteDeploymentSummary(site.SiteIdentifier, [], 0, 0));
+            return Result<SiteDeploymentSummary>.Success(new SiteDeploymentSummary(site.SiteIdentifier, [], 0, 0, []));
+
+        // Deliberately-Disabled instances are NOT part of a bulk deploy. Deploy is a
+        // legal transition out of Disabled and the single-instance path uses it to
+        // deploy-and-enable, so including them here would have one site-wide push
+        // silently START every piece of equipment an operator had deliberately
+        // stopped. They are reported as skipped rows instead — no flatten, no send,
+        // no state change — and remain individually deployable, which is the
+        // explicit "yes, re-enable this one" action.
+        var deployable = new List<Instance>(instances.Count);
+        var skipped = new List<SkippedInstanceResult>();
+        foreach (var instance in instances)
+        {
+            if (instance.State == InstanceState.Disabled)
+                skipped.Add(new SkippedInstanceResult(instance.Id, instance.UniqueName, DisabledSkipReason));
+            else
+                deployable.Add(instance);
+        }
 
         // ---- Phase 1: prepare, serially, on ONE shared flatten session. ----
         var session = _flatteningPipeline.CreateSession();
-        var prepared = new List<PreparedDeployment>(instances.Count);
-        var results = new List<InstanceDeploymentResult>(instances.Count);
+        var reconcileGate = new BulkReconcileGate(_options.SiteDeploymentReconcileTimeout);
+        var prepared = new List<PreparedDeployment>(deployable.Count);
+        var results = new List<InstanceDeploymentResult>(deployable.Count);
 
         // Index of the first prepared deployment phase 3 has NOT yet finalised.
         // Everything from here to the end of `prepared` still owns an operation
@@ -817,14 +897,15 @@ public class DeploymentService
         var finalizedUpTo = 0;
         try
         {
-            foreach (var instance in instances)
+            foreach (var instance in deployable)
             {
                 cancellationToken.ThrowIfCancellationRequested();
 
                 PreparedDeployment step;
                 try
                 {
-                    step = await PrepareDeploymentAsync(instance.Id, user, session, cancellationToken);
+                    step = await PrepareDeploymentAsync(
+                        instance.Id, user, session, reconcileGate, cancellationToken);
                 }
                 catch (Exception ex)
                 {
@@ -891,13 +972,26 @@ public class DeploymentService
             site.SiteIdentifier,
             results,
             results.Count(r => r.Success),
-            results.Count(r => !r.Success));
+            results.Count(r => !r.Success),
+            skipped);
+
+        // ONE summary line for the whole batch, never one per skipped reconcile:
+        // an unreachable site would otherwise log N warnings for a single fact.
+        if (reconcileGate.SkippedCount > 0)
+        {
+            _logger.LogWarning(
+                "Site {SiteId} did not answer the query-before-redeploy reconciliation within {Timeout}; " +
+                "skipped it for the remaining {SkippedCount} instance(s) of this bulk deploy — they proceed " +
+                "on central's recorded state, with the site's own stale-revision rejection as the safety net",
+                site.SiteIdentifier, reconcileGate.ReconcileTimeout, reconcileGate.SkippedCount);
+        }
 
         _logger.LogInformation(
-            "Bulk deployment of site {SiteId} complete: {SuccessCount} succeeded, {FailureCount} failed " +
+            "Bulk deployment of site {SiteId} complete: {SuccessCount} succeeded, {FailureCount} failed, " +
+            "{SkippedCount} skipped as Disabled " +
             "({ChainLoads} template-chain load(s) across {InstanceCount} instance(s))",
-            site.SiteIdentifier, summary.SuccessCount, summary.FailureCount,
-            session.ChainLoads, instances.Count);
+            site.SiteIdentifier, summary.SuccessCount, summary.FailureCount, skipped.Count,
+            session.ChainLoads, deployable.Count);
 
         return Result<SiteDeploymentSummary>.Success(summary);
     }
@@ -1671,6 +1765,73 @@ public class DeploymentService
         return null;
     }
 
+    /// <summary>
+    /// Runs <see cref="TryReconcileWithSiteAsync"/> for one instance of a BULK
+    /// deploy, under the batch's short per-reconcile deadline and its
+    /// once-per-batch reachability circuit-breaker.
+    ///
+    /// <para>
+    /// Phase 1 is strictly serial, so an unreachable site makes every instance pay
+    /// the full communication-layer <c>QueryTimeout</c> (30 s) in turn before a
+    /// single deploy is sent — 500 instances is roughly four hours, so the CLI's
+    /// 5-minute bulk-deploy deadline cancels the operation and the next attempt
+    /// re-arms the identical cost. A bulk redeploy could therefore never converge
+    /// while the site was down. Two bounds fix that: a short
+    /// <see cref="DeploymentManagerOptions.SiteDeploymentReconcileTimeout"/> per
+    /// reconcile, and — after the FIRST timeout — skipping the remaining reconciles
+    /// for the rest of the batch.
+    /// </para>
+    ///
+    /// <para>
+    /// Skipping loses nothing: a reconcile that times out already returns
+    /// <see langword="null"/> and the instance proceeds on DB state with the site's
+    /// own stale-revision rejection as the safety net — which is exactly what every
+    /// subsequent reconcile against a down site would have produced, one 30-second
+    /// wait at a time.
+    /// </para>
+    ///
+    /// <para>
+    /// The gate needs no synchronisation: it is touched only from the serial
+    /// prepare phase.
+    /// </para>
+    /// </summary>
+    /// <param name="instance">Instance being prepared.</param>
+    /// <param name="targetRevisionHash">Revision hash this deploy would apply.</param>
+    /// <param name="configJson">Serialized flattened config for the reconciled snapshot.</param>
+    /// <param name="currentUser">User attributed on the reconciliation audit row.</param>
+    /// <param name="gate">The batch's reconcile budget and reachability state.</param>
+    /// <param name="cancellationToken">Cancellation token for the batch.</param>
+    /// <returns>The reconciled prior record, or <see langword="null"/> to proceed with a normal deploy.</returns>
+    private async Task<DeploymentRecord?> ReconcileUnderBatchGateAsync(
+        Instance instance,
+        string targetRevisionHash,
+        string configJson,
+        string currentUser,
+        BulkReconcileGate gate,
+        CancellationToken cancellationToken)
+    {
+        if (gate.SiteUnreachable)
+        {
+            gate.CountSkipped();
+            return null;
+        }
+
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        cts.CancelAfter(gate.ReconcileTimeout);
+
+        var reconciled = await TryReconcileWithSiteAsync(
+            instance, targetRevisionHash, configJson, currentUser, cts.Token);
+
+        // TryReconcileWithSiteAsync swallows its own query faults (it must — a
+        // failed query never aborts a deploy), so the deadline is detected on the
+        // token rather than on an exception. An outer cancellation is the caller
+        // aborting the whole batch, not evidence about the site.
+        if (reconciled == null && cts.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
+            gate.MarkUnreachable();
+
+        return reconciled;
+    }
+
     /// <summary>
     /// The site is queried before a re-deploy only when a
     /// prior record is stuck <see cref="DeploymentStatus.InProgress"/>, or is
@@ -1901,6 +2062,18 @@ public record InstanceDeploymentResult(
     bool Success,
     string? ErrorMessage);
 
+/// <summary>
+/// One instance a bulk site deployment did not attempt at all — no flatten, no
+/// site round-trip, and no state change of any kind.
+/// </summary>
+/// <param name="InstanceId">Database id of the instance.</param>
+/// <param name="UniqueName">The instance's unique name.</param>
+/// <param name="Reason">Why it was skipped, phrased for the operator.</param>
+public record SkippedInstanceResult(
+    int InstanceId,
+    string UniqueName,
+    string Reason);
+
 /// <summary>
 /// Result matrix of a bulk site deployment. Mirrors the shape of
 /// <c>ArtifactDeploymentSummary</c>: successes are not rolled back when other
@@ -1911,8 +2084,16 @@ public record InstanceDeploymentResult(
 /// <param name="InstanceResults">Per-instance outcomes.</param>
 /// <param name="SuccessCount">Number of instances the site confirmed.</param>
 /// <param name="FailureCount">Number of instances that failed for any reason.</param>
+/// <param name="SkippedInstances">
+/// Instances the batch deliberately did not attempt — currently the
+/// <see cref="InstanceState.Disabled"/> ones, which a bulk push must not silently
+/// re-enable. They are neither successes nor failures, so they are counted in
+/// neither total. Additive and nullable so an older client deserialising this
+/// summary is unaffected.
+/// </param>
 public record SiteDeploymentSummary(
     string SiteIdentifier,
     IReadOnlyList<InstanceDeploymentResult> InstanceResults,
     int SuccessCount,
-    int FailureCount);
+    int FailureCount,
+    IReadOnlyList<SkippedInstanceResult>? SkippedInstances = null);

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/FlattenSession.cs

@@ -86,6 +86,23 @@ public sealed class FlattenSession
     /// </summary>
     public IReadOnlyCollection<int> VisitedTemplateIds => _templates.Keys;
 
+    /// <summary>
+    /// Every visited template id paired with the watermark version this session
+    /// READ it at — the pre-flatten watermark the staleness fast path must memoise
+    /// against.
+    ///
+    /// <para>
+    /// Re-reading the versions after the flatten instead is a correctness bug, not
+    /// an optimisation detail: an edit that lands mid-flatten pairs the PRE-edit
+    /// content (and therefore the pre-edit hash) with the POST-edit version, and
+    /// the memo then answers "not stale" until something else moves the watermark.
+    /// Recording the version each template was read at can only ever cause an extra
+    /// flatten, never a stale answer.
+    /// </para>
+    /// </summary>
+    public IReadOnlyDictionary<int, long> VisitedTemplateVersions =>
+        _templates.ToDictionary(entry => entry.Key, entry => entry.Value.Version);
+
     /// <summary>
     /// Returns the inheritance chain for <paramref name="templateId"/> (the
     /// template itself, then each ancestor), loading it through

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/StaleInstanceProbe.cs

@@ -62,7 +62,7 @@ public sealed class StaleInstanceProbe : IStaleInstanceProbe
     /// </summary>
     /// <param name="Structure">Graph structure version at flatten time.</param>
     /// <param name="InstanceVersion">The instance's own version at flatten time.</param>
-    /// <param name="TemplateVersions">Version of every template the flatten walked, by template id.</param>
+    /// <param name="TemplateVersions">Version every template the flatten walked was READ at, by template id.</param>
     /// <param name="RevisionHash">The hash that reading produced.</param>
     private sealed record ProbeMemo(
         long Structure,
@@ -85,10 +85,15 @@ public sealed class StaleInstanceProbe : IStaleInstanceProbe
         // script-compilation stage (this probe is called per-instance across a whole
         // bundle import in BundleImporter.ComputeStaleInstanceIdsAsync).
         //
-        // Watermark readings are taken BEFORE the flatten. A mutation that lands
-        // during the flatten therefore bumps a version past the recorded one and
-        // invalidates the memo on the next read, rather than being captured as if
-        // it had already been included.
+        // Watermark readings are taken BEFORE the flatten they describe. A mutation
+        // that lands during the flatten therefore bumps a version past the recorded
+        // one and invalidates the memo on the next read, rather than being captured
+        // as if it had already been included. The structure and instance versions
+        // are read here; the per-template versions come from the session, which
+        // records each template's version at the moment it read that template (see
+        // FlattenSession.VisitedTemplateVersions) — re-reading them after the
+        // flatten would pair a pre-edit hash with a post-edit version and make the
+        // memo serve "not stale" indefinitely.
         var structure = _watermark.StructureVersion;
         var instanceVersion = _watermark.GetInstanceVersion(instanceId);
 
@@ -103,12 +108,8 @@ public sealed class StaleInstanceProbe : IStaleInstanceProbe
         if (Memos.Count >= MaxCachedProbes)
             Memos.Clear();
 
-        var visited = session.VisitedTemplateIds;
-        var versions = new Dictionary<int, long>(visited.Count);
-        foreach (var templateId in visited)
-            versions[templateId] = _watermark.GetTemplateVersion(templateId);
-
-        Memos[instanceId] = new ProbeMemo(structure, instanceVersion, versions, result.Value.RevisionHash);
+        Memos[instanceId] = new ProbeMemo(
+            structure, instanceVersion, session.VisitedTemplateVersions, result.Value.RevisionHash);
         return result.Value.RevisionHash;
     }