dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(deploymentmanager): bulk deploy skips Disabled + reconcile circuit-breaker + probe captures pre-flatten versions (arch-review 2026-08-19 P7-2 + P7-3 + P7-4)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-19 14:52:00 -04:00
parent b68f553f1e
commit 5cbc143b01
10 changed files with 541 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs
+198 -17
View File
@@ -57,6 +57,13 @@ public class DeploymentService
/// </summary>
private const string TimeoutFailurePrefix = "Communication failure:";
/// <summary>
/// Reason reported on every <see cref="SkippedInstanceResult"/> a bulk site
/// deploy produces for a <see cref="InstanceState.Disabled"/> instance.
/// </summary>
internal const string DisabledSkipReason =
"Disabled — bulk deploy preserves disabled state; deploy individually to re-enable";
/// <summary>Rows deleted per terminal-record purge batch.</summary>
private const int TerminalPurgeBatchSize = 500;
@@ -185,8 +192,13 @@ public class DeploymentService
// out of the same phase helpers is what keeps deployment identity,
// idempotency, lock coverage and optimistic concurrency from drifting apart
// between them.
// reconcileGate is null on this path on purpose: the batch-wide reconcile
// budget exists to stop a serial phase-1 loop multiplying one unreachable
// site's query timeout by the instance count. A single instance pays it
// once, and this is the explicit operator path — it keeps the full
// communication-layer QueryTimeout.
var prepared = await PrepareDeploymentAsync(
instanceId, user, session: null, cancellationToken);
instanceId, user, session: null, reconcileGate: null, cancellationToken);
if (prepared.EarlyResult is { } early)
return early;
@@ -230,6 +242,11 @@ public class DeploymentService
/// one. Sharing it across a batch is what collapses N identical template-chain
/// walks into one.
/// </param>
/// <param name="reconcileGate">
/// Batch-wide reconcile budget + reachability circuit-breaker for a bulk
/// deploy, or <see langword="null"/> on the single-instance path (which keeps
/// the communication layer's own query timeout).
/// </param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>
/// A prepared deployment ready to send, or one carrying
@@ -240,6 +257,7 @@ public class DeploymentService
int instanceId,
string user,
FlattenSession? session,
BulkReconcileGate? reconcileGate,
CancellationToken cancellationToken)
{
// Load instance
@@ -258,7 +276,8 @@ public class DeploymentService
try
{
return await PrepareUnderLockAsync(instance, user, session, lockHandle, cancellationToken);
return await PrepareUnderLockAsync(
instance, user, session, reconcileGate, lockHandle, cancellationToken);
}
catch
{
@@ -279,6 +298,7 @@ public class DeploymentService
Instance instance,
string user,
FlattenSession? session,
BulkReconcileGate? reconcileGate,
IDisposable lockHandle,
CancellationToken cancellationToken)
{
@@ -329,8 +349,11 @@ public class DeploymentService
// duplicate deployment is not produced (design: "Deployment Identity &
// Idempotency"). A clean prior Success or a fresh first-time deploy
// skips this extra round-trip.
var reconciled = await TryReconcileWithSiteAsync(
instance, revisionHash, configJson, user, cancellationToken);
var reconciled = reconcileGate is null
? await TryReconcileWithSiteAsync(
instance, revisionHash, configJson, user, cancellationToken)
: await ReconcileUnderBatchGateAsync(
instance, revisionHash, configJson, user, reconcileGate, cancellationToken);
if (reconciled != null)
{
lockHandle.Dispose();
@@ -745,6 +768,36 @@ public class DeploymentService
/// </summary>
private readonly record struct SendOutcome(DeploymentStatusResponse? Response, Exception? Error);
/// <summary>
/// A bulk deploy's shared query-before-redeploy budget: the short per-reconcile
/// deadline every instance in the batch runs under, plus the once-per-batch
/// reachability circuit-breaker that skips the rest after the first timeout.
/// See <see cref="ReconcileUnderBatchGateAsync"/> for why both bounds exist.
///
/// <para>
/// Deliberately not thread-safe: it is only ever touched from the serial
/// prepare phase.
/// </para>
/// </summary>
/// <param name="reconcileTimeout">Deadline applied to each reconcile round-trip.</param>
private sealed class BulkReconcileGate(TimeSpan reconcileTimeout)
{
/// <summary>Deadline applied to each reconcile round-trip in this batch.</summary>
public TimeSpan ReconcileTimeout { get; } = reconcileTimeout;
/// <summary>True once a reconcile in this batch hit its deadline.</summary>
public bool SiteUnreachable { get; private set; }
/// <summary>Number of reconciles the circuit-breaker skipped, for the batch's one summary log line.</summary>
public int SkippedCount { get; private set; }
/// <summary>Opens the circuit for the remainder of the batch.</summary>
public void MarkUnreachable() => SiteUnreachable = true;
/// <summary>Counts one reconcile skipped because the circuit is open.</summary>
public void CountSkipped() => SkippedCount++;
}
/// <summary>
/// Deploy every deployable instance at a site in one operation.
///
@@ -783,9 +836,18 @@ public class DeploymentService
/// <see cref="DeployInstanceAsync"/> — its own deployment id and revision hash,
/// its own operation lock held across the whole operation, its own
/// optimistically-concurrent status record, and the same query-before-redeploy
/// idempotency check. Instances that cannot be deployed (wrong state, failed
/// validation, lock already held) are reported as failed rows; the batch is
/// NOT all-or-nothing, mirroring the artifact deployment path.
/// idempotency check — with two batch-level differences. Instances that cannot
/// be deployed (wrong state, failed validation, lock already held) are reported
/// as failed rows; the batch is NOT all-or-nothing, mirroring the artifact
/// deployment path.
/// </para>
///
/// <para>
/// <b>Disabled instances are skipped entirely</b> and reported in
/// <see cref="SiteDeploymentSummary.SkippedInstances"/> — see the filter below
/// for why. <b>Reconciliation is bounded per batch</b> by
/// <see cref="ReconcileUnderBatchGateAsync"/>, so an unreachable site costs the
/// batch one short timeout rather than one 30-second query per instance.
/// </para>
/// </summary>
/// <param name="siteId">The database ID of the site whose instances are deployed.</param>
@@ -803,12 +865,30 @@ public class DeploymentService
var instances = await _siteRepository.GetInstancesBySiteIdAsync(siteId, cancellationToken);
if (instances.Count == 0)
return Result<SiteDeploymentSummary>.Success(new SiteDeploymentSummary(site.SiteIdentifier, [], 0, 0));
return Result<SiteDeploymentSummary>.Success(new SiteDeploymentSummary(site.SiteIdentifier, [], 0, 0, []));
// Deliberately-Disabled instances are NOT part of a bulk deploy. Deploy is a
// legal transition out of Disabled and the single-instance path uses it to
// deploy-and-enable, so including them here would have one site-wide push
// silently START every piece of equipment an operator had deliberately
// stopped. They are reported as skipped rows instead — no flatten, no send,
// no state change — and remain individually deployable, which is the
// explicit "yes, re-enable this one" action.
var deployable = new List<Instance>(instances.Count);
var skipped = new List<SkippedInstanceResult>();
foreach (var instance in instances)
{
if (instance.State == InstanceState.Disabled)
skipped.Add(new SkippedInstanceResult(instance.Id, instance.UniqueName, DisabledSkipReason));
else
deployable.Add(instance);
}
// ---- Phase 1: prepare, serially, on ONE shared flatten session. ----
var session = _flatteningPipeline.CreateSession();
var prepared = new List<PreparedDeployment>(instances.Count);
var results = new List<InstanceDeploymentResult>(instances.Count);
var reconcileGate = new BulkReconcileGate(_options.SiteDeploymentReconcileTimeout);
var prepared = new List<PreparedDeployment>(deployable.Count);
var results = new List<InstanceDeploymentResult>(deployable.Count);
// Index of the first prepared deployment phase 3 has NOT yet finalised.
// Everything from here to the end of `prepared` still owns an operation
@@ -817,14 +897,15 @@ public class DeploymentService
var finalizedUpTo = 0;
try
{
foreach (var instance in instances)
foreach (var instance in deployable)
{
cancellationToken.ThrowIfCancellationRequested();
PreparedDeployment step;
try
{
step = await PrepareDeploymentAsync(instance.Id, user, session, cancellationToken);
step = await PrepareDeploymentAsync(
instance.Id, user, session, reconcileGate, cancellationToken);
}
catch (Exception ex)
{
@@ -891,13 +972,26 @@ public class DeploymentService
site.SiteIdentifier,
results,
results.Count(r => r.Success),
results.Count(r => !r.Success));
results.Count(r => !r.Success),
skipped);
// ONE summary line for the whole batch, never one per skipped reconcile:
// an unreachable site would otherwise log N warnings for a single fact.
if (reconcileGate.SkippedCount > 0)
{
_logger.LogWarning(
"Site {SiteId} did not answer the query-before-redeploy reconciliation within {Timeout}; " +
"skipped it for the remaining {SkippedCount} instance(s) of this bulk deploy — they proceed " +
"on central's recorded state, with the site's own stale-revision rejection as the safety net",
site.SiteIdentifier, reconcileGate.ReconcileTimeout, reconcileGate.SkippedCount);
}
_logger.LogInformation(
"Bulk deployment of site {SiteId} complete: {SuccessCount} succeeded, {FailureCount} failed " +
"Bulk deployment of site {SiteId} complete: {SuccessCount} succeeded, {FailureCount} failed, " +
"{SkippedCount} skipped as Disabled " +
"({ChainLoads} template-chain load(s) across {InstanceCount} instance(s))",
site.SiteIdentifier, summary.SuccessCount, summary.FailureCount,
session.ChainLoads, instances.Count);
site.SiteIdentifier, summary.SuccessCount, summary.FailureCount, skipped.Count,
session.ChainLoads, deployable.Count);
return Result<SiteDeploymentSummary>.Success(summary);
}
@@ -1671,6 +1765,73 @@ public class DeploymentService
return null;
}
/// <summary>
/// Runs <see cref="TryReconcileWithSiteAsync"/> for one instance of a BULK
/// deploy, under the batch's short per-reconcile deadline and its
/// once-per-batch reachability circuit-breaker.
///
/// <para>
/// Phase 1 is strictly serial, so an unreachable site makes every instance pay
/// the full communication-layer <c>QueryTimeout</c> (30 s) in turn before a
/// single deploy is sent — 500 instances is roughly four hours, so the CLI's
/// 5-minute bulk-deploy deadline cancels the operation and the next attempt
/// re-arms the identical cost. A bulk redeploy could therefore never converge
/// while the site was down. Two bounds fix that: a short
/// <see cref="DeploymentManagerOptions.SiteDeploymentReconcileTimeout"/> per
/// reconcile, and — after the FIRST timeout — skipping the remaining reconciles
/// for the rest of the batch.
/// </para>
///
/// <para>
/// Skipping loses nothing: a reconcile that times out already returns
/// <see langword="null"/> and the instance proceeds on DB state with the site's
/// own stale-revision rejection as the safety net — which is exactly what every
/// subsequent reconcile against a down site would have produced, one 30-second
/// wait at a time.
/// </para>
///
/// <para>
/// The gate needs no synchronisation: it is touched only from the serial
/// prepare phase.
/// </para>
/// </summary>
/// <param name="instance">Instance being prepared.</param>
/// <param name="targetRevisionHash">Revision hash this deploy would apply.</param>
/// <param name="configJson">Serialized flattened config for the reconciled snapshot.</param>
/// <param name="currentUser">User attributed on the reconciliation audit row.</param>
/// <param name="gate">The batch's reconcile budget and reachability state.</param>
/// <param name="cancellationToken">Cancellation token for the batch.</param>
/// <returns>The reconciled prior record, or <see langword="null"/> to proceed with a normal deploy.</returns>
private async Task<DeploymentRecord?> ReconcileUnderBatchGateAsync(
Instance instance,
string targetRevisionHash,
string configJson,
string currentUser,
BulkReconcileGate gate,
CancellationToken cancellationToken)
{
if (gate.SiteUnreachable)
{
gate.CountSkipped();
return null;
}
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(gate.ReconcileTimeout);
var reconciled = await TryReconcileWithSiteAsync(
instance, targetRevisionHash, configJson, currentUser, cts.Token);
// TryReconcileWithSiteAsync swallows its own query faults (it must — a
// failed query never aborts a deploy), so the deadline is detected on the
// token rather than on an exception. An outer cancellation is the caller
// aborting the whole batch, not evidence about the site.
if (reconciled == null && cts.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
gate.MarkUnreachable();
return reconciled;
}
/// <summary>
/// The site is queried before a re-deploy only when a
/// prior record is stuck <see cref="DeploymentStatus.InProgress"/>, or is
@@ -1901,6 +2062,18 @@ public record InstanceDeploymentResult(
bool Success,
string? ErrorMessage);
/// <summary>
/// One instance a bulk site deployment did not attempt at all — no flatten, no
/// site round-trip, and no state change of any kind.
/// </summary>
/// <param name="InstanceId">Database id of the instance.</param>
/// <param name="UniqueName">The instance's unique name.</param>
/// <param name="Reason">Why it was skipped, phrased for the operator.</param>
public record SkippedInstanceResult(
int InstanceId,
string UniqueName,
string Reason);
/// <summary>
/// Result matrix of a bulk site deployment. Mirrors the shape of
/// <c>ArtifactDeploymentSummary</c>: successes are not rolled back when other
@@ -1911,8 +2084,16 @@ public record InstanceDeploymentResult(
/// <param name="InstanceResults">Per-instance outcomes.</param>
/// <param name="SuccessCount">Number of instances the site confirmed.</param>
/// <param name="FailureCount">Number of instances that failed for any reason.</param>
/// <param name="SkippedInstances">
/// Instances the batch deliberately did not attempt — currently the
/// <see cref="InstanceState.Disabled"/> ones, which a bulk push must not silently
/// re-enable. They are neither successes nor failures, so they are counted in
/// neither total. Additive and nullable so an older client deserialising this
/// summary is unaffected.
/// </param>
public record SiteDeploymentSummary(
string SiteIdentifier,
IReadOnlyList<InstanceDeploymentResult> InstanceResults,
int SuccessCount,
int FailureCount);
int FailureCount,
IReadOnlyList<SkippedInstanceResult>? SkippedInstances = null);