dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(deploy): flatten-session caching, bulk DeploySiteAsync, paged management queries

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:14:22 -04:00
parent ee193cd2bb
commit 48b3c40a7f
59 changed files with 3051 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs
+590 -64
View File
@@ -57,6 +57,24 @@ public class DeploymentService
/// </summary>
private const string TimeoutFailurePrefix = "Communication failure:";
/// <summary>Rows deleted per terminal-record purge batch.</summary>
private const int TerminalPurgeBatchSize = 500;
/// <summary>Batches deleted per opportunistic sweep, bounding one caller's latency cost.</summary>
private const int TerminalPurgeBatchesPerSweep = 4;
/// <summary>Minimum interval between opportunistic terminal-record sweeps.</summary>
private static readonly TimeSpan TerminalPurgeMinInterval = TimeSpan.FromHours(6);
/// <summary>
/// Ticks of the last opportunistic terminal-record sweep, process-wide.
/// <see cref="DeploymentService"/> is scoped, so this must be static to
/// rate-limit across requests; <see cref="Interlocked"/> makes the
/// claim-the-sweep step a compare-and-swap so concurrent readers do not all
/// sweep at once.
/// </summary>
private static long _lastTerminalPurgeTicks;
/// <summary>
/// Initializes a new instance of <see cref="DeploymentService"/> with all required dependencies.
/// </summary>
@@ -161,28 +179,113 @@ public class DeploymentService
int instanceId,
string user,
CancellationToken cancellationToken = default)
{
// Single-instance deploy = the same three phases the bulk path runs, with a
// batch of one and a private flatten session. Composing both entry points
// out of the same phase helpers is what keeps deployment identity,
// idempotency, lock coverage and optimistic concurrency from drifting apart
// between them.
var prepared = await PrepareDeploymentAsync(
instanceId, user, session: null, cancellationToken);
if (prepared.EarlyResult is { } early)
return early;
using (prepared.LockHandle)
{
var outcome = await SendDeploymentAsync(prepared, cancellationToken);
return await FinalizeDeploymentAsync(prepared, outcome, user, cancellationToken);
}
}
/// <summary>
/// Phase 1 of a deployment (serial, database-bound): validate the state
/// transition, take the per-instance operation lock, mint the deployment id,
/// flatten + validate, run query-before-redeploy reconciliation, stage the
/// <c>PendingDeployment</c> row and insert the <c>InProgress</c>
/// <see cref="DeploymentRecord"/>.
///
/// <para>
/// Everything here touches the scoped, non-thread-safe <c>DbContext</c>, so it
/// must run serially — a bulk site deploy loops this phase before fanning out
/// the phase-2 network waits. On success the returned
/// <see cref="PreparedDeployment"/> owns the operation lock; the CALLER is
/// responsible for disposing it once phase 3 has completed.
/// </para>
/// </summary>
/// <param name="instanceId">Instance being deployed.</param>
/// <param name="user">User attributed on the deployment record and audit rows.</param>
/// <param name="session">
/// Shared flatten session for a batch, or <see langword="null"/> for a private
/// one. Sharing it across a batch is what collapses N identical template-chain
/// walks into one.
/// </param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>
/// A prepared deployment ready to send, or one carrying
/// <see cref="PreparedDeployment.EarlyResult"/> when the deploy resolved before
/// any site round-trip (validation failure, or a reconciled prior deployment).
/// </returns>
private async Task<PreparedDeployment> PrepareDeploymentAsync(
int instanceId,
string user,
FlattenSession? session,
CancellationToken cancellationToken)
{
// Load instance
var instance = await _repository.GetInstanceByIdAsync(instanceId, cancellationToken);
if (instance == null)
return Result<DeploymentRecord>.Failure($"Instance with ID {instanceId} not found.");
return PreparedDeployment.Resolved(Result<DeploymentRecord>.Failure($"Instance with ID {instanceId} not found."));
// Validate state transition
var transitionError = StateTransitionValidator.ValidateTransition(instance.State, "deploy");
if (transitionError != null)
return Result<DeploymentRecord>.Failure(transitionError);
return PreparedDeployment.Resolved(Result<DeploymentRecord>.Failure(transitionError));
// Acquire per-instance operation lock
using var lockHandle = await _lockManager.AcquireAsync(
var lockHandle = await _lockManager.AcquireAsync(
instance.UniqueName, _options.OperationLockTimeout, cancellationToken);
try
{
return await PrepareUnderLockAsync(instance, user, session, lockHandle, cancellationToken);
}
catch
{
// The lock is handed to the caller only on the success path; any fault
// during preparation must release it here or a failed prepare would
// wedge the instance until the process restarts.
lockHandle.Dispose();
throw;
}
}
/// <summary>
/// The body of <see cref="PrepareDeploymentAsync"/> that runs while the
/// per-instance operation lock is held. Split out so the lock can be released
/// on every fault path with one try/catch rather than a nested one per step.
/// </summary>
private async Task<PreparedDeployment> PrepareUnderLockAsync(
Instance instance,
string user,
FlattenSession? session,
IDisposable lockHandle,
CancellationToken cancellationToken)
{
var instanceId = instance.Id;
// Generate unique deployment ID
var deploymentId = Guid.NewGuid().ToString("N");
// Flatten configuration (captures template state at this point in time)
var flattenResult = await _flatteningPipeline.FlattenAndValidateAsync(instanceId, cancellationToken);
var flattenResult = await _flatteningPipeline.FlattenAndValidateAsync(
instanceId, cancellationToken, validateScripts: true, session);
if (flattenResult.IsFailure)
return Result<DeploymentRecord>.Failure($"Validation failed: {flattenResult.Error}");
{
lockHandle.Dispose();
return PreparedDeployment.Resolved(
Result<DeploymentRecord>.Failure($"Validation failed: {flattenResult.Error}"));
}
var flattenedConfig = flattenResult.Value.Configuration;
var revisionHash = flattenResult.Value.RevisionHash;
@@ -200,8 +303,9 @@ public class DeploymentService
validationResult.Errors.Count,
string.Join("; ", validationResult.Errors.Select(e => e.Message)));
return Result<DeploymentRecord>.Failure(
$"Pre-deployment validation failed: {validationResult.SummarizeErrors()}");
lockHandle.Dispose();
return PreparedDeployment.Resolved(Result<DeploymentRecord>.Failure(
$"Pre-deployment validation failed: {validationResult.SummarizeErrors()}"));
}
// Serialize for transmission (also the payload stored in the deployed
@@ -218,7 +322,10 @@ public class DeploymentService
var reconciled = await TryReconcileWithSiteAsync(
instance, revisionHash, configJson, user, cancellationToken);
if (reconciled != null)
return Result<DeploymentRecord>.Success(reconciled);
{
lockHandle.Dispose();
return PreparedDeployment.Resolved(Result<DeploymentRecord>.Success(reconciled));
}
// Notify-and-fetch: the site fetches the staged config from
// CentralFetchBaseUrl, so a deploy is impossible without it. Fail fast
@@ -227,8 +334,11 @@ public class DeploymentService
// confusing downstream site-fetch failure (and no InProgress record is
// stranded).
if (string.IsNullOrEmpty(_commOptions.CentralFetchBaseUrl))
return Result<DeploymentRecord>.Failure(
"CentralFetchBaseUrl is not configured — required for deployment (notify-and-fetch).");
{
lockHandle.Dispose();
return PreparedDeployment.Resolved(Result<DeploymentRecord>.Failure(
"CentralFetchBaseUrl is not configured — required for deployment (notify-and-fetch)."));
}
// Create the deployment record directly in InProgress.
//
@@ -275,12 +385,8 @@ public class DeploymentService
deploymentId, instance.UniqueName, revisionHash, user, stagedAt,
_commOptions.CentralFetchBaseUrl, token);
_logger.LogInformation(
"Sending deployment {DeploymentId} for instance {Instance} to site {SiteId} (notify-and-fetch)",
deploymentId, instance.UniqueName, siteId);
// Cleanup of the staged PendingDeployment is TTL-based ONLY — the row
// is deliberately NOT deleted on success or in the catch. On a
// is deliberately NOT deleted on success or on failure. On a
// central-side Ask timeout the site may have applied AND told the
// standby node to fetch; deleting now would 404 that in-flight
// standby fetch and break failover. Supersession bounds pending rows
@@ -288,8 +394,89 @@ public class DeploymentService
// leaving rows for TTL purge is safe. Expired rows are swept by the
// central PendingDeploymentPurgeActor singleton on its
// CommunicationOptions.PendingDeploymentPurgeInterval cadence.
var response = await _communicationService.RefreshDeploymentAsync(siteId, command, cancellationToken);
return new PreparedDeployment(
instance, deploymentId, record, revisionHash, configJson,
siteId, command, lockHandle, EarlyResult: null);
}
catch (Exception ex)
{
// Staging failed before anything was sent. Record the failure exactly
// as the post-send path does (never leave the record InProgress) and
// release the lock — there is no phase 2 or 3 to run.
await MarkDeploymentFailedAsync(record, instance, deploymentId, user, ex);
lockHandle.Dispose();
return PreparedDeployment.Resolved(FailureResultFor(ex));
}
}
/// <summary>
/// Phase 2 of a deployment (parallelisable, network-bound): the
/// <c>RefreshDeploymentCommand</c> round-trip to the site. Touches no
/// repository and mutates no shared state, which is what makes it safe to fan
/// out across a batch while phases 1 and 3 stay serial.
/// </summary>
/// <param name="prepared">The prepared deployment to send.</param>
/// <param name="cancellationToken">Cancellation token, already carrying any per-instance deadline.</param>
/// <returns>The site's response, or the exception that prevented one.</returns>
private async Task<SendOutcome> SendDeploymentAsync(
PreparedDeployment prepared,
CancellationToken cancellationToken)
{
try
{
_logger.LogInformation(
"Sending deployment {DeploymentId} for instance {Instance} to site {SiteId} (notify-and-fetch)",
prepared.DeploymentId, prepared.Instance.UniqueName, prepared.SiteIdentifier);
var response = await _communicationService.RefreshDeploymentAsync(
prepared.SiteIdentifier, prepared.Command, cancellationToken);
return new SendOutcome(response, null);
}
catch (Exception ex)
{
return new SendOutcome(null, ex);
}
}
/// <summary>
/// Phase 3 of a deployment (serial, database-bound): commit the terminal
/// status, apply post-success side effects, and write the audit row.
/// </summary>
/// <param name="prepared">The prepared deployment whose send just completed.</param>
/// <param name="outcome">The site response or the fault from phase 2.</param>
/// <param name="user">User attributed on the audit rows.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The deployment result for the caller.</returns>
private async Task<Result<DeploymentRecord>> FinalizeDeploymentAsync(
PreparedDeployment prepared,
SendOutcome outcome,
string user,
CancellationToken cancellationToken)
{
var instance = prepared.Instance;
var record = prepared.Record;
var deploymentId = prepared.DeploymentId;
var instanceId = instance.Id;
if (outcome.Error is { } sendError)
{
// Any fault out of the site round-trip (timeout, cancellation,
// transport, serialization) must leave the deployment record as
// Failed -- the design requires an interrupted deployment to be
// treated as failed, never stuck in InProgress.
await MarkDeploymentFailedAsync(record, instance, deploymentId, user, sendError);
_logger.LogError(sendError,
"Deployment {DeploymentId} for instance {Instance} failed",
deploymentId, instance.UniqueName);
return FailureResultFor(sendError);
}
var response = outcome.Response!;
try
{
// Update status based on site response.
record.Status = response.Status;
record.ErrorMessage = response.ErrorMessage;
@@ -322,7 +509,7 @@ public class DeploymentService
// logged loudly for operator reconciliation but must not flip
// the already-committed Success record back to Failed.
await ApplyPostSuccessSideEffectsAsync(
instance, deploymentId, revisionHash, configJson,
instance, deploymentId, prepared.RevisionHash, prepared.ConfigJson,
forceEnabledState: true, cancellationToken);
}
@@ -345,61 +532,366 @@ public class DeploymentService
}
catch (Exception ex)
{
// Any exception out of the try (timeout,
// cancellation, transport, serialization, DB) must leave the
// deployment record as Failed -- the design requires an interrupted
// deployment to be treated as failed, never stuck in InProgress.
//
// The failure-status write must NOT use the
// operation's cancellation token. If the operation was cancelled or
// timed out, that token is already cancelled and the cleanup writes
// would themselves throw before the Failed status is persisted.
// Use CancellationToken.None so the failure is durably recorded.
var isTimeout = ex is TimeoutException or OperationCanceledException or Akka.Actor.AskTimeoutException;
record.Status = DeploymentStatus.Failed;
record.ErrorMessage = isTimeout
? $"{TimeoutFailurePrefix} {ex.Message}"
: $"Deployment error: {ex.Message}";
record.CompletedAt = DateTimeOffset.UtcNow;
try
{
await _repository.UpdateDeploymentRecordAsync(record, CancellationToken.None);
// Note: if the staging SaveChangesAsync above was interrupted, an
// Added PendingDeployment may still be tracked and will be
// committed by this cleanup save. That row is orphaned (no
// RefreshDeploymentCommand was sent, so no site holds its token)
// and is removed by TTL purge / superseded by the next deploy --
// harmless.
await _repository.SaveChangesAsync(CancellationToken.None);
NotifyStatusChange(record);
await _auditService.LogAsync(user, "DeployFailed", "Instance", instanceId.ToString(),
instance.UniqueName, new { DeploymentId = deploymentId, Error = ex.Message },
CancellationToken.None);
}
catch (Exception cleanupEx)
{
// The deployment already failed; a failed cleanup write must not
// mask the original error. Log loudly so an operator can reconcile.
_logger.LogError(cleanupEx,
"Failed to persist Failed status for deployment {DeploymentId} of instance {Instance} " +
"after deployment error: {Error}",
deploymentId, instance.UniqueName, ex.Message);
}
// A fault in the post-send persistence path is handled identically to
// a send fault: the record must never be left InProgress.
await MarkDeploymentFailedAsync(record, instance, deploymentId, user, ex);
_logger.LogError(ex,
"Deployment {DeploymentId} for instance {Instance} failed",
deploymentId, instance.UniqueName);
return Result<DeploymentRecord>.Failure(
isTimeout
? $"Deployment timed out: {ex.Message}"
: $"Deployment failed: {ex.Message}");
return FailureResultFor(ex);
}
}
/// <summary>
/// Writes the terminal <see cref="DeploymentStatus.Failed"/> status (plus the
/// failure audit row) for a deployment that faulted, on any phase.
///
/// <para>
/// The failure-status write deliberately uses <see cref="CancellationToken.None"/>:
/// if the operation was cancelled or timed out, the operation's own token is
/// already cancelled and these cleanup writes would themselves throw before
/// the Failed status was persisted, leaving the record stuck InProgress.
/// </para>
///
/// <para>
/// A fault DURING cleanup is logged loudly and swallowed — it must not mask
/// the original error the caller is about to report.
/// </para>
/// </summary>
private async Task MarkDeploymentFailedAsync(
DeploymentRecord record,
Instance instance,
string deploymentId,
string user,
Exception cause)
{
record.Status = DeploymentStatus.Failed;
record.ErrorMessage = IsTimeoutFault(cause)
? $"{TimeoutFailurePrefix} {cause.Message}"
: $"Deployment error: {cause.Message}";
record.CompletedAt = DateTimeOffset.UtcNow;
try
{
await _repository.UpdateDeploymentRecordAsync(record, CancellationToken.None);
// Note: if the staging SaveChangesAsync was interrupted, an Added
// PendingDeployment may still be tracked and will be committed by this
// cleanup save. That row is orphaned (no RefreshDeploymentCommand was
// sent, so no site holds its token) and is removed by TTL purge /
// superseded by the next deploy -- harmless.
await _repository.SaveChangesAsync(CancellationToken.None);
NotifyStatusChange(record);
await _auditService.LogAsync(user, "DeployFailed", "Instance", instance.Id.ToString(),
instance.UniqueName, new { DeploymentId = deploymentId, Error = cause.Message },
CancellationToken.None);
}
catch (Exception cleanupEx)
{
_logger.LogError(cleanupEx,
"Failed to persist Failed status for deployment {DeploymentId} of instance {Instance} " +
"after deployment error: {Error}",
deploymentId, instance.UniqueName, cause.Message);
}
}
/// <summary>
/// True when a deployment fault is a timeout/cancellation rather than a hard
/// error. Drives both the <see cref="TimeoutFailurePrefix"/> marker (which the
/// query-before-redeploy trigger reads) and the caller-facing wording.
/// </summary>
private static bool IsTimeoutFault(Exception ex) =>
ex is TimeoutException or OperationCanceledException or Akka.Actor.AskTimeoutException;
/// <summary>Maps a deployment fault to the caller-facing failure result.</summary>
private static Result<DeploymentRecord> FailureResultFor(Exception ex) =>
Result<DeploymentRecord>.Failure(
IsTimeoutFault(ex)
? $"Deployment timed out: {ex.Message}"
: $"Deployment failed: {ex.Message}");
/// <summary>
/// A deployment that has cleared phase 1 and is ready for its site round-trip,
/// OR one that resolved during phase 1 (carrying <see cref="EarlyResult"/>).
///
/// <para>
/// When <see cref="EarlyResult"/> is <see langword="null"/> this record OWNS the
/// per-instance operation lock in <see cref="LockHandle"/>, and the caller must
/// dispose it after phase 3. When <see cref="EarlyResult"/> is set the lock has
/// already been released and every other member is a placeholder.
/// </para>
/// </summary>
private sealed record PreparedDeployment(
Instance Instance,
string DeploymentId,
DeploymentRecord Record,
string RevisionHash,
string ConfigJson,
string SiteIdentifier,
RefreshDeploymentCommand Command,
IDisposable? LockHandle,
Result<DeploymentRecord>? EarlyResult)
{
/// <summary>Builds a phase-1-resolved deployment carrying the given result and holding no lock.</summary>
/// <param name="result">The result to hand back to the caller.</param>
/// <returns>A prepared deployment whose <see cref="EarlyResult"/> is set.</returns>
public static PreparedDeployment Resolved(Result<DeploymentRecord> result) =>
new(null!, string.Empty, null!, string.Empty, string.Empty, string.Empty, null!, null, result);
}
/// <summary>
/// Result of phase 2: either the site's response or the exception that
/// prevented one. Exactly one is non-null.
/// </summary>
private readonly record struct SendOutcome(DeploymentStatusResponse? Response, Exception? Error);
/// <summary>
/// Deploy every deployable instance at a site in one operation.
///
/// <para>
/// Structured as the three deployment phases with the middle one fanned out:
/// </para>
/// <list type="number">
/// <item>
/// <b>Prepare (serial).</b> Every instance is flattened, validated, staged
/// and given an <c>InProgress</c> record on the caller's single scoped
/// <c>DbContext</c>. All instances share ONE <see cref="FlattenSession"/>,
/// so a template chain common to N instances is walked once, and the
/// session-global queries (shared scripts, schema library, the site's data
/// connections) run once for the whole batch instead of once per instance.
/// </item>
/// <item>
/// <b>Send (bounded parallel).</b> Site round-trips run concurrently up to
/// <see cref="DeploymentManagerOptions.SiteDeploymentMaxParallelism"/>,
/// each under its own
/// <see cref="DeploymentManagerOptions.SiteDeploymentTimeoutPerInstance"/>
/// deadline, so one wedged instance cannot stall the batch. This phase
/// touches no repository — that is precisely why it is the only phase that
/// may run in parallel against a non-thread-safe <c>DbContext</c>.
/// </item>
/// <item>
/// <b>Finalize (serial).</b> Terminal statuses, post-success side effects
/// and audit rows are committed one instance at a time, then each
/// instance's operation lock is released.
/// </item>
/// </list>
///
/// <para>
/// Semantics per instance are byte-for-byte those of
/// <see cref="DeployInstanceAsync"/> — its own deployment id and revision hash,
/// its own operation lock held across the whole operation, its own
/// optimistically-concurrent status record, and the same query-before-redeploy
/// idempotency check. Instances that cannot be deployed (wrong state, failed
/// validation, lock already held) are reported as failed rows; the batch is
/// NOT all-or-nothing, mirroring the artifact deployment path.
/// </para>
/// </summary>
/// <param name="siteId">The database ID of the site whose instances are deployed.</param>
/// <param name="user">The username initiating the deployment, recorded on every record and audit row.</param>
/// <param name="cancellationToken">Cancellation token for the operation.</param>
/// <returns>A per-instance result matrix, or a failure result when the site itself cannot be resolved.</returns>
public async Task<Result<SiteDeploymentSummary>> DeploySiteAsync(
int siteId,
string user,
CancellationToken cancellationToken = default)
{
var site = await _siteRepository.GetSiteByIdAsync(siteId, cancellationToken);
if (site == null)
return Result<SiteDeploymentSummary>.Failure($"Site with ID {siteId} not found.");
var instances = await _siteRepository.GetInstancesBySiteIdAsync(siteId, cancellationToken);
if (instances.Count == 0)
return Result<SiteDeploymentSummary>.Success(new SiteDeploymentSummary(site.SiteIdentifier, [], 0, 0));
// ---- Phase 1: prepare, serially, on ONE shared flatten session. ----
var session = _flatteningPipeline.CreateSession();
var prepared = new List<PreparedDeployment>(instances.Count);
var results = new List<InstanceDeploymentResult>(instances.Count);
foreach (var instance in instances)
{
cancellationToken.ThrowIfCancellationRequested();
PreparedDeployment step;
try
{
step = await PrepareDeploymentAsync(instance.Id, user, session, cancellationToken);
}
catch (Exception ex)
{
// A prepare fault (most commonly a TimeoutException from the
// operation lock because another operation holds this instance)
// fails only this instance. Recording it and moving on is what
// makes a bulk deploy usable while individual instances are busy.
_logger.LogWarning(ex,
"Preparing instance {Instance} for bulk deployment of site {SiteId} failed",
instance.UniqueName, site.SiteIdentifier);
results.Add(new InstanceDeploymentResult(
instance.Id, instance.UniqueName, null, false, ex.Message));
continue;
}
if (step.EarlyResult is { } early)
{
results.Add(ToInstanceResult(instance.Id, instance.UniqueName, early));
continue;
}
prepared.Add(step);
}
// ---- Phase 2: bounded-parallel site round-trips. ----
var outcomes = await SendPreparedAsync(prepared, cancellationToken);
// ---- Phase 3: finalize, serially, releasing each lock as we go. ----
for (var i = 0; i < prepared.Count; i++)
{
var step = prepared[i];
try
{
var result = await FinalizeDeploymentAsync(step, outcomes[i], user, cancellationToken);
results.Add(ToInstanceResult(step.Instance.Id, step.Instance.UniqueName, result, step.DeploymentId));
}
finally
{
step.LockHandle?.Dispose();
}
}
var summary = new SiteDeploymentSummary(
site.SiteIdentifier,
results,
results.Count(r => r.Success),
results.Count(r => !r.Success));
_logger.LogInformation(
"Bulk deployment of site {SiteId} complete: {SuccessCount} succeeded, {FailureCount} failed " +
"({ChainLoads} template-chain load(s) across {InstanceCount} instance(s))",
site.SiteIdentifier, summary.SuccessCount, summary.FailureCount,
session.ChainLoads, instances.Count);
return Result<SiteDeploymentSummary>.Success(summary);
}
/// <summary>
/// Runs phase 2 for a whole batch: every prepared deployment's site round-trip,
/// concurrent up to <see cref="DeploymentManagerOptions.SiteDeploymentMaxParallelism"/>
/// and each bounded by
/// <see cref="DeploymentManagerOptions.SiteDeploymentTimeoutPerInstance"/>.
/// Outcomes are returned positionally so phase 3 can pair them back with their
/// prepared deployment.
/// </summary>
private async Task<SendOutcome[]> SendPreparedAsync(
IReadOnlyList<PreparedDeployment> prepared,
CancellationToken cancellationToken)
{
var outcomes = new SendOutcome[prepared.Count];
if (prepared.Count == 0)
return outcomes;
using var gate = new SemaphoreSlim(_options.SiteDeploymentMaxParallelism);
var sends = prepared.Select(async (step, index) =>
{
await gate.WaitAsync(cancellationToken);
try
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(_options.SiteDeploymentTimeoutPerInstance);
outcomes[index] = await SendDeploymentAsync(step, cts.Token);
}
finally
{
gate.Release();
}
});
await Task.WhenAll(sends);
return outcomes;
}
/// <summary>Maps a per-instance deployment result into the bulk summary row shape.</summary>
private static InstanceDeploymentResult ToInstanceResult(
int instanceId,
string uniqueName,
Result<DeploymentRecord> result,
string? deploymentId = null) =>
result.IsSuccess
? new InstanceDeploymentResult(
instanceId, uniqueName, result.Value.DeploymentId, true, null)
: new InstanceDeploymentResult(
instanceId, uniqueName, deploymentId, false, result.Error);
/// <summary>
/// Opportunistic retention sweep for TERMINAL deployment records, rate-limited
/// to once every <see cref="TerminalPurgeMinInterval"/> per process and bounded
/// to <see cref="TerminalPurgeBatchesPerSweep"/> batches per call.
///
/// <para>
/// Swept on read (the deployment-list path) rather than from a timer for the
/// same reason expired secured writes are: the component owns no scheduling
/// infrastructure of its own, and the read path is exactly the operation that
/// suffers when the table is allowed to grow without bound. Both central nodes
/// may sweep; the delete is idempotent, so a duplicate pass is a no-op.
/// </para>
///
/// <para>
/// Best-effort: a failed sweep is logged and swallowed. Retention maintenance
/// must never fail the operator's actual query.
/// </para>
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The number of records purged by this call (0 when the sweep was skipped).</returns>
public async Task<int> TryPurgeTerminalDeploymentRecordsAsync(CancellationToken cancellationToken = default)
{
var retention = _options.TerminalDeploymentRecordRetention;
if (retention <= TimeSpan.Zero)
return 0;
var now = DateTimeOffset.UtcNow;
var last = Interlocked.Read(ref _lastTerminalPurgeTicks);
if (last != 0 && now.UtcTicks - last < TerminalPurgeMinInterval.Ticks)
return 0;
// Claim the sweep. A loser of this CAS returns immediately rather than
// running a redundant second pass.
if (Interlocked.CompareExchange(ref _lastTerminalPurgeTicks, now.UtcTicks, last) != last)
return 0;
var cutoff = now - retention;
var purged = 0;
try
{
for (var batch = 0; batch < TerminalPurgeBatchesPerSweep; batch++)
{
var deleted = await _repository.PurgeTerminalDeploymentRecordsAsync(
cutoff, TerminalPurgeBatchSize, cancellationToken);
purged += deleted;
if (deleted < TerminalPurgeBatchSize)
break;
}
if (purged > 0)
{
_logger.LogInformation(
"Purged {Count} terminal deployment record(s) completed before {Cutoff} " +
"(retention {Retention}).",
purged, cutoff, retention);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Terminal deployment-record retention sweep failed; retrying after {Interval}.",
TerminalPurgeMinInterval);
}
return purged;
}
/// <summary>
/// Disable an instance. Stops Instance Actor, retains config, S&amp;F drains.
/// </summary>
@@ -1178,3 +1670,37 @@ public record DeploymentComparisonResult(
bool IsStale,
DateTimeOffset DeployedAt,
ConfigurationDiff? Diff = null);
/// <summary>
/// One instance's outcome inside a bulk site deployment.
/// </summary>
/// <param name="InstanceId">Database id of the instance.</param>
/// <param name="UniqueName">The instance's unique name (also its operation-lock key).</param>
/// <param name="DeploymentId">
/// The deployment id minted for this instance, or <see langword="null"/> when the
/// instance never got that far (state-transition rejection, lock contention).
/// </param>
/// <param name="Success">Whether the site confirmed the apply.</param>
/// <param name="ErrorMessage">Failure detail; <see langword="null"/> on success.</param>
public record InstanceDeploymentResult(
int InstanceId,
string UniqueName,
string? DeploymentId,
bool Success,
string? ErrorMessage);
/// <summary>
/// Result matrix of a bulk site deployment. Mirrors the shape of
/// <c>ArtifactDeploymentSummary</c>: successes are not rolled back when other
/// instances fail, and each failed instance is individually retryable through the
/// ordinary single-instance deploy.
/// </summary>
/// <param name="SiteIdentifier">The site's string identifier (e.g. <c>site-a</c>).</param>
/// <param name="InstanceResults">Per-instance outcomes.</param>
/// <param name="SuccessCount">Number of instances the site confirmed.</param>
/// <param name="FailureCount">Number of instances that failed for any reason.</param>
public record SiteDeploymentSummary(
string SiteIdentifier,
IReadOnlyList<InstanceDeploymentResult> InstanceResults,
int SuccessCount,
int FailureCount);