fix(management): purge terminal PendingSecuredWrites rows on the list sweep after a retention window (review 2026-10-21 08/G3)

This commit is contained in:
Joseph Doherty
2026-10-22 09:25:48 -04:00
parent baab6ac78e
commit d495494074
8 changed files with 237 additions and 1 deletions
@@ -153,4 +153,18 @@ public interface ISecuredWriteRepository
string? siteId,
IReadOnlyCollection<string>? permittedSiteIds = null,
CancellationToken ct = default);
/// <summary>
/// Deletes up to <paramref name="batchSize"/> rows in a post-Pending status
/// (<c>Approved</c>, <c>Rejected</c>, <c>Expired</c>, <c>Executed</c>, <c>Failed</c>) whose
/// last lifecycle instant — <c>ExecutedAtUtc</c>, else <c>DecidedAtUtc</c>, else
/// <c>SubmittedAtUtc</c> — is older than <paramref name="olderThanUtc"/>. <c>Pending</c>
/// rows are never purged (they are expired first). The table is operational state; the
/// append-only <c>AuditLog</c> keeps the lifecycle history (review 2026-10-21 08/G3).
/// </summary>
/// <param name="olderThanUtc">Rows whose last lifecycle instant is before this are purged.</param>
/// <param name="batchSize">Maximum rows deleted in this call.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the number of rows deleted.</returns>
Task<int> PurgeTerminalAsync(DateTime olderThanUtc, int batchSize, CancellationToken ct = default);
}
@@ -188,4 +188,25 @@ WHERE Id = {id}
return await query.CountAsync(ct);
}
/// <inheritdoc />
public async Task<int> PurgeTerminalAsync(DateTime olderThanUtc, int batchSize, CancellationToken ct = default)
{
if (batchSize <= 0)
{
return 0;
}
// Bounded set-based delete (DELETE ... WHERE Id IN (SELECT TOP(n) ...)): never a
// whole-table delete in one statement, so a first purge of a long-lived table cannot
// hold a large lock. Pending is excluded — it is expired first, then purged later.
return await _context.Set<PendingSecuredWrite>()
.Where(p => TerminalStatuses.Contains(p.Status)
&& (p.ExecutedAtUtc ?? p.DecidedAtUtc ?? p.SubmittedAtUtc) < olderThanUtc)
.OrderBy(p => p.Id)
.Take(batchSize)
.ExecuteDeleteAsync(ct);
}
private static readonly string[] TerminalStatuses = ["Approved", "Rejected", "Expired", "Executed", "Failed"];
}
@@ -1860,9 +1860,56 @@ public class ManagementActor : ReceiveActor
await TryExpireIfStaleAsync(sp, repo, row);
}
await TryPurgeTerminalSecuredWritesAsync(sp, repo);
return new SecuredWriteListResult(rows.Select(ToSecuredWriteDto).ToList(), totalCount);
}
// Last terminal-purge pass on this node (UTC ticks); throttles the purge that rides the
// list sweep to once per SecuredWriteTerminalPurgeInterval.
private static long _lastSecuredWritePurgeTicks;
private const int SecuredWritePurgeBatchSize = 1000;
/// <summary>
/// Purges decided/executed secured-write rows older than
/// <see cref="ManagementServiceOptions.SecuredWriteTerminalRetention"/> (review 2026-10-21
/// 08/G3 — the table used to grow forever and the list's offset paging scanned it all).
/// Rides the same opportunistic list sweep as Pending expiry rather than a dedicated
/// timer, throttled per node; one bounded batch per pass, so a backlog converges over
/// successive passes. Both central nodes may purge concurrently — the set-based delete is
/// idempotent. Best-effort: a failure is logged and never fails the list.
/// </summary>
private static async Task TryPurgeTerminalSecuredWritesAsync(IServiceProvider sp, ISecuredWriteRepository repo)
{
var options = sp.GetService<IOptions<ManagementServiceOptions>>()?.Value ?? new ManagementServiceOptions();
var retention = options.SecuredWriteTerminalRetention;
if (retention <= TimeSpan.Zero)
return;
var now = DateTime.UtcNow;
var last = Interlocked.Read(ref _lastSecuredWritePurgeTicks);
if (options.SecuredWriteTerminalPurgeInterval > TimeSpan.Zero
&& now.Ticks - last < options.SecuredWriteTerminalPurgeInterval.Ticks)
return;
// One winner per interval on this node; a concurrent list skips the pass.
if (Interlocked.CompareExchange(ref _lastSecuredWritePurgeTicks, now.Ticks, last) != last)
return;
try
{
var deleted = await repo.PurgeTerminalAsync(now - retention, SecuredWritePurgeBatchSize);
if (deleted > 0)
sp.GetService<ILogger<ManagementActor>>()?.LogInformation(
"Purged {Count} terminal secured-write row(s) older than {Retention}.", deleted, retention);
}
catch (Exception ex)
{
sp.GetService<ILogger<ManagementActor>>()?.LogWarning(
ex, "Best-effort terminal secured-write purge failed; the list is unaffected.");
}
}
// ========================================================================
// Site handlers
// ========================================================================
@@ -10,4 +10,10 @@ public class ManagementServiceOptions
/// <summary>Age after which a Pending secured write can no longer be approved; it is transitioned to Expired at the next decision/list touch. Default 24 hours. A non-positive value disables server-side expiry.</summary>
public TimeSpan SecuredWritePendingTtl { get; set; } = TimeSpan.FromHours(24);
/// <summary>Age after which a decided/executed secured-write row (Approved/Rejected/Expired/Executed/Failed, measured from its last lifecycle instant) is purged from <c>PendingSecuredWrites</c>; the AuditLog keeps the history. Default 90 days. A non-positive value disables the purge.</summary>
public TimeSpan SecuredWriteTerminalRetention { get; set; } = TimeSpan.FromDays(90);
/// <summary>Minimum interval between terminal secured-write purge passes on one node (the purge rides the Secured Writes list sweep). Default 1 hour; zero runs it on every list.</summary>
public TimeSpan SecuredWriteTerminalPurgeInterval { get; set; } = TimeSpan.FromHours(1);
}
@@ -11,7 +11,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
/// section fails fast at boot with a clear, key-naming message.
/// <para>
/// <see cref="ManagementServiceOptions.SecuredWritePendingTtl"/> is intentionally NOT required positive:
/// a non-positive value legitimately disables server-side secured-write expiry.
/// a non-positive value legitimately disables server-side secured-write expiry. The same holds for
/// <see cref="ManagementServiceOptions.SecuredWriteTerminalRetention"/> (non-positive disables the terminal-row purge).
/// </para>
/// </summary>
public sealed class ManagementServiceOptionsValidator : OptionsValidatorBase<ManagementServiceOptions>
@@ -26,5 +27,9 @@ public sealed class ManagementServiceOptionsValidator : OptionsValidatorBase<Man
builder.RequireThat(options.LongRunningCommandTimeout > FlowDebug.FlowDebugRunner.ResponseGrace,
$"ScadaBridge:ManagementService:LongRunningCommandTimeout must exceed the 5-second response grace " +
$"(was {options.LongRunningCommandTimeout}); it is the Ask timeout for transport/deploy commands.");
builder.RequireThat(options.SecuredWriteTerminalPurgeInterval >= TimeSpan.Zero,
$"ScadaBridge:ManagementService:SecuredWriteTerminalPurgeInterval must not be negative " +
$"(was {options.SecuredWriteTerminalPurgeInterval}); zero purges on every Secured Writes list.");
}
}