fix(audit-log): explicit long CommandTimeout on partition-switch and per-channel purge maintenance paths

This commit is contained in:
Joseph Doherty
2026-07-09 06:43:23 -04:00
parent 1b6b0aab2a
commit b59aa2d717
11 changed files with 231 additions and 17 deletions
@@ -182,7 +182,7 @@ public class AuditLogPurgeActor : ReceiveActor
try
{
var rowsDeleted = await repository
.SwitchOutPartitionAsync(boundary)
.SwitchOutPartitionAsync(boundary, _purgeOptions.ResolvedMaintenanceCommandTimeout)
.ConfigureAwait(false);
sw.Stop();
@@ -250,7 +250,11 @@ public class AuditLogPurgeActor : ReceiveActor
try
{
var rowsDeleted = await repository
.PurgeChannelOlderThanAsync(channel, channelThreshold, _purgeOptions.ChannelPurgeBatchSize)
.PurgeChannelOlderThanAsync(
channel,
channelThreshold,
_purgeOptions.ChannelPurgeBatchSize,
_purgeOptions.ResolvedMaintenanceCommandTimeout)
.ConfigureAwait(false);
sw.Stop();
@@ -56,6 +56,30 @@ public sealed class AuditLogPurgeOptions
/// </summary>
public int ChannelPurgeBatchSize => ChannelPurgeBatchSizeConfigured < 1 ? 1 : ChannelPurgeBatchSizeConfigured;
/// <summary>
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
/// both the partition switch-out drop-and-rebuild dance
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
/// </summary>
/// <remarks>
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
/// lets the metadata-only switch complete rather than self-locking. Resolved via
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
/// </remarks>
public int MaintenanceCommandTimeoutMinutes { get; set; } = 30;
/// <summary>
/// Resolves the effective maintenance command timeout, clamped to at least 1 minute so a
/// misconfigured <c>0</c>/negative value cannot yield a zero/negative timeout (which ADO.NET
/// interprets as "no timeout" for <c>0</c> and rejects for negatives).
/// </summary>
public TimeSpan ResolvedMaintenanceCommandTimeout =>
TimeSpan.FromMinutes(MaintenanceCommandTimeoutMinutes < 1 ? 1 : MaintenanceCommandTimeoutMinutes);
/// <summary>
/// Test-only override for finer control over the tick cadence than
/// whole-hour resolution allows. When non-null, takes precedence over
@@ -83,9 +83,17 @@ public interface IAuditLogRepository
/// </para>
/// </remarks>
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
/// <param name="commandTimeout">
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next
/// tick recovers (arch-review 04, S2).
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default);
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default);
/// <summary>
/// Per-channel retention override purge. Deletes <c>AuditLog</c> rows for a
@@ -115,12 +123,20 @@ public interface IAuditLogRepository
/// <param name="channel">Canonical channel name (the <c>Category</c> column value, e.g. <c>ApiOutbound</c>).</param>
/// <param name="threshold">Rows with <c>OccurredAtUtc</c> strictly older than this UTC datetime are deleted.</param>
/// <param name="batchSize">Maximum rows deleted per batch; must be &gt; 0.</param>
/// <param name="commandTimeout">
/// Optional per-batch <c>DELETE TOP</c> command timeout. When null the provider default applies.
/// The purge actor passes the same
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
/// it uses for the partition switch so a large single batch under lock contention is not aborted
/// by the ~30s ADO.NET default (arch-review 04, S2).
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the total number of rows deleted across all batches.</returns>
Task<long> PurgeChannelOlderThanAsync(
string channel,
DateTime threshold,
int batchSize,
TimeSpan? commandTimeout = null,
CancellationToken ct = default);
/// <summary>
@@ -227,8 +227,13 @@ VALUES
};
/// <inheritdoc />
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
var commandTimeoutSeconds = commandTimeout is { } ct2 ? (int)ct2.TotalSeconds : (int?)null;
// GUID-suffixed staging name: prevents collision with any concurrent
// purge attempt and avoids polluting the AuditLog object namespace with
// a predictable identifier.
@@ -351,6 +356,10 @@ VALUES
await using (var sampleCmd = conn.CreateCommand())
{
sampleCmd.CommandText = sampleSql;
if (commandTimeoutSeconds is { } sampleTimeout)
{
sampleCmd.CommandTimeout = sampleTimeout;
}
var sampleResult = await sampleCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
if (sampleResult is not null && sampleResult is not DBNull)
{
@@ -366,7 +375,27 @@ VALUES
}
}
await _context.Database.ExecuteSqlRawAsync(sql, ct);
// Apply the maintenance timeout to the drop-and-rebuild dance. The context is scoped per
// purge tick (see AuditLogPurgeActor.OnTickAsync's CreateAsyncScope), so a restore is not
// strictly required — but we restore the previous value in a finally for hygiene in case a
// future caller reuses the context for other work after the switch-out.
var previousTimeout = _context.Database.GetCommandTimeout();
if (commandTimeout is { } maintenanceTimeout)
{
_context.Database.SetCommandTimeout(maintenanceTimeout);
}
try
{
await _context.Database.ExecuteSqlRawAsync(sql, ct);
}
finally
{
if (commandTimeout is not null)
{
_context.Database.SetCommandTimeout(previousTimeout);
}
}
return rowsDeleted;
}
@@ -375,6 +404,7 @@ VALUES
string channel,
DateTime threshold,
int batchSize,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
if (string.IsNullOrWhiteSpace(channel))
@@ -389,6 +419,9 @@ VALUES
var thresholdUtc = DateTime.SpecifyKind(threshold.ToUniversalTime(), DateTimeKind.Utc);
// Maintenance timeout in whole seconds; null leaves the provider default (arch-review 04 S2).
var commandTimeoutSeconds = commandTimeout is { } cto ? (int)cto.TotalSeconds : (int?)null;
// Per-channel retention override purge. This is the ONLY DELETE
// against dbo.AuditLog in the codebase and it runs on the purge/maintenance
// path, NOT the append-only writer role (which has INSERT + SELECT only — see
@@ -427,6 +460,10 @@ VALUES
await using var cmd = conn.CreateCommand();
cmd.CommandText = deleteBatchSql;
if (commandTimeoutSeconds is { } batchTimeout)
{
cmd.CommandTimeout = batchTimeout;
}
var pBatch = cmd.CreateParameter();
pBatch.ParameterName = "@batch";