fix(audit-log): explicit long CommandTimeout on partition-switch and per-channel purge maintenance paths
This commit is contained in:
@@ -182,7 +182,7 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
try
|
||||
{
|
||||
var rowsDeleted = await repository
|
||||
.SwitchOutPartitionAsync(boundary)
|
||||
.SwitchOutPartitionAsync(boundary, _purgeOptions.ResolvedMaintenanceCommandTimeout)
|
||||
.ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
@@ -250,7 +250,11 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
try
|
||||
{
|
||||
var rowsDeleted = await repository
|
||||
.PurgeChannelOlderThanAsync(channel, channelThreshold, _purgeOptions.ChannelPurgeBatchSize)
|
||||
.PurgeChannelOlderThanAsync(
|
||||
channel,
|
||||
channelThreshold,
|
||||
_purgeOptions.ChannelPurgeBatchSize,
|
||||
_purgeOptions.ResolvedMaintenanceCommandTimeout)
|
||||
.ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
|
||||
@@ -56,6 +56,30 @@ public sealed class AuditLogPurgeOptions
|
||||
/// </summary>
|
||||
public int ChannelPurgeBatchSize => ChannelPurgeBatchSizeConfigured < 1 ? 1 : ChannelPurgeBatchSizeConfigured;
|
||||
|
||||
/// <summary>
|
||||
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
|
||||
/// both the partition switch-out drop-and-rebuild dance
|
||||
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
|
||||
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
|
||||
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort
|
||||
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a
|
||||
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
|
||||
/// lets the metadata-only switch complete rather than self-locking. Resolved via
|
||||
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
|
||||
/// </remarks>
|
||||
public int MaintenanceCommandTimeoutMinutes { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the effective maintenance command timeout, clamped to at least 1 minute so a
|
||||
/// misconfigured <c>0</c>/negative value cannot yield a zero/negative timeout (which ADO.NET
|
||||
/// interprets as "no timeout" for <c>0</c> and rejects for negatives).
|
||||
/// </summary>
|
||||
public TimeSpan ResolvedMaintenanceCommandTimeout =>
|
||||
TimeSpan.FromMinutes(MaintenanceCommandTimeoutMinutes < 1 ? 1 : MaintenanceCommandTimeoutMinutes);
|
||||
|
||||
/// <summary>
|
||||
/// Test-only override for finer control over the tick cadence than
|
||||
/// whole-hour resolution allows. When non-null, takes precedence over
|
||||
|
||||
@@ -83,9 +83,17 @@ public interface IAuditLogRepository
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
|
||||
/// <param name="commandTimeout">
|
||||
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
|
||||
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
|
||||
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large
|
||||
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next
|
||||
/// tick recovers (arch-review 04, S2).
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
|
||||
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default);
|
||||
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Per-channel retention override purge. Deletes <c>AuditLog</c> rows for a
|
||||
@@ -115,12 +123,20 @@ public interface IAuditLogRepository
|
||||
/// <param name="channel">Canonical channel name (the <c>Category</c> column value, e.g. <c>ApiOutbound</c>).</param>
|
||||
/// <param name="threshold">Rows with <c>OccurredAtUtc</c> strictly older than this UTC datetime are deleted.</param>
|
||||
/// <param name="batchSize">Maximum rows deleted per batch; must be > 0.</param>
|
||||
/// <param name="commandTimeout">
|
||||
/// Optional per-batch <c>DELETE TOP</c> command timeout. When null the provider default applies.
|
||||
/// The purge actor passes the same
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
|
||||
/// it uses for the partition switch so a large single batch under lock contention is not aborted
|
||||
/// by the ~30s ADO.NET default (arch-review 04, S2).
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the total number of rows deleted across all batches.</returns>
|
||||
Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel,
|
||||
DateTime threshold,
|
||||
int batchSize,
|
||||
TimeSpan? commandTimeout = null,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -227,8 +227,13 @@ VALUES
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
|
||||
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
{
|
||||
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
|
||||
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
|
||||
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
|
||||
var commandTimeoutSeconds = commandTimeout is { } ct2 ? (int)ct2.TotalSeconds : (int?)null;
|
||||
|
||||
// GUID-suffixed staging name: prevents collision with any concurrent
|
||||
// purge attempt and avoids polluting the AuditLog object namespace with
|
||||
// a predictable identifier.
|
||||
@@ -351,6 +356,10 @@ VALUES
|
||||
await using (var sampleCmd = conn.CreateCommand())
|
||||
{
|
||||
sampleCmd.CommandText = sampleSql;
|
||||
if (commandTimeoutSeconds is { } sampleTimeout)
|
||||
{
|
||||
sampleCmd.CommandTimeout = sampleTimeout;
|
||||
}
|
||||
var sampleResult = await sampleCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
if (sampleResult is not null && sampleResult is not DBNull)
|
||||
{
|
||||
@@ -366,7 +375,27 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
await _context.Database.ExecuteSqlRawAsync(sql, ct);
|
||||
// Apply the maintenance timeout to the drop-and-rebuild dance. The context is scoped per
|
||||
// purge tick (see AuditLogPurgeActor.OnTickAsync's CreateAsyncScope), so a restore is not
|
||||
// strictly required — but we restore the previous value in a finally for hygiene in case a
|
||||
// future caller reuses the context for other work after the switch-out.
|
||||
var previousTimeout = _context.Database.GetCommandTimeout();
|
||||
if (commandTimeout is { } maintenanceTimeout)
|
||||
{
|
||||
_context.Database.SetCommandTimeout(maintenanceTimeout);
|
||||
}
|
||||
try
|
||||
{
|
||||
await _context.Database.ExecuteSqlRawAsync(sql, ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (commandTimeout is not null)
|
||||
{
|
||||
_context.Database.SetCommandTimeout(previousTimeout);
|
||||
}
|
||||
}
|
||||
|
||||
return rowsDeleted;
|
||||
}
|
||||
|
||||
@@ -375,6 +404,7 @@ VALUES
|
||||
string channel,
|
||||
DateTime threshold,
|
||||
int batchSize,
|
||||
TimeSpan? commandTimeout = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(channel))
|
||||
@@ -389,6 +419,9 @@ VALUES
|
||||
|
||||
var thresholdUtc = DateTime.SpecifyKind(threshold.ToUniversalTime(), DateTimeKind.Utc);
|
||||
|
||||
// Maintenance timeout in whole seconds; null leaves the provider default (arch-review 04 S2).
|
||||
var commandTimeoutSeconds = commandTimeout is { } cto ? (int)cto.TotalSeconds : (int?)null;
|
||||
|
||||
// Per-channel retention override purge. This is the ONLY DELETE
|
||||
// against dbo.AuditLog in the codebase and it runs on the purge/maintenance
|
||||
// path, NOT the append-only writer role (which has INSERT + SELECT only — see
|
||||
@@ -427,6 +460,10 @@ VALUES
|
||||
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = deleteBatchSql;
|
||||
if (commandTimeoutSeconds is { } batchTimeout)
|
||||
{
|
||||
cmd.CommandTimeout = batchTimeout;
|
||||
}
|
||||
|
||||
var pBatch = cmd.CreateParameter();
|
||||
pBatch.ParameterName = "@batch";
|
||||
|
||||
Reference in New Issue
Block a user