feat(audit-log): surface partition-purge failure as health event + counter

This commit is contained in:
Joseph Doherty
2026-07-09 06:49:28 -04:00
parent b59aa2d717
commit 14c4df54f9
9 changed files with 202 additions and 5 deletions
@@ -149,7 +149,14 @@ public class AuditLogPurgeActor : ReceiveActor
return;
}
// Optional purge-failure counter — resolved from the same per-tick scope the repository
// comes from. NoOp default on site/test roots; the central composition root binds the
// AuditCentralHealthSnapshot so failures surface on the central health dashboard. Mirrors
// AuditLogIngestActor's ICentralAuditWriteFailureCounter resolution shape.
var purgeFailureCounter = scope.ServiceProvider.GetService<IAuditPurgeFailureCounter>();
IReadOnlyList<DateTime> boundaries;
var enumSw = Stopwatch.StartNew();
try
{
boundaries = await repository
@@ -158,10 +165,14 @@ public class AuditLogPurgeActor : ReceiveActor
}
catch (Exception ex)
{
enumSw.Stop();
_logger.LogError(
ex,
"Failed to enumerate eligible AuditLog partition boundaries (threshold {ThresholdUtc:o}); skipping purge tick.",
threshold);
// Enumeration threw before any boundary was selected — publish with the
// DateTime.MinValue sentinel so the failure is still observable on the health surface.
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, enumSw.ElapsedMilliseconds);
return;
}
@@ -169,7 +180,7 @@ public class AuditLogPurgeActor : ReceiveActor
{
// No whole-month partitions are eligible, but per-channel overrides may
// still expire rows earlier than the global window — run them below.
await RunPerChannelOverridesAsync(repository).ConfigureAwait(false);
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
return;
}
@@ -203,6 +214,7 @@ public class AuditLogPurgeActor : ReceiveActor
"Failed to purge AuditLog partition {MonthBoundary:yyyy-MM-dd}; other partitions continue. Elapsed {DurationMs} ms.",
boundary,
sw.ElapsedMilliseconds);
PublishPurgeFailure(eventStream, purgeFailureCounter, boundary, ex, sw.ElapsedMilliseconds);
}
}
@@ -212,7 +224,24 @@ public class AuditLogPurgeActor : ReceiveActor
// switch-out has already dropped whole months older than RetentionDays; these
// deletes only ever expire rows EARLIER than that, so they run last and are a
// strict tightening.
await RunPerChannelOverridesAsync(repository).ConfigureAwait(false);
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
}
/// <summary>
/// Publishes an <see cref="AuditLogPurgeFailedEvent"/> on the EventStream and increments the
/// central purge-failure health counter (when wired). Best-effort — the symmetric failure twin
/// of the success-path <see cref="AuditLogPurgedEvent"/> publish, so a silently failing
/// retention job is a health-surface condition, not just a log line (arch-review 04, S2).
/// </summary>
private static void PublishPurgeFailure(
Akka.Event.EventStream eventStream,
IAuditPurgeFailureCounter? counter,
DateTime monthBoundary,
Exception ex,
long elapsedMilliseconds)
{
eventStream.Publish(new AuditLogPurgeFailedEvent(monthBoundary, ex.Message, elapsedMilliseconds));
counter?.Increment();
}
/// <summary>
@@ -224,7 +253,12 @@ public class AuditLogPurgeActor : ReceiveActor
/// per-boundary error isolation of the partition switch-out loop.
/// </summary>
/// <param name="repository">The repository resolved for this tick's DI scope.</param>
private async Task RunPerChannelOverridesAsync(IAuditLogRepository repository)
/// <param name="eventStream">The actor-system EventStream captured before the first await.</param>
/// <param name="purgeFailureCounter">Optional central purge-failure health counter (NoOp on site/test roots).</param>
private async Task RunPerChannelOverridesAsync(
IAuditLogRepository repository,
Akka.Event.EventStream eventStream,
IAuditPurgeFailureCounter? purgeFailureCounter)
{
var overrides = _auditOptions.PerChannelRetentionDays;
if (overrides is null || overrides.Count == 0)
@@ -281,6 +315,10 @@ public class AuditLogPurgeActor : ReceiveActor
channel,
days,
sw.ElapsedMilliseconds);
// A per-channel DELETE failure is a purge failure too — surface it alongside the
// switch-out failures. MonthBoundary is DateTime.MinValue (this is not a
// whole-month switch), matching the enumeration-phase sentinel.
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, sw.ElapsedMilliseconds);
}
}
}