feat(audit-log): permanently-abandoned reconciliation rows leave a durable ReconciliationAbandoned audit record

When a pulled AuditEvent fails to insert on every retry up to the permanent-
abandon threshold, the reconciliation actor now writes ONE synthetic
ReconciliationAbandoned row (new AuditKind) alongside the Critical log line —
fresh EventId, Status=Failed, channel preserved from the lost row, Extra carrying
the abandoned EventId + source site + final error — via the same
ScadaBridgeAuditEventFactory the ingest path uses. Best-effort in its own
try/catch so it never blocks the cursor twice. The permanent loss is now
queryable in the Audit Log, not only in a rotating log file.
This commit is contained in:
Joseph Doherty
2026-07-09 08:50:44 -04:00
parent 20098c6108
commit 0385942c3f
4 changed files with 207 additions and 1 deletions
@@ -1,9 +1,11 @@
using System.Text.Json;
using Akka.Actor;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
@@ -290,6 +292,13 @@ public class SiteAuditReconciliationActor : ReceiveActor
attempts);
_failedInsertAttempts.Remove(evt.EventId);
advanceForThisRow = true;
// Task 17: the permanent loss must be queryable in the Audit Log
// itself, not only in a rotating log file. Leave a durable
// synthetic ReconciliationAbandoned row recording the lost id,
// the site, and the final error.
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
.ConfigureAwait(false);
}
else
{
@@ -324,6 +333,68 @@ public class SiteAuditReconciliationActor : ReceiveActor
UpdateStalledState(site.SiteId, draining: !nonDraining, eventStream);
}
/// <summary>
/// Writes ONE synthetic <see cref="AuditKind.ReconciliationAbandoned"/> audit
/// row when a pulled event is permanently abandoned (Task 17) — so the loss is
/// queryable in the Audit Log rather than living only in the Critical log line.
/// The synthetic row is built through the same
/// <see cref="ScadaBridgeAuditEventFactory"/> the ingest path uses (fresh
/// <c>EventId</c>, <c>Actor="system"</c>, channel preserved from the lost row's
/// <c>Category</c> where parseable) and carries the abandoned <c>EventId</c>,
/// the source site, and the final error in <c>Extra</c>. It runs in its own
/// try/catch: the synthetic row is small and well-formed, so the row-specific
/// fault that killed the original will not recur; if it somehow does, log and
/// continue — the abandonment cursor must never be blocked a second time.
/// </summary>
private async Task EmitAbandonmentRecordAsync(
IAuditLogRepository repository,
ZB.MOM.WW.Audit.AuditEvent lost,
string siteId,
Exception error,
DateTime nowUtc)
{
try
{
// Category is the channel name (BuildCategory); preserve it where it
// parses, else ApiOutbound so the row still lands (mirrors the Task-14
// fallback convention).
var channel = Enum.TryParse<AuditChannel>(lost.Category, out var parsed)
? parsed
: AuditChannel.ApiOutbound;
var extra = JsonSerializer.Serialize(new
{
abandonedEventId = lost.EventId,
sourceSiteId = siteId,
error = error.Message,
});
var synthetic = ScadaBridgeAuditEventFactory.Create(
channel: channel,
kind: AuditKind.ReconciliationAbandoned,
status: AuditStatus.Failed,
occurredAtUtc: nowUtc,
actor: "system",
sourceNode: lost.SourceNode,
sourceSiteId: siteId,
errorMessage: error.Message,
extra: extra,
ingestedAtUtc: nowUtc);
await repository.InsertIfNotExistsAsync(synthetic).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Failed to write the synthetic ReconciliationAbandoned audit row for the "
+ "permanently-abandoned AuditEvent {EventId} (site {SiteId}); the loss is recorded "
+ "only in the Critical log line.",
lost.EventId,
siteId);
}
}
/// <summary>
/// Flips the per-site stalled flag based on whether this tick drained the
/// queue. A "draining" cycle is one where the server reported no more rows
@@ -22,5 +22,15 @@ public enum AuditKind
SecuredWriteSubmit,
SecuredWriteApprove,
SecuredWriteReject,
SecuredWriteExecute
SecuredWriteExecute,
/// <summary>
/// A reconciliation pull row that failed to insert on every retry up to the
/// permanent-abandon threshold, so central advanced its cursor past it and
/// the row is permanently lost from the mirror. The reconciliation actor
/// emits ONE synthetic audit row of this kind (carrying the abandoned
/// <c>EventId</c>, source site, and final error) so the loss is queryable in
/// the Audit Log itself, not only in a rotating log file. (arch-review 04, Task 17)
/// </summary>
ReconciliationAbandoned
}