feat(audit-log): permanently-abandoned reconciliation rows leave a durable ReconciliationAbandoned audit record
When a pulled AuditEvent fails to insert on every retry up to the permanent- abandon threshold, the reconciliation actor now writes ONE synthetic ReconciliationAbandoned row (new AuditKind) alongside the Critical log line — fresh EventId, Status=Failed, channel preserved from the lost row, Extra carrying the abandoned EventId + source site + final error — via the same ScadaBridgeAuditEventFactory the ingest path uses. Best-effort in its own try/catch so it never blocks the cursor twice. The permanent loss is now queryable in the Audit Log, not only in a rotating log file.
This commit is contained in:
@@ -1,9 +1,11 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
@@ -290,6 +292,13 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
attempts);
|
||||
_failedInsertAttempts.Remove(evt.EventId);
|
||||
advanceForThisRow = true;
|
||||
|
||||
// Task 17: the permanent loss must be queryable in the Audit Log
|
||||
// itself, not only in a rotating log file. Leave a durable
|
||||
// synthetic ReconciliationAbandoned row recording the lost id,
|
||||
// the site, and the final error.
|
||||
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -324,6 +333,68 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
UpdateStalledState(site.SiteId, draining: !nonDraining, eventStream);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes ONE synthetic <see cref="AuditKind.ReconciliationAbandoned"/> audit
|
||||
/// row when a pulled event is permanently abandoned (Task 17) — so the loss is
|
||||
/// queryable in the Audit Log rather than living only in the Critical log line.
|
||||
/// The synthetic row is built through the same
|
||||
/// <see cref="ScadaBridgeAuditEventFactory"/> the ingest path uses (fresh
|
||||
/// <c>EventId</c>, <c>Actor="system"</c>, channel preserved from the lost row's
|
||||
/// <c>Category</c> where parseable) and carries the abandoned <c>EventId</c>,
|
||||
/// the source site, and the final error in <c>Extra</c>. It runs in its own
|
||||
/// try/catch: the synthetic row is small and well-formed, so the row-specific
|
||||
/// fault that killed the original will not recur; if it somehow does, log and
|
||||
/// continue — the abandonment cursor must never be blocked a second time.
|
||||
/// </summary>
|
||||
private async Task EmitAbandonmentRecordAsync(
|
||||
IAuditLogRepository repository,
|
||||
ZB.MOM.WW.Audit.AuditEvent lost,
|
||||
string siteId,
|
||||
Exception error,
|
||||
DateTime nowUtc)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Category is the channel name (BuildCategory); preserve it where it
|
||||
// parses, else ApiOutbound so the row still lands (mirrors the Task-14
|
||||
// fallback convention).
|
||||
var channel = Enum.TryParse<AuditChannel>(lost.Category, out var parsed)
|
||||
? parsed
|
||||
: AuditChannel.ApiOutbound;
|
||||
|
||||
var extra = JsonSerializer.Serialize(new
|
||||
{
|
||||
abandonedEventId = lost.EventId,
|
||||
sourceSiteId = siteId,
|
||||
error = error.Message,
|
||||
});
|
||||
|
||||
var synthetic = ScadaBridgeAuditEventFactory.Create(
|
||||
channel: channel,
|
||||
kind: AuditKind.ReconciliationAbandoned,
|
||||
status: AuditStatus.Failed,
|
||||
occurredAtUtc: nowUtc,
|
||||
actor: "system",
|
||||
sourceNode: lost.SourceNode,
|
||||
sourceSiteId: siteId,
|
||||
errorMessage: error.Message,
|
||||
extra: extra,
|
||||
ingestedAtUtc: nowUtc);
|
||||
|
||||
await repository.InsertIfNotExistsAsync(synthetic).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to write the synthetic ReconciliationAbandoned audit row for the "
|
||||
+ "permanently-abandoned AuditEvent {EventId} (site {SiteId}); the loss is recorded "
|
||||
+ "only in the Critical log line.",
|
||||
lost.EventId,
|
||||
siteId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flips the per-site stalled flag based on whether this tick drained the
|
||||
/// queue. A "draining" cycle is one where the server reported no more rows
|
||||
|
||||
@@ -22,5 +22,15 @@ public enum AuditKind
|
||||
SecuredWriteSubmit,
|
||||
SecuredWriteApprove,
|
||||
SecuredWriteReject,
|
||||
SecuredWriteExecute
|
||||
SecuredWriteExecute,
|
||||
|
||||
/// <summary>
|
||||
/// A reconciliation pull row that failed to insert on every retry up to the
|
||||
/// permanent-abandon threshold, so central advanced its cursor past it and
|
||||
/// the row is permanently lost from the mirror. The reconciliation actor
|
||||
/// emits ONE synthetic audit row of this kind (carrying the abandoned
|
||||
/// <c>EventId</c>, source site, and final error) so the loss is queryable in
|
||||
/// the Audit Log itself, not only in a rotating log file. (arch-review 04, Task 17)
|
||||
/// </summary>
|
||||
ReconciliationAbandoned
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user