dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(auditlog): reconciliation abandons only row-intrinsic failures and only when a sibling row inserted (review 2026-10-21 follow-up M1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 09:31:26 -04:00
parent 6b8a91cc83
commit 1f87942ce3
4 changed files with 225 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/SiteAuditReconciliationActor.cs
+91 -59
View File
@@ -391,7 +391,12 @@ public class SiteAuditReconciliationActor : ReceiveActor
/// when the batch throws does the page fall back to the per-row path, which keeps the
/// documented poison-row isolation: the failing row is retried up to
/// <see cref="MaxPermanentInsertAttempts"/> times with the cursor held back, then
/// permanently abandoned. Re-running rows the failed batch may already have committed is
/// permanently abandoned. Only a row-intrinsic failure
/// (<see cref="AuditIngestErrorClassifier.IsPermanentRowRejection"/>) counts toward that
/// threshold, and a row is abandoned only on a tick where at least one sibling row of the same
/// page inserted — a page where every row fails is a database problem (schema drift after a
/// missed migration, a narrowed column), and abandoning it would let the site mark every row
/// <c>Reconciled</c> and purge it (review 2026-10-21 follow-up M1). Re-running rows the failed batch may already have committed is
/// safe — every insert is idempotent on <see cref="AuditEvent.EventId"/>, which is also
/// what lets a pre-PERF-1 site that still re-serves <c>Forwarded</c> rows be pulled
/// unchanged.
@@ -449,6 +454,39 @@ public class SiteAuditReconciliationActor : ReceiveActor
}
}
// Per-row fallback (only when the set-based insert threw). Every row is attempted FIRST and
// its outcome recorded, so the abandonment decision below can see whether ANY sibling in
// this page inserted (review 2026-10-21 follow-up M1).
var rowFailures = new Dictionary<Guid, Exception>();
var anyRowInserted = false;
if (!batchCommitted)
{
foreach (var evt in response.Events)
{
if (_abandonedEventIds.Contains(evt.EventId))
{
continue;
}
try
{
// Idempotent repository write: duplicate EventIds (from a
// concurrent push, or a retry of this very pull) collapse to
// a no-op on InsertIfNotExistsAsync.
// IngestedAtUtc is a DetailsJson field on the canonical record —
// stamp it via the projection helper.
var ingested = AuditRowProjection.WithIngestedAtUtc(evt, nowUtc);
await repository.InsertIfNotExistsAsync(ingested).ConfigureAwait(false);
_failedInsertAttempts.Remove(evt.EventId);
anyRowInserted = true;
}
catch (Exception ex)
{
rowFailures[evt.EventId] = ex;
}
}
}
foreach (var evt in response.Events)
{
var advanceForThisRow = false;
@@ -464,78 +502,72 @@ public class SiteAuditReconciliationActor : ReceiveActor
// row collapse into the existing one anyway.
advanceForThisRow = true;
}
else if (batchCommitted)
else if (batchCommitted || !rowFailures.TryGetValue(evt.EventId, out var ex))
{
advanceForThisRow = true;
}
else
{
try
// Per-row verdict so one bad event does not abandon the rest of the batch.
// Only a ROW-INTRINSIC failure (AuditIngestErrorClassifier.IsPermanentRowRejection
// — the same narrow set the push path reports back as rejected) counts toward
// MaxPermanentInsertAttempts; below the threshold the cursor is HELD BACK so the
// next tick re-pulls and retries; at the threshold the row is permanently
// abandoned (LogCritical + cursor advances past) to keep a truly broken row from
// blocking all subsequent progress for the site.
if (!AuditIngestErrorClassifier.IsPermanentRowRejection(ex))
{
// Idempotent repository write: duplicate EventIds (from a
// concurrent push, or a retry of this very pull) collapse to
// a no-op on InsertIfNotExistsAsync.
// IngestedAtUtc is a DetailsJson field on the canonical record —
// stamp it via the projection helper.
var ingested = AuditRowProjection.WithIngestedAtUtc(evt, nowUtc);
await repository.InsertIfNotExistsAsync(ingested).ConfigureAwait(false);
_failedInsertAttempts.Remove(evt.EventId);
advanceForThisRow = true;
// A database problem (timeout, connection loss, deadlock, throttling, a
// maintenance window) or anything not provably about THIS row — e.g. schema
// drift after a missed central migration (207 / 208 invalid column/object) —
// is not a verdict about the row: hold the cursor and retry, but never count
// it toward permanent abandonment. Abandoning such rows flips the site's
// cursor past them, the site marks them Reconciled, and its retention purge
// deletes them — a silent loss of every row of the outage (review 2026-10-21
// 06/PERF-1 adversarial follow-up; follow-up M1).
_logger.LogWarning(
ex,
"Reconciliation ingest of AuditEvent {EventId} from site {SiteId} failed with a non-row-specific error; cursor held back, not counted toward abandonment.",
evt.EventId,
site.SiteId);
hasUnresolvedFailure = true;
}
catch (Exception ex)
else
{
// Per-row catch so one bad event does not abandon
// the rest of the batch. Track the failure count per EventId —
// below MaxPermanentInsertAttempts the cursor is HELD BACK so
// the next tick re-pulls and retries; at the threshold the row
// is permanently abandoned (LogCritical + cursor advances past)
// to keep a truly broken row from blocking all subsequent
// progress for the site.
if (AuditIngestErrorClassifier.IsTransient(ex))
var attempts = _failedInsertAttempts.GetValueOrDefault(evt.EventId) + 1;
_failedInsertAttempts[evt.EventId] = attempts;
if (attempts >= MaxPermanentInsertAttempts && anyRowInserted)
{
// A database problem (timeout, connection loss, deadlock, throttling, a
// maintenance window) is not a verdict about this row: hold the cursor
// and retry, but never count it toward permanent abandonment — a long
// outage must not turn healthy rows into ReconciliationAbandoned losses
// (review 2026-10-21 06/PERF-1 adversarial follow-up).
_logger.LogWarning(
_logger.LogCritical(
ex,
"Reconciliation ingest of AuditEvent {EventId} from site {SiteId} hit a transient database fault; cursor held back, not counted toward abandonment.",
"Permanently abandoning AuditEvent {EventId} from site {SiteId} after {Attempts} consecutive insert failures; cursor will advance past it.",
evt.EventId,
site.SiteId);
hasUnresolvedFailure = true;
site.SiteId,
attempts);
_failedInsertAttempts.Remove(evt.EventId);
RememberAbandoned(evt.EventId);
advanceForThisRow = true;
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
.ConfigureAwait(false);
}
else
{
var attempts = _failedInsertAttempts.GetValueOrDefault(evt.EventId) + 1;
_failedInsertAttempts[evt.EventId] = attempts;
if (attempts >= MaxPermanentInsertAttempts)
{
_logger.LogCritical(
ex,
"Permanently abandoning AuditEvent {EventId} from site {SiteId} after {Attempts} consecutive insert failures; cursor will advance past it.",
evt.EventId,
site.SiteId,
attempts);
_failedInsertAttempts.Remove(evt.EventId);
RememberAbandoned(evt.EventId);
advanceForThisRow = true;
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
.ConfigureAwait(false);
}
else
{
_logger.LogError(
ex,
"Reconciliation ingest failed for AuditEvent {EventId} from site {SiteId} (attempt {Attempts}/{Max}); cursor held back for retry.",
evt.EventId,
site.SiteId,
attempts,
MaxPermanentInsertAttempts);
hasUnresolvedFailure = true;
}
// At the threshold but no sibling in this page inserted: every row failing
// identically is a DATABASE problem (e.g. a column narrowed by a migration
// that 245/8114s every row), not one poison row — hold, never abandon. The
// row is abandoned on a later tick whose page also carries a row that
// inserts, which proves the database itself accepts writes.
_logger.LogError(
ex,
"Reconciliation ingest failed for AuditEvent {EventId} from site {SiteId} (attempt {Attempts}/{Max}{NoSibling}); cursor held back for retry.",
evt.EventId,
site.SiteId,
attempts,
MaxPermanentInsertAttempts,
anyRowInserted ? string.Empty : ", no sibling row inserted this tick");
hasUnresolvedFailure = true;
}
}
}