dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(auditlog): reconciliation pull serves only unacked rows (PushAbandoned/Rejected push-exit states), central pages within a tick with set-based inserts; refused push rows leave the queue after N attempts (review 2026-10-21 06/PERF-1, 06/COR-3, 06/IMPL-1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 06:38:02 -04:00
parent 1da5d0bac4
commit 9e7e19c3c1
13 changed files with 873 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/Telemetry/SiteAuditTelemetryActor.cs
+96 -25
View File
@@ -179,6 +179,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
}
var batch = BuildBatch(pending);
var sentIds = batch.Events.Select(e => e.EventId).ToList();
IngestAck ack;
try
@@ -203,6 +204,8 @@ public class SiteAuditTelemetryActor : ReceiveActor
await _queue.MarkForwardedAsync(acceptedIds, ct)
.ConfigureAwait(false);
}
await RecordRefusalsAsync(sentIds, ack, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
@@ -269,7 +272,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
var wireBytes = AuditWireBudget.InitialBytes;
var emittedEventIds = new List<Guid>(pending.Count);
// Rows whose operational half can never be built. They are marked
// Forwarded so they LEAVE this queue — see AbandonUnresolvableAsync
// PushAbandoned so they LEAVE this queue — see AbandonUnresolvableAsync
// for why that is safe and why leaving them Pending is not.
var abandonedEventIds = new List<Guid>();
// Counted, not logged per row: a whole batch can fail identically
@@ -362,7 +365,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
if (batch.Packets.Count == 0)
{
// Nothing resolvable in this read. Any permanently-unresolvable
// rows have just been marked Forwarded above, so the next drain
// rows have just been marked PushAbandoned above, so the next drain
// sees past them rather than re-reading the same head of queue.
return;
}
@@ -387,6 +390,9 @@ public class SiteAuditTelemetryActor : ReceiveActor
await _queue.MarkForwardedAsync(acceptedIds, ct)
.ConfigureAwait(false);
}
await RecordRefusalsAsync(
batch.Packets.Select(p => p.AuditEvent.EventId).ToList(), ack, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
@@ -408,33 +414,35 @@ public class SiteAuditTelemetryActor : ReceiveActor
}
/// <summary>
/// Marks cached rows whose operational half can never be built as
/// Forwarded, so they leave the cached-drain queue.
/// Moves cached rows whose operational half can never be built out of the
/// cached-drain queue, as <see cref="SiteAuditForwardStates.PushAbandoned"/>.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why they must leave.</b> Leaving an unresolvable row Pending — the
/// previous behaviour — is not a harmless skip. The queue is read
/// oldest-first with a fixed <c>BatchSize</c>, so once a batch's worth of
/// unresolvable rows collects at the head, every drain re-reads exactly
/// those rows, fails identically, and never sees the newer rows behind
/// them. The cached-telemetry path stalls permanently, and each pass logs
/// once per row (measured at ~2 800 warnings/minute on a rig).
/// <b>Why they must leave.</b> Leaving an unresolvable row Pending is not a
/// harmless skip. The queue is read oldest-first with a fixed
/// <c>BatchSize</c>, so once a batch's worth of unresolvable rows collects at
/// the head, every drain re-reads exactly those rows, fails identically, and
/// never sees the newer rows behind them. The cached-telemetry path stalls
/// permanently, and each pass logs once per row (measured at ~2 800
/// warnings/minute on a rig).
/// </para>
/// <para>
/// <b>Why Forwarded is the honest state.</b> Its role in this state machine
/// is "no longer owed by the drain, still eligible for reconciliation" —
/// which is exactly the situation here. The audit half is NOT dropped:
/// <c>ReadPendingSinceAsync</c> covers Forwarded rows as well as Pending
/// ones and central dedups on EventId, so the reconciliation pull still
/// delivers them. What is genuinely lost is the operational
/// (<c>SiteCalls</c>) half — and that is unrecoverable regardless, because
/// the tracking row it would have been built from no longer exists.
/// <b>Why PushAbandoned, not Forwarded.</b> These rows were marked Forwarded
/// until review 2026-10-21 (06/PERF-1), which only worked because the
/// reconciliation pull used to re-serve every Forwarded row — the coupling
/// that capped the pull's throughput. Forwarded now means "central acked it"
/// and the pull no longer serves it, so an un-acked row needs a state of its
/// own: <c>PushAbandoned</c> is served by the pull, retired by a covering
/// cursor and never age-purged. The audit half therefore still reaches central;
/// what is genuinely lost is the operational (<c>SiteCalls</c>) half, which is
/// unrecoverable regardless because the tracking row no longer exists.
/// </para>
/// <para>
/// A failure to mark is swallowed: the rows simply stay Pending and the
/// next drain retries. Escalating here would take down the audit drain over
/// a best-effort cleanup.
/// A queue that does not implement <see cref="ISiteAuditPushLifecycle"/> (a
/// test double) falls back to the old Forwarded flip. A failure to mark is
/// swallowed: the rows simply stay Pending and the next drain retries.
/// Escalating here would take down the audit drain over a best-effort cleanup.
/// </para>
/// </remarks>
private async Task AbandonUnresolvableAsync(
@@ -442,24 +450,87 @@ public class SiteAuditTelemetryActor : ReceiveActor
{
try
{
await _queue.MarkForwardedAsync(eventIds, ct).ConfigureAwait(false);
if (_queue is ISiteAuditPushLifecycle lifecycle)
{
await lifecycle.MarkPushAbandonedAsync(eventIds, ct).ConfigureAwait(false);
}
else
{
await _queue.MarkForwardedAsync(eventIds, ct).ConfigureAwait(false);
}
_logger.LogWarning(
"Cached-telemetry drain: abandoned the operational half of {Count} row(s) with no " +
"resolvable tracking snapshot after {Grace}s; marked Forwarded so they no longer block " +
"resolvable tracking snapshot after {Grace}s; marked PushAbandoned so they no longer block " +
"the queue. The audit half still reaches central via the reconciliation pull. This is " +
"expected after tracking retention elapses or the tracking store is reset independently " +
"of the audit store.",
eventIds.Count, graceSeconds);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Cached-telemetry drain: could not mark {Count} unresolvable row(s) Forwarded " +
"Cached-telemetry drain: could not mark {Count} unresolvable row(s) PushAbandoned " +
"(sqlite {SqliteError}); they stay Pending and will be retried.",
eventIds.Count, SqliteErrorCodes.Describe(ex));
}
}
/// <summary>
/// Records a refused push attempt for every row that was SENT in an answered batch but is
/// missing from the ack's <c>AcceptedEventIds</c>; rows refused
/// <see cref="SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/> times leave the push
/// queue as <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1).
/// </summary>
/// <remarks>
/// Called only after the push RETURNED — a transport fault is not a verdict about any row
/// and never counts. Best-effort: a bookkeeping failure leaves the rows Pending, exactly as
/// before this existed.
/// </remarks>
private async Task RecordRefusalsAsync(IReadOnlyList<string> sentIds, IngestAck ack, CancellationToken ct)
{
if (_queue is not ISiteAuditPushLifecycle lifecycle || sentIds.Count == 0)
{
return;
}
var accepted = new HashSet<string>(ack.AcceptedEventIds, StringComparer.OrdinalIgnoreCase);
var refused = new List<Guid>();
foreach (var raw in sentIds)
{
if (!accepted.Contains(raw) && Guid.TryParse(raw, out var id))
{
refused.Add(id);
}
}
if (refused.Count == 0)
{
return;
}
try
{
await lifecycle
.RecordRejectedPushAttemptsAsync(refused, Math.Max(1, _options.MaxRejectedPushAttempts), ct)
.ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Could not record {Count} refused push attempt(s) (sqlite {SqliteError}); the rows stay Pending.",
refused.Count, SqliteErrorCodes.Describe(ex));
}
}
private static AuditEventBatch BuildBatch(IReadOnlyList<AuditEvent> events)
{
var batch = new AuditEventBatch();