dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(auditlog): reconciliation pull serves only unacked rows (PushAbandoned/Rejected push-exit states), central pages within a tick with set-based inserts; refused push rows leave the queue after N attempts (review 2026-10-21 06/PERF-1, 06/COR-3, 06/IMPL-1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 06:38:02 -04:00
parent 1da5d0bac4
commit 9e7e19c3c1
13 changed files with 873 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SqliteAuditWriter.cs
+204 -24
View File
@@ -43,7 +43,7 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
/// stamped by central on ingest, not a site column.
/// </para>
/// </remarks>
public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable, IDisposable
public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLifecycle, IAsyncDisposable, IDisposable
{
// Microsoft.Data.Sqlite reports a generic SQLITE_CONSTRAINT (error code 19)
// on a PRIMARY KEY violation; the extended subcode 1555 (SQLITE_CONSTRAINT_PRIMARYKEY)
@@ -78,6 +78,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
private readonly SqliteAuditWriterOptions _options;
private readonly ILogger<SqliteAuditWriter> _logger;
private readonly INodeIdentityProvider _nodeIdentity;
private readonly IAuditWriteFailureCounter? _failureCounter;
private readonly object _writeLock = new();
private readonly Channel<PendingAuditEvent> _writeQueue;
private readonly Task _writerLoop;
@@ -88,11 +89,17 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
/// <param name="logger">Logger instance.</param>
/// <param name="nodeIdentity">Node identity provider.</param>
/// <param name="connectionStringOverride">Optional connection string override.</param>
/// <param name="failureCounter">
/// Optional <c>SiteAuditWriteFailures</c> sink, bumped once per row the push gives up on as
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1). Null on
/// composition roots without Site Health Monitoring.
/// </param>
public SqliteAuditWriter(
IOptions<SqliteAuditWriterOptions> options,
ILogger<SqliteAuditWriter> logger,
INodeIdentityProvider nodeIdentity,
string? connectionStringOverride = null)
string? connectionStringOverride = null,
IAuditWriteFailureCounter? failureCounter = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(logger);
@@ -101,6 +108,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
_options = options.Value;
_logger = logger;
_nodeIdentity = nodeIdentity;
_failureCounter = failureCounter;
var connectionString = connectionStringOverride
?? $"Data Source={_options.DatabasePath};Cache=Shared";
@@ -801,9 +809,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
}
/// <summary>
/// Returns up to <paramref name="batchSize"/> pending or forwarded audit events
/// with <see cref="AuditEvent.OccurredAtUtc"/> &gt;= <paramref name="sinceUtc"/>, oldest first.
/// Used by the reconciliation-pull handler.
/// Returns up to <paramref name="batchSize"/> audit events central has not yet acknowledged
/// (<see cref="AuditForwardState.Pending"/>, <see cref="SiteAuditForwardStates.PushAbandoned"/>,
/// <see cref="SiteAuditForwardStates.Rejected"/>) with <see cref="AuditEvent.OccurredAtUtc"/>
/// &gt;= <paramref name="sinceUtc"/>, oldest first. <see cref="AuditForwardState.Forwarded"/> rows
/// are NOT served (review 2026-10-21 06/PERF-1). Used by the reconciliation-pull handler.
/// </summary>
/// <param name="sinceUtc">Lower bound timestamp (UTC) for event occurrence.</param>
/// <param name="batchSize">Maximum number of rows to return.</param>
@@ -827,9 +837,14 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
// Read via _readConnection / _readLock — same lock-
// decoupling as ReadPendingAsync. JOIN the sidecar; the range scan
// is on the sidecar's duplicated OccurredAtUtc so it stays on IX_fwd.
// Both Pending and Forwarded rows are returned (the central reconciliation
// puller dedups on EventId; re-shipping a Forwarded-but-not-yet-ingested
// row is safe).
//
// ONLY rows central has NOT acknowledged are served: Pending, plus the
// push-exit states PushAbandoned / Rejected (review 2026-10-21 06/PERF-1).
// Forwarded rows used to be served too, which made the pull re-ship every
// row the push had already delivered — central re-inserted each one, and the
// pull's 256-rows-per-tick budget was spent on acked rows, so any site above
// ~0.85 audit rows/s fell permanently behind and latched the stalled signal.
// A Forwarded row is acked by central; the pull has nothing to repair there.
lock (_readLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
@@ -851,13 +866,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
fs.rowid
FROM audit_event ae
JOIN audit_forward_state fs ON fs.EventId = ae.EventId
WHERE fs.ForwardState IN ($pending, $forwarded)
WHERE fs.ForwardState IN ($pending, $pushAbandoned, $rejected)
AND {cursorPredicate}
ORDER BY fs.OccurredAtUtc ASC, ae.EventId ASC
LIMIT $limit;
""";
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
AddOwedStateParameters(cmd);
// Normalise to UTC ISO-8601 round-trip format to match how OccurredAtUtc
// is stored on insert ("o" format) — string comparison is monotonic for
// that encoding so we can index-scan against it.
@@ -874,6 +888,165 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
}
}
/// <summary>
/// Binds the three "owed to central" states — <c>$pending</c>, <c>$pushAbandoned</c>,
/// <c>$rejected</c> — that the reconciliation pull serves and retires.
/// </summary>
private static void AddOwedStateParameters(SqliteCommand cmd)
{
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
cmd.Parameters.AddWithValue("$pushAbandoned", SiteAuditForwardStates.PushAbandoned);
cmd.Parameters.AddWithValue("$rejected", SiteAuditForwardStates.Rejected);
}
/// <inheritdoc />
public Task MarkPushAbandonedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(eventIds);
if (eventIds.Count == 0)
{
return Task.CompletedTask;
}
lock (_writeLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
// Pending → PushAbandoned only. Unlike the old "mark Forwarded" this is honest:
// central never acked these rows, so they stay owed to the reconciliation pull
// (served, retired by a covering cursor, never age-purged) — review 2026-10-21
// 06/PERF-1. Attempt bookkeeping is bumped like any drain pass that touched them.
var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
using var transaction = _connection.BeginTransaction();
for (int start = 0; start < eventIds.Count; start += MarkForwardStateChunkSize)
{
var chunk = Math.Min(MarkForwardStateChunkSize, eventIds.Count - start);
using var cmd = _connection.CreateCommand();
cmd.Transaction = transaction;
var sb = new System.Text.StringBuilder();
sb.Append("UPDATE audit_forward_state SET ForwardState = $abandoned, ")
.Append("AttemptCount = AttemptCount + 1, LastAttemptUtc = $now ")
.Append("WHERE ForwardState = $pending AND EventId IN (");
AppendIdParameters(cmd, sb, eventIds, start, chunk);
sb.Append(");");
cmd.CommandText = sb.ToString();
cmd.Parameters.AddWithValue("$abandoned", SiteAuditForwardStates.PushAbandoned);
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
cmd.Parameters.AddWithValue("$now", now);
cmd.ExecuteNonQuery();
}
transaction.Commit();
return Task.CompletedTask;
}
}
/// <inheritdoc />
public Task<int> RecordRejectedPushAttemptsAsync(
IReadOnlyList<Guid> eventIds, int maxAttempts, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(eventIds);
if (eventIds.Count == 0)
{
return Task.FromResult(0);
}
var threshold = Math.Max(1, maxAttempts);
lock (_writeLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
// Review 2026-10-21 06/IMPL-1: AttemptCount/LastAttemptUtc used to move only on
// the Pending→Forwarded flip, so a row central kept refusing had no history and
// no give-up — and since the push reads oldest-first, it sat at the head of every
// batch forever. Bump on every refused attempt, then move rows that reached the
// threshold out of the push queue into Rejected (still owed to the pull).
var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
var flipped = 0;
using var transaction = _connection.BeginTransaction();
for (int start = 0; start < eventIds.Count; start += MarkForwardStateChunkSize)
{
var chunk = Math.Min(MarkForwardStateChunkSize, eventIds.Count - start);
using (var bump = _connection.CreateCommand())
{
bump.Transaction = transaction;
var sb = new System.Text.StringBuilder();
sb.Append("UPDATE audit_forward_state SET AttemptCount = AttemptCount + 1, LastAttemptUtc = $now ")
.Append("WHERE ForwardState = $pending AND EventId IN (");
AppendIdParameters(bump, sb, eventIds, start, chunk);
sb.Append(");");
bump.CommandText = sb.ToString();
bump.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
bump.Parameters.AddWithValue("$now", now);
bump.ExecuteNonQuery();
}
using (var reject = _connection.CreateCommand())
{
reject.Transaction = transaction;
var sb = new System.Text.StringBuilder();
sb.Append("UPDATE audit_forward_state SET ForwardState = $rejected ")
.Append("WHERE ForwardState = $pending AND AttemptCount >= $max AND EventId IN (");
AppendIdParameters(reject, sb, eventIds, start, chunk);
sb.Append(");");
reject.CommandText = sb.ToString();
reject.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
reject.Parameters.AddWithValue("$rejected", SiteAuditForwardStates.Rejected);
reject.Parameters.AddWithValue("$max", threshold);
flipped += reject.ExecuteNonQuery();
}
}
transaction.Commit();
if (flipped > 0)
{
_logger.LogWarning(
"{Count} site audit row(s) were refused by central on {Max} consecutive pushes and moved out " +
"of the push queue as Rejected; the reconciliation pull still serves them (central records " +
"a ReconciliationAbandoned row if they can never be inserted).",
flipped, threshold);
for (var i = 0; i < flipped; i++)
{
try { _failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
}
}
return Task.FromResult(flipped);
}
}
private static void AppendIdParameters(
SqliteCommand cmd, System.Text.StringBuilder sb, IReadOnlyList<Guid> eventIds, int start, int chunk)
{
for (int i = 0; i < chunk; i++)
{
if (i > 0) sb.Append(',');
var p = $"$id{i}";
sb.Append(p);
cmd.Parameters.AddWithValue(p, eventIds[start + i].ToString());
}
}
/// <summary>
/// Test/diagnostic read of one row's sidecar state: <c>(ForwardState, AttemptCount)</c>, or
/// null when the row does not exist.
/// </summary>
/// <param name="eventId">The event to look up.</param>
/// <returns>The row's forward state and attempt count, or null.</returns>
internal (string State, int AttemptCount)? GetForwardStateForTests(Guid eventId)
{
lock (_readLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
using var cmd = _readConnection.CreateCommand();
cmd.CommandText = "SELECT ForwardState, AttemptCount FROM audit_forward_state WHERE EventId = $id;";
cmd.Parameters.AddWithValue("$id", eventId.ToString());
using var reader = cmd.ExecuteReader();
return reader.Read() ? (reader.GetString(0), reader.GetInt32(1)) : null;
}
}
/// <summary>Flow Debugger PullTraceEvents: hard cap on ids per read (design §5.4).</summary>
public const int MaxExecutionIdsPerRead = 64;
@@ -1042,12 +1215,15 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
//
// fs.rowid is insertion order, so bounding the flip at the highest rowid this
// instance has ever served restores the invariant exactly: rows inserted after
// that point are out of the flip's reach no matter where the cursor sits. A
// FORWARDED row is exempt from the bound — central ACKED it through the
// telemetry push path, which is receipt-proof independent of the pull.
// that point are out of the flip's reach no matter where the cursor sits.
//
// Only the OWED states flip (Pending / PushAbandoned / Rejected — exactly what
// ReadPendingSinceAsync serves). Forwarded rows are no longer touched: central
// acked them over the push path, they are already purge-eligible, and flipping
// them was one UPDATE per pushed row for no benefit (review 2026-10-21 06/PERF-1).
//
// Bound state is per-process: after a restart it is 0 until this instance serves
// a batch, so the first pull retires only Forwarded rows and the pull after it
// a batch, so the first pull retires nothing and the pull after it
// resumes normal retirement. Conservative in the safe direction (delays
// retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
// interface doc + Component-AuditLog.md covers what a permanently halted
@@ -1056,13 +1232,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
cmd.CommandText = $"""
UPDATE audit_forward_state AS fs
SET ForwardState = $reconciled
WHERE fs.ForwardState IN ($pending, $forwarded)
AND (fs.ForwardState = $forwarded OR fs.rowid <= $maxServedRowId)
WHERE fs.ForwardState IN ($pending, $pushAbandoned, $rejected)
AND fs.rowid <= $maxServedRowId
AND {predicate};
""";
cmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
AddOwedStateParameters(cmd);
cmd.Parameters.AddWithValue("$maxServedRowId", Volatile.Read(ref _maxServedRowId));
cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
"o", System.Globalization.CultureInfo.InvariantCulture));
@@ -1088,8 +1263,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
{
ObjectDisposedException.ThrowIf(_disposed, this);
// Flip the sidecar from Pending/Forwarded → Reconciled. Rows
// already Reconciled are left untouched (idempotent re-call), and the
// Flip the sidecar from Pending/Forwarded (or a push-exit state) → Reconciled.
// Rows already Reconciled are left untouched (idempotent re-call), and the
// canonical audit_event row is never modified.
var reconciled = AuditForwardState.Reconciled.ToString();
var pending = AuditForwardState.Pending.ToString();
@@ -1105,7 +1280,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
cmd.Transaction = transaction;
var sb = new System.Text.StringBuilder();
sb.Append("UPDATE audit_forward_state SET ForwardState = $reconciled ")
.Append("WHERE ForwardState IN ($pending, $forwarded) AND EventId IN (");
.Append("WHERE ForwardState IN ($pending, $forwarded, $pushAbandoned, $rejected) AND EventId IN (");
for (int i = 0; i < chunk; i++)
{
if (i > 0) sb.Append(',');
@@ -1118,6 +1293,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
cmd.Parameters.AddWithValue("$reconciled", reconciled);
cmd.Parameters.AddWithValue("$pending", pending);
cmd.Parameters.AddWithValue("$forwarded", forwarded);
cmd.Parameters.AddWithValue("$pushAbandoned", SiteAuditForwardStates.PushAbandoned);
cmd.Parameters.AddWithValue("$rejected", SiteAuditForwardStates.Rejected);
cmd.ExecuteNonQuery();
}
transaction.Commit();
@@ -1145,12 +1322,15 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
ObjectDisposedException.ThrowIf(_disposed, this);
using var cmd = _readConnection.CreateCommand();
// "Pending" on the health report means OWED to central: the push-exit states
// (PushAbandoned / Rejected) still await the reconciliation pull and are never
// purged, so they count toward the backlog and its oldest-row age.
cmd.CommandText = """
SELECT COUNT(*), MIN(OccurredAtUtc)
FROM audit_forward_state
WHERE ForwardState = $pending;
WHERE ForwardState IN ($pending, $pushAbandoned, $rejected);
""";
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
AddOwedStateParameters(cmd);
using var reader = cmd.ExecuteReader();
reader.Read();