dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 08:40:42 -04:00
parent 38167bca88
commit b0a08bb63b
23 changed files with 790 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SqliteAuditWriter.cs
+104 -21
View File
@@ -78,7 +78,6 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
private readonly SqliteAuditWriterOptions _options;
private readonly ILogger<SqliteAuditWriter> _logger;
private readonly INodeIdentityProvider _nodeIdentity;
private readonly IAuditWriteFailureCounter? _failureCounter;
private readonly object _writeLock = new();
private readonly Channel<PendingAuditEvent> _writeQueue;
private readonly Task _writerLoop;
@@ -89,17 +88,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
/// <param name="logger">Logger instance.</param>
/// <param name="nodeIdentity">Node identity provider.</param>
/// <param name="connectionStringOverride">Optional connection string override.</param>
/// <param name="failureCounter">
/// Optional <c>SiteAuditWriteFailures</c> sink, bumped once per row the push gives up on as
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1). Null on
/// composition roots without Site Health Monitoring.
/// </param>
public SqliteAuditWriter(
IOptions<SqliteAuditWriterOptions> options,
ILogger<SqliteAuditWriter> logger,
INodeIdentityProvider nodeIdentity,
string? connectionStringOverride = null,
IAuditWriteFailureCounter? failureCounter = null)
string? connectionStringOverride = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(logger);
@@ -108,7 +101,6 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
_options = options.Value;
_logger = logger;
_nodeIdentity = nodeIdentity;
_failureCounter = failureCounter;
var connectionString = connectionStringOverride
?? $"Data Source={_options.DatabasePath};Cache=Shared";
@@ -116,6 +108,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
_connection.Open();
InitializeSchema();
LoadMaxServedRowId();
// Open a second connection for read-only callers
// (GetBacklogStatsAsync, ReadPendingAsync, ReadPendingSinceAsync,
@@ -275,6 +268,16 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
CREATE INDEX IF NOT EXISTS IX_fwd
ON audit_forward_state (ForwardState, IsCachedKind, OccurredAtUtc);
-- Node-local key/value state that must survive a process restart. Holds the
-- reconciliation pull's served-rows high-water mark ('max_served_rowid'), so a
-- restart no longer resets the bound that lets a covering cursor retire rows the
-- pull already delivered (review 2026-10-21 06/PERF-1 adversarial follow-up).
-- Additive and idempotent: an older binary simply ignores the table.
CREATE TABLE IF NOT EXISTS audit_site_state (
Key TEXT NOT NULL PRIMARY KEY,
Value INTEGER NOT NULL
);
-- Flow Debugger (design 2026-09-05 §5.4): PullTraceEvents reads audit_event by
-- time window + json_extract on ExecutionId/ParentExecutionId; this bounds the scan
-- to the window. Additive and idempotent.
@@ -345,6 +348,13 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
// TryWrite can tell "disposed" from "full" without racing the channel's state.
private bool _writerCompleted;
/// <summary>
/// Rows moved to <see cref="SiteAuditForwardStates.Rejected"/> since construction (also exported
/// as the <c>scadabridge.site.audit.rejected_rows</c> counter).
/// </summary>
public long RejectedPushRowCount => Interlocked.Read(ref _rejectedPushRows);
private long _rejectedPushRows;
private async Task ProcessWriteQueueAsync()
{
var batch = new List<PendingAuditEvent>(_options.BatchSize);
@@ -845,6 +855,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
// pull's 256-rows-per-tick budget was spent on acked rows, so any site above
// ~0.85 audit rows/s fell permanently behind and latched the stalled signal.
// A Forwarded row is acked by central; the pull has nothing to repair there.
IReadOnlyList<AuditEvent> rows;
var before = Volatile.Read(ref _maxServedRowId);
lock (_readLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
@@ -884,8 +896,33 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
}
cmd.Parameters.AddWithValue("$limit", batchSize);
return Task.FromResult(ReadServedRows(cmd, batchSize));
rows = ReadServedRows(cmd, batchSize);
}
// Persist the (possibly advanced) mark OUTSIDE _readLock so the two locks are never
// nested in read→write order. Only when it moved — at most one tiny upsert per pull.
var mark = Volatile.Read(ref _maxServedRowId);
if (mark > before)
{
try
{
lock (_writeLock)
{
if (!_disposed)
{
PersistMaxServedRowId(mark);
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"SqliteAuditWriter could not persist the served high-water mark {Mark}; a restart re-learns it from the next pull.",
mark);
}
}
return Task.FromResult(rows);
}
/// <summary>
@@ -1005,11 +1042,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
"of the push queue as Rejected; the reconciliation pull still serves them (central records " +
"a ReconciliationAbandoned row if they can never be inserted).",
flipped, threshold);
for (var i = 0; i < flipped; i++)
{
try { _failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
}
// Its OWN counter, not SiteAuditWriteFailures: the local write succeeded, and
// the refusal is central's verdict on the row — blaming the site's writer for it
// would be wrong (review 2026-10-21 adversarial follow-up).
Interlocked.Add(ref _rejectedPushRows, flipped);
SiteAuditTelemetryMetrics.RecordRejectedRows(flipped);
}
return Task.FromResult(flipped);
@@ -1154,6 +1191,49 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
/// </summary>
private long _maxServedRowId;
private const string MaxServedRowIdKey = "max_served_rowid";
/// <summary>
/// Restores <see cref="_maxServedRowId"/> from <c>audit_site_state</c>, clamped to the live
/// <c>MAX(rowid)</c> so a stale value can never vouch for a reused rowid. Before this the
/// bound was per-process: after a site-node restart it was 0, and because a healthy site's
/// pull now often serves nothing, owed rows central had already ingested (e.g. PushAbandoned)
/// could stay unretired — and counted on the backlog — indefinitely.
/// </summary>
private void LoadMaxServedRowId()
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = """
SELECT MIN(
IFNULL((SELECT Value FROM audit_site_state WHERE Key = $key), 0),
IFNULL((SELECT MAX(rowid) FROM audit_forward_state), 0));
""";
cmd.Parameters.AddWithValue("$key", MaxServedRowIdKey);
var value = Convert.ToInt64(cmd.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture);
Volatile.Write(ref _maxServedRowId, value);
}
/// <summary>
/// Durably records the served high-water mark. Runs on the WRITE connection under
/// <c>_writeLock</c> (never nested inside <c>_readLock</c>). Best-effort: a failure leaves
/// the in-memory bound intact and only means a restart re-learns it from the next pull.
/// </summary>
private void PersistMaxServedRowId(long value, SqliteTransaction? transaction = null)
{
using var cmd = _connection.CreateCommand();
cmd.Transaction = transaction;
cmd.CommandText = """
INSERT INTO audit_site_state (Key, Value) VALUES ($key, $value)
ON CONFLICT(Key) DO UPDATE SET Value = excluded.Value;
""";
cmd.Parameters.AddWithValue("$key", MaxServedRowIdKey);
cmd.Parameters.AddWithValue("$value", value);
cmd.ExecuteNonQuery();
}
/// <summary>Test seam: the current in-memory served high-water mark.</summary>
internal long MaxServedRowIdForTests => Volatile.Read(ref _maxServedRowId);
/// <summary>
/// Executes a reconciliation-pull read whose projection carries <c>fs.rowid</c> as an
/// 11th column, materialising the canonical rows while advancing
@@ -1222,12 +1302,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
// acked them over the push path, they are already purge-eligible, and flipping
// them was one UPDATE per pushed row for no benefit (review 2026-10-21 06/PERF-1).
//
// Bound state is per-process: after a restart it is 0 until this instance serves
// a batch, so the first pull retires nothing and the pull after it
// resumes normal retirement. Conservative in the safe direction (delays
// retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
// interface doc + Component-AuditLog.md covers what a permanently halted
// reconciliation means for such rows.
// The bound is persisted in audit_site_state (loaded at construction, clamped to
// the live MAX(rowid)), so a site-node restart no longer resets it to 0 and strands
// rows the pull had already served. Conservative in the safe direction (it can
// only delay retirement, never lose a row); the liveness note in
// MarkReconciledUpToAsync's interface doc + Component-AuditLog.md covers what a
// permanently halted reconciliation means for such rows.
using var cmd = _connection.CreateCommand();
cmd.CommandText = $"""
UPDATE audit_forward_state AS fs
@@ -1506,7 +1586,10 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
var liveMax = Convert.ToInt64(maxCmd.ExecuteScalar(),
System.Globalization.CultureInfo.InvariantCulture);
if (liveMax < Volatile.Read(ref _maxServedRowId))
{
Volatile.Write(ref _maxServedRowId, liveMax);
PersistMaxServedRowId(liveMax, transaction);
}
}
transaction.Commit();