fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)

This commit is contained in:
Joseph Doherty
2026-10-22 08:40:42 -04:00
parent 38167bca88
commit b0a08bb63b
23 changed files with 790 additions and 168 deletions
@@ -0,0 +1,125 @@
using System.Data.Common;
using System.Net.Sockets;
using Microsoft.Data.SqlClient;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// <summary>
/// Classifies a central audit-insert failure (review 2026-10-21 06/IMPL-1 follow-up) so that a
/// DATABASE problem — a maintenance window, a timeout, a dropped connection, an exhausted budget —
/// is never mistaken for a verdict about the ROW. Two questions, two predicates:
/// <list type="bullet">
/// <item><see cref="IsPermanentRowRejection"/> — narrow: the row itself can never be inserted
/// (string truncation, NULL into NOT NULL, a conversion / overflow, a constraint). Only these ids
/// go back to the site as <c>rejected_event_ids</c>, which is what lets the site give a row up as
/// <c>Rejected</c>.</item>
/// <item><see cref="IsTransient"/> — broad: anything that may clear on its own. The reconciliation
/// puller never counts these toward permanently abandoning a row.</item>
/// </list>
/// The exception's whole <see cref="Exception.InnerException"/> chain is inspected, because EF Core
/// and the execution strategy wrap the provider exception.
/// </summary>
internal static class AuditIngestErrorClassifier
{
/// <summary>
/// SQL Server errors that describe the ROW, not the server: 8152 / 2628 string or binary data
/// would be truncated, 515 NULL into a NOT NULL column, 245 / 241 / 242 / 8114 / 8115 / 220 / 232
/// conversion and arithmetic overflow, 547 CHECK / FOREIGN KEY constraint.
/// </summary>
private static readonly HashSet<int> PermanentRowErrorNumbers = new()
{
8152, 2628, 515, 245, 241, 242, 8114, 8115, 220, 232, 547,
};
/// <summary>
/// SQL Server errors that may clear on their own (timeout, connection loss, deadlock, lock
/// timeout, resource / Azure throttling, database unavailable or in a restricted state).
/// </summary>
private static readonly HashSet<int> TransientErrorNumbers = new()
{
-2, -1, 0, 2, 20, 53, 64, 121, 233, 1205, 1222, 3960, 4060, 4221, 8645, 8651,
10053, 10054, 10060, 10928, 10929, 40143, 40197, 40501, 40540, 40613,
49918, 49919, 49920, 942, 952, 983, 976, 978, 3906, 9001, 9002,
};
/// <summary>
/// True when the failure is a SQL Server error that says the row itself can never be inserted.
/// Everything else — including every transient error and every non-SQL exception — is false,
/// so the site keeps such rows Pending and simply retries them.
/// </summary>
/// <param name="ex">The insert failure.</param>
/// <returns>Whether the row should be reported back to the site as permanently rejected.</returns>
public static bool IsPermanentRowRejection(Exception ex)
{
if (IsTransient(ex))
{
return false;
}
for (var e = ex; e is not null; e = e.InnerException)
{
if (e is SqlException sql)
{
foreach (SqlError error in sql.Errors)
{
if (PermanentRowErrorNumbers.Contains(error.Number))
{
return true;
}
}
return PermanentRowErrorNumbers.Contains(sql.Number);
}
}
return false;
}
/// <summary>
/// True when any exception in the chain is a timeout, cancellation, socket / IO / connection
/// failure, an EF retry-limit exhaustion, a provider error flagged transient, or a SQL Server
/// error number in the transient set.
/// </summary>
/// <param name="ex">The insert failure.</param>
/// <returns>Whether the failure may clear on its own.</returns>
public static bool IsTransient(Exception ex)
{
for (var e = ex; e is not null; e = e.InnerException)
{
switch (e)
{
case TimeoutException:
case OperationCanceledException:
case SocketException:
case IOException:
case HttpRequestException:
return true;
case SqlException sql:
if (TransientErrorNumbers.Contains(sql.Number))
{
return true;
}
foreach (SqlError error in sql.Errors)
{
if (TransientErrorNumbers.Contains(error.Number))
{
return true;
}
}
if (sql.IsTransient)
{
return true;
}
break;
case DbException db when db.IsTransient:
return true;
}
// EF Core's RetryLimitExceededException (an InvalidOperationException) wraps a
// transient provider error once the execution strategy gives up.
if (e.GetType().Name == "RetryLimitExceededException")
{
return true;
}
}
return false;
}
}
@@ -152,6 +152,7 @@ public class AuditLogIngestActor : ReceiveActor
var replyTo = Sender;
var nowUtc = DateTime.UtcNow;
var accepted = new List<Guid>(cmd.Events.Count);
var rejected = new List<Guid>();
// Resolve the repository for the whole batch — one DbContext per
// message, mirroring NotificationOutboxActor.
@@ -184,7 +185,7 @@ public class AuditLogIngestActor : ReceiveActor
// unregistered (test composition roots) means the per-row catch
// simply logs without surfacing on the health dashboard.
var failureCounter = scope.ServiceProvider.GetService<ICentralAuditWriteFailureCounter>();
await IngestWithRepositoryAsync(repository, redactor, failureCounter, cmd, nowUtc, accepted)
await IngestWithRepositoryAsync(repository, redactor, failureCounter, cmd, nowUtc, accepted, rejected)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -201,7 +202,7 @@ public class AuditLogIngestActor : ReceiveActor
accepted.Count);
}
replyTo.Tell(new IngestAuditEventsReply(accepted));
replyTo.Tell(new IngestAuditEventsReply(accepted) { RejectedEventIds = rejected });
}
private async Task IngestWithRepositoryAsync(
@@ -210,7 +211,8 @@ public class AuditLogIngestActor : ReceiveActor
ICentralAuditWriteFailureCounter? failureCounter,
IngestAuditEventsCommand cmd,
DateTime nowUtc,
List<Guid> accepted)
List<Guid> accepted,
List<Guid> rejected)
{
// Stamp the ingest timestamp here, not at the site. Redact BEFORE the
// IngestedAtUtc stamp so the redacted copy carries the central-side
@@ -303,6 +305,15 @@ public class AuditLogIngestActor : ReceiveActor
catch { /* counter must never throw — defence in depth */ }
}
// Review 2026-10-21 06/IMPL-1: tell the site ONLY about rows the server
// refused for a reason intrinsic to the row (truncation, NOT NULL, conversion,
// constraint). A timeout / connection loss / maintenance window leaves the id
// in neither list, so the site retries it without counting a refusal.
if (AuditIngestErrorClassifier.IsPermanentRowRejection(rowEx))
{
rejected.Add(cmd.Events[i].EventId);
}
_logger.LogError(rowEx,
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
cmd.Events[i].EventId);
@@ -336,6 +347,7 @@ public class AuditLogIngestActor : ReceiveActor
{
var replyTo = Sender;
var accepted = new List<Guid>(cmd.Entries.Count);
var rejected = new List<Guid>();
try
{
@@ -370,7 +382,7 @@ public class AuditLogIngestActor : ReceiveActor
strategy, dbContext, auditRepo, siteCallRepo, redactor, cmd, accepted, budget.Token)
.ConfigureAwait(false))
{
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
replyTo.Tell(new IngestCachedTelemetryReply(accepted) { RejectedEventIds = rejected });
return;
}
@@ -445,6 +457,11 @@ public class AuditLogIngestActor : ReceiveActor
// sustained dual-write failure surfaces on the dashboard.
try { failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
if (AuditIngestErrorClassifier.IsPermanentRowRejection(ex))
{
rejected.Add(entry.Audit.EventId);
}
_logger.LogError(
ex,
"Combined telemetry dual-write failed for AuditEvent {EventId} / TrackedOperationId {TrackedOpId}; rolled back.",
@@ -463,7 +480,7 @@ public class AuditLogIngestActor : ReceiveActor
"Combined telemetry batch ingest failed before per-entry processing.");
}
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
replyTo.Tell(new IngestCachedTelemetryReply(accepted) { RejectedEventIds = rejected });
}
/// <summary>
@@ -302,13 +302,15 @@ public class SiteAuditReconciliationActor : ReceiveActor
return;
}
PruneUnknownSites(sites, eventStream);
if (sites.Count == 0)
{
return;
}
// After the empty guard: an empty enumeration is as likely a transient lookup glitch
// as a fleet with no sites, and pruning on it would throw away every cursor.
PruneUnknownSites(sites, eventStream);
// Use CreateAsyncScope + await using so scoped EF Core
// services (IAsyncDisposable DbContexts) dispose asynchronously
// without blocking on sync Dispose() of pending connection cleanup.
@@ -489,34 +491,51 @@ public class SiteAuditReconciliationActor : ReceiveActor
// is permanently abandoned (LogCritical + cursor advances past)
// to keep a truly broken row from blocking all subsequent
// progress for the site.
var attempts = _failedInsertAttempts.GetValueOrDefault(evt.EventId) + 1;
_failedInsertAttempts[evt.EventId] = attempts;
if (attempts >= MaxPermanentInsertAttempts)
if (AuditIngestErrorClassifier.IsTransient(ex))
{
_logger.LogCritical(
// A database problem (timeout, connection loss, deadlock, throttling, a
// maintenance window) is not a verdict about this row: hold the cursor
// and retry, but never count it toward permanent abandonment — a long
// outage must not turn healthy rows into ReconciliationAbandoned losses
// (review 2026-10-21 06/PERF-1 adversarial follow-up).
_logger.LogWarning(
ex,
"Permanently abandoning AuditEvent {EventId} from site {SiteId} after {Attempts} consecutive insert failures; cursor will advance past it.",
"Reconciliation ingest of AuditEvent {EventId} from site {SiteId} hit a transient database fault; cursor held back, not counted toward abandonment.",
evt.EventId,
site.SiteId,
attempts);
_failedInsertAttempts.Remove(evt.EventId);
RememberAbandoned(evt.EventId);
advanceForThisRow = true;
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
.ConfigureAwait(false);
site.SiteId);
hasUnresolvedFailure = true;
}
else
{
_logger.LogError(
ex,
"Reconciliation ingest failed for AuditEvent {EventId} from site {SiteId} (attempt {Attempts}/{Max}); cursor held back for retry.",
evt.EventId,
site.SiteId,
attempts,
MaxPermanentInsertAttempts);
hasUnresolvedFailure = true;
var attempts = _failedInsertAttempts.GetValueOrDefault(evt.EventId) + 1;
_failedInsertAttempts[evt.EventId] = attempts;
if (attempts >= MaxPermanentInsertAttempts)
{
_logger.LogCritical(
ex,
"Permanently abandoning AuditEvent {EventId} from site {SiteId} after {Attempts} consecutive insert failures; cursor will advance past it.",
evt.EventId,
site.SiteId,
attempts);
_failedInsertAttempts.Remove(evt.EventId);
RememberAbandoned(evt.EventId);
advanceForThisRow = true;
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
.ConfigureAwait(false);
}
else
{
_logger.LogError(
ex,
"Reconciliation ingest failed for AuditEvent {EventId} from site {SiteId} (attempt {Attempts}/{Max}); cursor held back for retry.",
evt.EventId,
site.SiteId,
attempts,
MaxPermanentInsertAttempts);
hasUnresolvedFailure = true;
}
}
}
}
@@ -25,7 +25,7 @@ public static class SiteAuditForwardStates
public const string PushAbandoned = "PushAbandoned";
/// <summary>
/// A row central answered but refused (not in <c>AcceptedEventIds</c>) on
/// A row central explicitly and permanently refused (listed in the ack's <c>rejected_event_ids</c>) on
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/>
/// consecutive drains. Moved out of the push queue so one poison row cannot wedge every push
/// batch behind it; the pull then delivers it, or central's permanent-abandon path records the
@@ -53,11 +53,11 @@ public interface ISiteAuditPushLifecycle
Task MarkPushAbandonedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default);
/// <summary>
/// Records one refused push attempt for each supplied <see cref="AuditForwardState.Pending"/>
/// Records one explicitly-refused push attempt for each supplied <see cref="AuditForwardState.Pending"/>
/// row (bumps <c>AttemptCount</c>, stamps <c>LastAttemptUtc</c>) and flips every row that has
/// now reached <paramref name="maxAttempts"/> to <see cref="SiteAuditForwardStates.Rejected"/>.
/// </summary>
/// <param name="eventIds">Event IDs central answered for but did not accept.</param>
/// <param name="eventIds">Event IDs central listed as permanently rejected.</param>
/// <param name="maxAttempts">Refused attempts after which a row leaves the push queue.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the number of rows flipped to Rejected by this call.</returns>
@@ -0,0 +1,28 @@
using System.Diagnostics.Metrics;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
/// <summary>
/// Site audit forwarding instruments. Published on a meter named
/// <see cref="ScadaBridgeTelemetry.MeterName"/> — OpenTelemetry matches meters by NAME, so these
/// export through the existing <c>ObservedMeters</c> allowlist entry without a Host change.
/// </summary>
internal static class SiteAuditTelemetryMetrics
{
private static readonly Meter Meter = new(ScadaBridgeTelemetry.MeterName);
private static readonly Counter<long> RejectedRows =
Meter.CreateCounter<long>("scadabridge.site.audit.rejected_rows", unit: "1",
description: "Site audit rows central permanently refused MaxRejectedPushAttempts times, moved out of the push queue as Rejected (still served by the reconciliation pull).");
/// <summary>Records <paramref name="count"/> rows flipped to Rejected.</summary>
/// <param name="count">Rows flipped.</param>
public static void RecordRejectedRows(long count)
{
if (count > 0)
{
RejectedRows.Add(count);
}
}
}
@@ -78,7 +78,6 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
private readonly SqliteAuditWriterOptions _options;
private readonly ILogger<SqliteAuditWriter> _logger;
private readonly INodeIdentityProvider _nodeIdentity;
private readonly IAuditWriteFailureCounter? _failureCounter;
private readonly object _writeLock = new();
private readonly Channel<PendingAuditEvent> _writeQueue;
private readonly Task _writerLoop;
@@ -89,17 +88,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
/// <param name="logger">Logger instance.</param>
/// <param name="nodeIdentity">Node identity provider.</param>
/// <param name="connectionStringOverride">Optional connection string override.</param>
/// <param name="failureCounter">
/// Optional <c>SiteAuditWriteFailures</c> sink, bumped once per row the push gives up on as
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1). Null on
/// composition roots without Site Health Monitoring.
/// </param>
public SqliteAuditWriter(
IOptions<SqliteAuditWriterOptions> options,
ILogger<SqliteAuditWriter> logger,
INodeIdentityProvider nodeIdentity,
string? connectionStringOverride = null,
IAuditWriteFailureCounter? failureCounter = null)
string? connectionStringOverride = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(logger);
@@ -108,7 +101,6 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
_options = options.Value;
_logger = logger;
_nodeIdentity = nodeIdentity;
_failureCounter = failureCounter;
var connectionString = connectionStringOverride
?? $"Data Source={_options.DatabasePath};Cache=Shared";
@@ -116,6 +108,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
_connection.Open();
InitializeSchema();
LoadMaxServedRowId();
// Open a second connection for read-only callers
// (GetBacklogStatsAsync, ReadPendingAsync, ReadPendingSinceAsync,
@@ -275,6 +268,16 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
CREATE INDEX IF NOT EXISTS IX_fwd
ON audit_forward_state (ForwardState, IsCachedKind, OccurredAtUtc);
-- Node-local key/value state that must survive a process restart. Holds the
-- reconciliation pull's served-rows high-water mark ('max_served_rowid'), so a
-- restart no longer resets the bound that lets a covering cursor retire rows the
-- pull already delivered (review 2026-10-21 06/PERF-1 adversarial follow-up).
-- Additive and idempotent: an older binary simply ignores the table.
CREATE TABLE IF NOT EXISTS audit_site_state (
Key TEXT NOT NULL PRIMARY KEY,
Value INTEGER NOT NULL
);
-- Flow Debugger (design 2026-09-05 §5.4): PullTraceEvents reads audit_event by
-- time window + json_extract on ExecutionId/ParentExecutionId; this bounds the scan
-- to the window. Additive and idempotent.
@@ -345,6 +348,13 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
// TryWrite can tell "disposed" from "full" without racing the channel's state.
private bool _writerCompleted;
/// <summary>
/// Rows moved to <see cref="SiteAuditForwardStates.Rejected"/> since construction (also exported
/// as the <c>scadabridge.site.audit.rejected_rows</c> counter).
/// </summary>
public long RejectedPushRowCount => Interlocked.Read(ref _rejectedPushRows);
private long _rejectedPushRows;
private async Task ProcessWriteQueueAsync()
{
var batch = new List<PendingAuditEvent>(_options.BatchSize);
@@ -845,6 +855,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
// pull's 256-rows-per-tick budget was spent on acked rows, so any site above
// ~0.85 audit rows/s fell permanently behind and latched the stalled signal.
// A Forwarded row is acked by central; the pull has nothing to repair there.
IReadOnlyList<AuditEvent> rows;
var before = Volatile.Read(ref _maxServedRowId);
lock (_readLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
@@ -884,8 +896,33 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
}
cmd.Parameters.AddWithValue("$limit", batchSize);
return Task.FromResult(ReadServedRows(cmd, batchSize));
rows = ReadServedRows(cmd, batchSize);
}
// Persist the (possibly advanced) mark OUTSIDE _readLock so the two locks are never
// nested in read→write order. Only when it moved — at most one tiny upsert per pull.
var mark = Volatile.Read(ref _maxServedRowId);
if (mark > before)
{
try
{
lock (_writeLock)
{
if (!_disposed)
{
PersistMaxServedRowId(mark);
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"SqliteAuditWriter could not persist the served high-water mark {Mark}; a restart re-learns it from the next pull.",
mark);
}
}
return Task.FromResult(rows);
}
/// <summary>
@@ -1005,11 +1042,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
"of the push queue as Rejected; the reconciliation pull still serves them (central records " +
"a ReconciliationAbandoned row if they can never be inserted).",
flipped, threshold);
for (var i = 0; i < flipped; i++)
{
try { _failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
}
// Its OWN counter, not SiteAuditWriteFailures: the local write succeeded, and
// the refusal is central's verdict on the row — blaming the site's writer for it
// would be wrong (review 2026-10-21 adversarial follow-up).
Interlocked.Add(ref _rejectedPushRows, flipped);
SiteAuditTelemetryMetrics.RecordRejectedRows(flipped);
}
return Task.FromResult(flipped);
@@ -1154,6 +1191,49 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
/// </summary>
private long _maxServedRowId;
private const string MaxServedRowIdKey = "max_served_rowid";
/// <summary>
/// Restores <see cref="_maxServedRowId"/> from <c>audit_site_state</c>, clamped to the live
/// <c>MAX(rowid)</c> so a stale value can never vouch for a reused rowid. Before this the
/// bound was per-process: after a site-node restart it was 0, and because a healthy site's
/// pull now often serves nothing, owed rows central had already ingested (e.g. PushAbandoned)
/// could stay unretired — and counted on the backlog — indefinitely.
/// </summary>
private void LoadMaxServedRowId()
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = """
SELECT MIN(
IFNULL((SELECT Value FROM audit_site_state WHERE Key = $key), 0),
IFNULL((SELECT MAX(rowid) FROM audit_forward_state), 0));
""";
cmd.Parameters.AddWithValue("$key", MaxServedRowIdKey);
var value = Convert.ToInt64(cmd.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture);
Volatile.Write(ref _maxServedRowId, value);
}
/// <summary>
/// Durably records the served high-water mark. Runs on the WRITE connection under
/// <c>_writeLock</c> (never nested inside <c>_readLock</c>). Best-effort: a failure leaves
/// the in-memory bound intact and only means a restart re-learns it from the next pull.
/// </summary>
private void PersistMaxServedRowId(long value, SqliteTransaction? transaction = null)
{
using var cmd = _connection.CreateCommand();
cmd.Transaction = transaction;
cmd.CommandText = """
INSERT INTO audit_site_state (Key, Value) VALUES ($key, $value)
ON CONFLICT(Key) DO UPDATE SET Value = excluded.Value;
""";
cmd.Parameters.AddWithValue("$key", MaxServedRowIdKey);
cmd.Parameters.AddWithValue("$value", value);
cmd.ExecuteNonQuery();
}
/// <summary>Test seam: the current in-memory served high-water mark.</summary>
internal long MaxServedRowIdForTests => Volatile.Read(ref _maxServedRowId);
/// <summary>
/// Executes a reconciliation-pull read whose projection carries <c>fs.rowid</c> as an
/// 11th column, materialising the canonical rows while advancing
@@ -1222,12 +1302,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
// acked them over the push path, they are already purge-eligible, and flipping
// them was one UPDATE per pushed row for no benefit (review 2026-10-21 06/PERF-1).
//
// Bound state is per-process: after a restart it is 0 until this instance serves
// a batch, so the first pull retires nothing and the pull after it
// resumes normal retirement. Conservative in the safe direction (delays
// retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
// interface doc + Component-AuditLog.md covers what a permanently halted
// reconciliation means for such rows.
// The bound is persisted in audit_site_state (loaded at construction, clamped to
// the live MAX(rowid)), so a site-node restart no longer resets it to 0 and strands
// rows the pull had already served. Conservative in the safe direction (it can
// only delay retirement, never lose a row); the liveness note in
// MarkReconciledUpToAsync's interface doc + Component-AuditLog.md covers what a
// permanently halted reconciliation means for such rows.
using var cmd = _connection.CreateCommand();
cmd.CommandText = $"""
UPDATE audit_forward_state AS fs
@@ -1506,7 +1586,10 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
var liveMax = Convert.ToInt64(maxCmd.ExecuteScalar(),
System.Globalization.CultureInfo.InvariantCulture);
if (liveMax < Volatile.Read(ref _maxServedRowId))
{
Volatile.Write(ref _maxServedRowId, liveMax);
PersistMaxServedRowId(liveMax, transaction);
}
}
transaction.Commit();
@@ -481,28 +481,30 @@ public class SiteAuditTelemetryActor : ReceiveActor
}
/// <summary>
/// Records a refused push attempt for every row that was SENT in an answered batch but is
/// missing from the ack's <c>AcceptedEventIds</c>; rows refused
/// <see cref="SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/> times leave the push
/// queue as <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1).
/// Records a refused push attempt for every SENT row central explicitly listed in the ack's
/// <c>RejectedEventIds</c> — a permanent, row-intrinsic insert failure. Rows refused
/// <see cref="SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/> times leave the push queue as
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1).
/// </summary>
/// <remarks>
/// Called only after the push RETURNED — a transport fault is not a verdict about any row
/// and never counts. Best-effort: a bookkeeping failure leaves the rows Pending, exactly as
/// before this existed.
/// "Not accepted" is NOT a refusal: central answers with an empty or partial ack during a SQL
/// maintenance window, a timeout, or an exhausted ingest budget, and counting those would flip
/// healthy rows to Rejected. Only the explicit list counts; an older central never sends it, so
/// against it this degrades to the pre-IMPL-1 behaviour (rows stay Pending). Best-effort: a
/// bookkeeping failure leaves the rows Pending.
/// </remarks>
private async Task RecordRefusalsAsync(IReadOnlyList<string> sentIds, IngestAck ack, CancellationToken ct)
{
if (_queue is not ISiteAuditPushLifecycle lifecycle || sentIds.Count == 0)
if (_queue is not ISiteAuditPushLifecycle lifecycle || sentIds.Count == 0 || ack.RejectedEventIds.Count == 0)
{
return;
}
var accepted = new HashSet<string>(ack.AcceptedEventIds, StringComparer.OrdinalIgnoreCase);
var sent = new HashSet<string>(sentIds, StringComparer.OrdinalIgnoreCase);
var refused = new List<Guid>();
foreach (var raw in sentIds)
foreach (var raw in ack.RejectedEventIds)
{
if (!accepted.Contains(raw) && Guid.TryParse(raw, out var id))
if (sent.Contains(raw) && Guid.TryParse(raw, out var id))
{
refused.Add(id);
}
@@ -57,18 +57,22 @@ public sealed class SiteAuditTelemetryOptions
public int CachedTrackingGraceSeconds { get; set; } = 300;
/// <summary>
/// Number of drains in which central ANSWERED a push but refused a row (the row is
/// missing from <c>AcceptedEventIds</c>) after which the row leaves the push queue as
/// <c>Rejected</c>. Default: 10 (≈ 50 s at the busy cadence).
/// Number of drains in which central EXPLICITLY refused a row (listed it in the ack's
/// additive <c>rejected_event_ids</c> — a permanent, row-intrinsic insert failure such as an
/// over-width value) after which the row leaves the push queue as <c>Rejected</c>.
/// Default: 10 (≈ 50 s at the busy cadence).
/// </summary>
/// <remarks>
/// The push reads oldest-first, so a row central can never insert (e.g. an over-width
/// column) used to sit at the head of every batch forever, failing each set-based insert
/// at central and dragging the batch into the per-row fallback (review 2026-10-21
/// 06/IMPL-1). A transport fault (no answer) never counts. <c>Rejected</c> is NOT a drop:
/// 06/IMPL-1). A transport fault, and a row merely ABSENT from the accepted list (a database
/// outage, a timeout, an exhausted ingest budget), never counts; an older central that does not
/// send the field therefore never triggers it. <c>Rejected</c> is NOT a drop:
/// the reconciliation pull still serves the row, so a transient central refusal still
/// lands, and a genuinely poison row is recorded centrally as <c>ReconciliationAbandoned</c>
/// before it retires. Each flip bumps <c>SiteAuditWriteFailures</c> on the site health report.
/// before it retires. Each flip is counted on <c>scadabridge.site.audit.rejected_rows</c> (NOT
/// <c>SiteAuditWriteFailures</c> — the site's own write succeeded).
/// </remarks>
public int MaxRejectedPushAttempts { get; set; } = 10;
}
@@ -82,7 +82,7 @@ public sealed class SiteCommunicationAuditClient : ISiteStreamAuditClient
.Ask<IngestAuditEventsReply>(new IngestAuditEventsCommand(events), _askTimeout, ct)
.ConfigureAwait(false);
return ToAck(reply.AcceptedEventIds);
return ToAck(reply.AcceptedEventIds, reply.RejectedEventIds);
}
/// <inheritdoc/>
@@ -105,16 +105,21 @@ public sealed class SiteCommunicationAuditClient : ISiteStreamAuditClient
.Ask<IngestCachedTelemetryReply>(new IngestCachedTelemetryCommand(entries), _askTimeout, ct)
.ConfigureAwait(false);
return ToAck(reply.AcceptedEventIds);
return ToAck(reply.AcceptedEventIds, reply.RejectedEventIds);
}
private static IngestAck ToAck(IReadOnlyList<Guid> acceptedEventIds)
private static IngestAck ToAck(IReadOnlyList<Guid> acceptedEventIds, IReadOnlyList<Guid> rejectedEventIds)
{
var ack = new IngestAck();
foreach (var id in acceptedEventIds)
{
ack.AcceptedEventIds.Add(id.ToString());
}
// Explicit permanent refusals only (review 2026-10-21 06/IMPL-1); empty from an older central.
foreach (var id in rejectedEventIds)
{
ack.RejectedEventIds.Add(id.ToString());
}
return ack;
}
}