fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
using System.Data.Common;
|
||||
using System.Net.Sockets;
|
||||
using Microsoft.Data.SqlClient;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Classifies a central audit-insert failure (review 2026-10-21 06/IMPL-1 follow-up) so that a
|
||||
/// DATABASE problem — a maintenance window, a timeout, a dropped connection, an exhausted budget —
|
||||
/// is never mistaken for a verdict about the ROW. Two questions, two predicates:
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="IsPermanentRowRejection"/> — narrow: the row itself can never be inserted
|
||||
/// (string truncation, NULL into NOT NULL, a conversion / overflow, a constraint). Only these ids
|
||||
/// go back to the site as <c>rejected_event_ids</c>, which is what lets the site give a row up as
|
||||
/// <c>Rejected</c>.</item>
|
||||
/// <item><see cref="IsTransient"/> — broad: anything that may clear on its own. The reconciliation
|
||||
/// puller never counts these toward permanently abandoning a row.</item>
|
||||
/// </list>
|
||||
/// The exception's whole <see cref="Exception.InnerException"/> chain is inspected, because EF Core
|
||||
/// and the execution strategy wrap the provider exception.
|
||||
/// </summary>
|
||||
internal static class AuditIngestErrorClassifier
|
||||
{
|
||||
/// <summary>
|
||||
/// SQL Server errors that describe the ROW, not the server: 8152 / 2628 string or binary data
|
||||
/// would be truncated, 515 NULL into a NOT NULL column, 245 / 241 / 242 / 8114 / 8115 / 220 / 232
|
||||
/// conversion and arithmetic overflow, 547 CHECK / FOREIGN KEY constraint.
|
||||
/// </summary>
|
||||
private static readonly HashSet<int> PermanentRowErrorNumbers = new()
|
||||
{
|
||||
8152, 2628, 515, 245, 241, 242, 8114, 8115, 220, 232, 547,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// SQL Server errors that may clear on their own (timeout, connection loss, deadlock, lock
|
||||
/// timeout, resource / Azure throttling, database unavailable or in a restricted state).
|
||||
/// </summary>
|
||||
private static readonly HashSet<int> TransientErrorNumbers = new()
|
||||
{
|
||||
-2, -1, 0, 2, 20, 53, 64, 121, 233, 1205, 1222, 3960, 4060, 4221, 8645, 8651,
|
||||
10053, 10054, 10060, 10928, 10929, 40143, 40197, 40501, 40540, 40613,
|
||||
49918, 49919, 49920, 942, 952, 983, 976, 978, 3906, 9001, 9002,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// True when the failure is a SQL Server error that says the row itself can never be inserted.
|
||||
/// Everything else — including every transient error and every non-SQL exception — is false,
|
||||
/// so the site keeps such rows Pending and simply retries them.
|
||||
/// </summary>
|
||||
/// <param name="ex">The insert failure.</param>
|
||||
/// <returns>Whether the row should be reported back to the site as permanently rejected.</returns>
|
||||
public static bool IsPermanentRowRejection(Exception ex)
|
||||
{
|
||||
if (IsTransient(ex))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
{
|
||||
if (e is SqlException sql)
|
||||
{
|
||||
foreach (SqlError error in sql.Errors)
|
||||
{
|
||||
if (PermanentRowErrorNumbers.Contains(error.Number))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return PermanentRowErrorNumbers.Contains(sql.Number);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when any exception in the chain is a timeout, cancellation, socket / IO / connection
|
||||
/// failure, an EF retry-limit exhaustion, a provider error flagged transient, or a SQL Server
|
||||
/// error number in the transient set.
|
||||
/// </summary>
|
||||
/// <param name="ex">The insert failure.</param>
|
||||
/// <returns>Whether the failure may clear on its own.</returns>
|
||||
public static bool IsTransient(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
{
|
||||
switch (e)
|
||||
{
|
||||
case TimeoutException:
|
||||
case OperationCanceledException:
|
||||
case SocketException:
|
||||
case IOException:
|
||||
case HttpRequestException:
|
||||
return true;
|
||||
case SqlException sql:
|
||||
if (TransientErrorNumbers.Contains(sql.Number))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
foreach (SqlError error in sql.Errors)
|
||||
{
|
||||
if (TransientErrorNumbers.Contains(error.Number))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (sql.IsTransient)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
case DbException db when db.IsTransient:
|
||||
return true;
|
||||
}
|
||||
|
||||
// EF Core's RetryLimitExceededException (an InvalidOperationException) wraps a
|
||||
// transient provider error once the execution strategy gives up.
|
||||
if (e.GetType().Name == "RetryLimitExceededException")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -152,6 +152,7 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
var replyTo = Sender;
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var accepted = new List<Guid>(cmd.Events.Count);
|
||||
var rejected = new List<Guid>();
|
||||
|
||||
// Resolve the repository for the whole batch — one DbContext per
|
||||
// message, mirroring NotificationOutboxActor.
|
||||
@@ -184,7 +185,7 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
// unregistered (test composition roots) means the per-row catch
|
||||
// simply logs without surfacing on the health dashboard.
|
||||
var failureCounter = scope.ServiceProvider.GetService<ICentralAuditWriteFailureCounter>();
|
||||
await IngestWithRepositoryAsync(repository, redactor, failureCounter, cmd, nowUtc, accepted)
|
||||
await IngestWithRepositoryAsync(repository, redactor, failureCounter, cmd, nowUtc, accepted, rejected)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -201,7 +202,7 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
accepted.Count);
|
||||
}
|
||||
|
||||
replyTo.Tell(new IngestAuditEventsReply(accepted));
|
||||
replyTo.Tell(new IngestAuditEventsReply(accepted) { RejectedEventIds = rejected });
|
||||
}
|
||||
|
||||
private async Task IngestWithRepositoryAsync(
|
||||
@@ -210,7 +211,8 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
ICentralAuditWriteFailureCounter? failureCounter,
|
||||
IngestAuditEventsCommand cmd,
|
||||
DateTime nowUtc,
|
||||
List<Guid> accepted)
|
||||
List<Guid> accepted,
|
||||
List<Guid> rejected)
|
||||
{
|
||||
// Stamp the ingest timestamp here, not at the site. Redact BEFORE the
|
||||
// IngestedAtUtc stamp so the redacted copy carries the central-side
|
||||
@@ -303,6 +305,15 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
catch { /* counter must never throw — defence in depth */ }
|
||||
}
|
||||
|
||||
// Review 2026-10-21 06/IMPL-1: tell the site ONLY about rows the server
|
||||
// refused for a reason intrinsic to the row (truncation, NOT NULL, conversion,
|
||||
// constraint). A timeout / connection loss / maintenance window leaves the id
|
||||
// in neither list, so the site retries it without counting a refusal.
|
||||
if (AuditIngestErrorClassifier.IsPermanentRowRejection(rowEx))
|
||||
{
|
||||
rejected.Add(cmd.Events[i].EventId);
|
||||
}
|
||||
|
||||
_logger.LogError(rowEx,
|
||||
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
|
||||
cmd.Events[i].EventId);
|
||||
@@ -336,6 +347,7 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
{
|
||||
var replyTo = Sender;
|
||||
var accepted = new List<Guid>(cmd.Entries.Count);
|
||||
var rejected = new List<Guid>();
|
||||
|
||||
try
|
||||
{
|
||||
@@ -370,7 +382,7 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
strategy, dbContext, auditRepo, siteCallRepo, redactor, cmd, accepted, budget.Token)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
|
||||
replyTo.Tell(new IngestCachedTelemetryReply(accepted) { RejectedEventIds = rejected });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -445,6 +457,11 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
// sustained dual-write failure surfaces on the dashboard.
|
||||
try { failureCounter?.Increment(); }
|
||||
catch { /* counter must never throw — defence in depth */ }
|
||||
if (AuditIngestErrorClassifier.IsPermanentRowRejection(ex))
|
||||
{
|
||||
rejected.Add(entry.Audit.EventId);
|
||||
}
|
||||
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Combined telemetry dual-write failed for AuditEvent {EventId} / TrackedOperationId {TrackedOpId}; rolled back.",
|
||||
@@ -463,7 +480,7 @@ public class AuditLogIngestActor : ReceiveActor
|
||||
"Combined telemetry batch ingest failed before per-entry processing.");
|
||||
}
|
||||
|
||||
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
|
||||
replyTo.Tell(new IngestCachedTelemetryReply(accepted) { RejectedEventIds = rejected });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -302,13 +302,15 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
return;
|
||||
}
|
||||
|
||||
PruneUnknownSites(sites, eventStream);
|
||||
|
||||
if (sites.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// After the empty guard: an empty enumeration is as likely a transient lookup glitch
|
||||
// as a fleet with no sites, and pruning on it would throw away every cursor.
|
||||
PruneUnknownSites(sites, eventStream);
|
||||
|
||||
// Use CreateAsyncScope + await using so scoped EF Core
|
||||
// services (IAsyncDisposable DbContexts) dispose asynchronously
|
||||
// without blocking on sync Dispose() of pending connection cleanup.
|
||||
@@ -489,34 +491,51 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
// is permanently abandoned (LogCritical + cursor advances past)
|
||||
// to keep a truly broken row from blocking all subsequent
|
||||
// progress for the site.
|
||||
var attempts = _failedInsertAttempts.GetValueOrDefault(evt.EventId) + 1;
|
||||
_failedInsertAttempts[evt.EventId] = attempts;
|
||||
|
||||
if (attempts >= MaxPermanentInsertAttempts)
|
||||
if (AuditIngestErrorClassifier.IsTransient(ex))
|
||||
{
|
||||
_logger.LogCritical(
|
||||
// A database problem (timeout, connection loss, deadlock, throttling, a
|
||||
// maintenance window) is not a verdict about this row: hold the cursor
|
||||
// and retry, but never count it toward permanent abandonment — a long
|
||||
// outage must not turn healthy rows into ReconciliationAbandoned losses
|
||||
// (review 2026-10-21 06/PERF-1 adversarial follow-up).
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Permanently abandoning AuditEvent {EventId} from site {SiteId} after {Attempts} consecutive insert failures; cursor will advance past it.",
|
||||
"Reconciliation ingest of AuditEvent {EventId} from site {SiteId} hit a transient database fault; cursor held back, not counted toward abandonment.",
|
||||
evt.EventId,
|
||||
site.SiteId,
|
||||
attempts);
|
||||
_failedInsertAttempts.Remove(evt.EventId);
|
||||
RememberAbandoned(evt.EventId);
|
||||
advanceForThisRow = true;
|
||||
|
||||
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
|
||||
.ConfigureAwait(false);
|
||||
site.SiteId);
|
||||
hasUnresolvedFailure = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Reconciliation ingest failed for AuditEvent {EventId} from site {SiteId} (attempt {Attempts}/{Max}); cursor held back for retry.",
|
||||
evt.EventId,
|
||||
site.SiteId,
|
||||
attempts,
|
||||
MaxPermanentInsertAttempts);
|
||||
hasUnresolvedFailure = true;
|
||||
var attempts = _failedInsertAttempts.GetValueOrDefault(evt.EventId) + 1;
|
||||
_failedInsertAttempts[evt.EventId] = attempts;
|
||||
|
||||
if (attempts >= MaxPermanentInsertAttempts)
|
||||
{
|
||||
_logger.LogCritical(
|
||||
ex,
|
||||
"Permanently abandoning AuditEvent {EventId} from site {SiteId} after {Attempts} consecutive insert failures; cursor will advance past it.",
|
||||
evt.EventId,
|
||||
site.SiteId,
|
||||
attempts);
|
||||
_failedInsertAttempts.Remove(evt.EventId);
|
||||
RememberAbandoned(evt.EventId);
|
||||
advanceForThisRow = true;
|
||||
|
||||
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Reconciliation ingest failed for AuditEvent {EventId} from site {SiteId} (attempt {Attempts}/{Max}); cursor held back for retry.",
|
||||
evt.EventId,
|
||||
site.SiteId,
|
||||
attempts,
|
||||
MaxPermanentInsertAttempts);
|
||||
hasUnresolvedFailure = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ public static class SiteAuditForwardStates
|
||||
public const string PushAbandoned = "PushAbandoned";
|
||||
|
||||
/// <summary>
|
||||
/// A row central answered but refused (not in <c>AcceptedEventIds</c>) on
|
||||
/// A row central explicitly and permanently refused (listed in the ack's <c>rejected_event_ids</c>) on
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/>
|
||||
/// consecutive drains. Moved out of the push queue so one poison row cannot wedge every push
|
||||
/// batch behind it; the pull then delivers it, or central's permanent-abandon path records the
|
||||
@@ -53,11 +53,11 @@ public interface ISiteAuditPushLifecycle
|
||||
Task MarkPushAbandonedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Records one refused push attempt for each supplied <see cref="AuditForwardState.Pending"/>
|
||||
/// Records one explicitly-refused push attempt for each supplied <see cref="AuditForwardState.Pending"/>
|
||||
/// row (bumps <c>AttemptCount</c>, stamps <c>LastAttemptUtc</c>) and flips every row that has
|
||||
/// now reached <paramref name="maxAttempts"/> to <see cref="SiteAuditForwardStates.Rejected"/>.
|
||||
/// </summary>
|
||||
/// <param name="eventIds">Event IDs central answered for but did not accept.</param>
|
||||
/// <param name="eventIds">Event IDs central listed as permanently rejected.</param>
|
||||
/// <param name="maxAttempts">Refused attempts after which a row leaves the push queue.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the number of rows flipped to Rejected by this call.</returns>
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Site audit forwarding instruments. Published on a meter named
|
||||
/// <see cref="ScadaBridgeTelemetry.MeterName"/> — OpenTelemetry matches meters by NAME, so these
|
||||
/// export through the existing <c>ObservedMeters</c> allowlist entry without a Host change.
|
||||
/// </summary>
|
||||
internal static class SiteAuditTelemetryMetrics
|
||||
{
|
||||
private static readonly Meter Meter = new(ScadaBridgeTelemetry.MeterName);
|
||||
|
||||
private static readonly Counter<long> RejectedRows =
|
||||
Meter.CreateCounter<long>("scadabridge.site.audit.rejected_rows", unit: "1",
|
||||
description: "Site audit rows central permanently refused MaxRejectedPushAttempts times, moved out of the push queue as Rejected (still served by the reconciliation pull).");
|
||||
|
||||
/// <summary>Records <paramref name="count"/> rows flipped to Rejected.</summary>
|
||||
/// <param name="count">Rows flipped.</param>
|
||||
public static void RecordRejectedRows(long count)
|
||||
{
|
||||
if (count > 0)
|
||||
{
|
||||
RejectedRows.Add(count);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,7 +78,6 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
private readonly SqliteAuditWriterOptions _options;
|
||||
private readonly ILogger<SqliteAuditWriter> _logger;
|
||||
private readonly INodeIdentityProvider _nodeIdentity;
|
||||
private readonly IAuditWriteFailureCounter? _failureCounter;
|
||||
private readonly object _writeLock = new();
|
||||
private readonly Channel<PendingAuditEvent> _writeQueue;
|
||||
private readonly Task _writerLoop;
|
||||
@@ -89,17 +88,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="nodeIdentity">Node identity provider.</param>
|
||||
/// <param name="connectionStringOverride">Optional connection string override.</param>
|
||||
/// <param name="failureCounter">
|
||||
/// Optional <c>SiteAuditWriteFailures</c> sink, bumped once per row the push gives up on as
|
||||
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1). Null on
|
||||
/// composition roots without Site Health Monitoring.
|
||||
/// </param>
|
||||
public SqliteAuditWriter(
|
||||
IOptions<SqliteAuditWriterOptions> options,
|
||||
ILogger<SqliteAuditWriter> logger,
|
||||
INodeIdentityProvider nodeIdentity,
|
||||
string? connectionStringOverride = null,
|
||||
IAuditWriteFailureCounter? failureCounter = null)
|
||||
string? connectionStringOverride = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
@@ -108,7 +101,6 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_nodeIdentity = nodeIdentity;
|
||||
_failureCounter = failureCounter;
|
||||
|
||||
var connectionString = connectionStringOverride
|
||||
?? $"Data Source={_options.DatabasePath};Cache=Shared";
|
||||
@@ -116,6 +108,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
_connection.Open();
|
||||
|
||||
InitializeSchema();
|
||||
LoadMaxServedRowId();
|
||||
|
||||
// Open a second connection for read-only callers
|
||||
// (GetBacklogStatsAsync, ReadPendingAsync, ReadPendingSinceAsync,
|
||||
@@ -275,6 +268,16 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
CREATE INDEX IF NOT EXISTS IX_fwd
|
||||
ON audit_forward_state (ForwardState, IsCachedKind, OccurredAtUtc);
|
||||
|
||||
-- Node-local key/value state that must survive a process restart. Holds the
|
||||
-- reconciliation pull's served-rows high-water mark ('max_served_rowid'), so a
|
||||
-- restart no longer resets the bound that lets a covering cursor retire rows the
|
||||
-- pull already delivered (review 2026-10-21 06/PERF-1 adversarial follow-up).
|
||||
-- Additive and idempotent: an older binary simply ignores the table.
|
||||
CREATE TABLE IF NOT EXISTS audit_site_state (
|
||||
Key TEXT NOT NULL PRIMARY KEY,
|
||||
Value INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Flow Debugger (design 2026-09-05 §5.4): PullTraceEvents reads audit_event by
|
||||
-- time window + json_extract on ExecutionId/ParentExecutionId; this bounds the scan
|
||||
-- to the window. Additive and idempotent.
|
||||
@@ -345,6 +348,13 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
// TryWrite can tell "disposed" from "full" without racing the channel's state.
|
||||
private bool _writerCompleted;
|
||||
|
||||
/// <summary>
|
||||
/// Rows moved to <see cref="SiteAuditForwardStates.Rejected"/> since construction (also exported
|
||||
/// as the <c>scadabridge.site.audit.rejected_rows</c> counter).
|
||||
/// </summary>
|
||||
public long RejectedPushRowCount => Interlocked.Read(ref _rejectedPushRows);
|
||||
private long _rejectedPushRows;
|
||||
|
||||
private async Task ProcessWriteQueueAsync()
|
||||
{
|
||||
var batch = new List<PendingAuditEvent>(_options.BatchSize);
|
||||
@@ -845,6 +855,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
// pull's 256-rows-per-tick budget was spent on acked rows, so any site above
|
||||
// ~0.85 audit rows/s fell permanently behind and latched the stalled signal.
|
||||
// A Forwarded row is acked by central; the pull has nothing to repair there.
|
||||
IReadOnlyList<AuditEvent> rows;
|
||||
var before = Volatile.Read(ref _maxServedRowId);
|
||||
lock (_readLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
@@ -884,8 +896,33 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
}
|
||||
cmd.Parameters.AddWithValue("$limit", batchSize);
|
||||
|
||||
return Task.FromResult(ReadServedRows(cmd, batchSize));
|
||||
rows = ReadServedRows(cmd, batchSize);
|
||||
}
|
||||
|
||||
// Persist the (possibly advanced) mark OUTSIDE _readLock so the two locks are never
|
||||
// nested in read→write order. Only when it moved — at most one tiny upsert per pull.
|
||||
var mark = Volatile.Read(ref _maxServedRowId);
|
||||
if (mark > before)
|
||||
{
|
||||
try
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
PersistMaxServedRowId(mark);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"SqliteAuditWriter could not persist the served high-water mark {Mark}; a restart re-learns it from the next pull.",
|
||||
mark);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(rows);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1005,11 +1042,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
"of the push queue as Rejected; the reconciliation pull still serves them (central records " +
|
||||
"a ReconciliationAbandoned row if they can never be inserted).",
|
||||
flipped, threshold);
|
||||
for (var i = 0; i < flipped; i++)
|
||||
{
|
||||
try { _failureCounter?.Increment(); }
|
||||
catch { /* counter must never throw — defence in depth */ }
|
||||
}
|
||||
// Its OWN counter, not SiteAuditWriteFailures: the local write succeeded, and
|
||||
// the refusal is central's verdict on the row — blaming the site's writer for it
|
||||
// would be wrong (review 2026-10-21 adversarial follow-up).
|
||||
Interlocked.Add(ref _rejectedPushRows, flipped);
|
||||
SiteAuditTelemetryMetrics.RecordRejectedRows(flipped);
|
||||
}
|
||||
|
||||
return Task.FromResult(flipped);
|
||||
@@ -1154,6 +1191,49 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
/// </summary>
|
||||
private long _maxServedRowId;
|
||||
|
||||
private const string MaxServedRowIdKey = "max_served_rowid";
|
||||
|
||||
/// <summary>
|
||||
/// Restores <see cref="_maxServedRowId"/> from <c>audit_site_state</c>, clamped to the live
|
||||
/// <c>MAX(rowid)</c> so a stale value can never vouch for a reused rowid. Before this the
|
||||
/// bound was per-process: after a site-node restart it was 0, and because a healthy site's
|
||||
/// pull now often serves nothing, owed rows central had already ingested (e.g. PushAbandoned)
|
||||
/// could stay unretired — and counted on the backlog — indefinitely.
|
||||
/// </summary>
|
||||
private void LoadMaxServedRowId()
|
||||
{
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
SELECT MIN(
|
||||
IFNULL((SELECT Value FROM audit_site_state WHERE Key = $key), 0),
|
||||
IFNULL((SELECT MAX(rowid) FROM audit_forward_state), 0));
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$key", MaxServedRowIdKey);
|
||||
var value = Convert.ToInt64(cmd.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture);
|
||||
Volatile.Write(ref _maxServedRowId, value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Durably records the served high-water mark. Runs on the WRITE connection under
|
||||
/// <c>_writeLock</c> (never nested inside <c>_readLock</c>). Best-effort: a failure leaves
|
||||
/// the in-memory bound intact and only means a restart re-learns it from the next pull.
|
||||
/// </summary>
|
||||
private void PersistMaxServedRowId(long value, SqliteTransaction? transaction = null)
|
||||
{
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.Transaction = transaction;
|
||||
cmd.CommandText = """
|
||||
INSERT INTO audit_site_state (Key, Value) VALUES ($key, $value)
|
||||
ON CONFLICT(Key) DO UPDATE SET Value = excluded.Value;
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$key", MaxServedRowIdKey);
|
||||
cmd.Parameters.AddWithValue("$value", value);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
/// <summary>Test seam: the current in-memory served high-water mark.</summary>
|
||||
internal long MaxServedRowIdForTests => Volatile.Read(ref _maxServedRowId);
|
||||
|
||||
/// <summary>
|
||||
/// Executes a reconciliation-pull read whose projection carries <c>fs.rowid</c> as an
|
||||
/// 11th column, materialising the canonical rows while advancing
|
||||
@@ -1222,12 +1302,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
// acked them over the push path, they are already purge-eligible, and flipping
|
||||
// them was one UPDATE per pushed row for no benefit (review 2026-10-21 06/PERF-1).
|
||||
//
|
||||
// Bound state is per-process: after a restart it is 0 until this instance serves
|
||||
// a batch, so the first pull retires nothing and the pull after it
|
||||
// resumes normal retirement. Conservative in the safe direction (delays
|
||||
// retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
|
||||
// interface doc + Component-AuditLog.md covers what a permanently halted
|
||||
// reconciliation means for such rows.
|
||||
// The bound is persisted in audit_site_state (loaded at construction, clamped to
|
||||
// the live MAX(rowid)), so a site-node restart no longer resets it to 0 and strands
|
||||
// rows the pull had already served. Conservative in the safe direction (it can
|
||||
// only delay retirement, never lose a row); the liveness note in
|
||||
// MarkReconciledUpToAsync's interface doc + Component-AuditLog.md covers what a
|
||||
// permanently halted reconciliation means for such rows.
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = $"""
|
||||
UPDATE audit_forward_state AS fs
|
||||
@@ -1506,7 +1586,10 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLi
|
||||
var liveMax = Convert.ToInt64(maxCmd.ExecuteScalar(),
|
||||
System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (liveMax < Volatile.Read(ref _maxServedRowId))
|
||||
{
|
||||
Volatile.Write(ref _maxServedRowId, liveMax);
|
||||
PersistMaxServedRowId(liveMax, transaction);
|
||||
}
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
|
||||
@@ -481,28 +481,30 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a refused push attempt for every row that was SENT in an answered batch but is
|
||||
/// missing from the ack's <c>AcceptedEventIds</c>; rows refused
|
||||
/// <see cref="SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/> times leave the push
|
||||
/// queue as <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1).
|
||||
/// Records a refused push attempt for every SENT row central explicitly listed in the ack's
|
||||
/// <c>RejectedEventIds</c> — a permanent, row-intrinsic insert failure. Rows refused
|
||||
/// <see cref="SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/> times leave the push queue as
|
||||
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Called only after the push RETURNED — a transport fault is not a verdict about any row
|
||||
/// and never counts. Best-effort: a bookkeeping failure leaves the rows Pending, exactly as
|
||||
/// before this existed.
|
||||
/// "Not accepted" is NOT a refusal: central answers with an empty or partial ack during a SQL
|
||||
/// maintenance window, a timeout, or an exhausted ingest budget, and counting those would flip
|
||||
/// healthy rows to Rejected. Only the explicit list counts; an older central never sends it, so
|
||||
/// against it this degrades to the pre-IMPL-1 behaviour (rows stay Pending). Best-effort: a
|
||||
/// bookkeeping failure leaves the rows Pending.
|
||||
/// </remarks>
|
||||
private async Task RecordRefusalsAsync(IReadOnlyList<string> sentIds, IngestAck ack, CancellationToken ct)
|
||||
{
|
||||
if (_queue is not ISiteAuditPushLifecycle lifecycle || sentIds.Count == 0)
|
||||
if (_queue is not ISiteAuditPushLifecycle lifecycle || sentIds.Count == 0 || ack.RejectedEventIds.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var accepted = new HashSet<string>(ack.AcceptedEventIds, StringComparer.OrdinalIgnoreCase);
|
||||
var sent = new HashSet<string>(sentIds, StringComparer.OrdinalIgnoreCase);
|
||||
var refused = new List<Guid>();
|
||||
foreach (var raw in sentIds)
|
||||
foreach (var raw in ack.RejectedEventIds)
|
||||
{
|
||||
if (!accepted.Contains(raw) && Guid.TryParse(raw, out var id))
|
||||
if (sent.Contains(raw) && Guid.TryParse(raw, out var id))
|
||||
{
|
||||
refused.Add(id);
|
||||
}
|
||||
|
||||
@@ -57,18 +57,22 @@ public sealed class SiteAuditTelemetryOptions
|
||||
public int CachedTrackingGraceSeconds { get; set; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Number of drains in which central ANSWERED a push but refused a row (the row is
|
||||
/// missing from <c>AcceptedEventIds</c>) after which the row leaves the push queue as
|
||||
/// <c>Rejected</c>. Default: 10 (≈ 50 s at the busy cadence).
|
||||
/// Number of drains in which central EXPLICITLY refused a row (listed it in the ack's
|
||||
/// additive <c>rejected_event_ids</c> — a permanent, row-intrinsic insert failure such as an
|
||||
/// over-width value) after which the row leaves the push queue as <c>Rejected</c>.
|
||||
/// Default: 10 (≈ 50 s at the busy cadence).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The push reads oldest-first, so a row central can never insert (e.g. an over-width
|
||||
/// column) used to sit at the head of every batch forever, failing each set-based insert
|
||||
/// at central and dragging the batch into the per-row fallback (review 2026-10-21
|
||||
/// 06/IMPL-1). A transport fault (no answer) never counts. <c>Rejected</c> is NOT a drop:
|
||||
/// 06/IMPL-1). A transport fault, and a row merely ABSENT from the accepted list (a database
|
||||
/// outage, a timeout, an exhausted ingest budget), never counts; an older central that does not
|
||||
/// send the field therefore never triggers it. <c>Rejected</c> is NOT a drop:
|
||||
/// the reconciliation pull still serves the row, so a transient central refusal still
|
||||
/// lands, and a genuinely poison row is recorded centrally as <c>ReconciliationAbandoned</c>
|
||||
/// before it retires. Each flip bumps <c>SiteAuditWriteFailures</c> on the site health report.
|
||||
/// before it retires. Each flip is counted on <c>scadabridge.site.audit.rejected_rows</c> (NOT
|
||||
/// <c>SiteAuditWriteFailures</c> — the site's own write succeeded).
|
||||
/// </remarks>
|
||||
public int MaxRejectedPushAttempts { get; set; } = 10;
|
||||
}
|
||||
|
||||
@@ -82,7 +82,7 @@ public sealed class SiteCommunicationAuditClient : ISiteStreamAuditClient
|
||||
.Ask<IngestAuditEventsReply>(new IngestAuditEventsCommand(events), _askTimeout, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return ToAck(reply.AcceptedEventIds);
|
||||
return ToAck(reply.AcceptedEventIds, reply.RejectedEventIds);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
@@ -105,16 +105,21 @@ public sealed class SiteCommunicationAuditClient : ISiteStreamAuditClient
|
||||
.Ask<IngestCachedTelemetryReply>(new IngestCachedTelemetryCommand(entries), _askTimeout, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return ToAck(reply.AcceptedEventIds);
|
||||
return ToAck(reply.AcceptedEventIds, reply.RejectedEventIds);
|
||||
}
|
||||
|
||||
private static IngestAck ToAck(IReadOnlyList<Guid> acceptedEventIds)
|
||||
private static IngestAck ToAck(IReadOnlyList<Guid> acceptedEventIds, IReadOnlyList<Guid> rejectedEventIds)
|
||||
{
|
||||
var ack = new IngestAck();
|
||||
foreach (var id in acceptedEventIds)
|
||||
{
|
||||
ack.AcceptedEventIds.Add(id.ToString());
|
||||
}
|
||||
// Explicit permanent refusals only (review 2026-10-21 06/IMPL-1); empty from an older central.
|
||||
foreach (var id in rejectedEventIds)
|
||||
{
|
||||
ack.RejectedEventIds.Add(id.ToString());
|
||||
}
|
||||
return ack;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user