fix(auditlog): reconciliation pull serves only unacked rows (PushAbandoned/Rejected push-exit states), central pages within a tick with set-based inserts; refused push rows leave the queue after N attempts (review 2026-10-21 06/PERF-1, 06/COR-3, 06/IMPL-1)
This commit is contained in:
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
@@ -24,8 +25,12 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// (<see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.SiteAuditTelemetryActor"/> +
|
||||
/// <c>IngestAuditEvents</c>) is the primary mechanism. This actor exists so a
|
||||
/// missed push (gRPC blip, central restart, site offline) is eventually
|
||||
/// repaired by central re-pulling whatever the site still has in
|
||||
/// <c>Pending</c>/<c>Forwarded</c> state. Idempotency on
|
||||
/// repaired by central re-pulling whatever the site has NOT yet had acknowledged —
|
||||
/// <c>Pending</c> rows plus rows that left the push queue without an ack
|
||||
/// (<c>PushAbandoned</c>, <c>Rejected</c>). <c>Forwarded</c> rows are no longer
|
||||
/// re-served: central already acked them, and re-shipping every pushed row capped the
|
||||
/// pull at a throughput no busy site could keep up with (review 2026-10-21 06/PERF-1).
|
||||
/// A pre-PERF-1 site still serves them; idempotency on
|
||||
/// <see cref="AuditEvent.EventId"/> makes duplicate
|
||||
/// arrivals from both paths a silent no-op.
|
||||
/// </para>
|
||||
@@ -34,15 +39,18 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// kept in-memory for the actor's lifetime. The cluster singleton normally
|
||||
/// survives the host process; on a deliberate failover OR a singleton restart
|
||||
/// the cursors reset to <see cref="DateTime.MinValue"/>. That is conservative
|
||||
/// but correct — the next tick simply asks for everything the site still has,
|
||||
/// and idempotent ingest swallows the dupes. Persisting cursors to MS SQL was
|
||||
/// but correct — the next tick simply asks for every row the site still owes,
|
||||
/// and idempotent ingest swallows the dupes. Because the site serves only
|
||||
/// unacknowledged rows, that re-pull is the small owed set, not a multi-day
|
||||
/// replay of every forwarded row (review 2026-10-21 06/COR-3). Persisting cursors to MS SQL was
|
||||
/// considered and rejected: the cost of a write per tick outweighs the
|
||||
/// rare benefit of avoiding one over-broad pull after a restart.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Stalled detection.</b> The brief calls a site "stalled" when two
|
||||
/// consecutive pull cycles BOTH return non-empty AND <c>MoreAvailable=true</c>
|
||||
/// — i.e. the backlog isn't draining. The actor publishes
|
||||
/// <b>Stalled detection.</b> Each tick pages through a site (up to
|
||||
/// <see cref="SiteAuditReconciliationOptions.MaxPagesPerTick"/> pulls, each ingested with
|
||||
/// one set-based insert). A site is "stalled" when two consecutive ticks BOTH still end
|
||||
/// non-empty AND <c>MoreAvailable=true</c> after that — i.e. the backlog isn't draining. The actor publishes
|
||||
/// <see cref="SiteAuditTelemetryStalledChanged"/> on the actor system's
|
||||
/// EventStream so a future <c>ICentralHealthCollector</c> bridge
|
||||
/// can flip the health metric without coupling this actor to the health
|
||||
@@ -173,6 +181,8 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
|
||||
private ICancelable? _timer;
|
||||
|
||||
private void RememberAbandoned(Guid eventId) => _abandonedEventIds.Add(eventId);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the reconciliation actor with its dependencies and registers the tick handler.
|
||||
/// </summary>
|
||||
@@ -288,23 +298,59 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drains one site for this tick: issues up to
|
||||
/// <see cref="SiteAuditReconciliationOptions.MaxPagesPerTick"/> <c>PullAuditEvents</c>
|
||||
/// RPCs back to back while the site reports <c>MoreAvailable</c> and the cursor keeps
|
||||
/// advancing, then updates the stalled signal from the LAST page.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Paging within a tick (review 2026-10-21 06/PERF-1, mirroring <c>SiteCallAuditActor</c>'s
|
||||
/// ≤50-page loop) is what makes the stalled latch mean something: one pull per tick capped
|
||||
/// the pull at <c>BatchSize</c> rows per <c>ReconciliationInterval</c> (0.85 rows/s at the
|
||||
/// defaults), so any busy-but-healthy site reported <c>MoreAvailable</c> on every tick and
|
||||
/// latched <c>SiteAuditTelemetryStalled</c> permanently. Now a site is non-draining only when
|
||||
/// it still has more rows after the page cap, or its cursor is held back by a row that keeps
|
||||
/// failing — a genuine non-drain.
|
||||
/// </remarks>
|
||||
private async Task PullSiteAsync(SiteEntry site, IAuditLogRepository repository, Akka.Event.EventStream eventStream)
|
||||
{
|
||||
var maxPages = Math.Max(1, _options.MaxPagesPerTick);
|
||||
var nonDraining = false;
|
||||
for (var page = 0; page < maxPages; page++)
|
||||
{
|
||||
var (response, cursorAdvanced) = await PullPageAsync(site, repository).ConfigureAwait(false);
|
||||
nonDraining = response.MoreAvailable && response.Events.Count > 0;
|
||||
if (!nonDraining || !cursorAdvanced)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
UpdateStalledState(site.SiteId, draining: !nonDraining, eventStream);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Issues one <c>PullAuditEvents</c> RPC against the site, ingests the
|
||||
/// returned rows idempotently into the central repository, and advances
|
||||
/// the composite <c>(<see cref="AuditEvent.OccurredAtUtc"/>, <see cref="AuditEvent.EventId"/>)</c>
|
||||
/// cursor to the maximum row observed. The brief's "saturate until backlog clears" intent is
|
||||
/// met by the natural cadence — each tick issues one pull, and a backed-up site
|
||||
/// drains across consecutive ticks. The stalled signal (two non-draining
|
||||
/// ticks in a row) surfaces when that drain isn't keeping up.
|
||||
/// cursor to the maximum row observed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Unlike <c>SiteCallAuditActor</c> this does NOT page within a tick; one pull per tick is
|
||||
/// deliberate (the stalled signal is how a lagging drain surfaces). The composite cursor is
|
||||
/// what makes that safe against a same-instant burst larger than one batch: the id tiebreak
|
||||
/// advances even when the timestamp cannot, so consecutive ticks make real progress instead
|
||||
/// of re-serving the same window forever.
|
||||
/// The page is ingested with ONE set-based
|
||||
/// <see cref="IAuditLogRepository.InsertManyIfNotExistsAsync"/> call (review 2026-10-21
|
||||
/// 06/PERF-1 — it used to cost one <c>InsertIfNotExistsAsync</c> round trip per row). Only
|
||||
/// when the batch throws does the page fall back to the per-row path, which keeps the
|
||||
/// documented poison-row isolation: the failing row is retried up to
|
||||
/// <see cref="MaxPermanentInsertAttempts"/> times with the cursor held back, then
|
||||
/// permanently abandoned. Re-running rows the failed batch may already have committed is
|
||||
/// safe — every insert is idempotent on <see cref="AuditEvent.EventId"/>, which is also
|
||||
/// what lets a pre-PERF-1 site that still re-serves <c>Forwarded</c> rows be pulled
|
||||
/// unchanged.
|
||||
/// </remarks>
|
||||
private async Task PullSiteAsync(SiteEntry site, IAuditLogRepository repository, Akka.Event.EventStream eventStream)
|
||||
/// <returns>The site's response and whether the cursor moved.</returns>
|
||||
private async Task<(PullAuditEventsResponse Response, bool CursorAdvanced)> PullPageAsync(
|
||||
SiteEntry site, IAuditLogRepository repository)
|
||||
{
|
||||
var cursor = _cursors.TryGetValue(site.SiteId, out var c)
|
||||
? c
|
||||
@@ -320,6 +366,41 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
var maxAfterId = afterId;
|
||||
var hasUnresolvedFailure = false;
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
|
||||
// Set-based fast path for every row not already known-abandoned.
|
||||
var batchCommitted = false;
|
||||
var toInsert = new List<ZB.MOM.WW.Audit.AuditEvent>(response.Events.Count);
|
||||
foreach (var evt in response.Events)
|
||||
{
|
||||
if (!_abandonedEventIds.Contains(evt.EventId))
|
||||
{
|
||||
toInsert.Add(AuditRowProjection.WithIngestedAtUtc(evt, nowUtc));
|
||||
}
|
||||
}
|
||||
|
||||
if (toInsert.Count == 0)
|
||||
{
|
||||
batchCommitted = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
await repository.InsertManyIfNotExistsAsync(toInsert).ConfigureAwait(false);
|
||||
batchCommitted = true;
|
||||
foreach (var evt in toInsert)
|
||||
{
|
||||
_failedInsertAttempts.Remove(evt.EventId);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Set-based reconciliation ingest of {Count} audit event(s) from site {SiteId} failed; falling back to per-row inserts so one bad row does not sink the page.",
|
||||
toInsert.Count, site.SiteId);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var evt in response.Events)
|
||||
{
|
||||
var advanceForThisRow = false;
|
||||
@@ -335,6 +416,10 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
// row collapse into the existing one anyway.
|
||||
advanceForThisRow = true;
|
||||
}
|
||||
else if (batchCommitted)
|
||||
{
|
||||
advanceForThisRow = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
@@ -370,7 +455,7 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
site.SiteId,
|
||||
attempts);
|
||||
_failedInsertAttempts.Remove(evt.EventId);
|
||||
_abandonedEventIds.Add(evt.EventId);
|
||||
RememberAbandoned(evt.EventId);
|
||||
advanceForThisRow = true;
|
||||
|
||||
await EmitAbandonmentRecordAsync(repository, evt, site.SiteId, ex, nowUtc)
|
||||
@@ -413,12 +498,13 @@ public class SiteAuditReconciliationActor : ReceiveActor
|
||||
// another attempt. Once it succeeds (or hits the permanent-abandon
|
||||
// threshold) the cursor unblocks naturally. Both halves move together: holding the
|
||||
// timestamp back while advancing the id would skip the very rows being retried.
|
||||
_cursors[site.SiteId] = hasUnresolvedFailure
|
||||
var next = hasUnresolvedFailure
|
||||
? (since, afterId)
|
||||
: (maxOccurred, maxAfterId);
|
||||
_cursors[site.SiteId] = next;
|
||||
|
||||
var nonDraining = response.MoreAvailable && response.Events.Count > 0;
|
||||
UpdateStalledState(site.SiteId, draining: !nonDraining, eventStream);
|
||||
var advanced = next.Item1 != since || !string.Equals(next.Item2, afterId, StringComparison.Ordinal);
|
||||
return (response, advanced);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -3,8 +3,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// <summary>
|
||||
/// Tuning knobs for the central <see cref="SiteAuditReconciliationActor"/> singleton.
|
||||
/// Defaults: pull every 5 minutes per site, 256 rows per
|
||||
/// batch, declare a site "stalled" after two consecutive pull cycles return non-empty
|
||||
/// AND <c>MoreAvailable=true</c> (the backlog is not draining).
|
||||
/// page and up to 50 pages per site per tick, declare a site "stalled" after two
|
||||
/// consecutive cycles still end non-empty AND <c>MoreAvailable=true</c> after the page cap
|
||||
/// (the backlog is not draining).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -77,6 +78,15 @@ public sealed class SiteAuditReconciliationOptions
|
||||
/// </summary>
|
||||
public int BatchSize { get; set; } = 256;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum <c>PullAuditEvents</c> pages issued against one site in one tick while the
|
||||
/// site keeps reporting <c>MoreAvailable</c> (review 2026-10-21 06/PERF-1, mirroring
|
||||
/// <c>SiteCallAuditActor</c>'s 50-page bound). Bounds a misbehaving site's per-tick cost
|
||||
/// while letting a busy-but-healthy site drain inside one tick, so the stalled signal
|
||||
/// only fires on a genuine non-drain.
|
||||
/// </summary>
|
||||
public int MaxPagesPerTick { get; set; } = 50;
|
||||
|
||||
/// <summary>
|
||||
/// Number of consecutive non-draining cycles (events returned AND
|
||||
/// <c>MoreAvailable=true</c>) that must accumulate for a site before the actor
|
||||
|
||||
@@ -24,6 +24,10 @@ public sealed class SiteAuditReconciliationOptionsValidator
|
||||
$"AuditLog:Reconciliation:{nameof(SiteAuditReconciliationOptions.BatchSize)} " +
|
||||
$"({options.BatchSize}) must be 1..2000.");
|
||||
|
||||
builder.RequireThat(options.MaxPagesPerTick is > 0 and <= 1000,
|
||||
$"AuditLog:Reconciliation:{nameof(SiteAuditReconciliationOptions.MaxPagesPerTick)} " +
|
||||
$"({options.MaxPagesPerTick}) must be 1..1000.");
|
||||
|
||||
builder.RequireThat(options.StalledAfterNonDrainingCycles > 0,
|
||||
$"AuditLog:Reconciliation:{nameof(SiteAuditReconciliationOptions.StalledAfterNonDrainingCycles)} " +
|
||||
$"({options.StalledAfterNonDrainingCycles}) must be > 0.");
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Site-only forwarding states that extend <see cref="AuditForwardState"/> inside the site
|
||||
/// SQLite sidecar (<c>audit_forward_state.ForwardState</c> is TEXT, so these need no schema
|
||||
/// change). Both mean "left the telemetry PUSH queue without a central ack, still owed to
|
||||
/// central through the reconciliation PULL": <c>ReadPendingSinceAsync</c> serves them,
|
||||
/// <c>MarkReconciledUpToAsync</c> retires them, and the retention purge never drops them
|
||||
/// (review 2026-10-21 06/PERF-1, 06/IMPL-1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// They are deliberately NOT members of the Commons <see cref="AuditForwardState"/> enum: they
|
||||
/// never leave the site store (the wire and the central table carry no forward state), and a
|
||||
/// downgraded node simply ignores rows in a state it does not know.
|
||||
/// </remarks>
|
||||
public static class SiteAuditForwardStates
|
||||
{
|
||||
/// <summary>
|
||||
/// A cached-lifecycle row whose operational (<c>SiteCalls</c>) half can never be built — its
|
||||
/// tracking snapshot is gone — so the combined-telemetry drain gave up pushing it. Its audit
|
||||
/// half still reaches central through the reconciliation pull.
|
||||
/// </summary>
|
||||
public const string PushAbandoned = "PushAbandoned";
|
||||
|
||||
/// <summary>
|
||||
/// A row central answered but refused (not in <c>AcceptedEventIds</c>) on
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/>
|
||||
/// consecutive drains. Moved out of the push queue so one poison row cannot wedge every push
|
||||
/// batch behind it; the pull then delivers it, or central's permanent-abandon path records the
|
||||
/// loss as a <c>ReconciliationAbandoned</c> row and retires it.
|
||||
/// </summary>
|
||||
public const string Rejected = "Rejected";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Push-side lifecycle transitions the <c>SiteAuditTelemetryActor</c> needs beyond the Commons
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ISiteAuditQueue"/> surface.
|
||||
/// Implemented by <see cref="SqliteAuditWriter"/>; the actor discovers it by type-testing the
|
||||
/// injected queue, so composition roots and test doubles that only implement
|
||||
/// <c>ISiteAuditQueue</c> keep working (the actor falls back to the pre-review behaviour).
|
||||
/// </summary>
|
||||
public interface ISiteAuditPushLifecycle
|
||||
{
|
||||
/// <summary>
|
||||
/// Flips the supplied EventIds from <see cref="AuditForwardState.Pending"/> to
|
||||
/// <see cref="SiteAuditForwardStates.PushAbandoned"/>. Rows in any other state are untouched.
|
||||
/// </summary>
|
||||
/// <param name="eventIds">Event IDs whose push is abandoned.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task MarkPushAbandonedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Records one refused push attempt for each supplied <see cref="AuditForwardState.Pending"/>
|
||||
/// row (bumps <c>AttemptCount</c>, stamps <c>LastAttemptUtc</c>) and flips every row that has
|
||||
/// now reached <paramref name="maxAttempts"/> to <see cref="SiteAuditForwardStates.Rejected"/>.
|
||||
/// </summary>
|
||||
/// <param name="eventIds">Event IDs central answered for but did not accept.</param>
|
||||
/// <param name="maxAttempts">Refused attempts after which a row leaves the push queue.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the number of rows flipped to Rejected by this call.</returns>
|
||||
Task<int> RecordRejectedPushAttemptsAsync(
|
||||
IReadOnlyList<Guid> eventIds, int maxAttempts, CancellationToken ct = default);
|
||||
}
|
||||
@@ -43,7 +43,7 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
/// stamped by central on ingest, not a site column.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable, IDisposable
|
||||
public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, ISiteAuditPushLifecycle, IAsyncDisposable, IDisposable
|
||||
{
|
||||
// Microsoft.Data.Sqlite reports a generic SQLITE_CONSTRAINT (error code 19)
|
||||
// on a PRIMARY KEY violation; the extended subcode 1555 (SQLITE_CONSTRAINT_PRIMARYKEY)
|
||||
@@ -78,6 +78,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
private readonly SqliteAuditWriterOptions _options;
|
||||
private readonly ILogger<SqliteAuditWriter> _logger;
|
||||
private readonly INodeIdentityProvider _nodeIdentity;
|
||||
private readonly IAuditWriteFailureCounter? _failureCounter;
|
||||
private readonly object _writeLock = new();
|
||||
private readonly Channel<PendingAuditEvent> _writeQueue;
|
||||
private readonly Task _writerLoop;
|
||||
@@ -88,11 +89,17 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="nodeIdentity">Node identity provider.</param>
|
||||
/// <param name="connectionStringOverride">Optional connection string override.</param>
|
||||
/// <param name="failureCounter">
|
||||
/// Optional <c>SiteAuditWriteFailures</c> sink, bumped once per row the push gives up on as
|
||||
/// <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1). Null on
|
||||
/// composition roots without Site Health Monitoring.
|
||||
/// </param>
|
||||
public SqliteAuditWriter(
|
||||
IOptions<SqliteAuditWriterOptions> options,
|
||||
ILogger<SqliteAuditWriter> logger,
|
||||
INodeIdentityProvider nodeIdentity,
|
||||
string? connectionStringOverride = null)
|
||||
string? connectionStringOverride = null,
|
||||
IAuditWriteFailureCounter? failureCounter = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
@@ -101,6 +108,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_nodeIdentity = nodeIdentity;
|
||||
_failureCounter = failureCounter;
|
||||
|
||||
var connectionString = connectionStringOverride
|
||||
?? $"Data Source={_options.DatabasePath};Cache=Shared";
|
||||
@@ -801,9 +809,11 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns up to <paramref name="batchSize"/> pending or forwarded audit events
|
||||
/// with <see cref="AuditEvent.OccurredAtUtc"/> >= <paramref name="sinceUtc"/>, oldest first.
|
||||
/// Used by the reconciliation-pull handler.
|
||||
/// Returns up to <paramref name="batchSize"/> audit events central has not yet acknowledged
|
||||
/// (<see cref="AuditForwardState.Pending"/>, <see cref="SiteAuditForwardStates.PushAbandoned"/>,
|
||||
/// <see cref="SiteAuditForwardStates.Rejected"/>) with <see cref="AuditEvent.OccurredAtUtc"/>
|
||||
/// >= <paramref name="sinceUtc"/>, oldest first. <see cref="AuditForwardState.Forwarded"/> rows
|
||||
/// are NOT served (review 2026-10-21 06/PERF-1). Used by the reconciliation-pull handler.
|
||||
/// </summary>
|
||||
/// <param name="sinceUtc">Lower bound timestamp (UTC) for event occurrence.</param>
|
||||
/// <param name="batchSize">Maximum number of rows to return.</param>
|
||||
@@ -827,9 +837,14 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
// Read via _readConnection / _readLock — same lock-
|
||||
// decoupling as ReadPendingAsync. JOIN the sidecar; the range scan
|
||||
// is on the sidecar's duplicated OccurredAtUtc so it stays on IX_fwd.
|
||||
// Both Pending and Forwarded rows are returned (the central reconciliation
|
||||
// puller dedups on EventId; re-shipping a Forwarded-but-not-yet-ingested
|
||||
// row is safe).
|
||||
//
|
||||
// ONLY rows central has NOT acknowledged are served: Pending, plus the
|
||||
// push-exit states PushAbandoned / Rejected (review 2026-10-21 06/PERF-1).
|
||||
// Forwarded rows used to be served too, which made the pull re-ship every
|
||||
// row the push had already delivered — central re-inserted each one, and the
|
||||
// pull's 256-rows-per-tick budget was spent on acked rows, so any site above
|
||||
// ~0.85 audit rows/s fell permanently behind and latched the stalled signal.
|
||||
// A Forwarded row is acked by central; the pull has nothing to repair there.
|
||||
lock (_readLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
@@ -851,13 +866,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
fs.rowid
|
||||
FROM audit_event ae
|
||||
JOIN audit_forward_state fs ON fs.EventId = ae.EventId
|
||||
WHERE fs.ForwardState IN ($pending, $forwarded)
|
||||
WHERE fs.ForwardState IN ($pending, $pushAbandoned, $rejected)
|
||||
AND {cursorPredicate}
|
||||
ORDER BY fs.OccurredAtUtc ASC, ae.EventId ASC
|
||||
LIMIT $limit;
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
|
||||
AddOwedStateParameters(cmd);
|
||||
// Normalise to UTC ISO-8601 round-trip format to match how OccurredAtUtc
|
||||
// is stored on insert ("o" format) — string comparison is monotonic for
|
||||
// that encoding so we can index-scan against it.
|
||||
@@ -874,6 +888,165 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Binds the three "owed to central" states — <c>$pending</c>, <c>$pushAbandoned</c>,
|
||||
/// <c>$rejected</c> — that the reconciliation pull serves and retires.
|
||||
/// </summary>
|
||||
private static void AddOwedStateParameters(SqliteCommand cmd)
|
||||
{
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
cmd.Parameters.AddWithValue("$pushAbandoned", SiteAuditForwardStates.PushAbandoned);
|
||||
cmd.Parameters.AddWithValue("$rejected", SiteAuditForwardStates.Rejected);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task MarkPushAbandonedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(eventIds);
|
||||
if (eventIds.Count == 0)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
lock (_writeLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
// Pending → PushAbandoned only. Unlike the old "mark Forwarded" this is honest:
|
||||
// central never acked these rows, so they stay owed to the reconciliation pull
|
||||
// (served, retired by a covering cursor, never age-purged) — review 2026-10-21
|
||||
// 06/PERF-1. Attempt bookkeeping is bumped like any drain pass that touched them.
|
||||
var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
|
||||
using var transaction = _connection.BeginTransaction();
|
||||
for (int start = 0; start < eventIds.Count; start += MarkForwardStateChunkSize)
|
||||
{
|
||||
var chunk = Math.Min(MarkForwardStateChunkSize, eventIds.Count - start);
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.Transaction = transaction;
|
||||
var sb = new System.Text.StringBuilder();
|
||||
sb.Append("UPDATE audit_forward_state SET ForwardState = $abandoned, ")
|
||||
.Append("AttemptCount = AttemptCount + 1, LastAttemptUtc = $now ")
|
||||
.Append("WHERE ForwardState = $pending AND EventId IN (");
|
||||
AppendIdParameters(cmd, sb, eventIds, start, chunk);
|
||||
sb.Append(");");
|
||||
cmd.CommandText = sb.ToString();
|
||||
cmd.Parameters.AddWithValue("$abandoned", SiteAuditForwardStates.PushAbandoned);
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
cmd.Parameters.AddWithValue("$now", now);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
transaction.Commit();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<int> RecordRejectedPushAttemptsAsync(
|
||||
IReadOnlyList<Guid> eventIds, int maxAttempts, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(eventIds);
|
||||
if (eventIds.Count == 0)
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
var threshold = Math.Max(1, maxAttempts);
|
||||
lock (_writeLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
// Review 2026-10-21 06/IMPL-1: AttemptCount/LastAttemptUtc used to move only on
|
||||
// the Pending→Forwarded flip, so a row central kept refusing had no history and
|
||||
// no give-up — and since the push reads oldest-first, it sat at the head of every
|
||||
// batch forever. Bump on every refused attempt, then move rows that reached the
|
||||
// threshold out of the push queue into Rejected (still owed to the pull).
|
||||
var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
|
||||
var flipped = 0;
|
||||
using var transaction = _connection.BeginTransaction();
|
||||
for (int start = 0; start < eventIds.Count; start += MarkForwardStateChunkSize)
|
||||
{
|
||||
var chunk = Math.Min(MarkForwardStateChunkSize, eventIds.Count - start);
|
||||
|
||||
using (var bump = _connection.CreateCommand())
|
||||
{
|
||||
bump.Transaction = transaction;
|
||||
var sb = new System.Text.StringBuilder();
|
||||
sb.Append("UPDATE audit_forward_state SET AttemptCount = AttemptCount + 1, LastAttemptUtc = $now ")
|
||||
.Append("WHERE ForwardState = $pending AND EventId IN (");
|
||||
AppendIdParameters(bump, sb, eventIds, start, chunk);
|
||||
sb.Append(");");
|
||||
bump.CommandText = sb.ToString();
|
||||
bump.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
bump.Parameters.AddWithValue("$now", now);
|
||||
bump.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
using (var reject = _connection.CreateCommand())
|
||||
{
|
||||
reject.Transaction = transaction;
|
||||
var sb = new System.Text.StringBuilder();
|
||||
sb.Append("UPDATE audit_forward_state SET ForwardState = $rejected ")
|
||||
.Append("WHERE ForwardState = $pending AND AttemptCount >= $max AND EventId IN (");
|
||||
AppendIdParameters(reject, sb, eventIds, start, chunk);
|
||||
sb.Append(");");
|
||||
reject.CommandText = sb.ToString();
|
||||
reject.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
reject.Parameters.AddWithValue("$rejected", SiteAuditForwardStates.Rejected);
|
||||
reject.Parameters.AddWithValue("$max", threshold);
|
||||
flipped += reject.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
transaction.Commit();
|
||||
|
||||
if (flipped > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"{Count} site audit row(s) were refused by central on {Max} consecutive pushes and moved out " +
|
||||
"of the push queue as Rejected; the reconciliation pull still serves them (central records " +
|
||||
"a ReconciliationAbandoned row if they can never be inserted).",
|
||||
flipped, threshold);
|
||||
for (var i = 0; i < flipped; i++)
|
||||
{
|
||||
try { _failureCounter?.Increment(); }
|
||||
catch { /* counter must never throw — defence in depth */ }
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(flipped);
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendIdParameters(
|
||||
SqliteCommand cmd, System.Text.StringBuilder sb, IReadOnlyList<Guid> eventIds, int start, int chunk)
|
||||
{
|
||||
for (int i = 0; i < chunk; i++)
|
||||
{
|
||||
if (i > 0) sb.Append(',');
|
||||
var p = $"$id{i}";
|
||||
sb.Append(p);
|
||||
cmd.Parameters.AddWithValue(p, eventIds[start + i].ToString());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test/diagnostic read of one row's sidecar state: <c>(ForwardState, AttemptCount)</c>, or
|
||||
/// null when the row does not exist.
|
||||
/// </summary>
|
||||
/// <param name="eventId">The event to look up.</param>
|
||||
/// <returns>The row's forward state and attempt count, or null.</returns>
|
||||
internal (string State, int AttemptCount)? GetForwardStateForTests(Guid eventId)
|
||||
{
|
||||
lock (_readLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
using var cmd = _readConnection.CreateCommand();
|
||||
cmd.CommandText = "SELECT ForwardState, AttemptCount FROM audit_forward_state WHERE EventId = $id;";
|
||||
cmd.Parameters.AddWithValue("$id", eventId.ToString());
|
||||
using var reader = cmd.ExecuteReader();
|
||||
return reader.Read() ? (reader.GetString(0), reader.GetInt32(1)) : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Flow Debugger PullTraceEvents: hard cap on ids per read (design §5.4).</summary>
|
||||
public const int MaxExecutionIdsPerRead = 64;
|
||||
|
||||
@@ -1042,12 +1215,15 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
//
|
||||
// fs.rowid is insertion order, so bounding the flip at the highest rowid this
|
||||
// instance has ever served restores the invariant exactly: rows inserted after
|
||||
// that point are out of the flip's reach no matter where the cursor sits. A
|
||||
// FORWARDED row is exempt from the bound — central ACKED it through the
|
||||
// telemetry push path, which is receipt-proof independent of the pull.
|
||||
// that point are out of the flip's reach no matter where the cursor sits.
|
||||
//
|
||||
// Only the OWED states flip (Pending / PushAbandoned / Rejected — exactly what
|
||||
// ReadPendingSinceAsync serves). Forwarded rows are no longer touched: central
|
||||
// acked them over the push path, they are already purge-eligible, and flipping
|
||||
// them was one UPDATE per pushed row for no benefit (review 2026-10-21 06/PERF-1).
|
||||
//
|
||||
// Bound state is per-process: after a restart it is 0 until this instance serves
|
||||
// a batch, so the first pull retires only Forwarded rows and the pull after it
|
||||
// a batch, so the first pull retires nothing and the pull after it
|
||||
// resumes normal retirement. Conservative in the safe direction (delays
|
||||
// retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
|
||||
// interface doc + Component-AuditLog.md covers what a permanently halted
|
||||
@@ -1056,13 +1232,12 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
cmd.CommandText = $"""
|
||||
UPDATE audit_forward_state AS fs
|
||||
SET ForwardState = $reconciled
|
||||
WHERE fs.ForwardState IN ($pending, $forwarded)
|
||||
AND (fs.ForwardState = $forwarded OR fs.rowid <= $maxServedRowId)
|
||||
WHERE fs.ForwardState IN ($pending, $pushAbandoned, $rejected)
|
||||
AND fs.rowid <= $maxServedRowId
|
||||
AND {predicate};
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
|
||||
AddOwedStateParameters(cmd);
|
||||
cmd.Parameters.AddWithValue("$maxServedRowId", Volatile.Read(ref _maxServedRowId));
|
||||
cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
|
||||
"o", System.Globalization.CultureInfo.InvariantCulture));
|
||||
@@ -1088,8 +1263,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
// Flip the sidecar from Pending/Forwarded → Reconciled. Rows
|
||||
// already Reconciled are left untouched (idempotent re-call), and the
|
||||
// Flip the sidecar from Pending/Forwarded (or a push-exit state) → Reconciled.
|
||||
// Rows already Reconciled are left untouched (idempotent re-call), and the
|
||||
// canonical audit_event row is never modified.
|
||||
var reconciled = AuditForwardState.Reconciled.ToString();
|
||||
var pending = AuditForwardState.Pending.ToString();
|
||||
@@ -1105,7 +1280,7 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
cmd.Transaction = transaction;
|
||||
var sb = new System.Text.StringBuilder();
|
||||
sb.Append("UPDATE audit_forward_state SET ForwardState = $reconciled ")
|
||||
.Append("WHERE ForwardState IN ($pending, $forwarded) AND EventId IN (");
|
||||
.Append("WHERE ForwardState IN ($pending, $forwarded, $pushAbandoned, $rejected) AND EventId IN (");
|
||||
for (int i = 0; i < chunk; i++)
|
||||
{
|
||||
if (i > 0) sb.Append(',');
|
||||
@@ -1118,6 +1293,8 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
cmd.Parameters.AddWithValue("$reconciled", reconciled);
|
||||
cmd.Parameters.AddWithValue("$pending", pending);
|
||||
cmd.Parameters.AddWithValue("$forwarded", forwarded);
|
||||
cmd.Parameters.AddWithValue("$pushAbandoned", SiteAuditForwardStates.PushAbandoned);
|
||||
cmd.Parameters.AddWithValue("$rejected", SiteAuditForwardStates.Rejected);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
transaction.Commit();
|
||||
@@ -1145,12 +1322,15 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
using var cmd = _readConnection.CreateCommand();
|
||||
// "Pending" on the health report means OWED to central: the push-exit states
|
||||
// (PushAbandoned / Rejected) still await the reconciliation pull and are never
|
||||
// purged, so they count toward the backlog and its oldest-row age.
|
||||
cmd.CommandText = """
|
||||
SELECT COUNT(*), MIN(OccurredAtUtc)
|
||||
FROM audit_forward_state
|
||||
WHERE ForwardState = $pending;
|
||||
WHERE ForwardState IN ($pending, $pushAbandoned, $rejected);
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
AddOwedStateParameters(cmd);
|
||||
|
||||
using var reader = cmd.ExecuteReader();
|
||||
reader.Read();
|
||||
|
||||
@@ -179,6 +179,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
}
|
||||
|
||||
var batch = BuildBatch(pending);
|
||||
var sentIds = batch.Events.Select(e => e.EventId).ToList();
|
||||
|
||||
IngestAck ack;
|
||||
try
|
||||
@@ -203,6 +204,8 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
await _queue.MarkForwardedAsync(acceptedIds, ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await RecordRefusalsAsync(sentIds, ack, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
@@ -269,7 +272,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
var wireBytes = AuditWireBudget.InitialBytes;
|
||||
var emittedEventIds = new List<Guid>(pending.Count);
|
||||
// Rows whose operational half can never be built. They are marked
|
||||
// Forwarded so they LEAVE this queue — see AbandonUnresolvableAsync
|
||||
// PushAbandoned so they LEAVE this queue — see AbandonUnresolvableAsync
|
||||
// for why that is safe and why leaving them Pending is not.
|
||||
var abandonedEventIds = new List<Guid>();
|
||||
// Counted, not logged per row: a whole batch can fail identically
|
||||
@@ -362,7 +365,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
if (batch.Packets.Count == 0)
|
||||
{
|
||||
// Nothing resolvable in this read. Any permanently-unresolvable
|
||||
// rows have just been marked Forwarded above, so the next drain
|
||||
// rows have just been marked PushAbandoned above, so the next drain
|
||||
// sees past them rather than re-reading the same head of queue.
|
||||
return;
|
||||
}
|
||||
@@ -387,6 +390,9 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
await _queue.MarkForwardedAsync(acceptedIds, ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await RecordRefusalsAsync(
|
||||
batch.Packets.Select(p => p.AuditEvent.EventId).ToList(), ack, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
@@ -408,33 +414,35 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks cached rows whose operational half can never be built as
|
||||
/// Forwarded, so they leave the cached-drain queue.
|
||||
/// Moves cached rows whose operational half can never be built out of the
|
||||
/// cached-drain queue, as <see cref="SiteAuditForwardStates.PushAbandoned"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why they must leave.</b> Leaving an unresolvable row Pending — the
|
||||
/// previous behaviour — is not a harmless skip. The queue is read
|
||||
/// oldest-first with a fixed <c>BatchSize</c>, so once a batch's worth of
|
||||
/// unresolvable rows collects at the head, every drain re-reads exactly
|
||||
/// those rows, fails identically, and never sees the newer rows behind
|
||||
/// them. The cached-telemetry path stalls permanently, and each pass logs
|
||||
/// once per row (measured at ~2 800 warnings/minute on a rig).
|
||||
/// <b>Why they must leave.</b> Leaving an unresolvable row Pending is not a
|
||||
/// harmless skip. The queue is read oldest-first with a fixed
|
||||
/// <c>BatchSize</c>, so once a batch's worth of unresolvable rows collects at
|
||||
/// the head, every drain re-reads exactly those rows, fails identically, and
|
||||
/// never sees the newer rows behind them. The cached-telemetry path stalls
|
||||
/// permanently, and each pass logs once per row (measured at ~2 800
|
||||
/// warnings/minute on a rig).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why Forwarded is the honest state.</b> Its role in this state machine
|
||||
/// is "no longer owed by the drain, still eligible for reconciliation" —
|
||||
/// which is exactly the situation here. The audit half is NOT dropped:
|
||||
/// <c>ReadPendingSinceAsync</c> covers Forwarded rows as well as Pending
|
||||
/// ones and central dedups on EventId, so the reconciliation pull still
|
||||
/// delivers them. What is genuinely lost is the operational
|
||||
/// (<c>SiteCalls</c>) half — and that is unrecoverable regardless, because
|
||||
/// the tracking row it would have been built from no longer exists.
|
||||
/// <b>Why PushAbandoned, not Forwarded.</b> These rows were marked Forwarded
|
||||
/// until review 2026-10-21 (06/PERF-1), which only worked because the
|
||||
/// reconciliation pull used to re-serve every Forwarded row — the coupling
|
||||
/// that capped the pull's throughput. Forwarded now means "central acked it"
|
||||
/// and the pull no longer serves it, so an un-acked row needs a state of its
|
||||
/// own: <c>PushAbandoned</c> is served by the pull, retired by a covering
|
||||
/// cursor and never age-purged. The audit half therefore still reaches central;
|
||||
/// what is genuinely lost is the operational (<c>SiteCalls</c>) half, which is
|
||||
/// unrecoverable regardless because the tracking row no longer exists.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A failure to mark is swallowed: the rows simply stay Pending and the
|
||||
/// next drain retries. Escalating here would take down the audit drain over
|
||||
/// a best-effort cleanup.
|
||||
/// A queue that does not implement <see cref="ISiteAuditPushLifecycle"/> (a
|
||||
/// test double) falls back to the old Forwarded flip. A failure to mark is
|
||||
/// swallowed: the rows simply stay Pending and the next drain retries.
|
||||
/// Escalating here would take down the audit drain over a best-effort cleanup.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task AbandonUnresolvableAsync(
|
||||
@@ -442,24 +450,87 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
{
|
||||
try
|
||||
{
|
||||
await _queue.MarkForwardedAsync(eventIds, ct).ConfigureAwait(false);
|
||||
if (_queue is ISiteAuditPushLifecycle lifecycle)
|
||||
{
|
||||
await lifecycle.MarkPushAbandonedAsync(eventIds, ct).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _queue.MarkForwardedAsync(eventIds, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Cached-telemetry drain: abandoned the operational half of {Count} row(s) with no " +
|
||||
"resolvable tracking snapshot after {Grace}s; marked Forwarded so they no longer block " +
|
||||
"resolvable tracking snapshot after {Grace}s; marked PushAbandoned so they no longer block " +
|
||||
"the queue. The audit half still reaches central via the reconciliation pull. This is " +
|
||||
"expected after tracking retention elapses or the tracking store is reset independently " +
|
||||
"of the audit store.",
|
||||
eventIds.Count, graceSeconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Cached-telemetry drain: could not mark {Count} unresolvable row(s) Forwarded " +
|
||||
"Cached-telemetry drain: could not mark {Count} unresolvable row(s) PushAbandoned " +
|
||||
"(sqlite {SqliteError}); they stay Pending and will be retried.",
|
||||
eventIds.Count, SqliteErrorCodes.Describe(ex));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a refused push attempt for every row that was SENT in an answered batch but is
|
||||
/// missing from the ack's <c>AcceptedEventIds</c>; rows refused
|
||||
/// <see cref="SiteAuditTelemetryOptions.MaxRejectedPushAttempts"/> times leave the push
|
||||
/// queue as <see cref="SiteAuditForwardStates.Rejected"/> (review 2026-10-21 06/IMPL-1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Called only after the push RETURNED — a transport fault is not a verdict about any row
|
||||
/// and never counts. Best-effort: a bookkeeping failure leaves the rows Pending, exactly as
|
||||
/// before this existed.
|
||||
/// </remarks>
|
||||
private async Task RecordRefusalsAsync(IReadOnlyList<string> sentIds, IngestAck ack, CancellationToken ct)
|
||||
{
|
||||
if (_queue is not ISiteAuditPushLifecycle lifecycle || sentIds.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var accepted = new HashSet<string>(ack.AcceptedEventIds, StringComparer.OrdinalIgnoreCase);
|
||||
var refused = new List<Guid>();
|
||||
foreach (var raw in sentIds)
|
||||
{
|
||||
if (!accepted.Contains(raw) && Guid.TryParse(raw, out var id))
|
||||
{
|
||||
refused.Add(id);
|
||||
}
|
||||
}
|
||||
|
||||
if (refused.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await lifecycle
|
||||
.RecordRejectedPushAttemptsAsync(refused, Math.Max(1, _options.MaxRejectedPushAttempts), ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Could not record {Count} refused push attempt(s) (sqlite {SqliteError}); the rows stay Pending.",
|
||||
refused.Count, SqliteErrorCodes.Describe(ex));
|
||||
}
|
||||
}
|
||||
|
||||
private static AuditEventBatch BuildBatch(IReadOnlyList<AuditEvent> events)
|
||||
{
|
||||
var batch = new AuditEventBatch();
|
||||
|
||||
@@ -29,7 +29,7 @@ public sealed class SiteAuditTelemetryOptions
|
||||
/// <summary>
|
||||
/// How long a cached-telemetry audit row may go without a resolvable
|
||||
/// <c>OperationTracking</c> snapshot before the drain ABANDONS its
|
||||
/// operational half and marks the row Forwarded. Default: 300 s.
|
||||
/// operational half and marks the row <c>PushAbandoned</c>. Default: 300 s.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -45,8 +45,9 @@ public sealed class SiteAuditTelemetryOptions
|
||||
/// <para>
|
||||
/// Abandoning costs only the operational (<c>SiteCalls</c>) half, which is
|
||||
/// unrecoverable anyway once the tracking row is gone. The audit half is
|
||||
/// NOT lost: the reconciliation pull covers Forwarded rows as well as
|
||||
/// Pending ones, and central dedups on EventId.
|
||||
/// NOT lost: the reconciliation pull serves <c>PushAbandoned</c> rows (central
|
||||
/// never acked them) exactly as it serves Pending ones, and central dedups on
|
||||
/// EventId (review 2026-10-21 06/PERF-1).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Set to 0 to abandon on the first failed lookup. Negative values are
|
||||
@@ -54,4 +55,20 @@ public sealed class SiteAuditTelemetryOptions
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public int CachedTrackingGraceSeconds { get; set; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Number of drains in which central ANSWERED a push but refused a row (the row is
|
||||
/// missing from <c>AcceptedEventIds</c>) after which the row leaves the push queue as
|
||||
/// <c>Rejected</c>. Default: 10 (≈ 50 s at the busy cadence).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The push reads oldest-first, so a row central can never insert (e.g. an over-width
|
||||
/// column) used to sit at the head of every batch forever, failing each set-based insert
|
||||
/// at central and dragging the batch into the per-row fallback (review 2026-10-21
|
||||
/// 06/IMPL-1). A transport fault (no answer) never counts. <c>Rejected</c> is NOT a drop:
|
||||
/// the reconciliation pull still serves the row, so a transient central refusal still
|
||||
/// lands, and a genuinely poison row is recorded centrally as <c>ReconciliationAbandoned</c>
|
||||
/// before it retires. Each flip bumps <c>SiteAuditWriteFailures</c> on the site health report.
|
||||
/// </remarks>
|
||||
public int MaxRejectedPushAttempts { get; set; } = 10;
|
||||
}
|
||||
|
||||
+4
@@ -27,6 +27,10 @@ public sealed class SiteAuditTelemetryOptionsValidator : OptionsValidatorBase<Si
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.IdleIntervalSeconds)} " +
|
||||
$"({options.IdleIntervalSeconds}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.MaxRejectedPushAttempts > 0,
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.MaxRejectedPushAttempts)} " +
|
||||
$"({options.MaxRejectedPushAttempts}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.IdleIntervalSeconds >= options.BusyIntervalSeconds,
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.IdleIntervalSeconds)} " +
|
||||
$"({options.IdleIntervalSeconds}) must be >= " +
|
||||
|
||||
Reference in New Issue
Block a user