fix(comms): review findings — consumer-based debug orphan net, foreign-cancel triad, honest onConnected, served-row-exact retirement, full-rate reconcile
F1 (HIGH) DebugStreamBridgeActor: the 5-minute orphan net measured the MAILBOX (SetReceiveTimeout), and once stream events were correctly marked INotInfluenceReceiveTimeout nothing recurring reset it — the snapshot lands once and GrpcStreamStable once — so every healthy session self-terminated at ~6 min with a false "Site disconnected". Replaced with a periodic self-tick (ConsumerLivenessCheckInterval, 30s) over a consumer-last-seen stamp renewed only by DebugStreamConsumerAlive, which DebugStreamService Tells on a shared timer to every session still in its registry (holding a session there IS "a consumer is attached" — both the Blazor view and the SignalR hub release it on dispose/disconnect, and it works headless). Reverting the wrapper was rejected: it would restore the quiet-instance orphan bug. F2 (MED) SiteStreamGrpcClient: the RpcException(Cancelled) filter now requires cts.IsCancellationRequested. A peer-originated / channel-dispose Cancelled fired none of onError/onCompleted/onConnected, leaving SiteAlarmAggregatorActor with _streamDown=false forever (IsLive stuck true, reconcile reopen guard never fired). F3 (MED) SiteStreamGrpcClient: a header TIMEOUT is no longer reported as connected — that shape is exactly what an unreachable site produces, and it cleared _streamDown, consumed _seedOnConnect and launched a full snapshot fan-out at a dead site. AwaitHeadersAsync returns bool; the first received event is the fallback connected signal, fired at most once from headers OR first event. F4 (LOW-MED) SqliteAuditWriter.MarkReconciledUpToAsync: the blanket below-cursor UPDATE retired late-stamped inserts that were never served (then age-purged — silent loss). The flip is now bounded by insertion order: a Pending row retires only if its rowid is at or below the high-water mark of rows this instance has served from ReadPendingSinceAsync (clamped on purge, since SQLite reuses rowids); Forwarded rows are exempt (central ACKed them over the push path). At-least-once is unchanged. F5 (LOW) Documented the liveness dependency (a served row never covered by a later cursor stays Pending forever; PurgeExpiredAsync never purges Pending) in ISiteAuditQueue + Component-AuditLog.md, and added a cheap site-health signal: SiteAuditBacklogReporter logs a rate-limited warning when the existing oldest-pending metric exceeds 24h. F6 (MED) SiteAlarmAggregatorActor: _fanoutSinceLastTick was armed by the reconcile's OWN fan-out, so steady state ran fan-out→skip→fan-out→skip — one reconcile per 2x interval (120s), halving the not-reporting refresh and the alarm reconcile backstop. The skip is now armed only by connect/failover-driven seeds (initial, _seedOnConnect, and a re-seed queued behind one). Tests: Communication.Tests 691 passed (+13), AuditLog.Tests 382 passed (+5).
This commit is contained in:
Joseph Doherty
@@ -45,12 +45,27 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan DefaultRefreshInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Age at which the oldest still-Pending row is called out in the log. A Pending row is
|
||||
/// one central has not acknowledged through EITHER path (telemetry push or reconciliation
|
||||
/// pull), and <c>PurgeExpiredAsync</c> deliberately never purges Pending — so the site
|
||||
/// store's floor depends on reconciliation actually running. Rows served by a pull but
|
||||
/// never covered by a later cursor sit in exactly this state, which is why the age is
|
||||
/// worth a signal rather than only a dashboard number. One day is comfortably longer than
|
||||
/// any normal drain/reconcile outage and well inside the ~7-day site retention window.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan StalePendingThreshold = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>How often the stale-pending warning may repeat (it is a standing condition).</summary>
|
||||
internal static readonly TimeSpan StalePendingWarnInterval = TimeSpan.FromHours(1);
|
||||
|
||||
private readonly ISiteAuditQueue _queue;
|
||||
private readonly ISiteHealthCollector _collector;
|
||||
private readonly ILogger<SiteAuditBacklogReporter> _logger;
|
||||
private readonly TimeSpan _refreshInterval;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _loop;
|
||||
private DateTime _lastStalePendingWarnUtc = DateTime.MinValue;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="SiteAuditBacklogReporter"/>.</summary>
|
||||
/// <param name="queue">The site audit queue used to probe the backlog count.</param>
|
||||
@@ -151,6 +166,7 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
|
||||
{
|
||||
var snapshot = await _queue.GetBacklogStatsAsync(ct).ConfigureAwait(false);
|
||||
_collector.UpdateSiteAuditBacklog(snapshot);
|
||||
WarnIfPendingIsStale(snapshot.OldestPendingUtc, snapshot.PendingCount);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -166,6 +182,34 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs a rate-limited warning when the oldest still-Pending audit row is older than
|
||||
/// <see cref="StalePendingThreshold"/>. Pending rows are exempt from the retention purge
|
||||
/// by design, so a standing Pending backlog is the one site-store condition that does not
|
||||
/// self-heal on age — it clears only when central acknowledges the rows (telemetry ack or
|
||||
/// a reconciliation cursor that covers them). Internal so tests can drive it directly.
|
||||
/// </summary>
|
||||
/// <param name="oldestPendingUtc">Oldest pending row's occurrence instant, or null when none.</param>
|
||||
/// <param name="pendingCount">Number of rows currently pending.</param>
|
||||
internal void WarnIfPendingIsStale(DateTime? oldestPendingUtc, int pendingCount)
|
||||
{
|
||||
if (oldestPendingUtc is null) return;
|
||||
|
||||
var age = DateTime.UtcNow - DateTime.SpecifyKind(oldestPendingUtc.Value, DateTimeKind.Utc);
|
||||
if (age < StalePendingThreshold) return;
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
if (now - _lastStalePendingWarnUtc < StalePendingWarnInterval) return;
|
||||
_lastStalePendingWarnUtc = now;
|
||||
|
||||
_logger.LogWarning(
|
||||
"Site audit backlog: oldest pending row is {AgeHours:F1}h old ({PendingCount} pending). " +
|
||||
"Pending rows are never purged on age, so this backlog only clears when central " +
|
||||
"acknowledges them — check the site→central audit telemetry drain and the central " +
|
||||
"reconciliation pull.",
|
||||
age.TotalHours, pendingCount);
|
||||
}
|
||||
|
||||
/// <summary>Signals the polling loop to stop and waits for it to complete.</summary>
|
||||
/// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
|
||||
@@ -792,9 +792,14 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
// ordering the query applies, so it is a strict "everything after this row".
|
||||
: "(fs.OccurredAtUtc > $since OR (fs.OccurredAtUtc = $since AND ae.EventId > $afterId))";
|
||||
|
||||
// fs.rowid rides along as an 11th column purely to maintain
|
||||
// _maxServedRowId — the insertion-order high-water mark that bounds
|
||||
// MarkReconciledUpToAsync (see that method). It is never mapped onto
|
||||
// the returned AuditEvent; rowid stays a storage-layer concept.
|
||||
cmd.CommandText = $"""
|
||||
SELECT ae.EventId, ae.OccurredAtUtc, ae.Actor, ae.Action, ae.Outcome,
|
||||
ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson
|
||||
ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson,
|
||||
fs.rowid
|
||||
FROM audit_event ae
|
||||
JOIN audit_forward_state fs ON fs.EventId = ae.EventId
|
||||
WHERE fs.ForwardState IN ($pending, $forwarded)
|
||||
@@ -816,10 +821,47 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
}
|
||||
cmd.Parameters.AddWithValue("$limit", batchSize);
|
||||
|
||||
return Task.FromResult(ReadRows(cmd, batchSize));
|
||||
return Task.FromResult(ReadServedRows(cmd, batchSize));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Highest <c>audit_forward_state.rowid</c> this instance has ever SERVED from
|
||||
/// <see cref="ReadPendingSinceAsync"/> — i.e. the insertion-order high-water mark of
|
||||
/// rows central could possibly have received through the pull path. Bounds
|
||||
/// <see cref="MarkReconciledUpToAsync"/> so a row inserted after that point can never be
|
||||
/// retired by a cursor that happens to sit above its timestamp. Monotonic (a later, smaller
|
||||
/// batch never lowers it) except for the purge clamp. Written under <c>_readLock</c> (the
|
||||
/// read path) and read under <c>_writeLock</c> (the flip/purge paths) — two different
|
||||
/// locks, so the accesses go through <see cref="Volatile"/> for visibility. A stale read
|
||||
/// can only make the bound smaller, i.e. more conservative, never unsafe.
|
||||
/// </summary>
|
||||
private long _maxServedRowId;
|
||||
|
||||
/// <summary>
|
||||
/// Executes a reconciliation-pull read whose projection carries <c>fs.rowid</c> as an
|
||||
/// 11th column, materialising the canonical rows while advancing
|
||||
/// <see cref="_maxServedRowId"/>. Serving a row is what makes it eligible for
|
||||
/// cursor-proved retirement, so the two must happen in the same step.
|
||||
/// </summary>
|
||||
private IReadOnlyList<AuditEvent> ReadServedRows(SqliteCommand cmd, int capacityHint)
|
||||
{
|
||||
var rows = new List<AuditEvent>(Math.Min(capacityHint, 256));
|
||||
var maxRowId = Volatile.Read(ref _maxServedRowId);
|
||||
using (var reader = cmd.ExecuteReader())
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
rows.Add(MapRow(reader));
|
||||
var rowId = reader.GetInt64(10);
|
||||
if (rowId > maxRowId) maxRowId = rowId;
|
||||
}
|
||||
}
|
||||
|
||||
Volatile.Write(ref _maxServedRowId, maxRowId);
|
||||
return rows;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalises a wire-supplied event id to the exact textual form stored in
|
||||
/// <c>audit_event.EventId</c> so the keyset comparison is apples-to-apples. An
|
||||
@@ -845,16 +887,40 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
? "fs.OccurredAtUtc < $since"
|
||||
: "(fs.OccurredAtUtc < $since OR (fs.OccurredAtUtc = $since AND fs.EventId <= $afterId))";
|
||||
|
||||
// INSERTION-ORDER BOUND — "only rows we actually served can retire".
|
||||
//
|
||||
// The timestamp cursor alone is not enough. OccurredAtUtc is stamped by the
|
||||
// caller, so a row can be INSERTED after a batch was served yet carry a
|
||||
// timestamp BELOW central's (by then advanced) cursor — a late-stamped insert.
|
||||
// The blanket cursor UPDATE retired exactly those rows: never served, never
|
||||
// servable again (ReadPendingSinceAsync's keyset has moved past them), and,
|
||||
// being Reconciled, purged on age — silent audit loss, a failure mode the old
|
||||
// explicit id-set flip could not produce.
|
||||
//
|
||||
// fs.rowid is insertion order, so bounding the flip at the highest rowid this
|
||||
// instance has ever served restores the invariant exactly: rows inserted after
|
||||
// that point are out of the flip's reach no matter where the cursor sits. A
|
||||
// FORWARDED row is exempt from the bound — central ACKED it through the
|
||||
// telemetry push path, which is receipt-proof independent of the pull.
|
||||
//
|
||||
// Bound state is per-process: after a restart it is 0 until this instance serves
|
||||
// a batch, so the first pull retires only Forwarded rows and the pull after it
|
||||
// resumes normal retirement. Conservative in the safe direction (delays
|
||||
// retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
|
||||
// interface doc + Component-AuditLog.md covers what a permanently halted
|
||||
// reconciliation means for such rows.
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = $"""
|
||||
UPDATE audit_forward_state AS fs
|
||||
SET ForwardState = $reconciled
|
||||
WHERE fs.ForwardState IN ($pending, $forwarded)
|
||||
AND (fs.ForwardState = $forwarded OR fs.rowid <= $maxServedRowId)
|
||||
AND {predicate};
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
|
||||
cmd.Parameters.AddWithValue("$maxServedRowId", Volatile.Read(ref _maxServedRowId));
|
||||
cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
|
||||
"o", System.Globalization.CultureInfo.InvariantCulture));
|
||||
if (afterId is not null)
|
||||
@@ -1063,6 +1129,22 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
dropCmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// Clamp the served-rows high-water mark to what is still in the table.
|
||||
// SQLite hands a fresh insert MAX(rowid)+1, so a purge that empties the
|
||||
// sidecar (or removes its top rows) makes rowids REUSABLE — a stale-high
|
||||
// _maxServedRowId would then vouch for rows nobody has served. Cheap:
|
||||
// MAX(rowid) is an index-less O(1) lookup.
|
||||
if (purged > 0)
|
||||
{
|
||||
using var maxCmd = _connection.CreateCommand();
|
||||
maxCmd.Transaction = transaction;
|
||||
maxCmd.CommandText = "SELECT IFNULL(MAX(rowid), 0) FROM audit_forward_state;";
|
||||
var liveMax = Convert.ToInt64(maxCmd.ExecuteScalar(),
|
||||
System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (liveMax < Volatile.Read(ref _maxServedRowId))
|
||||
Volatile.Write(ref _maxServedRowId, liveMax);
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
catch
|
||||
|
||||
Reference in New Issue
Block a user