Merge branch 'worktree-agent-a465fb3cd6ec48cd3' into arch-review-remediation

2026-08-14 23:53:00 -04:00
parent 11abda1d1f fd5e023d08
commit a9ca51e008
16 changed files with 1192 additions and 84 deletions
@@ -45,12 +45,27 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
    /// </summary>
    internal static readonly TimeSpan DefaultRefreshInterval = TimeSpan.FromSeconds(30);

+    /// <summary>
+    /// Age at which the oldest still-Pending row is called out in the log. A Pending row is
+    /// one central has not acknowledged through EITHER path (telemetry push or reconciliation
+    /// pull), and <c>PurgeExpiredAsync</c> deliberately never purges Pending — so the site
+    /// store's floor depends on reconciliation actually running. Rows served by a pull but
+    /// never covered by a later cursor sit in exactly this state, which is why the age is
+    /// worth a signal rather than only a dashboard number. One day is comfortably longer than
+    /// any normal drain/reconcile outage and well inside the ~7-day site retention window.
+    /// </summary>
+    internal static readonly TimeSpan StalePendingThreshold = TimeSpan.FromHours(24);
+
+    /// <summary>How often the stale-pending warning may repeat (it is a standing condition).</summary>
+    internal static readonly TimeSpan StalePendingWarnInterval = TimeSpan.FromHours(1);
+
    private readonly ISiteAuditQueue _queue;
    private readonly ISiteHealthCollector _collector;
    private readonly ILogger<SiteAuditBacklogReporter> _logger;
    private readonly TimeSpan _refreshInterval;
    private CancellationTokenSource? _cts;
    private Task? _loop;
+    private DateTime _lastStalePendingWarnUtc = DateTime.MinValue;

    /// <summary>Initializes a new instance of <see cref="SiteAuditBacklogReporter"/>.</summary>
    /// <param name="queue">The site audit queue used to probe the backlog count.</param>
@@ -151,6 +166,7 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
        {
            var snapshot = await _queue.GetBacklogStatsAsync(ct).ConfigureAwait(false);
            _collector.UpdateSiteAuditBacklog(snapshot);
+            WarnIfPendingIsStale(snapshot.OldestPendingUtc, snapshot.PendingCount);
        }
        catch (OperationCanceledException)
        {
@@ -166,6 +182,34 @@ public sealed class SiteAuditBacklogReporter : IHostedService, IDisposable
        }
    }

+    /// <summary>
+    /// Logs a rate-limited warning when the oldest still-Pending audit row is older than
+    /// <see cref="StalePendingThreshold"/>. Pending rows are exempt from the retention purge
+    /// by design, so a standing Pending backlog is the one site-store condition that does not
+    /// self-heal on age — it clears only when central acknowledges the rows (telemetry ack or
+    /// a reconciliation cursor that covers them). Internal so tests can drive it directly.
+    /// </summary>
+    /// <param name="oldestPendingUtc">Oldest pending row's occurrence instant, or null when none.</param>
+    /// <param name="pendingCount">Number of rows currently pending.</param>
+    internal void WarnIfPendingIsStale(DateTime? oldestPendingUtc, int pendingCount)
+    {
+        if (oldestPendingUtc is null) return;
+
+        var age = DateTime.UtcNow - DateTime.SpecifyKind(oldestPendingUtc.Value, DateTimeKind.Utc);
+        if (age < StalePendingThreshold) return;
+
+        var now = DateTime.UtcNow;
+        if (now - _lastStalePendingWarnUtc < StalePendingWarnInterval) return;
+        _lastStalePendingWarnUtc = now;
+
+        _logger.LogWarning(
+            "Site audit backlog: oldest pending row is {AgeHours:F1}h old ({PendingCount} pending). " +
+            "Pending rows are never purged on age, so this backlog only clears when central " +
+            "acknowledges them — check the site→central audit telemetry drain and the central " +
+            "reconciliation pull.",
+            age.TotalHours, pendingCount);
+    }
+
    /// <summary>Signals the polling loop to stop and waits for it to complete.</summary>
    /// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
@@ -792,9 +792,14 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
                // ordering the query applies, so it is a strict "everything after this row".
                : "(fs.OccurredAtUtc > $since OR (fs.OccurredAtUtc = $since AND ae.EventId > $afterId))";

+            // fs.rowid rides along as an 11th column purely to maintain
+            // _maxServedRowId — the insertion-order high-water mark that bounds
+            // MarkReconciledUpToAsync (see that method). It is never mapped onto
+            // the returned AuditEvent; rowid stays a storage-layer concept.
            cmd.CommandText = $"""
                SELECT ae.EventId, ae.OccurredAtUtc, ae.Actor, ae.Action, ae.Outcome,
-                       ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson
+                       ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson,
+                       fs.rowid
                FROM   audit_event ae
                JOIN   audit_forward_state fs ON fs.EventId = ae.EventId
                WHERE  fs.ForwardState IN ($pending, $forwarded)
@@ -816,10 +821,47 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
            }
            cmd.Parameters.AddWithValue("$limit", batchSize);

-            return Task.FromResult(ReadRows(cmd, batchSize));
+            return Task.FromResult(ReadServedRows(cmd, batchSize));
        }
    }

+    /// <summary>
+    /// Highest <c>audit_forward_state.rowid</c> this instance has ever SERVED from
+    /// <see cref="ReadPendingSinceAsync"/> — i.e. the insertion-order high-water mark of
+    /// rows central could possibly have received through the pull path. Bounds
+    /// <see cref="MarkReconciledUpToAsync"/> so a row inserted after that point can never be
+    /// retired by a cursor that happens to sit above its timestamp. Monotonic (a later, smaller
+    /// batch never lowers it) except for the purge clamp. Written under <c>_readLock</c> (the
+    /// read path) and read under <c>_writeLock</c> (the flip/purge paths) — two different
+    /// locks, so the accesses go through <see cref="Volatile"/> for visibility. A stale read
+    /// can only make the bound smaller, i.e. more conservative, never unsafe.
+    /// </summary>
+    private long _maxServedRowId;
+
+    /// <summary>
+    /// Executes a reconciliation-pull read whose projection carries <c>fs.rowid</c> as an
+    /// 11th column, materialising the canonical rows while advancing
+    /// <see cref="_maxServedRowId"/>. Serving a row is what makes it eligible for
+    /// cursor-proved retirement, so the two must happen in the same step.
+    /// </summary>
+    private IReadOnlyList<AuditEvent> ReadServedRows(SqliteCommand cmd, int capacityHint)
+    {
+        var rows = new List<AuditEvent>(Math.Min(capacityHint, 256));
+        var maxRowId = Volatile.Read(ref _maxServedRowId);
+        using (var reader = cmd.ExecuteReader())
+        {
+            while (reader.Read())
+            {
+                rows.Add(MapRow(reader));
+                var rowId = reader.GetInt64(10);
+                if (rowId > maxRowId) maxRowId = rowId;
+            }
+        }
+
+        Volatile.Write(ref _maxServedRowId, maxRowId);
+        return rows;
+    }
+
    /// <summary>
    /// Normalises a wire-supplied event id to the exact textual form stored in
    /// <c>audit_event.EventId</c> so the keyset comparison is apples-to-apples. An
@@ -845,16 +887,40 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
                ? "fs.OccurredAtUtc < $since"
                : "(fs.OccurredAtUtc < $since OR (fs.OccurredAtUtc = $since AND fs.EventId <= $afterId))";

+            // INSERTION-ORDER BOUND — "only rows we actually served can retire".
+            //
+            // The timestamp cursor alone is not enough. OccurredAtUtc is stamped by the
+            // caller, so a row can be INSERTED after a batch was served yet carry a
+            // timestamp BELOW central's (by then advanced) cursor — a late-stamped insert.
+            // The blanket cursor UPDATE retired exactly those rows: never served, never
+            // servable again (ReadPendingSinceAsync's keyset has moved past them), and,
+            // being Reconciled, purged on age — silent audit loss, a failure mode the old
+            // explicit id-set flip could not produce.
+            //
+            // fs.rowid is insertion order, so bounding the flip at the highest rowid this
+            // instance has ever served restores the invariant exactly: rows inserted after
+            // that point are out of the flip's reach no matter where the cursor sits. A
+            // FORWARDED row is exempt from the bound — central ACKED it through the
+            // telemetry push path, which is receipt-proof independent of the pull.
+            //
+            // Bound state is per-process: after a restart it is 0 until this instance serves
+            // a batch, so the first pull retires only Forwarded rows and the pull after it
+            // resumes normal retirement. Conservative in the safe direction (delays
+            // retirement, never loses a row); the liveness note in MarkReconciledUpToAsync's
+            // interface doc + Component-AuditLog.md covers what a permanently halted
+            // reconciliation means for such rows.
            using var cmd = _connection.CreateCommand();
            cmd.CommandText = $"""
                UPDATE audit_forward_state AS fs
                SET    ForwardState = $reconciled
                WHERE  fs.ForwardState IN ($pending, $forwarded)
+                  AND  (fs.ForwardState = $forwarded OR fs.rowid <= $maxServedRowId)
                  AND  {predicate};
                """;
            cmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
            cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
            cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
+            cmd.Parameters.AddWithValue("$maxServedRowId", Volatile.Read(ref _maxServedRowId));
            cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
                "o", System.Globalization.CultureInfo.InvariantCulture));
            if (afterId is not null)
@@ -1063,6 +1129,22 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
                        dropCmd.ExecuteNonQuery();
                    }

+                    // Clamp the served-rows high-water mark to what is still in the table.
+                    // SQLite hands a fresh insert MAX(rowid)+1, so a purge that empties the
+                    // sidecar (or removes its top rows) makes rowids REUSABLE — a stale-high
+                    // _maxServedRowId would then vouch for rows nobody has served. Cheap:
+                    // MAX(rowid) is an index-less O(1) lookup.
+                    if (purged > 0)
+                    {
+                        using var maxCmd = _connection.CreateCommand();
+                        maxCmd.Transaction = transaction;
+                        maxCmd.CommandText = "SELECT IFNULL(MAX(rowid), 0) FROM audit_forward_state;";
+                        var liveMax = Convert.ToInt64(maxCmd.ExecuteScalar(),
+                            System.Globalization.CultureInfo.InvariantCulture);
+                        if (liveMax < Volatile.Read(ref _maxServedRowId))
+                            Volatile.Write(ref _maxServedRowId, liveMax);
+                    }
+
                    transaction.Commit();
                }
                catch