feat(audit-log): permanently-abandoned reconciliation rows leave a durable ReconciliationAbandoned audit record

When a pulled AuditEvent fails to insert on every retry up to the permanent- abandon threshold, the reconciliation actor now writes ONE synthetic ReconciliationAbandoned row (new AuditKind) alongside the Critical log line — fresh EventId, Status=Failed, channel preserved from the lost row, Extra carrying the abandoned EventId + source site + final error — via the same ScadaBridgeAuditEventFactory the ingest path uses. Best-effort in its own try/catch so it never blocks the cursor twice. The permanent loss is now queryable in the Audit Log, not only in a rotating log file.
2026-07-09 08:50:44 -04:00
parent 20098c6108
commit 0385942c3f
4 changed files with 207 additions and 1 deletions
@@ -113,6 +113,69 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            Task.FromResult<IReadOnlyList<string>>(Array.Empty<string>());
    }

+    /// <summary>
+    /// Recording stub whose <see cref="InsertIfNotExistsAsync"/> throws for ONE
+    /// poison <see cref="AuditEvent.EventId"/> (every attempt) but records every
+    /// other row — including the synthetic ReconciliationAbandoned row the actor
+    /// emits once it gives up on the poison row (Task 17).
+    /// </summary>
+    private sealed class PoisonInsertRepo : IAuditLogRepository
+    {
+        private readonly Guid _poisonId;
+        private readonly Exception _fault;
+        public List<AuditEvent> Inserted { get; } = new();
+        private readonly HashSet<Guid> _seen = new();
+
+        public PoisonInsertRepo(Guid poisonId, Exception fault)
+        {
+            _poisonId = poisonId;
+            _fault = fault;
+        }
+
+        public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
+        {
+            if (evt.EventId == _poisonId)
+            {
+                throw _fault;
+            }
+            if (_seen.Add(evt.EventId))
+            {
+                Inserted.Add(evt);
+            }
+            return Task.CompletedTask;
+        }
+
+        public Task<IReadOnlyList<AuditEvent>> QueryAsync(
+            AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
+            Task.FromResult<IReadOnlyList<AuditEvent>>(Inserted);
+
+        public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
+            Task.FromResult(0L);
+
+        public Task<long> PurgeChannelOlderThanAsync(
+            string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
+            Task.FromResult(0L);
+
+        public Task<long> BackfillSourceNodeAsync(
+            string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
+            Task.FromResult(0L);
+
+        public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
+            DateTime threshold, CancellationToken ct = default) =>
+            Task.FromResult<IReadOnlyList<DateTime>>(Array.Empty<DateTime>());
+
+        public Task<ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot> GetKpiSnapshotAsync(
+            TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default) =>
+            Task.FromResult(new ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot(0L, 0L, 0L, nowUtc ?? DateTime.UtcNow));
+
+        public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
+            Guid executionId, CancellationToken ct = default) =>
+            Task.FromResult<IReadOnlyList<ExecutionTreeNode>>(Array.Empty<ExecutionTreeNode>());
+
+        public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default) =>
+            Task.FromResult<IReadOnlyList<string>>(Array.Empty<string>());
+    }
+
    /// <summary>
    /// In-memory enumerator returning a static list of sites.
    /// </summary>
@@ -457,4 +520,56 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        Assert.False(second.Stalled);
        Assert.Equal("siteA", second.SiteId);
    }
+
+    // ---------------------------------------------------------------------
+    // 9. Task 17: a permanently-abandoned row leaves a durable
+    //    ReconciliationAbandoned audit record (not just a Critical log line),
+    //    and the cursor advances past it.
+    // ---------------------------------------------------------------------
+
+    [Fact]
+    public void PermanentAbandonment_EmitsReconciliationAbandonedRow_AndAdvancesCursor()
+    {
+        var siteId = "siteAbandon";
+        var poisonTime = new DateTime(2026, 5, 20, 10, 5, 0, DateTimeKind.Utc);
+        var poison = NewEvent(siteId, poisonTime);
+
+        var sites = new StaticEnumerator(new SiteEntry(siteId, "http://x:8083"));
+        // Five pulls each returning the same poison row so its per-EventId attempt
+        // counter reaches MaxPermanentInsertAttempts (5); afterwards the scripted
+        // client returns empty and the actor settles.
+        var responses = Enumerable.Range(0, 5)
+            .Select(_ => new PullAuditEventsResponse(new[] { poison }, MoreAvailable: false))
+            .ToArray();
+        var client = new ScriptedPullClient().Script(siteId, responses);
+        var repo = new PoisonInsertRepo(
+            poison.EventId, new InvalidOperationException("poison row will never insert"));
+
+        CreateActor(sites, client, repo, FastTickOptions());
+
+        // After the 5th failed attempt the actor abandons the row and writes ONE
+        // synthetic ReconciliationAbandoned row (fresh EventId) whose details carry
+        // the abandoned id + the site.
+        AwaitAssert(
+            () =>
+            {
+                var synthetic = repo.Inserted.SingleOrDefault(
+                    e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal));
+                Assert.NotNull(synthetic);
+                Assert.NotEqual(poison.EventId, synthetic!.EventId);
+                Assert.Contains(poison.EventId.ToString(), synthetic.DetailsJson);
+                Assert.Contains(siteId, synthetic.DetailsJson);
+            },
+            duration: TimeSpan.FromSeconds(8),
+            interval: TimeSpan.FromMilliseconds(100));
+
+        // The cursor advanced past MinValue to the abandoned row's timestamp — the
+        // poison row no longer pins the site's reconciliation forever. A later tick
+        // must pull with since = poisonTime (poll for it; the abandonment tick and
+        // the first advanced-cursor pull are separate ticks).
+        AwaitAssert(
+            () => Assert.Contains(client.Calls, c => c.SinceUtc == poisonTime),
+            duration: TimeSpan.FromSeconds(5),
+            interval: TimeSpan.FromMilliseconds(100));
+    }
 }