fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)

2026-10-22 08:40:42 -04:00
parent 38167bca88
commit b0a08bb63b
23 changed files with 790 additions and 168 deletions
@@ -0,0 +1,132 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+using ZB.MOM.WW.Audit;
+using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
+using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+
+namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;
+
+/// <summary>
+/// Review 2026-10-21 06/IMPL-1 adversarial follow-up: central reports a row as REJECTED only for a
+/// permanent, row-intrinsic SQL failure; a database outage / timeout leaves the id in neither list,
+/// so the site never mistakes a maintenance window for a poison row. Also pins the classifier the
+/// reconciliation puller uses to decide what counts toward permanent abandonment (06/COR follow-up 3).
+/// </summary>
+public class AuditIngestRejectionTests : TestKit
+{
+    private static AuditEvent NewEvent() => ScadaBridgeAuditEventFactory.Create(
+        eventId: Guid.NewGuid(),
+        occurredAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
+        channel: AuditChannel.ApiOutbound,
+        kind: AuditKind.ApiCall,
+        status: AuditStatus.Delivered,
+        sourceSiteId: "site-a");
+
+    /// <summary>The batch reply lists only the over-width row as rejected; the timed-out row is in neither list.</summary>
+    [Fact]
+    public void Ingest_ReportsOnlyPermanentRowFailuresAsRejected()
+    {
+        var ok = NewEvent();
+        var overWidth = NewEvent();
+        var timedOut = NewEvent();
+        var repo = Substitute.For<IAuditLogRepository>();
+        repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
+            .ThrowsAsync(SqlExceptionFactory.Create(2628));
+        repo.InsertIfNotExistsAsync(Arg.Is<AuditEvent>(e => e.EventId == overWidth.EventId), Arg.Any<CancellationToken>())
+            .ThrowsAsync(SqlExceptionFactory.Create(2628));
+        repo.InsertIfNotExistsAsync(Arg.Is<AuditEvent>(e => e.EventId == timedOut.EventId), Arg.Any<CancellationToken>())
+            .ThrowsAsync(SqlExceptionFactory.Create(-2));
+
+        var actor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
+            StubAuditRepositoryProvider.Build(repo), NullLogger<AuditLogIngestActor>.Instance)));
+        actor.Tell(new IngestAuditEventsCommand(new[] { ok, overWidth, timedOut }), TestActor);
+
+        var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
+        Assert.Equal(new[] { ok.EventId }, reply.AcceptedEventIds);
+        Assert.Equal(new[] { overWidth.EventId }, reply.RejectedEventIds);
+    }
+
+    /// <summary>Classifier: row-intrinsic SQL errors are permanent; timeouts, connection loss and non-SQL faults are not.</summary>
+    [Fact]
+    public void Classifier_SeparatesRowVerdictsFromDatabaseProblems()
+    {
+        Assert.True(AuditIngestErrorClassifier.IsPermanentRowRejection(SqlExceptionFactory.Create(2628)));
+        Assert.True(AuditIngestErrorClassifier.IsPermanentRowRejection(SqlExceptionFactory.Create(515)));
+        Assert.True(AuditIngestErrorClassifier.IsPermanentRowRejection(
+            new InvalidOperationException("wrapped", SqlExceptionFactory.Create(8152))));
+        Assert.False(AuditIngestErrorClassifier.IsPermanentRowRejection(SqlExceptionFactory.Create(-2)));
+        Assert.False(AuditIngestErrorClassifier.IsPermanentRowRejection(new TimeoutException()));
+        Assert.False(AuditIngestErrorClassifier.IsPermanentRowRejection(new InvalidOperationException("bug")));
+
+        Assert.True(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(-2)));
+        Assert.True(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(1205)));
+        Assert.True(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(40613)));
+        Assert.True(AuditIngestErrorClassifier.IsTransient(new TimeoutException()));
+        Assert.True(AuditIngestErrorClassifier.IsTransient(new TaskCanceledException()));
+        Assert.True(AuditIngestErrorClassifier.IsTransient(
+            new InvalidOperationException("wrapped", new System.Net.Sockets.SocketException())));
+        Assert.False(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(2628)));
+        Assert.False(AuditIngestErrorClassifier.IsTransient(new InvalidOperationException("bug")));
+    }
+
+    private sealed class OneRowClient : IPullAuditEventsClient
+    {
+        private readonly AuditEvent _row;
+        public OneRowClient(AuditEvent row) => _row = row;
+        public System.Collections.Concurrent.ConcurrentQueue<DateTime> Since { get; } = new();
+        public Task<ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse> PullAsync(
+            string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
+        {
+            Since.Enqueue(sinceUtc);
+            return Task.FromResult(new ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse(
+                new[] { _row }, MoreAvailable: false));
+        }
+    }
+
+    private sealed class OneSite : ISiteEnumerator
+    {
+        public Task<IReadOnlyList<SiteEntry>> EnumerateAsync(CancellationToken ct = default) =>
+            Task.FromResult<IReadOnlyList<SiteEntry>>(new[] { new SiteEntry("site-a", "http://a:8083") });
+    }
+
+    /// <summary>
+    /// A row that keeps hitting a TRANSIENT database fault is retried with the cursor held back for
+    /// far longer than the permanent-abandon threshold, and is never abandoned.
+    /// </summary>
+    [Fact]
+    public void Reconciliation_TransientFaults_NeverCountTowardAbandonment()
+    {
+        var row = NewEvent();
+        var repo = Substitute.For<IAuditLogRepository>();
+        repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
+            .ThrowsAsync(SqlExceptionFactory.Create(-2));
+        repo.InsertIfNotExistsAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
+            .ThrowsAsync(SqlExceptionFactory.Create(-2));
+        var client = new OneRowClient(row);
+
+        var actor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
+            new OneSite(), client, StubAuditRepositoryProvider.Build(repo),
+            Microsoft.Extensions.Options.Options.Create(new SiteAuditReconciliationOptions
+            {
+                ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(30),
+            }),
+            NullLogger<SiteAuditReconciliationActor>.Instance)));
+
+        AwaitCondition(() => client.Since.Count >= 12, TimeSpan.FromSeconds(10));
+        Watch(actor);
+        Sys.Stop(actor);
+        ExpectTerminated(actor);
+
+        Assert.All(client.Since, s => Assert.Equal(DateTime.MinValue, s)); // cursor held every tick
+        repo.DidNotReceive().InsertIfNotExistsAsync(
+            Arg.Is<AuditEvent>(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
+            Arg.Any<CancellationToken>());
+    }
+}
@@ -116,9 +116,11 @@ public class ReconciliationStatePruningTests : TestKit

        Assert.True(probe.ExpectMsg<SiteAuditTelemetryStalledChanged>(TimeSpan.FromSeconds(5)).Stalled);

-        sites.Sites = Array.Empty<SiteEntry>();
+        // Not an empty list: an empty enumeration deliberately prunes nothing.
+        sites.Sites = new[] { new SiteEntry("survivor", "http://s:8083") };

-        var cleared = probe.ExpectMsg<SiteAuditTelemetryStalledChanged>(TimeSpan.FromSeconds(5));
+        var cleared = probe.FishForMessage<SiteAuditTelemetryStalledChanged>(
+            m => m.SiteId == "doomed", TimeSpan.FromSeconds(5));
        Assert.Equal("doomed", cleared.SiteId);
        Assert.False(cleared.Stalled);