ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditIngestRejectionTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;

namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;

/// <summary>
/// Review 2026-10-21 06/IMPL-1 adversarial follow-up: central reports a row as REJECTED only for a
/// permanent, row-intrinsic SQL failure; a database outage / timeout leaves the id in neither list,
/// so the site never mistakes a maintenance window for a poison row. Also pins the classifier the
/// reconciliation puller uses to decide what counts toward permanent abandonment (06/COR follow-up 3).
/// </summary>
public class AuditIngestRejectionTests : TestKit
{
    private static AuditEvent NewEvent() => ScadaBridgeAuditEventFactory.Create(
        eventId: Guid.NewGuid(),
        occurredAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
        channel: AuditChannel.ApiOutbound,
        kind: AuditKind.ApiCall,
        status: AuditStatus.Delivered,
        sourceSiteId: "site-a");

    /// <summary>The batch reply lists only the over-width row as rejected; the timed-out row is in neither list.</summary>
    [Fact]
    public void Ingest_ReportsOnlyPermanentRowFailuresAsRejected()
    {
        var ok = NewEvent();
        var overWidth = NewEvent();
        var timedOut = NewEvent();
        var repo = Substitute.For<IAuditLogRepository>();
        repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
            .ThrowsAsync(SqlExceptionFactory.Create(2628));
        repo.InsertIfNotExistsAsync(Arg.Is<AuditEvent>(e => e.EventId == overWidth.EventId), Arg.Any<CancellationToken>())
            .ThrowsAsync(SqlExceptionFactory.Create(2628));
        repo.InsertIfNotExistsAsync(Arg.Is<AuditEvent>(e => e.EventId == timedOut.EventId), Arg.Any<CancellationToken>())
            .ThrowsAsync(SqlExceptionFactory.Create(-2));

        var actor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
            StubAuditRepositoryProvider.Build(repo), NullLogger<AuditLogIngestActor>.Instance)));
        actor.Tell(new IngestAuditEventsCommand(new[] { ok, overWidth, timedOut }), TestActor);

        var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
        Assert.Equal(new[] { ok.EventId }, reply.AcceptedEventIds);
        Assert.Equal(new[] { overWidth.EventId }, reply.RejectedEventIds);
    }

    /// <summary>Classifier: row-intrinsic SQL errors are permanent; timeouts, connection loss and non-SQL faults are not.</summary>
    [Fact]
    public void Classifier_SeparatesRowVerdictsFromDatabaseProblems()
    {
        Assert.True(AuditIngestErrorClassifier.IsPermanentRowRejection(SqlExceptionFactory.Create(2628)));
        Assert.True(AuditIngestErrorClassifier.IsPermanentRowRejection(SqlExceptionFactory.Create(515)));
        Assert.True(AuditIngestErrorClassifier.IsPermanentRowRejection(
            new InvalidOperationException("wrapped", SqlExceptionFactory.Create(8152))));
        Assert.False(AuditIngestErrorClassifier.IsPermanentRowRejection(SqlExceptionFactory.Create(-2)));
        Assert.False(AuditIngestErrorClassifier.IsPermanentRowRejection(new TimeoutException()));
        Assert.False(AuditIngestErrorClassifier.IsPermanentRowRejection(new InvalidOperationException("bug")));

        Assert.True(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(-2)));
        Assert.True(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(1205)));
        Assert.True(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(40613)));
        Assert.True(AuditIngestErrorClassifier.IsTransient(new TimeoutException()));
        Assert.True(AuditIngestErrorClassifier.IsTransient(new TaskCanceledException()));
        Assert.True(AuditIngestErrorClassifier.IsTransient(
            new InvalidOperationException("wrapped", new System.Net.Sockets.SocketException())));
        Assert.False(AuditIngestErrorClassifier.IsTransient(SqlExceptionFactory.Create(2628)));
        Assert.False(AuditIngestErrorClassifier.IsTransient(new InvalidOperationException("bug")));
    }

    private sealed class OneRowClient : IPullAuditEventsClient
    {
        private readonly AuditEvent _row;
        public OneRowClient(AuditEvent row) => _row = row;
        public System.Collections.Concurrent.ConcurrentQueue<DateTime> Since { get; } = new();
        public Task<ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse> PullAsync(
            string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
        {
            Since.Enqueue(sinceUtc);
            return Task.FromResult(new ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration.PullAuditEventsResponse(
                new[] { _row }, MoreAvailable: false));
        }
    }

    private sealed class OneSite : ISiteEnumerator
    {
        public Task<IReadOnlyList<SiteEntry>> EnumerateAsync(CancellationToken ct = default) =>
            Task.FromResult<IReadOnlyList<SiteEntry>>(new[] { new SiteEntry("site-a", "http://a:8083") });
    }

    /// <summary>
    /// A row that keeps hitting a TRANSIENT database fault is retried with the cursor held back for
    /// far longer than the permanent-abandon threshold, and is never abandoned.
    /// </summary>
    [Fact]
    public void Reconciliation_TransientFaults_NeverCountTowardAbandonment()
    {
        var row = NewEvent();
        var repo = Substitute.For<IAuditLogRepository>();
        repo.InsertManyIfNotExistsAsync(Arg.Any<IReadOnlyList<AuditEvent>>(), Arg.Any<TimeSpan?>(), Arg.Any<CancellationToken>())
            .ThrowsAsync(SqlExceptionFactory.Create(-2));
        repo.InsertIfNotExistsAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
            .ThrowsAsync(SqlExceptionFactory.Create(-2));
        var client = new OneRowClient(row);

        var actor = Sys.ActorOf(Props.Create(() => new SiteAuditReconciliationActor(
            new OneSite(), client, StubAuditRepositoryProvider.Build(repo),
            Microsoft.Extensions.Options.Options.Create(new SiteAuditReconciliationOptions
            {
                ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(30),
            }),
            NullLogger<SiteAuditReconciliationActor>.Instance)));

        AwaitCondition(() => client.Since.Count >= 12, TimeSpan.FromSeconds(10));
        Watch(actor);
        Sys.Stop(actor);
        ExpectTerminated(actor);

        Assert.All(client.Since, s => Assert.Equal(DateTime.MinValue, s)); // cursor held every tick
        repo.DidNotReceive().InsertIfNotExistsAsync(
            Arg.Is<AuditEvent>(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
            Arg.Any<CancellationToken>());
    }
}