ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/SqliteAuditWriterForwardStateTests.cs

using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;

namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;

/// <summary>
/// Forward-state lifecycle regressions for review 2026-10-21 06/PERF-1 and 06/IMPL-1: the
/// reconciliation pull serves only rows central has NOT acknowledged (never Forwarded), the
/// push-exit states <see cref="SiteAuditForwardStates.PushAbandoned"/> and
/// <see cref="SiteAuditForwardStates.Rejected"/> stay owed to the pull until a covering cursor
/// retires them, and a row central keeps refusing leaves the push queue after N attempts.
/// </summary>
public class SqliteAuditWriterForwardStateTests : IDisposable
{
    private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"audit-fwdstate-{Guid.NewGuid():N}.db");

    /// <summary>Deletes the temporary database.</summary>
    public void Dispose()
    {
        foreach (var suffix in new[] { "", "-wal", "-shm" })
        {
            try { File.Delete(_dbPath + suffix); } catch { /* best-effort */ }
        }
    }

    private SqliteAuditWriter CreateWriter() => new(
        Options.Create(new SqliteAuditWriterOptions { DatabasePath = _dbPath }),
        NullLogger<SqliteAuditWriter>.Instance,
        new FakeNodeIdentityProvider());

    private static async Task<Guid> WriteAsync(SqliteAuditWriter writer, DateTime occurredAtUtc)
    {
        var id = Guid.NewGuid();
        await writer.WriteAsync(ScadaBridgeAuditEventFactory.Create(
            eventId: id,
            occurredAtUtc: occurredAtUtc,
            channel: AuditChannel.ApiOutbound,
            kind: AuditKind.ApiCall,
            status: AuditStatus.Delivered));
        return id;
    }

    /// <summary>The pull no longer re-ships rows the push already delivered.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ReadPendingSince_ExcludesForwardedRows()
    {
        await using var writer = CreateWriter();
        var t = DateTime.UtcNow.AddMinutes(-5);
        var forwarded = await WriteAsync(writer, t);
        var pending = await WriteAsync(writer, t.AddSeconds(1));
        await writer.MarkForwardedAsync(new[] { forwarded });

        var served = await writer.ReadPendingSinceAsync(DateTime.MinValue, 100);

        Assert.Equal(pending, Assert.Single(served).EventId);
    }

    /// <summary>A covering cursor retires only owed rows and leaves Forwarded rows alone.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task MarkReconciledUpTo_DoesNotTouchForwardedRows()
    {
        await using var writer = CreateWriter();
        var t = DateTime.UtcNow.AddMinutes(-5);
        var forwarded = await WriteAsync(writer, t);
        var pending = await WriteAsync(writer, t.AddSeconds(1));
        await writer.MarkForwardedAsync(new[] { forwarded });
        var served = await writer.ReadPendingSinceAsync(DateTime.MinValue, 100);

        var flipped = await writer.MarkReconciledUpToAsync(
            served[^1].OccurredAtUtc.UtcDateTime, served[^1].EventId.ToString());

        Assert.Equal(1, flipped);
        Assert.Equal("Forwarded", writer.GetForwardStateForTests(forwarded)!.Value.State);
        Assert.Equal("Reconciled", writer.GetForwardStateForTests(pending)!.Value.State);
    }

    /// <summary>A push-abandoned row leaves the push queue, is served by the pull, then retires.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task PushAbandoned_IsServedByThePull_ThenMarkedReconciled_AndNeverPurgedBeforeThat()
    {
        await using var writer = CreateWriter();
        var old = DateTime.UtcNow.AddDays(-10);
        var abandoned = await WriteAsync(writer, old);

        await writer.MarkPushAbandonedAsync(new[] { abandoned });

        Assert.Empty(await writer.ReadPendingAsync(100));
        // Still owed: never purged on age, counted on the backlog.
        Assert.Equal(0, await writer.PurgeExpiredAsync(DateTime.UtcNow));
        Assert.Equal(1, (await writer.GetBacklogStatsAsync()).PendingCount);

        var served = await writer.ReadPendingSinceAsync(DateTime.MinValue, 100);
        Assert.Equal(abandoned, Assert.Single(served).EventId);

        await writer.MarkReconciledUpToAsync(old, abandoned.ToString());

        Assert.Equal("Reconciled", writer.GetForwardStateForTests(abandoned)!.Value.State);
        Assert.Equal(1, await writer.PurgeExpiredAsync(DateTime.UtcNow));
    }

    /// <summary>
    /// A row central keeps refusing bumps its attempt history every time and, at the threshold,
    /// leaves the push queue as Rejected — unwedging the push head — while staying owed to the pull.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task RefusedRow_BumpsAttempts_ThenFlipsToRejected_UnwedgingThePushHead()
    {
        await using var writer = CreateWriter();
        var t = DateTime.UtcNow.AddMinutes(-5);
        var poison = await WriteAsync(writer, t);
        var behind = await WriteAsync(writer, t.AddSeconds(1));

        Assert.Equal(0, await writer.RecordRejectedPushAttemptsAsync(new[] { poison }, maxAttempts: 3));
        Assert.Equal(0, await writer.RecordRejectedPushAttemptsAsync(new[] { poison }, maxAttempts: 3));
        Assert.Equal(("Pending", 2), writer.GetForwardStateForTests(poison)!.Value);
        Assert.Equal(poison, (await writer.ReadPendingAsync(1))[0].EventId); // still the head

        Assert.Equal(1, await writer.RecordRejectedPushAttemptsAsync(new[] { poison }, maxAttempts: 3));

        Assert.Equal(("Rejected", 3), writer.GetForwardStateForTests(poison)!.Value);
        Assert.Equal(behind, Assert.Single(await writer.ReadPendingAsync(1)).EventId);
        Assert.Contains(await writer.ReadPendingSinceAsync(DateTime.MinValue, 100), e => e.EventId == poison);
        Assert.Equal(0, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(1)));
        Assert.Equal(1, writer.RejectedPushRowCount);
    }

    /// <summary>
    /// The served high-water mark survives a writer restart: rows served before the restart are
    /// retired by the first cursor that covers them after it, instead of being stranded owed
    /// (and counted on the backlog) until something happens to re-serve them.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ServedHighWaterMark_SurvivesRestart_SoTheNextCursorStillRetires()
    {
        var t = DateTime.UtcNow.AddMinutes(-5);
        Guid abandoned;
        IReadOnlyList<AuditEvent> served;
        await using (var writer = CreateWriter())
        {
            abandoned = await WriteAsync(writer, t);
            await writer.MarkPushAbandonedAsync(new[] { abandoned });
            served = await writer.ReadPendingSinceAsync(DateTime.MinValue, 100);
            Assert.Single(served);
        }

        await using (var restarted = CreateWriter())
        {
            Assert.True(restarted.MaxServedRowIdForTests > 0);

            var flipped = await restarted.MarkReconciledUpToAsync(
                served[0].OccurredAtUtc.UtcDateTime, served[0].EventId.ToString());

            Assert.Equal(1, flipped);
            Assert.Equal("Reconciled", restarted.GetForwardStateForTests(abandoned)!.Value.State);
            Assert.Equal(0, (await restarted.GetBacklogStatsAsync()).PendingCount);
        }
    }

    /// <summary>
    /// Follow-up L1: the served high-water mark is only ever MUTATED under the write lock — the
    /// same lock the purge's clamp runs under. Before the fix the read path wrote the mark under
    /// its own read lock from a value it sampled before the SELECT, so a purge that clamped the
    /// mark in between was overwritten with the stale-high value, re-vouching for reusable rowids.
    /// Holding the write lock across a pull must therefore leave the mark untouched until released.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ServedHighWaterMark_IsOnlyAdvancedUnderTheWriteLock()
    {
        await using var writer = CreateWriter();
        await WriteAsync(writer, DateTime.UtcNow.AddMinutes(-5));
        var before = writer.MaxServedRowIdForTests;

        Task<IReadOnlyList<AuditEvent>> pull;
        System.Threading.Monitor.Enter(writer.WriteLockForTests);
        try
        {
            pull = Task.Run(() => writer.ReadPendingSinceAsync(DateTime.MinValue, 100));
            // The SELECT itself needs only the read lock; give it ample time to run. A
            // synchronous sleep (not an await) so Monitor.Exit runs on the owning thread.
            Thread.Sleep(500);
            Assert.Equal(before, writer.MaxServedRowIdForTests);
        }
        finally
        {
            System.Threading.Monitor.Exit(writer.WriteLockForTests);
        }

        Assert.Single(await pull.WaitAsync(TimeSpan.FromSeconds(10)));
        Assert.True(writer.MaxServedRowIdForTests > before);
    }
}