ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests/SiteCallAuditActorTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;

namespace ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests;

/// <summary>
/// Tests for <see cref="SiteCallAuditActor"/>. Uses the
/// same <see cref="MsSqlMigrationFixture"/> as the repository tests
/// so the actor exercises the real monotonic-upsert SQL end to end against the
/// <c>SiteCalls</c> schema. Each test scopes its data by minting a fresh
/// <see cref="TrackedOperationId"/> (and a per-test <c>SourceSite</c> suffix)
/// so tests neither collide nor require teardown.
/// </summary>
public class SiteCallAuditActorTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
    private readonly MsSqlMigrationFixture _fixture;

    /// <summary>Initializes the test class with the shared MSSQL migration fixture.</summary>
    /// <param name="fixture">The shared MSSQL migration fixture.</param>
    public SiteCallAuditActorTests(MsSqlMigrationFixture fixture)
    {
        _fixture = fixture;
    }

    private ScadaBridgeDbContext CreateContext()
    {
        var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
            .UseSqlServer(_fixture.ConnectionString)
            .Options;
        return new ScadaBridgeDbContext(options);
    }

    private static string NewSiteId() =>
        "test-bundle-c1-" + Guid.NewGuid().ToString("N").Substring(0, 8);

    private static SiteCall NewRow(
        TrackedOperationId id,
        string sourceSite,
        string status = "Submitted",
        int retryCount = 0,
        string? lastError = null,
        DateTime? createdAtUtc = null,
        DateTime? updatedAtUtc = null,
        bool terminal = false,
        string? sourceNode = null)
    {
        var created = createdAtUtc ?? DateTime.UtcNow;
        var updated = updatedAtUtc ?? created;
        return new SiteCall
        {
            TrackedOperationId = id,
            Channel = "ApiOutbound",
            Target = "ERP.GetOrder",
            SourceSite = sourceSite,
            SourceNode = sourceNode,
            Status = status,
            RetryCount = retryCount,
            LastError = lastError,
            HttpStatus = null,
            CreatedAtUtc = created,
            UpdatedAtUtc = updated,
            TerminalAtUtc = terminal ? updated : null,
            IngestedAtUtc = DateTime.UtcNow,
        };
    }

    private IActorRef CreateActor(
        ISiteCallAuditRepository repository, SiteCallAuditOptions? options = null)
    {
        // Register row 61: the actor's repository-taking test ctor is gone. This MSSQL-backed
        // fixture supplies a stub provider that yields the same repository from every scope,
        // serialized (one DbContext cannot service concurrent operations) — see
        // StubRepositoryProvider. No reconciliation collaborators are registered, so the
        // reconciliation tick stays disarmed exactly as it did under the old ctor.
        var provider = TestSupport.StubRepositoryProvider.Build(repository);
        return Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
            provider,
            options ?? new SiteCallAuditOptions(),
            NullLogger<SiteCallAuditActor>.Instance)));
    }

    /// <summary>Verifies that an upsert command is persisted and acknowledged as accepted.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_UpsertSiteCallCommand_Persists_Replies_Accepted()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var id = TrackedOperationId.New();
        var row = NewRow(id, siteId, status: "Submitted", retryCount: 0);

        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        actor.Tell(new UpsertSiteCallCommand(row), TestActor);

        var reply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.True(reply.Accepted, "Actor should reply Accepted=true on a successful upsert.");
        Assert.Equal(id, reply.TrackedOperationId);

        // Verify the row landed in MSSQL via a fresh context (separate from the
        // actor's repository context).
        await using var readContext = CreateContext();
        var rows = await readContext.Set<SiteCall>()
            .Where(s => s.SourceSite == siteId)
            .ToListAsync();
        Assert.Single(rows);
        Assert.Equal("Submitted", rows[0].Status);
    }

    /// <summary>Verifies that a duplicate upsert with an older status changes nothing yet is still acknowledged as accepted.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_DuplicateUpsert_OlderStatus_NoOp_StillRepliesAccepted()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        // Idempotency contract: a stale/duplicate packet (lower rank than the
        // stored status) is a silent no-op at the repository — the actor must
        // still reply Accepted=true so the site is free to consider its
        // packet acked. Storage state is consistent either way.
        var siteId = NewSiteId();
        var id = TrackedOperationId.New();

        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        // Land Attempted (rank 2) first.
        actor.Tell(new UpsertSiteCallCommand(NewRow(id, siteId, status: "Attempted", retryCount: 1, lastError: "first")), TestActor);
        var firstReply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.True(firstReply.Accepted);

        // Late-arriving Submitted (rank 0) — must be no-op in storage and
        // still acked by the actor.
        actor.Tell(new UpsertSiteCallCommand(NewRow(id, siteId, status: "Submitted", retryCount: 0)), TestActor);
        var secondReply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.True(secondReply.Accepted, "Stale upsert must still be acked (idempotent contract).");

        // Storage must still show the rank 2 row, not rolled back.
        await using var readContext = CreateContext();
        var stored = await readContext.Set<SiteCall>()
            .Where(s => s.TrackedOperationId == id)
            .ToListAsync();
        Assert.Single(stored);
        Assert.Equal("Attempted", stored[0].Status);
        Assert.Equal(1, stored[0].RetryCount);
        Assert.Equal("first", stored[0].LastError);
    }

    /// <summary>Verifies that a transient repository failure replies not-accepted and leaves the actor alive.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_RepoThrowsTransient_RepliesAccepted_False_ActorStaysAlive()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        // Per CLAUDE.md: audit-write failure NEVER aborts the user-facing
        // action. The actor must catch the throw, reply Accepted=false, and
        // stay alive — a follow-up message on the same actor must still be
        // processed (the singleton cannot die on a transient repo error).
        var siteId = NewSiteId();
        var poisonId = TrackedOperationId.New();
        var healthyId = TrackedOperationId.New();

        await using var context = CreateContext();
        var realRepo = new SiteCallAuditRepository(context);
        var wrappedRepo = new ThrowingRepository(realRepo, poisonId);
        var actor = CreateActor(wrappedRepo);

        // Poison row — the wrapper throws when this id arrives.
        actor.Tell(new UpsertSiteCallCommand(NewRow(poisonId, siteId, status: "Submitted")), TestActor);
        var poisonReply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.False(poisonReply.Accepted, "Actor should reply Accepted=false when the repo throws.");
        Assert.Equal(poisonId, poisonReply.TrackedOperationId);

        // Healthy follow-up on the SAME actor — must still be processed
        // (singleton staying alive proves the actor did not crash).
        actor.Tell(new UpsertSiteCallCommand(NewRow(healthyId, siteId, status: "Submitted")), TestActor);
        var healthyReply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.True(healthyReply.Accepted, "Actor must stay alive after a transient repo failure.");
        Assert.Equal(healthyId, healthyReply.TrackedOperationId);

        // Verify storage: healthy row landed, poison row did not.
        await using var readContext = CreateContext();
        var rows = await readContext.Set<SiteCall>()
            .Where(s => s.SourceSite == siteId)
            .ToListAsync();
        Assert.Single(rows);
        Assert.Equal(healthyId, rows[0].TrackedOperationId);
    }

    // ── Read-side (query / detail / KPI) handlers ──

    /// <summary>Verifies that filtering a query by source site returns only the matching summaries.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_FilterBySourceSite_ReturnsMatchingSummaries()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Attempted", createdAtUtc: t0));
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: t0.AddMinutes(1), terminal: true));

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-q1", StatusFilter: null, SourceSiteFilter: siteId, ChannelFilter: null,
                TargetKeyword: null, StuckOnly: false, FromUtc: null, ToUtc: null,
                AfterCreatedAtUtc: null, AfterId: null, PageSize: 50),
            TestActor);

        var response = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Equal("corr-q1", response.CorrelationId);
        Assert.Equal(2, response.SiteCalls.Count);
        Assert.All(response.SiteCalls, s => Assert.Equal(siteId, s.SourceSite));
        // Newest first — ordered (CreatedAtUtc DESC).
        Assert.Equal("Delivered", response.SiteCalls[0].Status);
        // Cursor echoes the last (oldest) row of the page.
        Assert.Equal(t0, response.NextAfterCreatedAtUtc);
        Assert.Equal(response.SiteCalls[^1].TrackedOperationId, response.NextAfterId);
    }

    /// <summary>Verifies that filtering a query by source node returns only the matching summaries.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_FilterBySourceNode_ReturnsMatchingSummaries()
    {
        // The new Node filter input pushes a string into
        // SiteCallQueryRequest.SourceNodeFilter — the actor must thread it
        // onto SiteCallQueryFilter.SourceNode and the response summaries must
        // mirror the row's SourceNode column.
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        var t0 = new DateTime(2026, 5, 20, 14, 0, 0, DateTimeKind.Utc);
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Attempted",
            createdAtUtc: t0, sourceNode: "site-plant-a-node-a"));
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: t0.AddMinutes(1), terminal: true, sourceNode: "site-plant-a-node-b"));

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-node", StatusFilter: null, SourceSiteFilter: siteId, ChannelFilter: null,
                TargetKeyword: null, StuckOnly: false, FromUtc: null, ToUtc: null,
                AfterCreatedAtUtc: null, AfterId: null, PageSize: 50,
                SourceNodeFilter: "site-plant-a-node-a"),
            TestActor);

        var response = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Single(response.SiteCalls);
        Assert.Equal("site-plant-a-node-a", response.SiteCalls[0].SourceNode);
        Assert.Equal("Attempted", response.SiteCalls[0].Status);
    }

    /// <summary>Verifies that filtering a query by tracked operation id returns only that row.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_FilterByTrackedOperationId_ReturnsOnlyThatRow()
    {
        // Flow Debugger follow-up: the ?correlationId= deep link pushes
        // one operation id into SiteCallQueryRequest.TrackedOperationIdFilter —
        // the actor must thread it onto SiteCallQueryFilter.TrackedOperationId
        // so the page returns exactly that row and nothing else from the site.
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        var t0 = new DateTime(2026, 9, 6, 14, 0, 0, DateTimeKind.Utc);
        var rowA = NewRow(TrackedOperationId.New(), siteId, status: "Attempted", createdAtUtc: t0);
        var rowB = NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: t0.AddMinutes(1), terminal: true);
        await repo.UpsertAsync(rowA);
        await repo.UpsertAsync(rowB);

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-op", StatusFilter: null, SourceSiteFilter: siteId, ChannelFilter: null,
                TargetKeyword: null, StuckOnly: false, FromUtc: null, ToUtc: null,
                AfterCreatedAtUtc: null, AfterId: null, PageSize: 50,
                TrackedOperationIdFilter: rowA.TrackedOperationId.Value),
            TestActor);

        var response = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Single(response.SiteCalls);
        Assert.Equal(rowA.TrackedOperationId.Value, response.SiteCalls[0].TrackedOperationId);
        Assert.Equal("Attempted", response.SiteCalls[0].Status);
        // No further page — the exact-match filter yields at most one row.
        Assert.Equal(rowA.TrackedOperationId.Value, response.NextAfterId);
    }

    /// <summary>Verifies that keyset paging advances through the results via the cursor.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_KeysetPaging_AdvancesViaCursor()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        var t0 = new DateTime(2026, 5, 20, 13, 0, 0, DateTimeKind.Utc);
        for (var i = 0; i < 3; i++)
        {
            await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, createdAtUtc: t0.AddMinutes(i)));
        }

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-q2", null, siteId, null, null, false, null, null, null, null, PageSize: 2),
            TestActor);
        var page1 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.Equal(2, page1.SiteCalls.Count);

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-q3", null, siteId, null, null, false, null, null,
                page1.NextAfterCreatedAtUtc, page1.NextAfterId, PageSize: 2),
            TestActor);
        var page2 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.Single(page2.SiteCalls);

        // No overlap across the two pages.
        var allIds = page1.SiteCalls.Concat(page2.SiteCalls)
            .Select(s => s.TrackedOperationId).ToHashSet();
        Assert.Equal(3, allIds.Count);
    }

    /// <summary>Verifies that the stuck-only filter returns only old, non-terminal rows.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_StuckOnly_ReturnsOnlyOldNonTerminalRows()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        // 10-minute stuck threshold (the production default).
        var actor = CreateActor(repo, new SiteCallAuditOptions { StuckAgeThreshold = TimeSpan.FromMinutes(10) });

        var now = DateTime.UtcNow;
        // Stuck: non-terminal (Attempted, TerminalAtUtc null), created 30 min ago.
        var stuckId = TrackedOperationId.New();
        await repo.UpsertAsync(NewRow(stuckId, siteId, status: "Attempted", createdAtUtc: now.AddMinutes(-30)));
        // Not stuck: non-terminal but recent.
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Attempted", createdAtUtc: now.AddMinutes(-2)));
        // Not stuck: old but terminal (Delivered, TerminalAtUtc set).
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: now.AddMinutes(-40), terminal: true));

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-stuck", null, siteId, null, null, StuckOnly: true,
                null, null, null, null, PageSize: 50),
            TestActor);

        var response = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.Single(response.SiteCalls);
        Assert.Equal(stuckId.Value, response.SiteCalls[0].TrackedOperationId);
        Assert.True(response.SiteCalls[0].IsStuck);
    }

    /// <summary>Verifies that stuck-only pages are full and never come back under-filled with a next cursor.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_StuckOnly_PagesAreFull_NoEmptyPagesWithCursor()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo, new SiteCallAuditOptions { StuckAgeThreshold = TimeSpan.FromMinutes(10) });

        var now = DateTime.UtcNow;
        // Three stuck rows interleaved (by CreatedAtUtc) with three non-stuck
        // rows: recent non-terminal and old-but-terminal. With the StuckOnly
        // filter pushed into SQL, a page-size-of-two query must return exactly the
        // stuck rows two-per-page — never an under-filled page with a non-null
        // next cursor caused by post-filtering.
        for (var i = 0; i < 3; i++)
        {
            await repo.UpsertAsync(NewRow(
                TrackedOperationId.New(), siteId, status: "Attempted",
                createdAtUtc: now.AddMinutes(-30 - i)));
            await repo.UpsertAsync(NewRow(
                TrackedOperationId.New(), siteId, status: "Attempted",
                createdAtUtc: now.AddMinutes(-2 - i)));
            await repo.UpsertAsync(NewRow(
                TrackedOperationId.New(), siteId, status: "Delivered",
                createdAtUtc: now.AddMinutes(-40 - i), terminal: true));
        }

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-stuck-p1", null, siteId, null, null, StuckOnly: true,
                null, null, null, null, PageSize: 2),
            TestActor);
        var page1 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(page1.Success);
        // Page is full — two stuck rows, both honestly stuck.
        Assert.Equal(2, page1.SiteCalls.Count);
        Assert.All(page1.SiteCalls, s => Assert.True(s.IsStuck));
        Assert.NotNull(page1.NextAfterCreatedAtUtc);

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-stuck-p2", null, siteId, null, null, StuckOnly: true,
                null, null, page1.NextAfterCreatedAtUtc, page1.NextAfterId,
                PageSize: 2),
            TestActor);
        var page2 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(page2.Success);
        // Final page — the third stuck row, the only remaining match.
        Assert.Single(page2.SiteCalls);
        Assert.All(page2.SiteCalls, s => Assert.True(s.IsStuck));

        // No overlap, exactly the three stuck rows across both pages.
        var allIds = page1.SiteCalls.Concat(page2.SiteCalls)
            .Select(s => s.TrackedOperationId).ToHashSet();
        Assert.Equal(3, allIds.Count);
    }

    /// <summary>Verifies that a stuck-only cursor sitting on a non-stuck row skips to the next stuck row.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_StuckOnly_CursorAtNonStuckBoundary_SkipsToNextStuckRow()
    {
        // Boundary regression. The earlier paging test
        // interleaves stuck/non-stuck rows but the cursor between page 1 and
        // page 2 always lands on a stuck row. This test forces the cursor to
        // sit on a NON-stuck row (a page size of 1 over a strict
        // stuck-not_stuck-stuck-not_stuck-stuck-not_stuck pattern oldest-first)
        // so the SQL-side composition of the stuck predicate AND the keyset
        // cursor predicate (CreatedAtUtc < cursor OR =cursor AND id < ...) must
        // honestly skip the non-stuck rows between each page. Each page must
        // return exactly one stuck row, with no overlap and all three stuck
        // rows visited across three pages.
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo, new SiteCallAuditOptions { StuckAgeThreshold = TimeSpan.FromMinutes(10) });

        var now = DateTime.UtcNow;
        var stuckIds = new List<Guid>();
        // Insert pattern (relative-newest first, the order the actor returns
        // them in DESC-by-CreatedAtUtc):
        //   t=-1m  (non-stuck — Attempted but only 1m old, < 10m threshold),
        //   t=-15m (stuck     — Attempted, 15m > 10m: stuckA),
        //   t=-20m (non-stuck — terminal Delivered between stuckA & stuckB),
        //   t=-25m (stuck     — Attempted, 25m > 10m: stuckB),
        //   t=-30m (non-stuck — terminal Delivered between stuckB & stuckC),
        //   t=-35m (stuck     — Attempted, 35m > 10m: stuckC),
        //   t=-40m (non-stuck — terminal Delivered, oldest of all).
        // The non-stuck rows at -20m and -30m sit between consecutive stuck
        // rows, so a page-size-of-one cursor lands ON a non-stuck row between
        // pages — exactly the boundary the predicate composition must skip.
        var stuckA = TrackedOperationId.New();
        var stuckB = TrackedOperationId.New();
        var stuckC = TrackedOperationId.New();
        // Expected DESC-by-CreatedAtUtc order: A (-15m newest), B (-25m), C (-35m oldest).
        stuckIds.Add(stuckA.Value);
        stuckIds.Add(stuckB.Value);
        stuckIds.Add(stuckC.Value);

        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: now.AddMinutes(-40), terminal: true));
        await repo.UpsertAsync(NewRow(
            stuckC, siteId, status: "Attempted",
            createdAtUtc: now.AddMinutes(-35)));
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: now.AddMinutes(-30), terminal: true));
        await repo.UpsertAsync(NewRow(
            stuckB, siteId, status: "Attempted",
            createdAtUtc: now.AddMinutes(-25)));
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: now.AddMinutes(-20), terminal: true));
        await repo.UpsertAsync(NewRow(
            stuckA, siteId, status: "Attempted",
            createdAtUtc: now.AddMinutes(-15)));
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Attempted",
            createdAtUtc: now.AddMinutes(-1)));

        // Page 1: page size of 1, expect stuckA (newest stuck row).
        actor.Tell(
            new SiteCallQueryRequest(
                "corr-stuck-b1", null, siteId, null, null, StuckOnly: true,
                null, null, null, null, PageSize: 1),
            TestActor);
        var page1 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(page1.Success);
        Assert.Single(page1.SiteCalls);
        Assert.True(page1.SiteCalls[0].IsStuck);
        Assert.Equal(stuckA.Value, page1.SiteCalls[0].TrackedOperationId);
        Assert.NotNull(page1.NextAfterCreatedAtUtc);

        // Page 2: between stuckA and stuckB the non-stuck terminal row at -20m
        // sits at the cursor — the SQL must skip it, NOT return it.
        actor.Tell(
            new SiteCallQueryRequest(
                "corr-stuck-b2", null, siteId, null, null, StuckOnly: true,
                null, null, page1.NextAfterCreatedAtUtc, page1.NextAfterId,
                PageSize: 1),
            TestActor);
        var page2 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(page2.Success);
        Assert.Single(page2.SiteCalls);
        Assert.True(page2.SiteCalls[0].IsStuck);
        Assert.Equal(stuckB.Value, page2.SiteCalls[0].TrackedOperationId);
        Assert.NotNull(page2.NextAfterCreatedAtUtc);

        // Page 3: between stuckB and stuckC the non-stuck row at -30m sits at
        // the cursor — again must be skipped.
        actor.Tell(
            new SiteCallQueryRequest(
                "corr-stuck-b3", null, siteId, null, null, StuckOnly: true,
                null, null, page2.NextAfterCreatedAtUtc, page2.NextAfterId,
                PageSize: 1),
            TestActor);
        var page3 = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.True(page3.Success);
        Assert.Single(page3.SiteCalls);
        Assert.True(page3.SiteCalls[0].IsStuck);
        Assert.Equal(stuckC.Value, page3.SiteCalls[0].TrackedOperationId);

        // All three stuck rows visited exactly once, in DESC order, with no
        // non-stuck rows leaking through despite the cursor sitting on them.
        var visited = new[] { page1, page2, page3 }
            .SelectMany(p => p.SiteCalls)
            .Select(s => s.TrackedOperationId)
            .ToList();
        Assert.Equal(stuckIds, visited);
    }

    /// <summary>Verifies that a detail request for a known id returns the full detail.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallDetailRequest_KnownId_ReturnsFullDetail()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var id = TrackedOperationId.New();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        await repo.UpsertAsync(NewRow(
            id, siteId, status: "Attempted", retryCount: 2, lastError: "503", sourceNode: "node-a"));

        actor.Tell(new SiteCallDetailRequest("corr-d1", id.Value), TestActor);

        var response = ExpectMsg<SiteCallDetailResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        Assert.NotNull(response.Detail);
        Assert.Equal(id.Value, response.Detail!.TrackedOperationId);
        Assert.Equal("Attempted", response.Detail.Status);
        Assert.Equal(2, response.Detail.RetryCount);
        Assert.Equal("503", response.Detail.LastError);
        Assert.Equal(siteId, response.Detail.SourceSite);
        // SourceNode flows through ToDetail so the report detail modal binds
        // uniformly to the detail record (was previously read off the summary).
        Assert.Equal("node-a", response.Detail.SourceNode);
    }

    /// <summary>Verifies that a detail request for an unknown id replies not found.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallDetailRequest_UnknownId_RepliesNotFound()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        actor.Tell(new SiteCallDetailRequest("corr-d2", Guid.NewGuid()), TestActor);

        var response = ExpectMsg<SiteCallDetailResponse>(TimeSpan.FromSeconds(10));
        Assert.False(response.Success);
        Assert.Null(response.Detail);
        Assert.NotNull(response.ErrorMessage);
    }

    /// <summary>Verifies that a KPI request computes point-in-time counts.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallKpiRequest_ComputesPointInTimeCounts()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo, new SiteCallAuditOptions
        {
            StuckAgeThreshold = TimeSpan.FromMinutes(10),
            KpiInterval = TimeSpan.FromHours(1),
        });

        var now = DateTime.UtcNow;
        // Buffered (non-terminal Attempted) + stuck (created 30 min ago).
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Attempted", createdAtUtc: now.AddMinutes(-30)));
        // Buffered (non-terminal Attempted), not stuck.
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Attempted", createdAtUtc: now.AddMinutes(-2)));
        // Parked (terminal).
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Parked",
            createdAtUtc: now.AddMinutes(-5), terminal: true));
        // Delivered within the interval.
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Delivered",
            createdAtUtc: now.AddMinutes(-3), updatedAtUtc: now.AddMinutes(-1), terminal: true));

        actor.Tell(new SiteCallKpiRequest("corr-kpi"), TestActor);

        var response = ExpectMsg<SiteCallKpiResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);
        // Per-site rows are isolated by the unique siteId — but KPIs are global,
        // so assert the floor (>=) rather than exact counts: other tests' rows
        // may share the table.
        Assert.True(response.BufferedCount >= 2);
        Assert.True(response.ParkedCount >= 1);
        Assert.True(response.DeliveredLastInterval >= 1);
        Assert.True(response.StuckCount >= 1);
        Assert.NotNull(response.OldestPendingAge);
    }

    // ── Per-node KPI (per-node stuck-count KPIs) ──────────────────

    /// <summary>Verifies that the per-node KPI request scopes the counts to each node.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task PerNodeSiteCallKpiRequest_ScopesCountsToEachNode()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var nodeId = "node-" + Guid.NewGuid().ToString("N").Substring(0, 8);
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo, new SiteCallAuditOptions
        {
            StuckAgeThreshold = TimeSpan.FromMinutes(10),
            KpiInterval = TimeSpan.FromHours(1),
        });

        var now = DateTime.UtcNow;
        var siteId = NewSiteId();
        // Non-terminal Attempted, created 30 min ago — buffered + stuck.
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Attempted",
            createdAtUtc: now.AddMinutes(-30), sourceNode: nodeId));
        // Terminal Parked.
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Parked",
            createdAtUtc: now.AddMinutes(-5), terminal: true, sourceNode: nodeId));

        actor.Tell(new PerNodeSiteCallKpiRequest("corr-pnk"), TestActor);

        var response = ExpectMsg<PerNodeSiteCallKpiResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);

        var myNode = Assert.Single(response.Nodes, n => n.SourceNode == nodeId);
        Assert.Equal(1, myNode.BufferedCount);
        Assert.Equal(1, myNode.ParkedCount);
        Assert.Equal(1, myNode.StuckCount);
        Assert.NotNull(myNode.OldestPendingAge);
    }

    /// <summary>Verifies that the per-site KPI request scopes the counts to each site.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task PerSiteSiteCallKpiRequest_ScopesCountsToEachSite()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo, new SiteCallAuditOptions
        {
            StuckAgeThreshold = TimeSpan.FromMinutes(10),
            KpiInterval = TimeSpan.FromHours(1),
        });

        var now = DateTime.UtcNow;
        // Non-terminal Attempted, created 30 min ago — buffered + stuck.
        await repo.UpsertAsync(NewRow(TrackedOperationId.New(), siteId, status: "Attempted", createdAtUtc: now.AddMinutes(-30)));
        // Terminal Parked.
        await repo.UpsertAsync(NewRow(
            TrackedOperationId.New(), siteId, status: "Parked",
            createdAtUtc: now.AddMinutes(-5), terminal: true));

        actor.Tell(new PerSiteSiteCallKpiRequest("corr-psk"), TestActor);

        var response = ExpectMsg<PerSiteSiteCallKpiResponse>(TimeSpan.FromSeconds(10));
        Assert.True(response.Success);

        var mySite = Assert.Single(response.Sites, s => s.SourceSite == siteId);
        Assert.Equal(1, mySite.BufferedCount);
        Assert.Equal(1, mySite.ParkedCount);
        Assert.Equal(1, mySite.StuckCount);
        Assert.NotNull(mySite.OldestPendingAge);
    }

    /// <summary>Verifies that a repository failure during a query replies with a failure and leaves the actor alive.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task SiteCallQueryRequest_RepoThrows_RepliesFailure_ActorStaysAlive()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        await using var context = CreateContext();
        var realRepo = new SiteCallAuditRepository(context);
        var actor = CreateActor(new QueryThrowingRepository(realRepo));

        actor.Tell(
            new SiteCallQueryRequest(
                "corr-fault", null, siteId, null, null, false, null, null, null, null, 50),
            TestActor);

        var response = ExpectMsg<SiteCallQueryResponse>(TimeSpan.FromSeconds(10));
        Assert.False(response.Success);
        Assert.Empty(response.SiteCalls);
        Assert.NotNull(response.ErrorMessage);
        Assert.Equal("corr-fault", response.CorrelationId);
    }

    // ── SourceNode-stamping: end-to-end actor → repo persistence ──

    /// <summary>Verifies that an upsert command persists its source node end to end.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task UpsertSiteCallCommand_PersistsSourceNode_EndToEnd()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        // SourceNode-stamping: an UpsertSiteCallCommand carrying
        // SourceNode "node-a" must land in the SiteCalls row's SourceNode
        // column unchanged — verifies the actor's mapping path does not
        // strip the column AND the repository INSERT writes it.
        var siteId = NewSiteId();
        var id = TrackedOperationId.New();
        var row = NewRow(id, siteId, status: "Submitted", sourceNode: "node-a");

        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        actor.Tell(new UpsertSiteCallCommand(row), TestActor);
        var reply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.True(reply.Accepted);

        await using var readContext = CreateContext();
        var stored = await readContext.Set<SiteCall>()
            .Where(s => s.TrackedOperationId == id)
            .ToListAsync();
        Assert.Single(stored);
        Assert.Equal("node-a", stored[0].SourceNode);
    }

    /// <summary>Verifies that an upsert command with a null source node persists null.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task UpsertSiteCallCommand_NullSourceNode_PersistsNull()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        // Mirror of the above for unstamped packets — a command with null
        // SourceNode persists NULL on the row rather than falling back to a
        // placeholder. The first audit packet from a legacy host (or a node
        // without INodeIdentityProvider wired) must NOT inject a fabricated
        // value central-side.
        var siteId = NewSiteId();
        var id = TrackedOperationId.New();
        var row = NewRow(id, siteId, status: "Submitted", sourceNode: null);

        await using var context = CreateContext();
        var repo = new SiteCallAuditRepository(context);
        var actor = CreateActor(repo);

        actor.Tell(new UpsertSiteCallCommand(row), TestActor);
        var reply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
        Assert.True(reply.Accepted);

        await using var readContext = CreateContext();
        var stored = await readContext.Set<SiteCall>()
            .Where(s => s.TrackedOperationId == id)
            .ToListAsync();
        Assert.Single(stored);
        Assert.Null(stored[0].SourceNode);
    }

    /// <summary>
    /// Test double whose <see cref="ISiteCallAuditRepository.QueryAsync"/> always
    /// throws — used to verify the query handler's failure projection produces a
    /// <c>Success=false</c> response without crashing the actor.
    /// </summary>
    private sealed class QueryThrowingRepository : ISiteCallAuditRepository
    {
        private readonly ISiteCallAuditRepository _inner;

        /// <summary>Initializes the repository wrapper.</summary>
        /// <param name="inner">The real repository that every call except <c>QueryAsync</c> delegates to.</param>
        public QueryThrowingRepository(ISiteCallAuditRepository inner)
        {
            _inner = inner;
        }

        /// <inheritdoc />
        public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
            _inner.UpsertAsync(siteCall, ct);

        /// <inheritdoc />
        public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
            _inner.GetAsync(id, ct);

        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCall>> QueryAsync(
            SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
            throw new InvalidOperationException("simulated query failure");

        /// <inheritdoc />
        public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
            _inner.PurgeTerminalAsync(olderThanUtc, ct);

        /// <inheritdoc />
        public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputeKpisAsync(stuckCutoff, intervalSince, ct);

        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerSiteKpisAsync(stuckCutoff, intervalSince, ct);

        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerNodeKpisAsync(stuckCutoff, intervalSince, ct);
    }

    /// <summary>
    /// Tiny test double that delegates to a real repository but throws on a
    /// specified <see cref="TrackedOperationId"/>. Used to verify the actor's
    /// fault-isolation behaviour: a transient repository failure must produce
    /// <c>Accepted=false</c> without crashing the singleton.
    /// </summary>
    private sealed class ThrowingRepository : ISiteCallAuditRepository
    {
        private readonly ISiteCallAuditRepository _inner;
        private readonly TrackedOperationId _poisonId;

        /// <summary>Initializes the repository wrapper.</summary>
        /// <param name="inner">The real repository that non-poisoned calls delegate to.</param>
        /// <param name="poisonId">The tracked operation id whose upsert throws.</param>
        public ThrowingRepository(ISiteCallAuditRepository inner, TrackedOperationId poisonId)
        {
            _inner = inner;
            _poisonId = poisonId;
        }

        /// <inheritdoc />
        public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default)
        {
            if (siteCall.TrackedOperationId == _poisonId)
            {
                throw new InvalidOperationException("simulated transient repo failure for poison row");
            }
            return _inner.UpsertAsync(siteCall, ct);
        }

        /// <inheritdoc />
        public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
            _inner.GetAsync(id, ct);

        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCall>> QueryAsync(
            SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
            _inner.QueryAsync(filter, paging, ct);

        /// <inheritdoc />
        public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
            _inner.PurgeTerminalAsync(olderThanUtc, ct);

        /// <inheritdoc />
        public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputeKpisAsync(stuckCutoff, intervalSince, ct);

        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerSiteKpisAsync(stuckCutoff, intervalSince, ct);

        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerNodeKpisAsync(stuckCutoff, intervalSince, ct);
    }
}