ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogIngestActorCombinedTelemetryTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;

namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;

/// <summary>
/// Tests for <see cref="AuditLogIngestActor"/>'s combined-telemetry
/// dual-write transaction. Uses the same <see cref="MsSqlMigrationFixture"/>
/// as the audit-log and site-call repository tests so the actor exercises real
/// <see cref="AuditLogRepository.InsertIfNotExistsAsync"/> +
/// <see cref="SiteCallAuditRepository.UpsertAsync"/> against a per-test MSSQL
/// database. The transaction commits or rolls back inside one
/// <see cref="DbContext.Database"/>.
/// </summary>
public class AuditLogIngestActorCombinedTelemetryTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
    private readonly MsSqlMigrationFixture _fixture;

    /// <summary>Initializes the test class with the shared MSSQL migration fixture.</summary>
    /// <param name="fixture">The shared MSSQL migration fixture.</param>
    public AuditLogIngestActorCombinedTelemetryTests(MsSqlMigrationFixture fixture)
    {
        _fixture = fixture;
    }

    private ScadaBridgeDbContext CreateReadContext()
    {
        var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
            .UseSqlServer(_fixture.ConnectionString)
            .Options;
        return new ScadaBridgeDbContext(options);
    }

    private static string NewSiteId() =>
        "test-bundle-d2-cached-" + Guid.NewGuid().ToString("N").Substring(0, 8);

    private static (AuditEvent audit, SiteCall siteCall) NewEntry(
        string siteId,
        TrackedOperationId? trackedOperationId = null,
        Guid? eventId = null,
        string status = "Submitted",
        AuditStatus auditStatus = AuditStatus.Submitted)
    {
        var trackedId = trackedOperationId ?? TrackedOperationId.New();
        var now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);

        var audit = ScadaBridgeAuditEventFactory.Create(
            eventId: eventId ?? Guid.NewGuid(),
            occurredAtUtc: now,
            channel: AuditChannel.ApiOutbound,
            kind: AuditKind.CachedSubmit,
            status: auditStatus,
            sourceSiteId: siteId,
            correlationId: trackedId.Value);

        var siteCall = new SiteCall
        {
            TrackedOperationId = trackedId,
            Channel = "ApiOutbound",
            Target = "ERP.GetOrder",
            SourceSite = siteId,
            Status = status,
            RetryCount = 0,
            CreatedAtUtc = now,
            UpdatedAtUtc = now,
            IngestedAtUtc = now, // overwritten by the actor
        };

        return (audit, siteCall);
    }

    /// <summary>
    /// Builds a minimal DI container around the per-test MSSQL fixture's
    /// connection string — DbContext + the two repositories the dual-write
    /// handler resolves. Mirrors AddConfigurationDatabase without the
    /// DataProtection wiring (we never write secret columns in these tests).
    /// </summary>
    private IServiceProvider BuildServiceProvider(
        Func<ScadaBridgeDbContext, ISiteCallAuditRepository>? siteCallRepoFactory = null,
        Func<ScadaBridgeDbContext, IAuditLogRepository>? auditRepoFactory = null)
    {
        var services = new ServiceCollection();
        services.AddDbContext<ScadaBridgeDbContext>(opts =>
            opts.UseSqlServer(_fixture.ConnectionString)
                .ConfigureWarnings(w => w.Ignore(
                    Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
        if (auditRepoFactory is null)
        {
            services.AddScoped<IAuditLogRepository>(sp =>
                new AuditLogRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
        }
        else
        {
            services.AddScoped(sp =>
                auditRepoFactory(sp.GetRequiredService<ScadaBridgeDbContext>()));
        }
        if (siteCallRepoFactory is null)
        {
            services.AddScoped<ISiteCallAuditRepository>(sp =>
                new SiteCallAuditRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
        }
        else
        {
            services.AddScoped(sp =>
                siteCallRepoFactory(sp.GetRequiredService<ScadaBridgeDbContext>()));
        }
        return services.BuildServiceProvider();
    }

    private IActorRef CreateActor(IServiceProvider serviceProvider) =>
        Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
            serviceProvider,
            NullLogger<AuditLogIngestActor>.Instance)));

    /// <summary>Verifies that one cached-telemetry packet writes both an audit row and a site-call row and acknowledges the event id.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_OneCachedPacket_WritesAuditRow_AND_SiteCallRow_AcksId()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var (audit, siteCall) = NewEntry(siteId);

        var sp = BuildServiceProvider();
        var actor = CreateActor(sp);

        actor.Tell(
            new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }),
            TestActor);

        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));
        Assert.Single(reply.AcceptedEventIds);
        Assert.Equal(audit.EventId, reply.AcceptedEventIds[0]);

        // Verify rows landed in both tables.
        await using var read = CreateReadContext();
        var auditRow = await read.Set<AuditLogRow>().SingleOrDefaultAsync(e => e.EventId == audit.EventId);
        Assert.NotNull(auditRow);
        Assert.NotNull(auditRow!.IngestedAtUtc);

        var siteCallRow = await read.Set<SiteCall>()
            .SingleOrDefaultAsync(s => s.TrackedOperationId == siteCall.TrackedOperationId);
        Assert.NotNull(siteCallRow);
        Assert.Equal(siteCall.Status, siteCallRow!.Status);
    }

    /// <summary>Verifies that a duplicate event id with an unchanged status is a no-op that leaves the row counts unchanged and still acknowledges the id.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_DuplicateEventId_SameStatus_NoOp_RowCountUnchanged_AcksId()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var trackedId = TrackedOperationId.New();
        var eventId = Guid.NewGuid();
        var (audit, siteCall) = NewEntry(siteId, trackedId, eventId);

        var sp = BuildServiceProvider();
        var actor = CreateActor(sp);

        // First write
        actor.Tell(
            new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }),
            TestActor);
        ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));

        // Second write — same EventId and TrackedOperationId, same status. Both
        // the audit insert (idempotent) and the SiteCalls upsert (monotonic
        // same-rank → no-op) should silently do nothing while still acking.
        actor.Tell(
            new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }),
            TestActor);
        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));

        Assert.Single(reply.AcceptedEventIds);
        Assert.Equal(eventId, reply.AcceptedEventIds[0]);

        await using var read = CreateReadContext();
        var auditCount = await read.Set<AuditLogRow>().CountAsync(e => e.EventId == eventId);
        Assert.Equal(1, auditCount);

        var siteCallCount = await read.Set<SiteCall>()
            .CountAsync(s => s.TrackedOperationId == trackedId);
        Assert.Equal(1, siteCallCount);
    }

    /// <summary>Verifies that a duplicate event id carrying an advanced site-call status updates the existing site-call row.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_DuplicateEventId_AdvancedSiteCallStatus_UpdatesSiteCall()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var trackedId = TrackedOperationId.New();

        var sp = BuildServiceProvider();
        var actor = CreateActor(sp);

        // 1st packet — Submitted (first audit EventId, SiteCalls Status=Submitted).
        var (auditSubmit, siteCallSubmit) = NewEntry(
            siteId, trackedId, status: "Submitted", auditStatus: AuditStatus.Submitted);
        actor.Tell(
            new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(auditSubmit, siteCallSubmit) }),
            TestActor);
        ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));

        // 2nd packet — Attempted with retry count 1 (second audit EventId,
        // SiteCalls Status=Attempted — monotonic upsert wins). Same
        // TrackedOperationId throughout.
        var (auditAttempt, siteCallAttempt) = NewEntry(
            siteId, trackedId, status: "Attempted", auditStatus: AuditStatus.Attempted);
        var advanced = siteCallAttempt with { RetryCount = 1, UpdatedAtUtc = siteCallAttempt.UpdatedAtUtc.AddMinutes(1) };
        actor.Tell(
            new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(auditAttempt, advanced) }),
            TestActor);
        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));

        Assert.Single(reply.AcceptedEventIds);
        Assert.Equal(auditAttempt.EventId, reply.AcceptedEventIds[0]);

        // Both audit rows exist.
        await using var read = CreateReadContext();
        var auditRows = await read.Set<AuditLogRow>()
            .Where(e => e.SourceSiteId == siteId)
            .ToListAsync();
        Assert.Equal(2, auditRows.Count);

        // SiteCalls row advanced to Attempted with retry count 1.
        var siteCallRow = await read.Set<SiteCall>()
            .SingleAsync(s => s.TrackedOperationId == trackedId);
        Assert.Equal("Attempted", siteCallRow.Status);
        Assert.Equal(1, siteCallRow.RetryCount);
    }

    /// <summary>Verifies that when the site-call upsert throws after the audit insert succeeds, both writes are rolled back and no orphan audit row remains.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_AuditInsertSucceeds_SiteCallThrows_BothRolledBack_NoOrphanRow()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var (audit, siteCall) = NewEntry(siteId);

        // Wrap the SiteCalls repo so UpsertAsync always throws — the dual-write
        // transaction must roll back the AuditLog INSERT done in the same
        // transaction, leaving no orphan row.
        var sp = BuildServiceProvider(
            ctx => new ThrowingSiteCallRepo(new SiteCallAuditRepository(ctx)));
        var actor = CreateActor(sp);

        actor.Tell(
            new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }),
            TestActor);

        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));
        Assert.Empty(reply.AcceptedEventIds);

        await using var read = CreateReadContext();
        var auditRow = await read.Set<AuditLogRow>().SingleOrDefaultAsync(e => e.EventId == audit.EventId);
        Assert.Null(auditRow);

        var siteCallRow = await read.Set<SiteCall>()
            .SingleOrDefaultAsync(s => s.TrackedOperationId == siteCall.TrackedOperationId);
        Assert.Null(siteCallRow);
    }

    /// <summary>Verifies that five cached-telemetry packets are each persisted separately and all are acknowledged.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_FiveCachedPackets_AllPersistedSeparately_AllAcked()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var entries = Enumerable.Range(0, 5).Select(_ =>
        {
            var (audit, siteCall) = NewEntry(siteId);
            return new CachedTelemetryEntry(audit, siteCall);
        }).ToList();

        var sp = BuildServiceProvider();
        var actor = CreateActor(sp);

        actor.Tell(new IngestCachedTelemetryCommand(entries), TestActor);

        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));
        Assert.Equal(5, reply.AcceptedEventIds.Count);
        Assert.True(entries.Select(e => e.Audit.EventId).ToHashSet()
            .SetEquals(reply.AcceptedEventIds.ToHashSet()));

        await using var read = CreateReadContext();
        var auditCount = await read.Set<AuditLogRow>().CountAsync(e => e.SourceSiteId == siteId);
        Assert.Equal(5, auditCount);

        var siteCallCount = await read.Set<SiteCall>().CountAsync(s => s.SourceSite == siteId);
        Assert.Equal(5, siteCallCount);
    }

    /// <summary>Verifies that a packet failing after an earlier packet succeeded leaves the first one committed and lets the batch continue.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_OnePacketSucceeds_NextPacketThrows_FirstStillCommitted_BatchContinues()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var (audit1, siteCall1) = NewEntry(siteId);
        var (audit2, siteCall2) = NewEntry(siteId);
        var (audit3, siteCall3) = NewEntry(siteId);
        var poisonTrackedId = siteCall2.TrackedOperationId;

        // Throw only for the middle entry's TrackedOperationId — entries on
        // either side must commit their own transactions independently.
        var sp = BuildServiceProvider(
            ctx => new PoisonOnIdSiteCallRepo(new SiteCallAuditRepository(ctx), poisonTrackedId));
        var actor = CreateActor(sp);

        actor.Tell(
            new IngestCachedTelemetryCommand(new[]
            {
                new CachedTelemetryEntry(audit1, siteCall1),
                new CachedTelemetryEntry(audit2, siteCall2),
                new CachedTelemetryEntry(audit3, siteCall3),
            }),
            TestActor);

        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(15));

        // Two entries committed; poison entry rolled back.
        Assert.Equal(2, reply.AcceptedEventIds.Count);
        Assert.Contains(audit1.EventId, reply.AcceptedEventIds);
        Assert.Contains(audit3.EventId, reply.AcceptedEventIds);
        Assert.DoesNotContain(audit2.EventId, reply.AcceptedEventIds);

        await using var read = CreateReadContext();
        var auditRows = await read.Set<AuditLogRow>().Where(e => e.SourceSiteId == siteId).ToListAsync();
        Assert.Equal(2, auditRows.Count);
        Assert.DoesNotContain(auditRows, r => r.EventId == audit2.EventId);

        var siteCallRows = await read.Set<SiteCall>().Where(s => s.SourceSite == siteId).ToListAsync();
        Assert.Equal(2, siteCallRows.Count);
        Assert.DoesNotContain(siteCallRows, r => r.TrackedOperationId == poisonTrackedId);
    }

    /// <summary>
    /// The per-entry fallback loop (used when the
    /// batch fast path fails) must run under its own short
    /// <see cref="AuditLogIngestActor.IngestFallbackBudget"/>, not an unbounded
    /// wait — otherwise a single slow/wedged downstream call can freeze this
    /// ReceiveAsync handler, and with it the central ingest singleton's mailbox,
    /// for as long as the SQL retry ladder allows (minutes). Forces the batch
    /// fast path to fail immediately (via <see cref="BatchFailingAuditRepo"/>) so
    /// the per-entry fallback runs, then hangs the first entry's SiteCalls
    /// upsert (via <see cref="HangingSiteCallRepo"/>) to prove the fallback's own
    /// token — not <see cref="CancellationToken.None"/> — reaches the repository
    /// call and bounds the wait.
    /// </summary>
    /// <remarks>
    /// Pre-fix, the per-entry loop passed no token to
    /// <c>BeginTransactionAsync</c>/<c>InsertIfNotExistsAsync</c>/<c>UpsertAsync</c>,
    /// so <see cref="HangingSiteCallRepo"/>'s <c>Task.Delay(Timeout.InfiniteTimeSpan, ct)</c>
    /// would never observe a cancellation and hang forever — this test's bounded
    /// <c>ExpectMsg</c> below would time out and fail, which is the fail-unfixed
    /// proof for this fix.
    /// </remarks>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]
    public async Task Receive_CachedTelemetry_PerEntryFallbackHangs_BudgetBoundsWait_RemainderSkipped()
    {
        Skip.IfNot(_fixture.Available, _fixture.SkipReason);

        var siteId = NewSiteId();
        var entries = Enumerable.Range(0, 3).Select(_ =>
        {
            var (audit, siteCall) = NewEntry(siteId);
            return new CachedTelemetryEntry(audit, siteCall);
        }).ToList();

        var hangingRepo = new HangingSiteCallRepo();
        var sp = BuildServiceProvider(
            siteCallRepoFactory: _ => hangingRepo,
            auditRepoFactory: ctx => new BatchFailingAuditRepo(new AuditLogRepository(ctx)));
        var actor = CreateActor(sp);

        var stopwatch = System.Diagnostics.Stopwatch.StartNew();
        actor.Tell(new IngestCachedTelemetryCommand(entries), TestActor);

        // Budget is IngestFallbackBudget (5s). Bound the wait comfortably above
        // that but far below "hangs indefinitely" — on unfixed code (no token
        // passed to the fallback loop) this ExpectMsg times out and the test
        // fails, which is the point: it proves the mailbox would otherwise be
        // frozen for the life of the hang, not just for one short budget.
        var reply = ExpectMsg<IngestCachedTelemetryReply>(TimeSpan.FromSeconds(20));
        stopwatch.Stop();

        Assert.True(
            stopwatch.Elapsed < TimeSpan.FromSeconds(15),
            $"handler took {stopwatch.Elapsed} — the per-entry fallback budget should bound the wait to ~{AuditLogIngestActor.IngestFallbackBudget.TotalSeconds}s, not the full hang.");

        // Only the first entry pays the hang; once the budget is exhausted the
        // remaining two are skipped without even being attempted. None of the
        // three entries commit (the first was cancelled mid-transaction; the
        // rest were never started).
        Assert.Empty(reply.AcceptedEventIds);
        Assert.Equal(1, hangingRepo.CallCount);

        // The token reaching the repository call is the fallback's OWN fresh
        // token — non-default and cancellable — never CancellationToken.None.
        Assert.NotNull(hangingRepo.ReceivedToken);
        Assert.NotEqual(CancellationToken.None, hangingRepo.ReceivedToken!.Value);
        Assert.True(hangingRepo.ReceivedToken.Value.CanBeCanceled);

        await using var read = CreateReadContext();
        var siteCallCount = await read.Set<SiteCall>().CountAsync(s => s.SourceSite == siteId);
        Assert.Equal(0, siteCallCount);
    }

    /// <summary>
    /// Test double — forces <see cref="AuditLogIngestActor"/>'s cached-telemetry
    /// batch fast path (<c>TryIngestCachedBatchAsync</c>) to fail immediately by
    /// throwing from <see cref="InsertManyIfNotExistsAsync"/>, so tests can reach
    /// the per-entry fallback loop without waiting out the real 20s
    /// <see cref="AuditLogIngestActor.IngestBudget"/>. Every other member
    /// delegates to a real <see cref="AuditLogRepository"/> so the fallback path
    /// itself still exercises genuine SQL.
    /// </summary>
    private sealed class BatchFailingAuditRepo : IAuditLogRepository
    {
        /// <inheritdoc />
        public Task<IReadOnlyList<AuditEvent>> GetTraceEventsAsync(Guid executionId, CancellationToken ct = default, IReadOnlyList<string>? permittedSiteIds = null)
            => throw new NotSupportedException("GetTraceEventsAsync is not exercised by this test double.");

        private readonly IAuditLogRepository _inner;
        /// <summary>Initializes the double with the real repository that every non-failing member delegates to.</summary>
        /// <param name="inner">The real repository to delegate to.</param>
        public BatchFailingAuditRepo(IAuditLogRepository inner) { _inner = inner; }

        /// <inheritdoc />
        public Task<int> InsertManyIfNotExistsAsync(
            IReadOnlyList<AuditEvent> events, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
            throw new InvalidOperationException("simulated batch ingest failure — forces the per-entry fallback");

        /// <inheritdoc />
        public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default) =>
            _inner.InsertIfNotExistsAsync(evt, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<AuditEvent>> QueryAsync(
            AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
            _inner.QueryAsync(filter, paging, ct);
        /// <inheritdoc />
        public Task<long> SwitchOutPartitionAsync(
            DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
            _inner.SwitchOutPartitionAsync(monthBoundary, commandTimeout, ct);
        /// <inheritdoc />
        public Task<long> PurgeChannelOlderThanAsync(
            string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default, int maxRows = 50_000) =>
            _inner.PurgeChannelOlderThanAsync(channel, threshold, batchSize, commandTimeout, ct, maxRows);
        /// <inheritdoc />
        public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
            DateTime threshold, CancellationToken ct = default) =>
            _inner.GetPartitionBoundariesOlderThanAsync(threshold, ct);
        /// <inheritdoc />
        public Task<ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot> GetKpiSnapshotAsync(
            TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default) =>
            _inner.GetKpiSnapshotAsync(window, nowUtc, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
            Guid executionId, CancellationToken ct = default, IReadOnlyList<string>? permittedSiteIds = null) =>
            _inner.GetExecutionTreeAsync(executionId, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default) =>
            _inner.GetDistinctSourceNodesAsync(ct);
        /// <inheritdoc />
        public Task<long> BackfillSourceNodeAsync(
            string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
            _inner.BackfillSourceNodeAsync(sentinel, before, batchSize, ct);
    }

    /// <summary>
    /// Test double — <see cref="UpsertAsync"/> hangs on
    /// <c>Task.Delay(Timeout.InfiniteTimeSpan, ct)</c> until the caller's token
    /// cancels, recording the call count and the token it was handed. Used to
    /// prove the per-entry fallback loop's own budget bounds the wait rather blocking forever on a wedged downstream call.
    /// </summary>
    private sealed class HangingSiteCallRepo : ISiteCallAuditRepository
    {
        private int _callCount;
        /// <summary>Gets the number of times <see cref="UpsertAsync"/> has been invoked.</summary>
        public int CallCount => _callCount;
        /// <summary>Gets the cancellation token most recently passed to <see cref="UpsertAsync"/>.</summary>
        public CancellationToken? ReceivedToken { get; private set; }

        /// <inheritdoc />
        public async Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default)
        {
            Interlocked.Increment(ref _callCount);
            ReceivedToken = ct;
            await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false);
        }

        /// <inheritdoc />
        public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
            throw new NotSupportedException();
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCall>> QueryAsync(
            SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
            throw new NotSupportedException();
        /// <inheritdoc />
        public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
            throw new NotSupportedException();
        /// <inheritdoc />
        public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            throw new NotSupportedException();
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            throw new NotSupportedException();
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            throw new NotSupportedException();
    }

    /// <summary>
    /// Test double — throws unconditionally from <see cref="UpsertAsync"/> so
    /// the dual-write transaction is forced to roll back. Lets the AuditLog
    /// row insert succeed in-transaction; the rollback must remove it.
    /// </summary>
    private sealed class ThrowingSiteCallRepo : ISiteCallAuditRepository
    {
        private readonly ISiteCallAuditRepository _inner;
        /// <summary>Initializes the double with the real repository that read members delegate to.</summary>
        /// <param name="inner">The real repository to delegate to.</param>
        public ThrowingSiteCallRepo(ISiteCallAuditRepository inner) { _inner = inner; }
        /// <inheritdoc />
        public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
            throw new InvalidOperationException("simulated SiteCalls upsert failure");
        /// <inheritdoc />
        public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
            _inner.GetAsync(id, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCall>> QueryAsync(
            SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
            _inner.QueryAsync(filter, paging, ct);
        /// <inheritdoc />
        public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
            _inner.PurgeTerminalAsync(olderThanUtc, ct);
        /// <inheritdoc />
        public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputeKpisAsync(stuckCutoff, intervalSince, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerSiteKpisAsync(stuckCutoff, intervalSince, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerNodeKpisAsync(stuckCutoff, intervalSince, ct);
    }

    /// <summary>
    /// Test double — throws only when the supplied poison TrackedOperationId
    /// is the one being upserted. Demonstrates per-entry transaction isolation:
    /// one entry's failed transaction must not abort the batch's other entries.
    /// </summary>
    private sealed class PoisonOnIdSiteCallRepo : ISiteCallAuditRepository
    {
        private readonly ISiteCallAuditRepository _inner;
        private readonly TrackedOperationId _poisonId;
        /// <summary>Initializes the double with the real repository and the id whose upsert should fail.</summary>
        /// <param name="inner">The real repository to delegate to.</param>
        /// <param name="poisonId">The tracked operation id whose upsert throws.</param>
        public PoisonOnIdSiteCallRepo(ISiteCallAuditRepository inner, TrackedOperationId poisonId)
        {
            _inner = inner;
            _poisonId = poisonId;
        }
        /// <inheritdoc />
        public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default)
        {
            if (siteCall.TrackedOperationId == _poisonId)
            {
                throw new InvalidOperationException("simulated SiteCalls upsert failure for poison id");
            }
            return _inner.UpsertAsync(siteCall, ct);
        }
        /// <inheritdoc />
        public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
            _inner.GetAsync(id, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCall>> QueryAsync(
            SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
            _inner.QueryAsync(filter, paging, ct);
        /// <inheritdoc />
        public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
            _inner.PurgeTerminalAsync(olderThanUtc, ct);
        /// <inheritdoc />
        public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputeKpisAsync(stuckCutoff, intervalSince, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerSiteKpisAsync(stuckCutoff, intervalSince, ct);
        /// <inheritdoc />
        public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
            _inner.ComputePerNodeKpisAsync(stuckCutoff, intervalSince, ct);
    }
}