ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorDispatchTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;

/// <summary>
/// Tests for the <see cref="NotificationOutboxActor"/> dispatcher loop — the
/// periodic sweep that claims due notifications via
/// <see cref="INotificationOutboxRepository.GetDueAsync"/>, delivers each through the
/// matching <see cref="INotificationDeliveryAdapter"/>, and applies the resulting status
/// transition with <see cref="INotificationOutboxRepository.UpdateAsync"/>.
/// </summary>
public class NotificationOutboxActorDispatchTests : TestKit
{
    private readonly INotificationOutboxRepository _outboxRepository =
        OutboxRepositorySubstitute.Healthy();

    private readonly INotificationRepository _notificationRepository =
        Substitute.For<INotificationRepository>();

    private IServiceProvider BuildServiceProvider(
        IEnumerable<INotificationDeliveryAdapter> adapters)
    {
        var services = new ServiceCollection();
        services.AddScoped(_ => _outboxRepository);
        services.AddScoped(_ => _notificationRepository);
        // The actor resolves the channel adapters from its per-sweep DI scope; register
        // each stub adapter under the INotificationDeliveryAdapter service.
        foreach (var adapter in adapters)
        {
            services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
        }

        return services.BuildServiceProvider();
    }

    /// <summary>
    /// Stub adapter whose <see cref="DeliverAsync"/> returns a configurable outcome and
    /// optionally blocks for a delay — used to exercise the overlapping-tick guard.
    /// </summary>
    private sealed class StubAdapter : INotificationDeliveryAdapter
    {
        private readonly Func<DeliveryOutcome> _outcome;
        private readonly TimeSpan _delay;

        /// <summary>Initializes the adapter with the outcome factory, an optional delivery delay and the notification type it serves.</summary>
        /// <param name="outcome">Produces the outcome returned for each delivery.</param>
        /// <param name="delay">Optional delay applied to each delivery.</param>
        /// <param name="type">The notification type this adapter serves.</param>
        public StubAdapter(Func<DeliveryOutcome> outcome, TimeSpan? delay = null,
            NotificationType type = NotificationType.Email)
        {
            _outcome = outcome;
            _delay = delay ?? TimeSpan.Zero;
            Type = type;
        }

        public int CallCount;

        /// <inheritdoc />
        public NotificationType Type { get; }

        /// <inheritdoc />
        public async Task<DeliveryOutcome> DeliverAsync(
            Notification notification, CancellationToken cancellationToken = default)
        {
            Interlocked.Increment(ref CallCount);
            if (_delay > TimeSpan.Zero)
            {
                await Task.Delay(_delay, cancellationToken);
            }

            return _outcome();
        }
    }

    private IActorRef CreateActor(
        IEnumerable<INotificationDeliveryAdapter> adapters,
        NotificationOutboxOptions? options = null)
    {
        return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
            BuildServiceProvider(adapters),
            options ?? new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) },
            new NoOpCentralAuditWriter(),
            NullLogger<NotificationOutboxActor>.Instance)));
    }

    private static Notification MakeNotification(
        NotificationType type = NotificationType.Email, int retryCount = 0)
    {
        return new Notification(
            Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1")
        {
            RetryCount = retryCount,
            CreatedAt = DateTimeOffset.UtcNow,
        };
    }

    private void SetupSmtpRetryPolicy(int maxRetries, TimeSpan retryDelay)
    {
        var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
        {
            MaxRetries = maxRetries,
            RetryDelay = retryDelay,
        };
        _notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
            .Returns(new[] { config });
    }

    /// <summary>Verifies that a dispatch tick claims due notifications and invokes the adapter.</summary>
    [Fact]
    public void DispatchTick_ClaimsDueNotifications_AndInvokesAdapter()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification();
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).GetDueAsync(
                Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>());
            Assert.Equal(1, adapter.CallCount);
        });
    }

    /// <summary>Verifies that an SMS notification is routed to the SMS adapter rather than the email adapter.</summary>
    [Fact]
    public void Dispatch_SmsTypedNotification_RoutesToSmsAdapter_NotEmailAdapter()
    {
        // NotificationOutbox-NNN: the Type-keyed adapter selection must land an SMS-typed
        // notification on the SMS adapter (not the Email one) and apply its outcome.
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification(type: NotificationType.Sms);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var emailAdapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"),
            type: NotificationType.Email);
        var smsAdapter = new StubAdapter(() => DeliveryOutcome.Success("+15551234567"),
            type: NotificationType.Sms);
        var actor = CreateActor([emailAdapter, smsAdapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            Assert.Equal(1, smsAdapter.CallCount);
            Assert.Equal(0, emailAdapter.CallCount);
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Delivered &&
                    n.ResolvedTargets == "+15551234567"),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a successful delivery marks the notification delivered with the resolved targets.</summary>
    [Fact]
    public void Success_MarksNotificationDelivered_WithResolvedTargets()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification();
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Delivered &&
                    n.DeliveredAt != null &&
                    n.LastAttemptAt != null &&
                    n.ResolvedTargets == "ops@example.com" &&
                    n.LastError == null),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a transient failure below the maximum retries marks the notification as retrying and schedules the next attempt.</summary>
    [Fact]
    public void TransientFailure_BelowMaxRetries_MarksRetrying_AndSchedulesNextAttempt()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(3));
        var notification = MakeNotification(retryCount: 1);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Retrying &&
                    n.RetryCount == 2 &&
                    n.NextAttemptAt != null &&
                    n.LastError == "smtp timeout" &&
                    n.LastAttemptAt != null),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a transient failure that reaches the maximum retries parks the notification.</summary>
    [Fact]
    public void TransientFailure_ReachingMaxRetries_MarksParked()
    {
        SetupSmtpRetryPolicy(maxRetries: 3, retryDelay: TimeSpan.FromMinutes(1));
        // RetryCount starts one below the maximum; the failed attempt increments it to the maximum.
        var notification = MakeNotification(retryCount: 2);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Parked &&
                    n.RetryCount == 3),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a permanent failure parks the notification and records the last error.</summary>
    [Fact]
    public void PermanentFailure_MarksParked_WithLastError()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification();
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Permanent("invalid recipient address"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Parked &&
                    n.LastError == "invalid recipient address" &&
                    n.LastAttemptAt != null),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a notification with no adapter for its type is parked with an explanatory error.</summary>
    [Fact]
    public void NoAdapterForType_MarksParked_WithExplanatoryError()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification();
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        // No adapters registered: none resolves for the notification's type.
        var actor = CreateActor([]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Parked &&
                    n.LastError != null &&
                    n.LastError.Contains("no delivery adapter") &&
                    n.LastAttemptAt != null),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a faulted dispatch pass clears the in-flight guard so the next tick still runs.</summary>
    [Fact]
    public void FaultedDispatchPass_ClearsInFlightGuard_SoNextTickStillRuns()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        // GetDueAsync throws on every call: the dispatch pass's task could fault if the
        // failure were not handled, which would leave _dispatching stuck true forever.
        // Count every claim via a thread-safe counter so the assertions can reason about
        // the number of sweeps that actually ran without depending on NSubstitute's
        // exact-count matchers (see the de-race note below).
        var claimAttempts = 0;
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns<IReadOnlyList<Notification>>(_ =>
            {
                Interlocked.Increment(ref claimAttempts);
                throw new InvalidOperationException("db down");
            });
        var actor = CreateActor([]);

        // First tick: the pass faults internally but must still clear the in-flight guard.
        actor.Tell(InternalMessages.DispatchTick.Instance);
        AwaitAssert(
            () => Assert.True(Volatile.Read(ref claimAttempts) >= 1),
            duration: TimeSpan.FromSeconds(10));

        // The guard must clear after the faulted sweep so a fresh tick runs another sweep.
        //
        // De-race (S11): the in-flight guard (_dispatching) is only lowered when the
        // faulted sweep's DispatchComplete round-trips back to the actor — which happens
        // AFTER the GetDueAsync side-effect the barrier above observes. So a single second
        // tick sent right after that barrier can race the guard while it is still up and
        // be silently dropped (the dispatcher's intended overlap protection), leaving the
        // claim count stuck at one forever — a dropped tick is never retried, so widening
        // the timeout cannot recover it. Re-tell the tick on every poll iteration instead:
        // a tick that lands while the guard is still up is harmlessly dropped, and the
        // first tick that lands after the guard lowers runs a fresh sweep. The assertion
        // is "at least two sweeps ran" rather than "exactly two": that is strictly the
        // wedge invariant under test (a wedged guard would pin the count at one no matter
        // how many ticks arrive), and an at-least assertion is immune to the surplus ticks
        // the re-tell driver may land under CPU starvation.
        AwaitAssert(
            () =>
            {
                actor.Tell(InternalMessages.DispatchTick.Instance);
                Assert.True(
                    Volatile.Read(ref claimAttempts) >= 2,
                    "the in-flight guard wedged: a second dispatch tick never ran a fresh sweep");
            },
            duration: TimeSpan.FromSeconds(10));
    }

    /// <summary>Verifies that a transient failure with a maximum retry count of zero retries using the fallback instead of parking immediately.</summary>
    [Fact]
    public void TransientFailure_WithZeroMaxRetries_RetriesUsingFallback_DoesNotParkImmediately()
    {
        // SmtpConfiguration.MaxRetries=0 used to satisfy 1 >= 0 on the very first
        // transient failure and park the row without a single retry. ResolveRetryPolicyAsync
        // now clamps non-positive MaxRetries to the FallbackMaxRetries (10) so transient
        // failures actually retry before parking.
        SetupSmtpRetryPolicy(maxRetries: 0, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification(retryCount: 0);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Retrying &&
                    n.RetryCount == 1 &&
                    n.NextAttemptAt != null &&
                    n.LastError == "smtp timeout"),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a transient failure with a negative maximum retry count retries using the fallback instead of parking immediately.</summary>
    [Fact]
    public void TransientFailure_WithNegativeMaxRetries_RetriesUsingFallback_DoesNotParkImmediately()
    {
        // A negative MaxRetries reaches ResolveRetryPolicyAsync just as -1 — same
        // park-immediately bug. Clamp to FallbackMaxRetries.
        SetupSmtpRetryPolicy(maxRetries: -1, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification(retryCount: 0);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Retrying &&
                    n.RetryCount == 1 &&
                    n.NextAttemptAt != null &&
                    n.LastError == "smtp timeout"),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that a non-positive retry delay is replaced by the fallback delay rather than zero.</summary>
    [Fact]
    public void TransientFailure_WithNonPositiveRetryDelay_UsesFallbackDelay_NotZero()
    {
        // A non-positive RetryDelay would burn-loop the dispatcher because
        // NextAttemptAt would equal now. Clamp to FallbackRetryDelay (1 min) so the
        // schedule actually advances.
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.Zero);
        var before = DateTimeOffset.UtcNow;
        var notification = MakeNotification(retryCount: 0);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Retrying &&
                    n.NextAttemptAt != null &&
                    n.NextAttemptAt > before + TimeSpan.FromSeconds(30)),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that stopping the actor cancels an in-flight delivery and leaves the row non-terminal.</summary>
    [Fact]
    public void PostStop_CancelsInFlightDelivery_LeavesRowNonTerminal()
    {
        // The dispatcher used to drop the CancellationToken on its way into
        // the channel adapter, so a coordinated shutdown had to wait the full SMTP
        // connect/auth/send timeout per in-flight notification before the sweep
        // finished. The actor now passes a lifecycle-scoped token; cancelling it on
        // PostStop must abort the in-flight Task.Delay (standing in for an SMTP
        // send) and the row must NOT be updated to a terminal state — the next
        // active node picks it back up.
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification();
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        // Long delay simulates a slow SMTP send; the test triggers PostStop before
        // the delay would naturally elapse, so the only way the delay completes is
        // if the token wired through.
        var adapter = new StubAdapter(
            () => DeliveryOutcome.Success("ops@example.com"),
            delay: TimeSpan.FromSeconds(30));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);
        // Wait until the adapter is actually in flight before stopping.
        AwaitAssert(() => Assert.Equal(1, adapter.CallCount));

        var start = DateTimeOffset.UtcNow;
        Sys.Stop(actor);

        // The sweep should observe cancellation promptly (well under the 30s delay).
        AwaitAssert(
            () =>
            {
                // No UpdateAsync was issued — the row is untouched and will be re-claimed
                // by the next active node.
                _outboxRepository.DidNotReceive().UpdateAsync(
                    Arg.Any<Notification>(), Arg.Any<CancellationToken>());
            },
            duration: TimeSpan.FromSeconds(5));

        Assert.True(DateTimeOffset.UtcNow - start < TimeSpan.FromSeconds(5),
            "PostStop did not cancel the in-flight delivery promptly.");
    }

    // ── Register row 45: the delivery adapter is resolved PER DELIVERY ────────
    //
    // Supersedes an earlier optimisation ("resolve the adapter dictionary once for the
    // actor's lifetime"). That optimisation cached scoped adapter instances — which hold
    // a scoped INotificationRepository over an EF DbContext — in an actor-lifetime DI
    // scope, so MaxParallelDeliveries concurrent deliveries all ran on ONE DbContext and
    // the rig logged `InvalidOperationException: A second operation was started on this
    // context instance before a previous operation completed` continuously. The adapter
    // now resolves inside the per-delivery scope that DeliverGatedAsync already opens.

    /// <summary>
    /// Aggregates what the parallel deliveries observed: how many times two deliveries
    /// were simultaneously inside ONE simulated DbContext, and a rendezvous that only
    /// completes once every delivery is concurrently in flight.
    /// </summary>
    private sealed class DeliveryConcurrencyLedger
    {
        private readonly int _expectedArrivals;
        private readonly TaskCompletionSource _allArrived =
            new(TaskCreationOptions.RunContinuationsAsynchronously);

        private int _arrived;
        private int _overlaps;

        /// <summary>Initializes the ledger with the number of deliveries expected to reach the rendezvous.</summary>
        /// <param name="expectedArrivals">The number of arrivals that completes the rendezvous.</param>
        public DeliveryConcurrencyLedger(int expectedArrivals) => _expectedArrivals = expectedArrivals;

        /// <summary>Number of second-operation-on-one-context collisions observed.</summary>
        public int Overlaps => Volatile.Read(ref _overlaps);

        /// <summary>Records one overlapping use of the same simulated context.</summary>
        public void RecordOverlap() => Interlocked.Increment(ref _overlaps);

        /// <summary>
        /// Holds the caller's simulated DbContext open until every delivery has reached
        /// this point — guaranteeing real overlap — or a safety timeout elapses. The
        /// timeout matters for the shared-context (pre-fix) run, where the colliding
        /// delivery throws instead of arriving and the rendezvous can never complete.
        /// </summary>
        /// <param name="cancellationToken">Token that cancels the wait.</param>
        /// <returns>A task that completes once every delivery has arrived or the safety timeout elapses.</returns>
        public async Task ArriveAndWaitAsync(CancellationToken cancellationToken)
        {
            if (Interlocked.Increment(ref _arrived) >= _expectedArrivals)
            {
                _allArrived.TrySetResult();
            }

            await Task.WhenAny(
                    _allArrived.Task,
                    Task.Delay(TimeSpan.FromSeconds(2), cancellationToken))
                .ConfigureAwait(false);
        }
    }

    /// <summary>
    /// Stands in for the EF <c>DbContext</c> behind a scoped
    /// <see cref="INotificationRepository"/> — one probe per resolved repository. A second
    /// concurrent operation throws the exact <see cref="InvalidOperationException"/> EF
    /// raises, so a test that shares one repository across parallel deliveries fails the
    /// same way the rig did.
    /// </summary>
    private sealed class SimulatedDbContextProbe
    {
        private readonly DeliveryConcurrencyLedger _ledger;
        private int _inFlight;

        /// <summary>Initializes the probe with the ledger it reports to.</summary>
        /// <param name="ledger">The ledger that records overlaps and coordinates the rendezvous.</param>
        public SimulatedDbContextProbe(DeliveryConcurrencyLedger ledger) => _ledger = ledger;

        /// <summary>Enters the simulated context, failing if another operation is already in flight, and waits at the rendezvous.</summary>
        /// <param name="cancellationToken">Token that cancels the wait.</param>
        /// <returns>A task that represents the asynchronous operation.</returns>
        public async Task EnterAsync(CancellationToken cancellationToken)
        {
            if (Interlocked.Increment(ref _inFlight) > 1)
            {
                Interlocked.Decrement(ref _inFlight);
                _ledger.RecordOverlap();
                throw new InvalidOperationException(
                    "A second operation was started on this context instance before a "
                    + "previous operation completed.");
            }

            try
            {
                await _ledger.ArriveAndWaitAsync(cancellationToken);
            }
            finally
            {
                Interlocked.Decrement(ref _inFlight);
            }
        }
    }

    /// <summary>
    /// Adapter shaped like the real Email/SMS adapters: it does its work THROUGH the
    /// scoped <see cref="INotificationRepository"/> it was constructed with, so sharing
    /// the adapter across deliveries also shares that repository's DbContext.
    /// </summary>
    private sealed class RepositoryBackedAdapter : INotificationDeliveryAdapter
    {
        private readonly INotificationRepository _repository;

        /// <summary>Initializes the adapter with the scoped repository it works through.</summary>
        /// <param name="repository">The scoped notification repository.</param>
        public RepositoryBackedAdapter(INotificationRepository repository) => _repository = repository;

        /// <inheritdoc />
        public NotificationType Type => NotificationType.Email;

        /// <inheritdoc />
        public async Task<DeliveryOutcome> DeliverAsync(
            Notification notification, CancellationToken cancellationToken = default)
        {
            // Mirrors the real adapters' first act: resolve the target list off the
            // scoped repository (and therefore the scoped DbContext).
            await _repository.GetListByNameAsync(notification.ListName, cancellationToken);
            return DeliveryOutcome.Success("ops@example.com");
        }
    }

    private static Task<NotificationList?> EnterProbeAsync(
        SimulatedDbContextProbe probe, CancellationToken cancellationToken)
    {
        return EnterProbeCoreAsync(probe, cancellationToken);

        static async Task<NotificationList?> EnterProbeCoreAsync(
            SimulatedDbContextProbe probe, CancellationToken cancellationToken)
        {
            await probe.EnterAsync(cancellationToken);
            // A null list is enough: the stub adapter ignores the result, and the
            // dispatcher never sees it.
            return null;
        }
    }

    /// <summary>Verifies that parallel deliveries each get their own adapter and DbContext.</summary>
    [Fact]
    public void ParallelDeliveries_EachGetTheirOwnAdapterAndDbContext()
    {
        // Register row 45 (fail-before-fix). Two due notifications dispatched with
        // MaxParallelDeliveries = 4. Each delivery's adapter reaches its scoped
        // INotificationRepository, whose simulated DbContext throws EF's
        // second-operation exception if a second delivery enters it concurrently.
        //
        // Pre-fix: the actor-lifetime _adaptersScope hands BOTH deliveries the same
        // adapter -> the same repository -> one probe. The rendezvous holds the first
        // delivery inside the context while the second collides, so Overlaps == 1 and
        // the collided row is never persisted (DeliverGatedAsync logs and moves on).
        // Post-fix: two per-delivery scopes -> two adapters -> two probes, both arrive
        // at the rendezvous concurrently, nothing collides, both rows persist.
        const int dueCount = 2;
        var ledger = new DeliveryConcurrencyLedger(expectedArrivals: dueCount);

        var outboxRepository = OutboxRepositorySubstitute.Healthy();
        var batch = Enumerable.Range(0, dueCount).Select(_ => MakeNotification()).ToArray();
        var claimed = 0;
        outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty<Notification>());

        var smtpConfiguration = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
        {
            MaxRetries = 5,
            RetryDelay = TimeSpan.FromMinutes(1),
        };

        var services = new ServiceCollection();
        services.AddScoped(_ => outboxRepository);
        // Scoped, with its OWN simulated DbContext per resolution — exactly the
        // production lifetime of INotificationRepository.
        services.AddScoped<INotificationRepository>(_ =>
        {
            var probe = new SimulatedDbContextProbe(ledger);
            var repository = Substitute.For<INotificationRepository>();
            repository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
                .Returns(new[] { smtpConfiguration });
            repository.GetListByNameAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
                .Returns(call => EnterProbeAsync(probe, call.Arg<CancellationToken>()));
            return repository;
        });
        // Scoped adapter holding the scoped repository — the production shape of both
        // EmailNotificationDeliveryAdapter and SmsNotificationDeliveryAdapter.
        services.AddScoped<INotificationDeliveryAdapter>(sp =>
            new RepositoryBackedAdapter(sp.GetRequiredService<INotificationRepository>()));
        var serviceProvider = services.BuildServiceProvider();

        var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
            serviceProvider,
            new NotificationOutboxOptions
            {
                DispatchInterval = TimeSpan.FromHours(1),
                MaxParallelDeliveries = 4,
            },
            new NoOpCentralAuditWriter(),
            NullLogger<NotificationOutboxActor>.Instance)));

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () => outboxRepository.Received(dueCount).UpdateAsync(
                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(10));

        Assert.Equal(0, ledger.Overlaps);
    }

    /// <summary>Verifies that the adapter is resolved per delivery rather than once per actor lifetime.</summary>
    [Fact]
    public void Dispatch_ResolvesAdapterPerDelivery_NotOncePerActorLifetime()
    {
        // Register row 45 (isolation, direct form): every delivery must resolve its own
        // adapter instance from its own DI scope. Four due rows dispatched in one sweep
        // must therefore yield four DISTINCT adapter instances. Pre-fix the actor cached
        // a single instance for its whole lifetime and this collapses to 1.
        const int dueCount = 4;
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));

        var outboxRepository = OutboxRepositorySubstitute.Healthy();
        var batch = Enumerable.Range(0, dueCount).Select(_ => MakeNotification()).ToArray();
        var claimed = 0;
        outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty<Notification>());

        var deliveringInstances = new System.Collections.Concurrent.ConcurrentBag<INotificationDeliveryAdapter>();
        var services = new ServiceCollection();
        services.AddScoped(_ => outboxRepository);
        services.AddScoped(_ => _notificationRepository);
        services.AddScoped<INotificationDeliveryAdapter>(_ => new RecordingStubAdapter(deliveringInstances));
        var serviceProvider = services.BuildServiceProvider();

        var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
            serviceProvider,
            new NotificationOutboxOptions
            {
                DispatchInterval = TimeSpan.FromHours(1),
                MaxParallelDeliveries = 4,
            },
            new NoOpCentralAuditWriter(),
            NullLogger<NotificationOutboxActor>.Instance)));

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () => outboxRepository.Received(dueCount).UpdateAsync(
                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(10));

        // RecordingStubAdapter does not override Equals, so Distinct() is reference
        // identity — exactly the "one instance per delivery" question being asked.
        Assert.Equal(dueCount, deliveringInstances.Distinct().Count());
    }

    /// <summary>
    /// Stub adapter that records which instance actually served each delivery, so a
    /// test can prove one adapter instance per delivery.
    /// </summary>
    private sealed class RecordingStubAdapter : INotificationDeliveryAdapter
    {
        private readonly System.Collections.Concurrent.ConcurrentBag<INotificationDeliveryAdapter> _seen;

        /// <summary>Initializes the adapter with the bag that records each adapter instance used.</summary>
        /// <param name="seen">The bag that collects the adapter instances that performed deliveries.</param>
        public RecordingStubAdapter(
            System.Collections.Concurrent.ConcurrentBag<INotificationDeliveryAdapter> seen) => _seen = seen;

        /// <inheritdoc />
        public NotificationType Type => NotificationType.Email;

        /// <inheritdoc />
        public Task<DeliveryOutcome> DeliverAsync(
            Notification notification, CancellationToken cancellationToken = default)
        {
            _seen.Add(this);
            return Task.FromResult(DeliveryOutcome.Success("ops@example.com"));
        }
    }

    /// <summary>Verifies that overlapping ticks while a dispatch is in flight do not claim notifications concurrently.</summary>
    [Fact]
    public void OverlappingTicks_WhileDispatchInFlight_DoNotClaimConcurrently()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        var notification = MakeNotification();
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        // Slow adapter keeps the first sweep in flight while the second tick arrives.
        var adapter = new StubAdapter(
            () => DeliveryOutcome.Success("ops@example.com"),
            delay: TimeSpan.FromMilliseconds(800));
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);
        actor.Tell(InternalMessages.DispatchTick.Instance);

        // Second tick is dropped by the in-flight guard: only one sweep runs.
        AwaitAssert(
            () => _outboxRepository.Received(1).GetDueAsync(
                Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(2));
    }

    // ── Bounded parallel delivery ───────────────────────────────────

    /// <summary>
    /// Adapter that records the concurrent-invocation high-water mark. Each
    /// <see cref="DeliverAsync"/> increments a live counter, notes the peak, then blocks
    /// on a shared gate until <paramref name="releaseWhen"/> deliveries are concurrently
    /// in flight — proving real overlap. A safety timeout releases the gate so a
    /// run with a parallelism of one (which can never reach <c>releaseWhen</c> concurrently) still
    /// drains instead of deadlocking.
    /// </summary>
    private sealed class ConcurrencyHighWaterAdapter : INotificationDeliveryAdapter
    {
        private readonly int _releaseWhen;
        private readonly TaskCompletionSource _gate =
            new(TaskCreationOptions.RunContinuationsAsynchronously);
        private int _current;
        private int _highWater;
        private int _entered;

        /// <summary>Initializes the adapter with the concurrency level that releases the gate.</summary>
        /// <param name="releaseWhen">The number of concurrent deliveries that releases the gate.</param>
        public ConcurrencyHighWaterAdapter(int releaseWhen) => _releaseWhen = releaseWhen;

        /// <inheritdoc />
        public NotificationType Type => NotificationType.Email;

        /// <summary>Gets the highest number of deliveries observed in flight at once.</summary>
        public int HighWater => Volatile.Read(ref _highWater);

        /// <inheritdoc />
        public async Task<DeliveryOutcome> DeliverAsync(
            Notification notification, CancellationToken cancellationToken = default)
        {
            var current = Interlocked.Increment(ref _current);
            int hw;
            while ((hw = Volatile.Read(ref _highWater)) < current
                && Interlocked.CompareExchange(ref _highWater, current, hw) != hw)
            {
                // retry until our peak is recorded
            }

            if (Interlocked.Increment(ref _entered) >= _releaseWhen)
            {
                _gate.TrySetResult();
            }

            // Overlap window: wait until enough deliveries are concurrently in flight,
            // or a short safety timeout (so the strictly-sequential config still drains).
            await Task.WhenAny(_gate.Task, Task.Delay(TimeSpan.FromMilliseconds(400), cancellationToken))
                .ConfigureAwait(false);

            Interlocked.Decrement(ref _current);
            return DeliveryOutcome.Success("ops@example.com");
        }
    }

    private void SetupDueBatchOnce(int count)
    {
        var batch = Enumerable.Range(0, count).Select(_ => MakeNotification()).ToArray();
        var claimed = 0;
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(_ => Interlocked.Increment(ref claimed) == 1
                ? batch
                : Array.Empty<Notification>());
    }

    /// <summary>Verifies that with a maximum parallelism of four, deliveries run concurrently and all reach a terminal state.</summary>
    [Fact]
    public void Dispatch_WithMaxParallelFour_DeliversConcurrently_AndAllReachTerminal()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        SetupDueBatchOnce(8);
        var adapter = new ConcurrencyHighWaterAdapter(releaseWhen: 2);
        var actor = CreateActor(
            [adapter],
            new NotificationOutboxOptions
            {
                DispatchInterval = TimeSpan.FromHours(1),
                MaxParallelDeliveries = 4,
            });

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () =>
            {
                // All 8 notifications reached a terminal Delivered update.
                _outboxRepository.Received(8).UpdateAsync(
                    Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                    Arg.Any<CancellationToken>());
                // At least two deliveries overlapped — the sweep is no longer sequential.
                Assert.True(adapter.HighWater >= 2,
                    $"expected concurrent deliveries (high-water >= 2), got {adapter.HighWater}");
            },
            duration: TimeSpan.FromSeconds(10));
    }

    /// <summary>Verifies that with a maximum parallelism of one, deliveries run sequentially and all reach a terminal state.</summary>
    [Fact]
    public void Dispatch_WithMaxParallelOne_DeliversSequentially_AndAllReachTerminal()
    {
        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
        SetupDueBatchOnce(8);
        var adapter = new ConcurrencyHighWaterAdapter(releaseWhen: 2);
        var actor = CreateActor(
            [adapter],
            new NotificationOutboxOptions
            {
                DispatchInterval = TimeSpan.FromHours(1),
                MaxParallelDeliveries = 1,
            });

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () => _outboxRepository.Received(8).UpdateAsync(
                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(10));

        // Strictly sequential: never more than one delivery in flight at once.
        Assert.Equal(1, adapter.HighWater);
    }

    // ── Per-type retry policy (SMS honors SmsConfiguration) ──────────

    private void SetupSmsRetryPolicy(int maxRetries, TimeSpan retryDelay)
    {
        var config = new SmsConfiguration("ACtestaccountsid", "+15550000000")
        {
            MaxRetries = maxRetries,
            RetryDelay = retryDelay,
        };
        _notificationRepository.GetSmsConfigurationAsync(Arg.Any<CancellationToken>())
            .Returns(config);
    }

    /// <summary>Verifies that a transient failure on an SMS notification uses the SMS retry policy.</summary>
    [Fact]
    public void SmsNotification_TransientFailure_UsesSmsRetryPolicy()
    {
        // An SMS-typed notification resolves its retry policy from the SMS
        // configuration, NOT the shared SMTP policy. The SMTP policy (MaxRetries=10)
        // would keep this row Retrying; the SMS policy (MaxRetries=1) parks it after
        // this attempt (RetryCount 1 -> 2 >= 1). A pre-fix dispatcher (SMTP-for-all)
        // would leave it Retrying with RetryCount=2 — the assertion below fails.
        SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(1));
        SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5));
        var notification = MakeNotification(type: NotificationType.Sms, retryCount: 1);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("twilio 503"),
            type: NotificationType.Sms);
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Parked &&
                    n.RetryCount == 2 &&
                    n.LastError == "twilio 503"),
                Arg.Any<CancellationToken>());
        });
    }

    /// <summary>Verifies that an email notification still uses the SMTP retry policy.</summary>
    [Fact]
    public void EmailNotification_StillUsesSmtpPolicy()
    {
        // An Email row continues to resolve against the SMTP policy even when a
        // (tighter) SMS policy is present. SMTP MaxRetries=10 keeps it Retrying
        // (RetryCount 1 -> 2 < 10) and schedules the next attempt with the SMTP delay.
        SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(3));
        SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5));
        var before = DateTimeOffset.UtcNow;
        var notification = MakeNotification(type: NotificationType.Email, retryCount: 1);
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(new[] { notification });
        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"),
            type: NotificationType.Email);
        var actor = CreateActor([adapter]);

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() =>
        {
            _outboxRepository.Received(1).UpdateAsync(
                Arg.Is<Notification>(n =>
                    n.Status == NotificationStatus.Retrying &&
                    n.RetryCount == 2 &&
                    n.NextAttemptAt != null &&
                    n.NextAttemptAt > before + TimeSpan.FromMinutes(2) &&
                    n.LastError == "smtp timeout"),
                Arg.Any<CancellationToken>());
        });
    }
}