ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorSweepPacingTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;

/// <summary>
/// Regression tests for the dispatch sweep's pacing (review 2026-10-21): per-attempt
/// timestamps (06/COR-1), the per-sweep wall-clock budget (06/PERF-5), and the Email
/// concurrency clamp to the SMTP row's <c>MaxConcurrentConnections</c> (06/IMPL-2).
/// </summary>
public class NotificationOutboxActorSweepPacingTests : TestKit
{
    private readonly INotificationOutboxRepository _outboxRepository =
        OutboxRepositorySubstitute.Healthy();

    private readonly INotificationRepository _notificationRepository =
        Substitute.For<INotificationRepository>();

    private IActorRef CreateActor(
        IEnumerable<INotificationDeliveryAdapter> adapters, NotificationOutboxOptions options)
    {
        var services = new ServiceCollection();
        services.AddScoped(_ => _outboxRepository);
        services.AddScoped(_ => _notificationRepository);
        foreach (var adapter in adapters)
        {
            services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
        }

        return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
            services.BuildServiceProvider(),
            options,
            new NoOpCentralAuditWriter(),
            NullLogger<NotificationOutboxActor>.Instance)));
    }

    private static Notification MakeNotification(NotificationType type = NotificationType.Email) =>
        new(Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1")
        {
            CreatedAt = DateTimeOffset.UtcNow,
        };

    private void SetupSmtp(int maxConcurrentConnections = 0, string? transport = null)
    {
        var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
        {
            MaxRetries = 5,
            RetryDelay = TimeSpan.FromMinutes(1),
            MaxConcurrentConnections = maxConcurrentConnections,
            Transport = transport,
        };
        _notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
            .Returns(new[] { config });
    }

    private void SetupDueBatchOnce(params Notification[] batch)
    {
        var claimed = 0;
        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
            .Returns(_ => Interlocked.Increment(ref claimed) == 1
                ? batch
                : Array.Empty<Notification>());
    }

    /// <summary>
    /// Adapter that delays every delivery, records the concurrency high-water mark and the
    /// instant each delivery completed.
    /// </summary>
    private sealed class PacedAdapter(
        TimeSpan delay, Func<DeliveryOutcome> outcome, NotificationType type = NotificationType.Email)
        : INotificationDeliveryAdapter
    {
        private int _current;
        private int _highWater;
        private int _calls;

        public NotificationType Type => type;

        public int HighWater => Volatile.Read(ref _highWater);

        public int Calls => Volatile.Read(ref _calls);

        public DateTimeOffset LastCompletedAt;

        public async Task<DeliveryOutcome> DeliverAsync(
            Notification notification, CancellationToken cancellationToken = default)
        {
            Interlocked.Increment(ref _calls);
            var current = Interlocked.Increment(ref _current);
            int hw;
            while ((hw = Volatile.Read(ref _highWater)) < current
                && Interlocked.CompareExchange(ref _highWater, current, hw) != hw)
            {
                // retry until our peak is recorded
            }

            await Task.Delay(delay, cancellationToken);
            Interlocked.Decrement(ref _current);
            LastCompletedAt = DateTimeOffset.UtcNow;
            return outcome();
        }
    }

    /// <summary>
    /// 06/COR-1: a delivery that completes well after the sweep began is stamped with the
    /// attempt instant, not the sweep-start instant.
    /// </summary>
    [Fact]
    public void Success_StampsDeliveredAt_AfterTheAdapterReturns()
    {
        SetupSmtp();
        var notification = MakeNotification();
        SetupDueBatchOnce(notification);
        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x"));
        var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) });
        var before = DateTimeOffset.UtcNow;

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
            Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
            Arg.Any<CancellationToken>()));
        Assert.True(notification.DeliveredAt >= before + TimeSpan.FromMilliseconds(350),
            $"DeliveredAt {notification.DeliveredAt:O} should be after the 400 ms attempt (sweep began {before:O})");
        Assert.Equal(notification.DeliveredAt, notification.LastAttemptAt);
    }

    /// <summary>
    /// 06/COR-1: a transient failure schedules the retry relative to the attempt instant, so a
    /// slow attempt cannot collapse the retry delay.
    /// </summary>
    [Fact]
    public void TransientFailure_SchedulesNextAttempt_FromTheAttemptInstant()
    {
        SetupSmtp();
        var notification = MakeNotification();
        SetupDueBatchOnce(notification);
        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Transient("timeout"));
        var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) });
        var before = DateTimeOffset.UtcNow;

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
            Arg.Is<Notification>(n => n.Status == NotificationStatus.Retrying),
            Arg.Any<CancellationToken>()));
        Assert.True(notification.LastAttemptAt >= before + TimeSpan.FromMilliseconds(350));
        Assert.Equal(notification.LastAttemptAt + TimeSpan.FromMinutes(1), notification.NextAttemptAt);
    }

    /// <summary>
    /// 06/PERF-5: once the sweep's wall-clock budget is spent, deliveries still waiting for a
    /// permit are skipped (left due for the next sweep) instead of extending the sweep.
    /// </summary>
    [Fact]
    public void SweepBudget_Exhausted_SkipsDeliveriesThatHaveNotStarted()
    {
        SetupSmtp();
        SetupDueBatchOnce(MakeNotification(), MakeNotification(), MakeNotification());
        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(600), () => DeliveryOutcome.Success("ops@x"));
        var actor = CreateActor([adapter], new NotificationOutboxOptions
        {
            DispatchInterval = TimeSpan.FromHours(1),
            MaxParallelDeliveries = 1,
            DispatchSweepBudget = TimeSpan.FromMilliseconds(200),
        });

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
            Arg.Any<Notification>(), Arg.Any<CancellationToken>()));
        // Give the (skipped) rest of the sweep ample time to have run had it not been skipped.
        Thread.Sleep(1500);
        Assert.Equal(1, adapter.Calls);
        _outboxRepository.Received(1).UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
    }

    /// <summary>
    /// 06/IMPL-2: Email deliveries over the SMTP transport are clamped to the SMTP row's
    /// MaxConcurrentConnections even when MaxParallelDeliveries is higher.
    /// </summary>
    [Fact]
    public void SmtpTransport_ClampsEmailParallelism_ToMaxConcurrentConnections()
    {
        SetupSmtp(maxConcurrentConnections: 1);
        SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray());
        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(150), () => DeliveryOutcome.Success("ops@x"));
        var actor = CreateActor([adapter], new NotificationOutboxOptions
        {
            DispatchInterval = TimeSpan.FromHours(1),
            MaxParallelDeliveries = 4,
        });

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () => _outboxRepository.Received(4).UpdateAsync(
                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(10));
        Assert.Equal(1, adapter.HighWater);
    }

    /// <summary>
    /// 06/IMPL-2: the EWS transport does not use MaxConcurrentConnections, so its deliveries
    /// keep the full MaxParallelDeliveries.
    /// </summary>
    [Fact]
    public void EwsTransport_IsNotClampedByMaxConcurrentConnections()
    {
        SetupSmtp(maxConcurrentConnections: 1, transport: "Ews");
        SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray());
        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x"));
        var actor = CreateActor([adapter], new NotificationOutboxOptions
        {
            DispatchInterval = TimeSpan.FromHours(1),
            MaxParallelDeliveries = 4,
        });

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () => _outboxRepository.Received(4).UpdateAsync(
                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(10));
        Assert.True(adapter.HighWater >= 2, $"expected concurrent EWS deliveries, got {adapter.HighWater}");
    }

    /// <summary>
    /// 06/IMPL-2: the SMTP clamp applies to Email only — SMS deliveries in the same sweep keep
    /// the full MaxParallelDeliveries.
    /// </summary>
    [Fact]
    public void SmtpClamp_DoesNotThrottleSms()
    {
        SetupSmtp(maxConcurrentConnections: 1);
        SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification(NotificationType.Sms)).ToArray());
        var sms = new PacedAdapter(
            TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("+1555"), NotificationType.Sms);
        var actor = CreateActor([sms], new NotificationOutboxOptions
        {
            DispatchInterval = TimeSpan.FromHours(1),
            MaxParallelDeliveries = 4,
        });

        actor.Tell(InternalMessages.DispatchTick.Instance);

        AwaitAssert(
            () => _outboxRepository.Received(4).UpdateAsync(
                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
                Arg.Any<CancellationToken>()),
            duration: TimeSpan.FromSeconds(10));
        Assert.True(sms.HighWater >= 2, $"expected concurrent SMS deliveries, got {sms.HighWater}");
    }
}