fix(notification-outbox): per-attempt timestamps, Pending/Retrying split claim + sweep budget, SMTP MaxConcurrentConnections clamp (review 2026-10-21 06/COR-1, 06/PERF-5, 06/IMPL-2)

2026-10-22 09:22:10 -04:00
parent ebf4b96789
commit 811a561885
10 changed files with 547 additions and 37 deletions
@@ -567,6 +567,71 @@ public class NotificationOutboxRepositoryTests : IDisposable
        Assert.Equal(retryingDue.NotificationId, capped[1].NotificationId);
    }

+    /// <summary>
+    /// Verifies that a retrying backlog older than every fresh Pending row cannot monopolise
+    /// the claim: the batch is split between the two statuses, each oldest-first
+    /// (review 2026-10-21 06/PERF-5).
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task GetDueAsync_RetryingBacklog_SharesBatchWithPending()
+    {
+        var now = new DateTimeOffset(2026, 5, 19, 12, 0, 0, TimeSpan.Zero);
+        var retrying = Enumerable.Range(0, 4)
+            .Select(i => MakeNotification(Guid.NewGuid().ToString(), NotificationStatus.Retrying,
+                createdAt: now.AddHours(-10).AddMinutes(i), nextAttemptAt: now.AddMinutes(-1)))
+            .ToList();
+        var pending = Enumerable.Range(0, 4)
+            .Select(i => MakeNotification(Guid.NewGuid().ToString(), NotificationStatus.Pending,
+                createdAt: now.AddMinutes(-10 + i)))
+            .ToList();
+        _context.Notifications.AddRange(retrying);
+        _context.Notifications.AddRange(pending);
+        await _context.SaveChangesAsync();
+        _context.ChangeTracker.Clear();
+
+        var due = await _repository.GetDueAsync(now, batchSize: 4);
+
+        Assert.Equal(
+            new[]
+            {
+                retrying[0].NotificationId, retrying[1].NotificationId,
+                pending[0].NotificationId, pending[1].NotificationId,
+            },
+            due.Select(n => n.NotificationId).ToArray());
+    }
+
+    /// <summary>
+    /// Verifies that an unused share of the batch is back-filled from the other status, so a
+    /// sweep with a short Retrying (or Pending) set still claims a full batch.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task GetDueAsync_ShortRetryingSet_BackfillsFromPending()
+    {
+        var now = new DateTimeOffset(2026, 5, 19, 12, 0, 0, TimeSpan.Zero);
+        var retrying = MakeNotification(Guid.NewGuid().ToString(), NotificationStatus.Retrying,
+            createdAt: now.AddHours(-10), nextAttemptAt: now.AddMinutes(-1));
+        var pending = Enumerable.Range(0, 5)
+            .Select(i => MakeNotification(Guid.NewGuid().ToString(), NotificationStatus.Pending,
+                createdAt: now.AddMinutes(-10 + i)))
+            .ToList();
+        _context.Notifications.Add(retrying);
+        _context.Notifications.AddRange(pending);
+        await _context.SaveChangesAsync();
+        _context.ChangeTracker.Clear();
+
+        var due = await _repository.GetDueAsync(now, batchSize: 4);
+
+        Assert.Equal(
+            new[]
+            {
+                retrying.NotificationId,
+                pending[0].NotificationId, pending[1].NotificationId, pending[2].NotificationId,
+            },
+            due.Select(n => n.NotificationId).ToArray());
+    }
+
    /// <summary>Verifies that get by id returns the row when present and null otherwise.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
@@ -0,0 +1,265 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
+using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
+using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport;
+
+namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
+
+/// <summary>
+/// Regression tests for the dispatch sweep's pacing (review 2026-10-21): per-attempt
+/// timestamps (06/COR-1), the per-sweep wall-clock budget (06/PERF-5), and the Email
+/// concurrency clamp to the SMTP row's <c>MaxConcurrentConnections</c> (06/IMPL-2).
+/// </summary>
+public class NotificationOutboxActorSweepPacingTests : TestKit
+{
+    private readonly INotificationOutboxRepository _outboxRepository =
+        OutboxRepositorySubstitute.Healthy();
+
+    private readonly INotificationRepository _notificationRepository =
+        Substitute.For<INotificationRepository>();
+
+    private IActorRef CreateActor(
+        IEnumerable<INotificationDeliveryAdapter> adapters, NotificationOutboxOptions options)
+    {
+        var services = new ServiceCollection();
+        services.AddScoped(_ => _outboxRepository);
+        services.AddScoped(_ => _notificationRepository);
+        foreach (var adapter in adapters)
+        {
+            services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
+        }
+
+        return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
+            services.BuildServiceProvider(),
+            options,
+            new NoOpCentralAuditWriter(),
+            NullLogger<NotificationOutboxActor>.Instance)));
+    }
+
+    private static Notification MakeNotification(NotificationType type = NotificationType.Email) =>
+        new(Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1")
+        {
+            CreatedAt = DateTimeOffset.UtcNow,
+        };
+
+    private void SetupSmtp(int maxConcurrentConnections = 0, string? transport = null)
+    {
+        var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
+        {
+            MaxRetries = 5,
+            RetryDelay = TimeSpan.FromMinutes(1),
+            MaxConcurrentConnections = maxConcurrentConnections,
+            Transport = transport,
+        };
+        _notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
+            .Returns(new[] { config });
+    }
+
+    private void SetupDueBatchOnce(params Notification[] batch)
+    {
+        var claimed = 0;
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(_ => Interlocked.Increment(ref claimed) == 1
+                ? batch
+                : Array.Empty<Notification>());
+    }
+
+    /// <summary>
+    /// Adapter that delays every delivery, records the concurrency high-water mark and the
+    /// instant each delivery completed.
+    /// </summary>
+    private sealed class PacedAdapter(
+        TimeSpan delay, Func<DeliveryOutcome> outcome, NotificationType type = NotificationType.Email)
+        : INotificationDeliveryAdapter
+    {
+        private int _current;
+        private int _highWater;
+        private int _calls;
+
+        public NotificationType Type => type;
+
+        public int HighWater => Volatile.Read(ref _highWater);
+
+        public int Calls => Volatile.Read(ref _calls);
+
+        public DateTimeOffset LastCompletedAt;
+
+        public async Task<DeliveryOutcome> DeliverAsync(
+            Notification notification, CancellationToken cancellationToken = default)
+        {
+            Interlocked.Increment(ref _calls);
+            var current = Interlocked.Increment(ref _current);
+            int hw;
+            while ((hw = Volatile.Read(ref _highWater)) < current
+                && Interlocked.CompareExchange(ref _highWater, current, hw) != hw)
+            {
+                // retry until our peak is recorded
+            }
+
+            await Task.Delay(delay, cancellationToken);
+            Interlocked.Decrement(ref _current);
+            LastCompletedAt = DateTimeOffset.UtcNow;
+            return outcome();
+        }
+    }
+
+    /// <summary>
+    /// 06/COR-1: a delivery that completes well after the sweep began is stamped with the
+    /// attempt instant, not the sweep-start instant.
+    /// </summary>
+    [Fact]
+    public void Success_StampsDeliveredAt_AfterTheAdapterReturns()
+    {
+        SetupSmtp();
+        var notification = MakeNotification();
+        SetupDueBatchOnce(notification);
+        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x"));
+        var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) });
+        var before = DateTimeOffset.UtcNow;
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
+            Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
+            Arg.Any<CancellationToken>()));
+        Assert.True(notification.DeliveredAt >= before + TimeSpan.FromMilliseconds(350),
+            $"DeliveredAt {notification.DeliveredAt:O} should be after the 400 ms attempt (sweep began {before:O})");
+        Assert.Equal(notification.DeliveredAt, notification.LastAttemptAt);
+    }
+
+    /// <summary>
+    /// 06/COR-1: a transient failure schedules the retry relative to the attempt instant, so a
+    /// slow attempt cannot collapse the retry delay.
+    /// </summary>
+    [Fact]
+    public void TransientFailure_SchedulesNextAttempt_FromTheAttemptInstant()
+    {
+        SetupSmtp();
+        var notification = MakeNotification();
+        SetupDueBatchOnce(notification);
+        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Transient("timeout"));
+        var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) });
+        var before = DateTimeOffset.UtcNow;
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
+            Arg.Is<Notification>(n => n.Status == NotificationStatus.Retrying),
+            Arg.Any<CancellationToken>()));
+        Assert.True(notification.LastAttemptAt >= before + TimeSpan.FromMilliseconds(350));
+        Assert.Equal(notification.LastAttemptAt + TimeSpan.FromMinutes(1), notification.NextAttemptAt);
+    }
+
+    /// <summary>
+    /// 06/PERF-5: once the sweep's wall-clock budget is spent, deliveries still waiting for a
+    /// permit are skipped (left due for the next sweep) instead of extending the sweep.
+    /// </summary>
+    [Fact]
+    public void SweepBudget_Exhausted_SkipsDeliveriesThatHaveNotStarted()
+    {
+        SetupSmtp();
+        SetupDueBatchOnce(MakeNotification(), MakeNotification(), MakeNotification());
+        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(600), () => DeliveryOutcome.Success("ops@x"));
+        var actor = CreateActor([adapter], new NotificationOutboxOptions
+        {
+            DispatchInterval = TimeSpan.FromHours(1),
+            MaxParallelDeliveries = 1,
+            DispatchSweepBudget = TimeSpan.FromMilliseconds(200),
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
+            Arg.Any<Notification>(), Arg.Any<CancellationToken>()));
+        // Give the (skipped) rest of the sweep ample time to have run had it not been skipped.
+        Thread.Sleep(1500);
+        Assert.Equal(1, adapter.Calls);
+        _outboxRepository.Received(1).UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
+    }
+
+    /// <summary>
+    /// 06/IMPL-2: Email deliveries over the SMTP transport are clamped to the SMTP row's
+    /// MaxConcurrentConnections even when MaxParallelDeliveries is higher.
+    /// </summary>
+    [Fact]
+    public void SmtpTransport_ClampsEmailParallelism_ToMaxConcurrentConnections()
+    {
+        SetupSmtp(maxConcurrentConnections: 1);
+        SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray());
+        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(150), () => DeliveryOutcome.Success("ops@x"));
+        var actor = CreateActor([adapter], new NotificationOutboxOptions
+        {
+            DispatchInterval = TimeSpan.FromHours(1),
+            MaxParallelDeliveries = 4,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(
+            () => _outboxRepository.Received(4).UpdateAsync(
+                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
+                Arg.Any<CancellationToken>()),
+            duration: TimeSpan.FromSeconds(10));
+        Assert.Equal(1, adapter.HighWater);
+    }
+
+    /// <summary>
+    /// 06/IMPL-2: the EWS transport does not use MaxConcurrentConnections, so its deliveries
+    /// keep the full MaxParallelDeliveries.
+    /// </summary>
+    [Fact]
+    public void EwsTransport_IsNotClampedByMaxConcurrentConnections()
+    {
+        SetupSmtp(maxConcurrentConnections: 1, transport: "Ews");
+        SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray());
+        var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x"));
+        var actor = CreateActor([adapter], new NotificationOutboxOptions
+        {
+            DispatchInterval = TimeSpan.FromHours(1),
+            MaxParallelDeliveries = 4,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(
+            () => _outboxRepository.Received(4).UpdateAsync(
+                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
+                Arg.Any<CancellationToken>()),
+            duration: TimeSpan.FromSeconds(10));
+        Assert.True(adapter.HighWater >= 2, $"expected concurrent EWS deliveries, got {adapter.HighWater}");
+    }
+
+    /// <summary>
+    /// 06/IMPL-2: the SMTP clamp applies to Email only — SMS deliveries in the same sweep keep
+    /// the full MaxParallelDeliveries.
+    /// </summary>
+    [Fact]
+    public void SmtpClamp_DoesNotThrottleSms()
+    {
+        SetupSmtp(maxConcurrentConnections: 1);
+        SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification(NotificationType.Sms)).ToArray());
+        var sms = new PacedAdapter(
+            TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("+1555"), NotificationType.Sms);
+        var actor = CreateActor([sms], new NotificationOutboxOptions
+        {
+            DispatchInterval = TimeSpan.FromHours(1),
+            MaxParallelDeliveries = 4,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(
+            () => _outboxRepository.Received(4).UpdateAsync(
+                Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
+                Arg.Any<CancellationToken>()),
+            duration: TimeSpan.FromSeconds(10));
+        Assert.True(sms.HighWater >= 2, $"expected concurrent SMS deliveries, got {sms.HighWater}");
+    }
+}
@@ -98,4 +98,13 @@ public class NotificationOutboxOptionsValidatorTests
        Assert.True(result.Failed);
        Assert.Contains("MaxParallelDeliveries", result.FailureMessage);
    }
+
+    /// <summary>Verifies that a zero dispatch sweep budget is rejected (review 2026-10-21 06/PERF-5).</summary>
+    [Fact]
+    public void ZeroDispatchSweepBudget_IsRejected()
+    {
+        var result = Validate(new NotificationOutboxOptions { DispatchSweepBudget = TimeSpan.Zero });
+        Assert.True(result.Failed);
+        Assert.Contains("DispatchSweepBudget", result.FailureMessage);
+    }
 }