using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery; using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages; using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport; namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests; /// /// Regression tests for the dispatch sweep's pacing (review 2026-10-21): per-attempt /// timestamps (06/COR-1), the per-sweep wall-clock budget (06/PERF-5), and the Email /// concurrency clamp to the SMTP row's MaxConcurrentConnections (06/IMPL-2). /// public class NotificationOutboxActorSweepPacingTests : TestKit { private readonly INotificationOutboxRepository _outboxRepository = OutboxRepositorySubstitute.Healthy(); private readonly INotificationRepository _notificationRepository = Substitute.For(); private IActorRef CreateActor( IEnumerable adapters, NotificationOutboxOptions options) { var services = new ServiceCollection(); services.AddScoped(_ => _outboxRepository); services.AddScoped(_ => _notificationRepository); foreach (var adapter in adapters) { services.AddScoped(_ => adapter); } return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor( services.BuildServiceProvider(), options, new NoOpCentralAuditWriter(), NullLogger.Instance))); } private static Notification MakeNotification(NotificationType type = NotificationType.Email) => new(Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1") { CreatedAt = DateTimeOffset.UtcNow, }; private void SetupSmtp(int maxConcurrentConnections = 0, string? transport = null) { var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com") { MaxRetries = 5, RetryDelay = TimeSpan.FromMinutes(1), MaxConcurrentConnections = maxConcurrentConnections, Transport = transport, }; _notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any()) .Returns(new[] { config }); } private void SetupDueBatchOnce(params Notification[] batch) { var claimed = 0; _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty()); } /// /// Adapter that delays every delivery, records the concurrency high-water mark and the /// instant each delivery completed. /// private sealed class PacedAdapter( TimeSpan delay, Func outcome, NotificationType type = NotificationType.Email) : INotificationDeliveryAdapter { private int _current; private int _highWater; private int _calls; public NotificationType Type => type; public int HighWater => Volatile.Read(ref _highWater); public int Calls => Volatile.Read(ref _calls); public DateTimeOffset LastCompletedAt; public async Task DeliverAsync( Notification notification, CancellationToken cancellationToken = default) { Interlocked.Increment(ref _calls); var current = Interlocked.Increment(ref _current); int hw; while ((hw = Volatile.Read(ref _highWater)) < current && Interlocked.CompareExchange(ref _highWater, current, hw) != hw) { // retry until our peak is recorded } await Task.Delay(delay, cancellationToken); Interlocked.Decrement(ref _current); LastCompletedAt = DateTimeOffset.UtcNow; return outcome(); } } /// /// 06/COR-1: a delivery that completes well after the sweep began is stamped with the /// attempt instant, not the sweep-start instant. /// [Fact] public void Success_StampsDeliveredAt_AfterTheAdapterReturns() { SetupSmtp(); var notification = MakeNotification(); SetupDueBatchOnce(notification); var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x")); var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) }); var before = DateTimeOffset.UtcNow; actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any())); Assert.True(notification.DeliveredAt >= before + TimeSpan.FromMilliseconds(350), $"DeliveredAt {notification.DeliveredAt:O} should be after the 400 ms attempt (sweep began {before:O})"); Assert.Equal(notification.DeliveredAt, notification.LastAttemptAt); } /// /// 06/COR-1: a transient failure schedules the retry relative to the attempt instant, so a /// slow attempt cannot collapse the retry delay. /// [Fact] public void TransientFailure_SchedulesNextAttempt_FromTheAttemptInstant() { SetupSmtp(); var notification = MakeNotification(); SetupDueBatchOnce(notification); var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Transient("timeout")); var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) }); var before = DateTimeOffset.UtcNow; actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Retrying), Arg.Any())); Assert.True(notification.LastAttemptAt >= before + TimeSpan.FromMilliseconds(350)); Assert.Equal(notification.LastAttemptAt + TimeSpan.FromMinutes(1), notification.NextAttemptAt); } /// /// 06/PERF-5: once the sweep's wall-clock budget is spent, deliveries still waiting for a /// permit are skipped (left due for the next sweep) instead of extending the sweep. /// [Fact] public void SweepBudget_Exhausted_SkipsDeliveriesThatHaveNotStarted() { SetupSmtp(); SetupDueBatchOnce(MakeNotification(), MakeNotification(), MakeNotification()); var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(600), () => DeliveryOutcome.Success("ops@x")); var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 1, DispatchSweepBudget = TimeSpan.FromMilliseconds(200), }); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync( Arg.Any(), Arg.Any())); // Give the (skipped) rest of the sweep ample time to have run had it not been skipped. Thread.Sleep(1500); Assert.Equal(1, adapter.Calls); _outboxRepository.Received(1).UpdateAsync(Arg.Any(), Arg.Any()); } /// /// 06/IMPL-2: Email deliveries over the SMTP transport are clamped to the SMTP row's /// MaxConcurrentConnections even when MaxParallelDeliveries is higher. /// [Fact] public void SmtpTransport_ClampsEmailParallelism_ToMaxConcurrentConnections() { SetupSmtp(maxConcurrentConnections: 1); SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray()); var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(150), () => DeliveryOutcome.Success("ops@x")); var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 4, }); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => _outboxRepository.Received(4).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()), duration: TimeSpan.FromSeconds(10)); Assert.Equal(1, adapter.HighWater); } /// /// 06/IMPL-2: the EWS transport does not use MaxConcurrentConnections, so its deliveries /// keep the full MaxParallelDeliveries. /// [Fact] public void EwsTransport_IsNotClampedByMaxConcurrentConnections() { SetupSmtp(maxConcurrentConnections: 1, transport: "Ews"); SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray()); var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x")); var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 4, }); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => _outboxRepository.Received(4).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()), duration: TimeSpan.FromSeconds(10)); Assert.True(adapter.HighWater >= 2, $"expected concurrent EWS deliveries, got {adapter.HighWater}"); } /// /// 06/IMPL-2: the SMTP clamp applies to Email only — SMS deliveries in the same sweep keep /// the full MaxParallelDeliveries. /// [Fact] public void SmtpClamp_DoesNotThrottleSms() { SetupSmtp(maxConcurrentConnections: 1); SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification(NotificationType.Sms)).ToArray()); var sms = new PacedAdapter( TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("+1555"), NotificationType.Sms); var actor = CreateActor([sms], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 4, }); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => _outboxRepository.Received(4).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()), duration: TimeSpan.FromSeconds(10)); Assert.True(sms.HighWater >= 2, $"expected concurrent SMS deliveries, got {sms.HighWater}"); } }