using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery; using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages; using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport; namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests; /// /// Task 14: Tests for the dispatcher loop — the /// periodic sweep that claims due notifications via /// , delivers each through the /// matching , and applies the resulting status /// transition with . /// public class NotificationOutboxActorDispatchTests : TestKit { private readonly INotificationOutboxRepository _outboxRepository = OutboxRepositorySubstitute.Healthy(); private readonly INotificationRepository _notificationRepository = Substitute.For(); private IServiceProvider BuildServiceProvider( IEnumerable adapters) { var services = new ServiceCollection(); services.AddScoped(_ => _outboxRepository); services.AddScoped(_ => _notificationRepository); // The actor resolves the channel adapters from its per-sweep DI scope; register // each stub adapter under the INotificationDeliveryAdapter service. foreach (var adapter in adapters) { services.AddScoped(_ => adapter); } return services.BuildServiceProvider(); } /// /// Stub adapter whose returns a configurable outcome and /// optionally blocks for a delay — used to exercise the overlapping-tick guard. /// private sealed class StubAdapter : INotificationDeliveryAdapter { private readonly Func _outcome; private readonly TimeSpan _delay; public StubAdapter(Func outcome, TimeSpan? delay = null, NotificationType type = NotificationType.Email) { _outcome = outcome; _delay = delay ?? TimeSpan.Zero; Type = type; } public int CallCount; public NotificationType Type { get; } public async Task DeliverAsync( Notification notification, CancellationToken cancellationToken = default) { Interlocked.Increment(ref CallCount); if (_delay > TimeSpan.Zero) { await Task.Delay(_delay, cancellationToken); } return _outcome(); } } private IActorRef CreateActor( IEnumerable adapters, NotificationOutboxOptions? options = null) { return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor( BuildServiceProvider(adapters), options ?? new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) }, new NoOpCentralAuditWriter(), NullLogger.Instance))); } private static Notification MakeNotification( NotificationType type = NotificationType.Email, int retryCount = 0) { return new Notification( Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1") { RetryCount = retryCount, CreatedAt = DateTimeOffset.UtcNow, }; } private void SetupSmtpRetryPolicy(int maxRetries, TimeSpan retryDelay) { var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com") { MaxRetries = maxRetries, RetryDelay = retryDelay, }; _notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any()) .Returns(new[] { config }); } [Fact] public void DispatchTick_ClaimsDueNotifications_AndInvokesAdapter() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).GetDueAsync( Arg.Any(), Arg.Any(), Arg.Any()); Assert.Equal(1, adapter.CallCount); }); } [Fact] public void Dispatch_SmsTypedNotification_RoutesToSmsAdapter_NotEmailAdapter() { // NotificationOutbox-NNN: the Type-keyed adapter selection must land an SMS-typed // notification on the SMS adapter (not the Email one) and apply its outcome. SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(type: NotificationType.Sms); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var emailAdapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"), type: NotificationType.Email); var smsAdapter = new StubAdapter(() => DeliveryOutcome.Success("+15551234567"), type: NotificationType.Sms); var actor = CreateActor([emailAdapter, smsAdapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { Assert.Equal(1, smsAdapter.CallCount); Assert.Equal(0, emailAdapter.CallCount); _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered && n.ResolvedTargets == "+15551234567"), Arg.Any()); }); } [Fact] public void Success_MarksNotificationDelivered_WithResolvedTargets() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered && n.DeliveredAt != null && n.LastAttemptAt != null && n.ResolvedTargets == "ops@example.com" && n.LastError == null), Arg.Any()); }); } [Fact] public void TransientFailure_BelowMaxRetries_MarksRetrying_AndSchedulesNextAttempt() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(3)); var notification = MakeNotification(retryCount: 1); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Retrying && n.RetryCount == 2 && n.NextAttemptAt != null && n.LastError == "smtp timeout" && n.LastAttemptAt != null), Arg.Any()); }); } [Fact] public void TransientFailure_ReachingMaxRetries_MarksParked() { SetupSmtpRetryPolicy(maxRetries: 3, retryDelay: TimeSpan.FromMinutes(1)); // RetryCount starts at max-1; the failed attempt increments it to max. var notification = MakeNotification(retryCount: 2); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Parked && n.RetryCount == 3), Arg.Any()); }); } [Fact] public void PermanentFailure_MarksParked_WithLastError() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Permanent("invalid recipient address")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Parked && n.LastError == "invalid recipient address" && n.LastAttemptAt != null), Arg.Any()); }); } [Fact] public void NoAdapterForType_MarksParked_WithExplanatoryError() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); // No adapters registered: none resolves for the notification's type. var actor = CreateActor([]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Parked && n.LastError != null && n.LastError.Contains("no delivery adapter") && n.LastAttemptAt != null), Arg.Any()); }); } [Fact] public void FaultedDispatchPass_ClearsInFlightGuard_SoNextTickStillRuns() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); // GetDueAsync throws on every call: the dispatch pass's task could fault if the // failure were not handled, which would leave _dispatching stuck true forever. // Count every claim via a thread-safe counter so the assertions can reason about // the number of sweeps that actually ran without depending on NSubstitute's // exact-count matchers (see the de-race note below). var claimAttempts = 0; _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns>(_ => { Interlocked.Increment(ref claimAttempts); throw new InvalidOperationException("db down"); }); var actor = CreateActor([]); // First tick: the pass faults internally but must still clear the in-flight guard. actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => Assert.True(Volatile.Read(ref claimAttempts) >= 1), duration: TimeSpan.FromSeconds(10)); // The guard must clear after the faulted sweep so a fresh tick runs another sweep. // // De-race (S11): the in-flight guard (_dispatching) is only lowered when the // faulted sweep's DispatchComplete round-trips back to the actor — which happens // AFTER the GetDueAsync side-effect the barrier above observes. So a single second // tick sent right after that barrier can race the guard while it is still up and // be silently dropped (the dispatcher's intended overlap protection), leaving the // claim count stuck at one forever — a dropped tick is never retried, so widening // the timeout cannot recover it. Re-tell the tick on every poll iteration instead: // a tick that lands while the guard is still up is harmlessly dropped, and the // first tick that lands after the guard lowers runs a fresh sweep. The assertion // is "at least two sweeps ran" rather than "exactly two": that is strictly the // wedge invariant under test (a wedged guard would pin the count at one no matter // how many ticks arrive), and an at-least assertion is immune to the surplus ticks // the re-tell driver may land under CPU starvation. AwaitAssert( () => { actor.Tell(InternalMessages.DispatchTick.Instance); Assert.True( Volatile.Read(ref claimAttempts) >= 2, "the in-flight guard wedged: a second dispatch tick never ran a fresh sweep"); }, duration: TimeSpan.FromSeconds(10)); } [Fact] public void TransientFailure_WithZeroMaxRetries_RetriesUsingFallback_DoesNotParkImmediately() { // NO-002: SmtpConfiguration.MaxRetries=0 used to satisfy 1 >= 0 on the very first // transient failure and park the row without a single retry. ResolveRetryPolicyAsync // now clamps non-positive MaxRetries to the FallbackMaxRetries (10) so transient // failures actually retry before parking. SetupSmtpRetryPolicy(maxRetries: 0, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(retryCount: 0); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Retrying && n.RetryCount == 1 && n.NextAttemptAt != null && n.LastError == "smtp timeout"), Arg.Any()); }); } [Fact] public void TransientFailure_WithNegativeMaxRetries_RetriesUsingFallback_DoesNotParkImmediately() { // NO-002: a negative MaxRetries reaches ResolveRetryPolicyAsync just as -1 — same // park-immediately bug. Clamp to FallbackMaxRetries. SetupSmtpRetryPolicy(maxRetries: -1, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(retryCount: 0); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Retrying && n.RetryCount == 1 && n.NextAttemptAt != null && n.LastError == "smtp timeout"), Arg.Any()); }); } [Fact] public void TransientFailure_WithNonPositiveRetryDelay_UsesFallbackDelay_NotZero() { // NO-002: a non-positive RetryDelay would burn-loop the dispatcher because // NextAttemptAt would equal now. Clamp to FallbackRetryDelay (1 min) so the // schedule actually advances. SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.Zero); var before = DateTimeOffset.UtcNow; var notification = MakeNotification(retryCount: 0); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout")); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Retrying && n.NextAttemptAt != null && n.NextAttemptAt > before + TimeSpan.FromSeconds(30)), Arg.Any()); }); } [Fact] public void PostStop_CancelsInFlightDelivery_LeavesRowNonTerminal() { // NO-003: the dispatcher used to drop the CancellationToken on its way into // the channel adapter, so a coordinated shutdown had to wait the full SMTP // connect/auth/send timeout per in-flight notification before the sweep // finished. The actor now passes a lifecycle-scoped token; cancelling it on // PostStop must abort the in-flight Task.Delay (standing in for an SMTP // send) and the row must NOT be updated to a terminal state — the next // active node picks it back up. SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); // Long delay simulates a slow SMTP send; the test triggers PostStop before // the delay would naturally elapse, so the only way the delay completes is // if the token wired through. var adapter = new StubAdapter( () => DeliveryOutcome.Success("ops@example.com"), delay: TimeSpan.FromSeconds(30)); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); // Wait until the adapter is actually in flight before stopping. AwaitAssert(() => Assert.Equal(1, adapter.CallCount)); var start = DateTimeOffset.UtcNow; Sys.Stop(actor); // The sweep should observe cancellation promptly (well under the 30s delay). AwaitAssert( () => { // No UpdateAsync was issued — the row is untouched and will be re-claimed // by the next active node. _outboxRepository.DidNotReceive().UpdateAsync( Arg.Any(), Arg.Any()); }, duration: TimeSpan.FromSeconds(5)); Assert.True(DateTimeOffset.UtcNow - start < TimeSpan.FromSeconds(5), "PostStop did not cancel the in-flight delivery promptly."); } // ── Register row 45: the delivery adapter is resolved PER DELIVERY ──────── // // Supersedes NotificationOutbox-006 ("resolve the adapter dictionary once for the // actor's lifetime"). That optimisation cached scoped adapter instances — which hold // a scoped INotificationRepository over an EF DbContext — in an actor-lifetime DI // scope, so MaxParallelDeliveries concurrent deliveries all ran on ONE DbContext and // the rig logged `InvalidOperationException: A second operation was started on this // context instance before a previous operation completed` continuously. The adapter // now resolves inside the per-delivery scope that DeliverGatedAsync already opens. /// /// Aggregates what the parallel deliveries observed: how many times two deliveries /// were simultaneously inside ONE simulated DbContext, and a rendezvous that only /// completes once every delivery is concurrently in flight. /// private sealed class DeliveryConcurrencyLedger { private readonly int _expectedArrivals; private readonly TaskCompletionSource _allArrived = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _arrived; private int _overlaps; public DeliveryConcurrencyLedger(int expectedArrivals) => _expectedArrivals = expectedArrivals; /// Number of second-operation-on-one-context collisions observed. public int Overlaps => Volatile.Read(ref _overlaps); public void RecordOverlap() => Interlocked.Increment(ref _overlaps); /// /// Holds the caller's simulated DbContext open until every delivery has reached /// this point — guaranteeing real overlap — or a safety timeout elapses. The /// timeout matters for the shared-context (pre-fix) run, where the colliding /// delivery throws instead of arriving and the rendezvous can never complete. /// public async Task ArriveAndWaitAsync(CancellationToken cancellationToken) { if (Interlocked.Increment(ref _arrived) >= _expectedArrivals) { _allArrived.TrySetResult(); } await Task.WhenAny( _allArrived.Task, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken)) .ConfigureAwait(false); } } /// /// Stands in for the EF DbContext behind a scoped /// — one probe per resolved repository. A second /// concurrent operation throws the exact EF /// raises, so a test that shares one repository across parallel deliveries fails the /// same way the rig did. /// private sealed class SimulatedDbContextProbe { private readonly DeliveryConcurrencyLedger _ledger; private int _inFlight; public SimulatedDbContextProbe(DeliveryConcurrencyLedger ledger) => _ledger = ledger; public async Task EnterAsync(CancellationToken cancellationToken) { if (Interlocked.Increment(ref _inFlight) > 1) { Interlocked.Decrement(ref _inFlight); _ledger.RecordOverlap(); throw new InvalidOperationException( "A second operation was started on this context instance before a " + "previous operation completed."); } try { await _ledger.ArriveAndWaitAsync(cancellationToken); } finally { Interlocked.Decrement(ref _inFlight); } } } /// /// Adapter shaped like the real Email/SMS adapters: it does its work THROUGH the /// scoped it was constructed with, so sharing /// the adapter across deliveries also shares that repository's DbContext. /// private sealed class RepositoryBackedAdapter : INotificationDeliveryAdapter { private readonly INotificationRepository _repository; public RepositoryBackedAdapter(INotificationRepository repository) => _repository = repository; public NotificationType Type => NotificationType.Email; public async Task DeliverAsync( Notification notification, CancellationToken cancellationToken = default) { // Mirrors the real adapters' first act: resolve the target list off the // scoped repository (and therefore the scoped DbContext). await _repository.GetListByNameAsync(notification.ListName, cancellationToken); return DeliveryOutcome.Success("ops@example.com"); } } private static Task EnterProbeAsync( SimulatedDbContextProbe probe, CancellationToken cancellationToken) { return EnterProbeCoreAsync(probe, cancellationToken); static async Task EnterProbeCoreAsync( SimulatedDbContextProbe probe, CancellationToken cancellationToken) { await probe.EnterAsync(cancellationToken); // A null list is enough: the stub adapter ignores the result, and the // dispatcher never sees it. return null; } } [Fact] public void ParallelDeliveries_EachGetTheirOwnAdapterAndDbContext() { // Register row 45 (fail-before-fix). Two due notifications dispatched with // MaxParallelDeliveries = 4. Each delivery's adapter reaches its scoped // INotificationRepository, whose simulated DbContext throws EF's // second-operation exception if a second delivery enters it concurrently. // // Pre-fix: the actor-lifetime _adaptersScope hands BOTH deliveries the same // adapter -> the same repository -> one probe. The rendezvous holds the first // delivery inside the context while the second collides, so Overlaps == 1 and // the collided row is never persisted (DeliverGatedAsync logs and moves on). // Post-fix: two per-delivery scopes -> two adapters -> two probes, both arrive // at the rendezvous concurrently, nothing collides, both rows persist. const int dueCount = 2; var ledger = new DeliveryConcurrencyLedger(expectedArrivals: dueCount); var outboxRepository = OutboxRepositorySubstitute.Healthy(); var batch = Enumerable.Range(0, dueCount).Select(_ => MakeNotification()).ToArray(); var claimed = 0; outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty()); var smtpConfiguration = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com") { MaxRetries = 5, RetryDelay = TimeSpan.FromMinutes(1), }; var services = new ServiceCollection(); services.AddScoped(_ => outboxRepository); // Scoped, with its OWN simulated DbContext per resolution — exactly the // production lifetime of INotificationRepository. services.AddScoped(_ => { var probe = new SimulatedDbContextProbe(ledger); var repository = Substitute.For(); repository.GetAllSmtpConfigurationsAsync(Arg.Any()) .Returns(new[] { smtpConfiguration }); repository.GetListByNameAsync(Arg.Any(), Arg.Any()) .Returns(call => EnterProbeAsync(probe, call.Arg())); return repository; }); // Scoped adapter holding the scoped repository — the production shape of both // EmailNotificationDeliveryAdapter and SmsNotificationDeliveryAdapter. services.AddScoped(sp => new RepositoryBackedAdapter(sp.GetRequiredService())); var serviceProvider = services.BuildServiceProvider(); var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor( serviceProvider, new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 4, }, new NoOpCentralAuditWriter(), NullLogger.Instance))); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => outboxRepository.Received(dueCount).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()), duration: TimeSpan.FromSeconds(10)); Assert.Equal(0, ledger.Overlaps); } [Fact] public void Dispatch_ResolvesAdapterPerDelivery_NotOncePerActorLifetime() { // Register row 45 (isolation, direct form): every delivery must resolve its own // adapter instance from its own DI scope. Four due rows dispatched in one sweep // must therefore yield four DISTINCT adapter instances. Pre-fix the actor cached // a single instance for its whole lifetime and this collapses to 1. const int dueCount = 4; SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var outboxRepository = OutboxRepositorySubstitute.Healthy(); var batch = Enumerable.Range(0, dueCount).Select(_ => MakeNotification()).ToArray(); var claimed = 0; outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty()); var deliveringInstances = new System.Collections.Concurrent.ConcurrentBag(); var services = new ServiceCollection(); services.AddScoped(_ => outboxRepository); services.AddScoped(_ => _notificationRepository); services.AddScoped(_ => new RecordingStubAdapter(deliveringInstances)); var serviceProvider = services.BuildServiceProvider(); var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor( serviceProvider, new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 4, }, new NoOpCentralAuditWriter(), NullLogger.Instance))); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => outboxRepository.Received(dueCount).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()), duration: TimeSpan.FromSeconds(10)); // RecordingStubAdapter does not override Equals, so Distinct() is reference // identity — exactly the "one instance per delivery" question being asked. Assert.Equal(dueCount, deliveringInstances.Distinct().Count()); } /// /// Stub adapter that records which instance actually served each delivery, so a /// test can prove one adapter instance per delivery. /// private sealed class RecordingStubAdapter : INotificationDeliveryAdapter { private readonly System.Collections.Concurrent.ConcurrentBag _seen; public RecordingStubAdapter( System.Collections.Concurrent.ConcurrentBag seen) => _seen = seen; public NotificationType Type => NotificationType.Email; public Task DeliverAsync( Notification notification, CancellationToken cancellationToken = default) { _seen.Add(this); return Task.FromResult(DeliveryOutcome.Success("ops@example.com")); } } [Fact] public void OverlappingTicks_WhileDispatchInFlight_DoNotClaimConcurrently() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); var notification = MakeNotification(); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); // Slow adapter keeps the first sweep in flight while the second tick arrives. var adapter = new StubAdapter( () => DeliveryOutcome.Success("ops@example.com"), delay: TimeSpan.FromMilliseconds(800)); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); actor.Tell(InternalMessages.DispatchTick.Instance); // Second tick is dropped by the in-flight guard: only one sweep runs. AwaitAssert( () => _outboxRepository.Received(1).GetDueAsync( Arg.Any(), Arg.Any(), Arg.Any()), duration: TimeSpan.FromSeconds(2)); } // ── Task 25: bounded parallel delivery ─────────────────────────────────── /// /// Adapter that records the concurrent-invocation high-water mark. Each /// increments a live counter, notes the peak, then blocks /// on a shared gate until deliveries are concurrently /// in flight — proving real overlap. A safety timeout releases the gate so a /// parallelism-1 run (which can never reach releaseWhen concurrently) still /// drains instead of deadlocking. /// private sealed class ConcurrencyHighWaterAdapter : INotificationDeliveryAdapter { private readonly int _releaseWhen; private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _current; private int _highWater; private int _entered; public ConcurrencyHighWaterAdapter(int releaseWhen) => _releaseWhen = releaseWhen; public NotificationType Type => NotificationType.Email; public int HighWater => Volatile.Read(ref _highWater); public async Task DeliverAsync( Notification notification, CancellationToken cancellationToken = default) { var current = Interlocked.Increment(ref _current); int hw; while ((hw = Volatile.Read(ref _highWater)) < current && Interlocked.CompareExchange(ref _highWater, current, hw) != hw) { // retry until our peak is recorded } if (Interlocked.Increment(ref _entered) >= _releaseWhen) { _gate.TrySetResult(); } // Overlap window: wait until enough deliveries are concurrently in flight, // or a short safety timeout (so the strictly-sequential config still drains). await Task.WhenAny(_gate.Task, Task.Delay(TimeSpan.FromMilliseconds(400), cancellationToken)) .ConfigureAwait(false); Interlocked.Decrement(ref _current); return DeliveryOutcome.Success("ops@example.com"); } } private void SetupDueBatchOnce(int count) { var batch = Enumerable.Range(0, count).Select(_ => MakeNotification()).ToArray(); var claimed = 0; _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty()); } [Fact] public void Dispatch_WithMaxParallelFour_DeliversConcurrently_AndAllReachTerminal() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); SetupDueBatchOnce(8); var adapter = new ConcurrencyHighWaterAdapter(releaseWhen: 2); var actor = CreateActor( [adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 4, }); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => { // All 8 notifications reached a terminal Delivered update. _outboxRepository.Received(8).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()); // At least two deliveries overlapped — the sweep is no longer sequential. Assert.True(adapter.HighWater >= 2, $"expected concurrent deliveries (high-water >= 2), got {adapter.HighWater}"); }, duration: TimeSpan.FromSeconds(10)); } [Fact] public void Dispatch_WithMaxParallelOne_DeliversSequentially_AndAllReachTerminal() { SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1)); SetupDueBatchOnce(8); var adapter = new ConcurrencyHighWaterAdapter(releaseWhen: 2); var actor = CreateActor( [adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1), MaxParallelDeliveries = 1, }); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert( () => _outboxRepository.Received(8).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Delivered), Arg.Any()), duration: TimeSpan.FromSeconds(10)); // Strictly sequential: never more than one delivery in flight at once. Assert.Equal(1, adapter.HighWater); } // ── Task 18: per-type retry policy (SMS honors SmsConfiguration) ────────── private void SetupSmsRetryPolicy(int maxRetries, TimeSpan retryDelay) { var config = new SmsConfiguration("ACtestaccountsid", "+15550000000") { MaxRetries = maxRetries, RetryDelay = retryDelay, }; _notificationRepository.GetSmsConfigurationAsync(Arg.Any()) .Returns(config); } [Fact] public void SmsNotification_TransientFailure_UsesSmsRetryPolicy() { // Task 18: an SMS-typed notification resolves its retry policy from the SMS // configuration, NOT the shared SMTP policy. The SMTP policy (MaxRetries=10) // would keep this row Retrying; the SMS policy (MaxRetries=1) parks it after // this attempt (RetryCount 1 -> 2 >= 1). A pre-fix dispatcher (SMTP-for-all) // would leave it Retrying with RetryCount=2 — the assertion below fails. SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(1)); SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5)); var notification = MakeNotification(type: NotificationType.Sms, retryCount: 1); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("twilio 503"), type: NotificationType.Sms); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Parked && n.RetryCount == 2 && n.LastError == "twilio 503"), Arg.Any()); }); } [Fact] public void EmailNotification_StillUsesSmtpPolicy() { // Task 18: an Email row continues to resolve against the SMTP policy even when a // (tighter) SMS policy is present. SMTP MaxRetries=10 keeps it Retrying // (RetryCount 1 -> 2 < 10) and schedules the next attempt with the SMTP delay. SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(3)); SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5)); var before = DateTimeOffset.UtcNow; var notification = MakeNotification(type: NotificationType.Email, retryCount: 1); _outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { notification }); var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"), type: NotificationType.Email); var actor = CreateActor([adapter]); actor.Tell(InternalMessages.DispatchTick.Instance); AwaitAssert(() => { _outboxRepository.Received(1).UpdateAsync( Arg.Is(n => n.Status == NotificationStatus.Retrying && n.RetryCount == 2 && n.NextAttemptAt != null && n.NextAttemptAt > before + TimeSpan.FromMinutes(2) && n.LastError == "smtp timeout"), Arg.Any()); }); } }