using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
///
/// Task 14: Tests for the dispatcher loop — the
/// periodic sweep that claims due notifications via
/// , delivers each through the
/// matching , and applies the resulting status
/// transition with .
///
public class NotificationOutboxActorDispatchTests : TestKit
{
private readonly INotificationOutboxRepository _outboxRepository =
OutboxRepositorySubstitute.Healthy();
private readonly INotificationRepository _notificationRepository =
Substitute.For();
private IServiceProvider BuildServiceProvider(
IEnumerable adapters)
{
var services = new ServiceCollection();
services.AddScoped(_ => _outboxRepository);
services.AddScoped(_ => _notificationRepository);
// The actor resolves the channel adapters from its per-sweep DI scope; register
// each stub adapter under the INotificationDeliveryAdapter service.
foreach (var adapter in adapters)
{
services.AddScoped(_ => adapter);
}
return services.BuildServiceProvider();
}
///
/// Stub adapter whose returns a configurable outcome and
/// optionally blocks for a delay — used to exercise the overlapping-tick guard.
///
private sealed class StubAdapter : INotificationDeliveryAdapter
{
private readonly Func _outcome;
private readonly TimeSpan _delay;
public StubAdapter(Func outcome, TimeSpan? delay = null,
NotificationType type = NotificationType.Email)
{
_outcome = outcome;
_delay = delay ?? TimeSpan.Zero;
Type = type;
}
public int CallCount;
public NotificationType Type { get; }
public async Task DeliverAsync(
Notification notification, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref CallCount);
if (_delay > TimeSpan.Zero)
{
await Task.Delay(_delay, cancellationToken);
}
return _outcome();
}
}
private IActorRef CreateActor(
IEnumerable adapters,
NotificationOutboxOptions? options = null)
{
return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
BuildServiceProvider(adapters),
options ?? new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) },
new NoOpCentralAuditWriter(),
NullLogger.Instance)));
}
private static Notification MakeNotification(
NotificationType type = NotificationType.Email, int retryCount = 0)
{
return new Notification(
Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1")
{
RetryCount = retryCount,
CreatedAt = DateTimeOffset.UtcNow,
};
}
private void SetupSmtpRetryPolicy(int maxRetries, TimeSpan retryDelay)
{
var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
{
MaxRetries = maxRetries,
RetryDelay = retryDelay,
};
_notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any())
.Returns(new[] { config });
}
[Fact]
public void DispatchTick_ClaimsDueNotifications_AndInvokesAdapter()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification();
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).GetDueAsync(
Arg.Any(), Arg.Any(), Arg.Any());
Assert.Equal(1, adapter.CallCount);
});
}
[Fact]
public void Dispatch_SmsTypedNotification_RoutesToSmsAdapter_NotEmailAdapter()
{
// NotificationOutbox-NNN: the Type-keyed adapter selection must land an SMS-typed
// notification on the SMS adapter (not the Email one) and apply its outcome.
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification(type: NotificationType.Sms);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var emailAdapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"),
type: NotificationType.Email);
var smsAdapter = new StubAdapter(() => DeliveryOutcome.Success("+15551234567"),
type: NotificationType.Sms);
var actor = CreateActor([emailAdapter, smsAdapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
Assert.Equal(1, smsAdapter.CallCount);
Assert.Equal(0, emailAdapter.CallCount);
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Delivered &&
n.ResolvedTargets == "+15551234567"),
Arg.Any());
});
}
[Fact]
public void Success_MarksNotificationDelivered_WithResolvedTargets()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification();
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Delivered &&
n.DeliveredAt != null &&
n.LastAttemptAt != null &&
n.ResolvedTargets == "ops@example.com" &&
n.LastError == null),
Arg.Any());
});
}
[Fact]
public void TransientFailure_BelowMaxRetries_MarksRetrying_AndSchedulesNextAttempt()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(3));
var notification = MakeNotification(retryCount: 1);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Retrying &&
n.RetryCount == 2 &&
n.NextAttemptAt != null &&
n.LastError == "smtp timeout" &&
n.LastAttemptAt != null),
Arg.Any());
});
}
[Fact]
public void TransientFailure_ReachingMaxRetries_MarksParked()
{
SetupSmtpRetryPolicy(maxRetries: 3, retryDelay: TimeSpan.FromMinutes(1));
// RetryCount starts at max-1; the failed attempt increments it to max.
var notification = MakeNotification(retryCount: 2);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Parked &&
n.RetryCount == 3),
Arg.Any());
});
}
[Fact]
public void PermanentFailure_MarksParked_WithLastError()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification();
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Permanent("invalid recipient address"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Parked &&
n.LastError == "invalid recipient address" &&
n.LastAttemptAt != null),
Arg.Any());
});
}
[Fact]
public void NoAdapterForType_MarksParked_WithExplanatoryError()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification();
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
// No adapters registered: none resolves for the notification's type.
var actor = CreateActor([]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Parked &&
n.LastError != null &&
n.LastError.Contains("no delivery adapter") &&
n.LastAttemptAt != null),
Arg.Any());
});
}
[Fact]
public void FaultedDispatchPass_ClearsInFlightGuard_SoNextTickStillRuns()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
// GetDueAsync throws on every call: the dispatch pass's task could fault if the
// failure were not handled, which would leave _dispatching stuck true forever.
// Count every claim via a thread-safe counter so the assertions can reason about
// the number of sweeps that actually ran without depending on NSubstitute's
// exact-count matchers (see the de-race note below).
var claimAttempts = 0;
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns>(_ =>
{
Interlocked.Increment(ref claimAttempts);
throw new InvalidOperationException("db down");
});
var actor = CreateActor([]);
// First tick: the pass faults internally but must still clear the in-flight guard.
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(
() => Assert.True(Volatile.Read(ref claimAttempts) >= 1),
duration: TimeSpan.FromSeconds(10));
// The guard must clear after the faulted sweep so a fresh tick runs another sweep.
//
// De-race (S11): the in-flight guard (_dispatching) is only lowered when the
// faulted sweep's DispatchComplete round-trips back to the actor — which happens
// AFTER the GetDueAsync side-effect the barrier above observes. So a single second
// tick sent right after that barrier can race the guard while it is still up and
// be silently dropped (the dispatcher's intended overlap protection), leaving the
// claim count stuck at one forever — a dropped tick is never retried, so widening
// the timeout cannot recover it. Re-tell the tick on every poll iteration instead:
// a tick that lands while the guard is still up is harmlessly dropped, and the
// first tick that lands after the guard lowers runs a fresh sweep. The assertion
// is "at least two sweeps ran" rather than "exactly two": that is strictly the
// wedge invariant under test (a wedged guard would pin the count at one no matter
// how many ticks arrive), and an at-least assertion is immune to the surplus ticks
// the re-tell driver may land under CPU starvation.
AwaitAssert(
() =>
{
actor.Tell(InternalMessages.DispatchTick.Instance);
Assert.True(
Volatile.Read(ref claimAttempts) >= 2,
"the in-flight guard wedged: a second dispatch tick never ran a fresh sweep");
},
duration: TimeSpan.FromSeconds(10));
}
[Fact]
public void TransientFailure_WithZeroMaxRetries_RetriesUsingFallback_DoesNotParkImmediately()
{
// NO-002: SmtpConfiguration.MaxRetries=0 used to satisfy 1 >= 0 on the very first
// transient failure and park the row without a single retry. ResolveRetryPolicyAsync
// now clamps non-positive MaxRetries to the FallbackMaxRetries (10) so transient
// failures actually retry before parking.
SetupSmtpRetryPolicy(maxRetries: 0, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification(retryCount: 0);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Retrying &&
n.RetryCount == 1 &&
n.NextAttemptAt != null &&
n.LastError == "smtp timeout"),
Arg.Any());
});
}
[Fact]
public void TransientFailure_WithNegativeMaxRetries_RetriesUsingFallback_DoesNotParkImmediately()
{
// NO-002: a negative MaxRetries reaches ResolveRetryPolicyAsync just as -1 — same
// park-immediately bug. Clamp to FallbackMaxRetries.
SetupSmtpRetryPolicy(maxRetries: -1, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification(retryCount: 0);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Retrying &&
n.RetryCount == 1 &&
n.NextAttemptAt != null &&
n.LastError == "smtp timeout"),
Arg.Any());
});
}
[Fact]
public void TransientFailure_WithNonPositiveRetryDelay_UsesFallbackDelay_NotZero()
{
// NO-002: a non-positive RetryDelay would burn-loop the dispatcher because
// NextAttemptAt would equal now. Clamp to FallbackRetryDelay (1 min) so the
// schedule actually advances.
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.Zero);
var before = DateTimeOffset.UtcNow;
var notification = MakeNotification(retryCount: 0);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Retrying &&
n.NextAttemptAt != null &&
n.NextAttemptAt > before + TimeSpan.FromSeconds(30)),
Arg.Any());
});
}
[Fact]
public void PostStop_CancelsInFlightDelivery_LeavesRowNonTerminal()
{
// NO-003: the dispatcher used to drop the CancellationToken on its way into
// the channel adapter, so a coordinated shutdown had to wait the full SMTP
// connect/auth/send timeout per in-flight notification before the sweep
// finished. The actor now passes a lifecycle-scoped token; cancelling it on
// PostStop must abort the in-flight Task.Delay (standing in for an SMTP
// send) and the row must NOT be updated to a terminal state — the next
// active node picks it back up.
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification();
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
// Long delay simulates a slow SMTP send; the test triggers PostStop before
// the delay would naturally elapse, so the only way the delay completes is
// if the token wired through.
var adapter = new StubAdapter(
() => DeliveryOutcome.Success("ops@example.com"),
delay: TimeSpan.FromSeconds(30));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
// Wait until the adapter is actually in flight before stopping.
AwaitAssert(() => Assert.Equal(1, adapter.CallCount));
var start = DateTimeOffset.UtcNow;
Sys.Stop(actor);
// The sweep should observe cancellation promptly (well under the 30s delay).
AwaitAssert(
() =>
{
// No UpdateAsync was issued — the row is untouched and will be re-claimed
// by the next active node.
_outboxRepository.DidNotReceive().UpdateAsync(
Arg.Any(), Arg.Any());
},
duration: TimeSpan.FromSeconds(5));
Assert.True(DateTimeOffset.UtcNow - start < TimeSpan.FromSeconds(5),
"PostStop did not cancel the in-flight delivery promptly.");
}
// ── Register row 45: the delivery adapter is resolved PER DELIVERY ────────
//
// Supersedes NotificationOutbox-006 ("resolve the adapter dictionary once for the
// actor's lifetime"). That optimisation cached scoped adapter instances — which hold
// a scoped INotificationRepository over an EF DbContext — in an actor-lifetime DI
// scope, so MaxParallelDeliveries concurrent deliveries all ran on ONE DbContext and
// the rig logged `InvalidOperationException: A second operation was started on this
// context instance before a previous operation completed` continuously. The adapter
// now resolves inside the per-delivery scope that DeliverGatedAsync already opens.
///
/// Aggregates what the parallel deliveries observed: how many times two deliveries
/// were simultaneously inside ONE simulated DbContext, and a rendezvous that only
/// completes once every delivery is concurrently in flight.
///
private sealed class DeliveryConcurrencyLedger
{
private readonly int _expectedArrivals;
private readonly TaskCompletionSource _allArrived =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _arrived;
private int _overlaps;
public DeliveryConcurrencyLedger(int expectedArrivals) => _expectedArrivals = expectedArrivals;
/// Number of second-operation-on-one-context collisions observed.
public int Overlaps => Volatile.Read(ref _overlaps);
public void RecordOverlap() => Interlocked.Increment(ref _overlaps);
///
/// Holds the caller's simulated DbContext open until every delivery has reached
/// this point — guaranteeing real overlap — or a safety timeout elapses. The
/// timeout matters for the shared-context (pre-fix) run, where the colliding
/// delivery throws instead of arriving and the rendezvous can never complete.
///
public async Task ArriveAndWaitAsync(CancellationToken cancellationToken)
{
if (Interlocked.Increment(ref _arrived) >= _expectedArrivals)
{
_allArrived.TrySetResult();
}
await Task.WhenAny(
_allArrived.Task,
Task.Delay(TimeSpan.FromSeconds(2), cancellationToken))
.ConfigureAwait(false);
}
}
///
/// Stands in for the EF DbContext behind a scoped
/// — one probe per resolved repository. A second
/// concurrent operation throws the exact EF
/// raises, so a test that shares one repository across parallel deliveries fails the
/// same way the rig did.
///
private sealed class SimulatedDbContextProbe
{
private readonly DeliveryConcurrencyLedger _ledger;
private int _inFlight;
public SimulatedDbContextProbe(DeliveryConcurrencyLedger ledger) => _ledger = ledger;
public async Task EnterAsync(CancellationToken cancellationToken)
{
if (Interlocked.Increment(ref _inFlight) > 1)
{
Interlocked.Decrement(ref _inFlight);
_ledger.RecordOverlap();
throw new InvalidOperationException(
"A second operation was started on this context instance before a "
+ "previous operation completed.");
}
try
{
await _ledger.ArriveAndWaitAsync(cancellationToken);
}
finally
{
Interlocked.Decrement(ref _inFlight);
}
}
}
///
/// Adapter shaped like the real Email/SMS adapters: it does its work THROUGH the
/// scoped it was constructed with, so sharing
/// the adapter across deliveries also shares that repository's DbContext.
///
private sealed class RepositoryBackedAdapter : INotificationDeliveryAdapter
{
private readonly INotificationRepository _repository;
public RepositoryBackedAdapter(INotificationRepository repository) => _repository = repository;
public NotificationType Type => NotificationType.Email;
public async Task DeliverAsync(
Notification notification, CancellationToken cancellationToken = default)
{
// Mirrors the real adapters' first act: resolve the target list off the
// scoped repository (and therefore the scoped DbContext).
await _repository.GetListByNameAsync(notification.ListName, cancellationToken);
return DeliveryOutcome.Success("ops@example.com");
}
}
private static Task EnterProbeAsync(
SimulatedDbContextProbe probe, CancellationToken cancellationToken)
{
return EnterProbeCoreAsync(probe, cancellationToken);
static async Task EnterProbeCoreAsync(
SimulatedDbContextProbe probe, CancellationToken cancellationToken)
{
await probe.EnterAsync(cancellationToken);
// A null list is enough: the stub adapter ignores the result, and the
// dispatcher never sees it.
return null;
}
}
[Fact]
public void ParallelDeliveries_EachGetTheirOwnAdapterAndDbContext()
{
// Register row 45 (fail-before-fix). Two due notifications dispatched with
// MaxParallelDeliveries = 4. Each delivery's adapter reaches its scoped
// INotificationRepository, whose simulated DbContext throws EF's
// second-operation exception if a second delivery enters it concurrently.
//
// Pre-fix: the actor-lifetime _adaptersScope hands BOTH deliveries the same
// adapter -> the same repository -> one probe. The rendezvous holds the first
// delivery inside the context while the second collides, so Overlaps == 1 and
// the collided row is never persisted (DeliverGatedAsync logs and moves on).
// Post-fix: two per-delivery scopes -> two adapters -> two probes, both arrive
// at the rendezvous concurrently, nothing collides, both rows persist.
const int dueCount = 2;
var ledger = new DeliveryConcurrencyLedger(expectedArrivals: dueCount);
var outboxRepository = OutboxRepositorySubstitute.Healthy();
var batch = Enumerable.Range(0, dueCount).Select(_ => MakeNotification()).ToArray();
var claimed = 0;
outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty());
var smtpConfiguration = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
{
MaxRetries = 5,
RetryDelay = TimeSpan.FromMinutes(1),
};
var services = new ServiceCollection();
services.AddScoped(_ => outboxRepository);
// Scoped, with its OWN simulated DbContext per resolution — exactly the
// production lifetime of INotificationRepository.
services.AddScoped(_ =>
{
var probe = new SimulatedDbContextProbe(ledger);
var repository = Substitute.For();
repository.GetAllSmtpConfigurationsAsync(Arg.Any())
.Returns(new[] { smtpConfiguration });
repository.GetListByNameAsync(Arg.Any(), Arg.Any())
.Returns(call => EnterProbeAsync(probe, call.Arg()));
return repository;
});
// Scoped adapter holding the scoped repository — the production shape of both
// EmailNotificationDeliveryAdapter and SmsNotificationDeliveryAdapter.
services.AddScoped(sp =>
new RepositoryBackedAdapter(sp.GetRequiredService()));
var serviceProvider = services.BuildServiceProvider();
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
serviceProvider,
new NotificationOutboxOptions
{
DispatchInterval = TimeSpan.FromHours(1),
MaxParallelDeliveries = 4,
},
new NoOpCentralAuditWriter(),
NullLogger.Instance)));
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(
() => outboxRepository.Received(dueCount).UpdateAsync(
Arg.Is(n => n.Status == NotificationStatus.Delivered),
Arg.Any()),
duration: TimeSpan.FromSeconds(10));
Assert.Equal(0, ledger.Overlaps);
}
[Fact]
public void Dispatch_ResolvesAdapterPerDelivery_NotOncePerActorLifetime()
{
// Register row 45 (isolation, direct form): every delivery must resolve its own
// adapter instance from its own DI scope. Four due rows dispatched in one sweep
// must therefore yield four DISTINCT adapter instances. Pre-fix the actor cached
// a single instance for its whole lifetime and this collapses to 1.
const int dueCount = 4;
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var outboxRepository = OutboxRepositorySubstitute.Healthy();
var batch = Enumerable.Range(0, dueCount).Select(_ => MakeNotification()).ToArray();
var claimed = 0;
outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(_ => Interlocked.Increment(ref claimed) == 1 ? batch : Array.Empty());
var deliveringInstances = new System.Collections.Concurrent.ConcurrentBag();
var services = new ServiceCollection();
services.AddScoped(_ => outboxRepository);
services.AddScoped(_ => _notificationRepository);
services.AddScoped(_ => new RecordingStubAdapter(deliveringInstances));
var serviceProvider = services.BuildServiceProvider();
var actor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
serviceProvider,
new NotificationOutboxOptions
{
DispatchInterval = TimeSpan.FromHours(1),
MaxParallelDeliveries = 4,
},
new NoOpCentralAuditWriter(),
NullLogger.Instance)));
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(
() => outboxRepository.Received(dueCount).UpdateAsync(
Arg.Is(n => n.Status == NotificationStatus.Delivered),
Arg.Any()),
duration: TimeSpan.FromSeconds(10));
// RecordingStubAdapter does not override Equals, so Distinct() is reference
// identity — exactly the "one instance per delivery" question being asked.
Assert.Equal(dueCount, deliveringInstances.Distinct().Count());
}
///
/// Stub adapter that records which instance actually served each delivery, so a
/// test can prove one adapter instance per delivery.
///
private sealed class RecordingStubAdapter : INotificationDeliveryAdapter
{
private readonly System.Collections.Concurrent.ConcurrentBag _seen;
public RecordingStubAdapter(
System.Collections.Concurrent.ConcurrentBag seen) => _seen = seen;
public NotificationType Type => NotificationType.Email;
public Task DeliverAsync(
Notification notification, CancellationToken cancellationToken = default)
{
_seen.Add(this);
return Task.FromResult(DeliveryOutcome.Success("ops@example.com"));
}
}
[Fact]
public void OverlappingTicks_WhileDispatchInFlight_DoNotClaimConcurrently()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
var notification = MakeNotification();
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
// Slow adapter keeps the first sweep in flight while the second tick arrives.
var adapter = new StubAdapter(
() => DeliveryOutcome.Success("ops@example.com"),
delay: TimeSpan.FromMilliseconds(800));
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
actor.Tell(InternalMessages.DispatchTick.Instance);
// Second tick is dropped by the in-flight guard: only one sweep runs.
AwaitAssert(
() => _outboxRepository.Received(1).GetDueAsync(
Arg.Any(), Arg.Any(), Arg.Any()),
duration: TimeSpan.FromSeconds(2));
}
// ── Task 25: bounded parallel delivery ───────────────────────────────────
///
/// Adapter that records the concurrent-invocation high-water mark. Each
/// increments a live counter, notes the peak, then blocks
/// on a shared gate until deliveries are concurrently
/// in flight — proving real overlap. A safety timeout releases the gate so a
/// parallelism-1 run (which can never reach releaseWhen concurrently) still
/// drains instead of deadlocking.
///
private sealed class ConcurrencyHighWaterAdapter : INotificationDeliveryAdapter
{
private readonly int _releaseWhen;
private readonly TaskCompletionSource _gate =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _current;
private int _highWater;
private int _entered;
public ConcurrencyHighWaterAdapter(int releaseWhen) => _releaseWhen = releaseWhen;
public NotificationType Type => NotificationType.Email;
public int HighWater => Volatile.Read(ref _highWater);
public async Task DeliverAsync(
Notification notification, CancellationToken cancellationToken = default)
{
var current = Interlocked.Increment(ref _current);
int hw;
while ((hw = Volatile.Read(ref _highWater)) < current
&& Interlocked.CompareExchange(ref _highWater, current, hw) != hw)
{
// retry until our peak is recorded
}
if (Interlocked.Increment(ref _entered) >= _releaseWhen)
{
_gate.TrySetResult();
}
// Overlap window: wait until enough deliveries are concurrently in flight,
// or a short safety timeout (so the strictly-sequential config still drains).
await Task.WhenAny(_gate.Task, Task.Delay(TimeSpan.FromMilliseconds(400), cancellationToken))
.ConfigureAwait(false);
Interlocked.Decrement(ref _current);
return DeliveryOutcome.Success("ops@example.com");
}
}
private void SetupDueBatchOnce(int count)
{
var batch = Enumerable.Range(0, count).Select(_ => MakeNotification()).ToArray();
var claimed = 0;
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(_ => Interlocked.Increment(ref claimed) == 1
? batch
: Array.Empty());
}
[Fact]
public void Dispatch_WithMaxParallelFour_DeliversConcurrently_AndAllReachTerminal()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
SetupDueBatchOnce(8);
var adapter = new ConcurrencyHighWaterAdapter(releaseWhen: 2);
var actor = CreateActor(
[adapter],
new NotificationOutboxOptions
{
DispatchInterval = TimeSpan.FromHours(1),
MaxParallelDeliveries = 4,
});
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(
() =>
{
// All 8 notifications reached a terminal Delivered update.
_outboxRepository.Received(8).UpdateAsync(
Arg.Is(n => n.Status == NotificationStatus.Delivered),
Arg.Any());
// At least two deliveries overlapped — the sweep is no longer sequential.
Assert.True(adapter.HighWater >= 2,
$"expected concurrent deliveries (high-water >= 2), got {adapter.HighWater}");
},
duration: TimeSpan.FromSeconds(10));
}
[Fact]
public void Dispatch_WithMaxParallelOne_DeliversSequentially_AndAllReachTerminal()
{
SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
SetupDueBatchOnce(8);
var adapter = new ConcurrencyHighWaterAdapter(releaseWhen: 2);
var actor = CreateActor(
[adapter],
new NotificationOutboxOptions
{
DispatchInterval = TimeSpan.FromHours(1),
MaxParallelDeliveries = 1,
});
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(
() => _outboxRepository.Received(8).UpdateAsync(
Arg.Is(n => n.Status == NotificationStatus.Delivered),
Arg.Any()),
duration: TimeSpan.FromSeconds(10));
// Strictly sequential: never more than one delivery in flight at once.
Assert.Equal(1, adapter.HighWater);
}
// ── Task 18: per-type retry policy (SMS honors SmsConfiguration) ──────────
private void SetupSmsRetryPolicy(int maxRetries, TimeSpan retryDelay)
{
var config = new SmsConfiguration("ACtestaccountsid", "+15550000000")
{
MaxRetries = maxRetries,
RetryDelay = retryDelay,
};
_notificationRepository.GetSmsConfigurationAsync(Arg.Any())
.Returns(config);
}
[Fact]
public void SmsNotification_TransientFailure_UsesSmsRetryPolicy()
{
// Task 18: an SMS-typed notification resolves its retry policy from the SMS
// configuration, NOT the shared SMTP policy. The SMTP policy (MaxRetries=10)
// would keep this row Retrying; the SMS policy (MaxRetries=1) parks it after
// this attempt (RetryCount 1 -> 2 >= 1). A pre-fix dispatcher (SMTP-for-all)
// would leave it Retrying with RetryCount=2 — the assertion below fails.
SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(1));
SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5));
var notification = MakeNotification(type: NotificationType.Sms, retryCount: 1);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("twilio 503"),
type: NotificationType.Sms);
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Parked &&
n.RetryCount == 2 &&
n.LastError == "twilio 503"),
Arg.Any());
});
}
[Fact]
public void EmailNotification_StillUsesSmtpPolicy()
{
// Task 18: an Email row continues to resolve against the SMTP policy even when a
// (tighter) SMS policy is present. SMTP MaxRetries=10 keeps it Retrying
// (RetryCount 1 -> 2 < 10) and schedules the next attempt with the SMTP delay.
SetupSmtpRetryPolicy(maxRetries: 10, retryDelay: TimeSpan.FromMinutes(3));
SetupSmsRetryPolicy(maxRetries: 1, retryDelay: TimeSpan.FromMinutes(5));
var before = DateTimeOffset.UtcNow;
var notification = MakeNotification(type: NotificationType.Email, retryCount: 1);
_outboxRepository.GetDueAsync(Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new[] { notification });
var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"),
type: NotificationType.Email);
var actor = CreateActor([adapter]);
actor.Tell(InternalMessages.DispatchTick.Instance);
AwaitAssert(() =>
{
_outboxRepository.Received(1).UpdateAsync(
Arg.Is(n =>
n.Status == NotificationStatus.Retrying &&
n.RetryCount == 2 &&
n.NextAttemptAt != null &&
n.NextAttemptAt > before + TimeSpan.FromMinutes(2) &&
n.LastError == "smtp timeout"),
Arg.Any());
});
}
}