fix(notification-outbox): per-attempt timestamps, Pending/Retrying split claim + sweep budget, SMTP MaxConcurrentConnections clamp (review 2026-10-21 06/COR-1, 06/PERF-5, 06/IMPL-2)
This commit is contained in:
Joseph Doherty
+265
@@ -0,0 +1,265 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Regression tests for the dispatch sweep's pacing (review 2026-10-21): per-attempt
|
||||
/// timestamps (06/COR-1), the per-sweep wall-clock budget (06/PERF-5), and the Email
|
||||
/// concurrency clamp to the SMTP row's <c>MaxConcurrentConnections</c> (06/IMPL-2).
|
||||
/// </summary>
|
||||
public class NotificationOutboxActorSweepPacingTests : TestKit
|
||||
{
|
||||
private readonly INotificationOutboxRepository _outboxRepository =
|
||||
OutboxRepositorySubstitute.Healthy();
|
||||
|
||||
private readonly INotificationRepository _notificationRepository =
|
||||
Substitute.For<INotificationRepository>();
|
||||
|
||||
private IActorRef CreateActor(
|
||||
IEnumerable<INotificationDeliveryAdapter> adapters, NotificationOutboxOptions options)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => _outboxRepository);
|
||||
services.AddScoped(_ => _notificationRepository);
|
||||
foreach (var adapter in adapters)
|
||||
{
|
||||
services.AddScoped<INotificationDeliveryAdapter>(_ => adapter);
|
||||
}
|
||||
|
||||
return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
||||
services.BuildServiceProvider(),
|
||||
options,
|
||||
new NoOpCentralAuditWriter(),
|
||||
NullLogger<NotificationOutboxActor>.Instance)));
|
||||
}
|
||||
|
||||
private static Notification MakeNotification(NotificationType type = NotificationType.Email) =>
|
||||
new(Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1")
|
||||
{
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
private void SetupSmtp(int maxConcurrentConnections = 0, string? transport = null)
|
||||
{
|
||||
var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
|
||||
{
|
||||
MaxRetries = 5,
|
||||
RetryDelay = TimeSpan.FromMinutes(1),
|
||||
MaxConcurrentConnections = maxConcurrentConnections,
|
||||
Transport = transport,
|
||||
};
|
||||
_notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new[] { config });
|
||||
}
|
||||
|
||||
private void SetupDueBatchOnce(params Notification[] batch)
|
||||
{
|
||||
var claimed = 0;
|
||||
_outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(_ => Interlocked.Increment(ref claimed) == 1
|
||||
? batch
|
||||
: Array.Empty<Notification>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adapter that delays every delivery, records the concurrency high-water mark and the
|
||||
/// instant each delivery completed.
|
||||
/// </summary>
|
||||
private sealed class PacedAdapter(
|
||||
TimeSpan delay, Func<DeliveryOutcome> outcome, NotificationType type = NotificationType.Email)
|
||||
: INotificationDeliveryAdapter
|
||||
{
|
||||
private int _current;
|
||||
private int _highWater;
|
||||
private int _calls;
|
||||
|
||||
public NotificationType Type => type;
|
||||
|
||||
public int HighWater => Volatile.Read(ref _highWater);
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
public DateTimeOffset LastCompletedAt;
|
||||
|
||||
public async Task<DeliveryOutcome> DeliverAsync(
|
||||
Notification notification, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
var current = Interlocked.Increment(ref _current);
|
||||
int hw;
|
||||
while ((hw = Volatile.Read(ref _highWater)) < current
|
||||
&& Interlocked.CompareExchange(ref _highWater, current, hw) != hw)
|
||||
{
|
||||
// retry until our peak is recorded
|
||||
}
|
||||
|
||||
await Task.Delay(delay, cancellationToken);
|
||||
Interlocked.Decrement(ref _current);
|
||||
LastCompletedAt = DateTimeOffset.UtcNow;
|
||||
return outcome();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 06/COR-1: a delivery that completes well after the sweep began is stamped with the
|
||||
/// attempt instant, not the sweep-start instant.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Success_StampsDeliveredAt_AfterTheAdapterReturns()
|
||||
{
|
||||
SetupSmtp();
|
||||
var notification = MakeNotification();
|
||||
SetupDueBatchOnce(notification);
|
||||
var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x"));
|
||||
var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) });
|
||||
var before = DateTimeOffset.UtcNow;
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
|
||||
Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
|
||||
Arg.Any<CancellationToken>()));
|
||||
Assert.True(notification.DeliveredAt >= before + TimeSpan.FromMilliseconds(350),
|
||||
$"DeliveredAt {notification.DeliveredAt:O} should be after the 400 ms attempt (sweep began {before:O})");
|
||||
Assert.Equal(notification.DeliveredAt, notification.LastAttemptAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 06/COR-1: a transient failure schedules the retry relative to the attempt instant, so a
|
||||
/// slow attempt cannot collapse the retry delay.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void TransientFailure_SchedulesNextAttempt_FromTheAttemptInstant()
|
||||
{
|
||||
SetupSmtp();
|
||||
var notification = MakeNotification();
|
||||
SetupDueBatchOnce(notification);
|
||||
var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Transient("timeout"));
|
||||
var actor = CreateActor([adapter], new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) });
|
||||
var before = DateTimeOffset.UtcNow;
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
|
||||
Arg.Is<Notification>(n => n.Status == NotificationStatus.Retrying),
|
||||
Arg.Any<CancellationToken>()));
|
||||
Assert.True(notification.LastAttemptAt >= before + TimeSpan.FromMilliseconds(350));
|
||||
Assert.Equal(notification.LastAttemptAt + TimeSpan.FromMinutes(1), notification.NextAttemptAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 06/PERF-5: once the sweep's wall-clock budget is spent, deliveries still waiting for a
|
||||
/// permit are skipped (left due for the next sweep) instead of extending the sweep.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void SweepBudget_Exhausted_SkipsDeliveriesThatHaveNotStarted()
|
||||
{
|
||||
SetupSmtp();
|
||||
SetupDueBatchOnce(MakeNotification(), MakeNotification(), MakeNotification());
|
||||
var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(600), () => DeliveryOutcome.Success("ops@x"));
|
||||
var actor = CreateActor([adapter], new NotificationOutboxOptions
|
||||
{
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
MaxParallelDeliveries = 1,
|
||||
DispatchSweepBudget = TimeSpan.FromMilliseconds(200),
|
||||
});
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
AwaitAssert(() => _outboxRepository.Received(1).UpdateAsync(
|
||||
Arg.Any<Notification>(), Arg.Any<CancellationToken>()));
|
||||
// Give the (skipped) rest of the sweep ample time to have run had it not been skipped.
|
||||
Thread.Sleep(1500);
|
||||
Assert.Equal(1, adapter.Calls);
|
||||
_outboxRepository.Received(1).UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 06/IMPL-2: Email deliveries over the SMTP transport are clamped to the SMTP row's
|
||||
/// MaxConcurrentConnections even when MaxParallelDeliveries is higher.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void SmtpTransport_ClampsEmailParallelism_ToMaxConcurrentConnections()
|
||||
{
|
||||
SetupSmtp(maxConcurrentConnections: 1);
|
||||
SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray());
|
||||
var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(150), () => DeliveryOutcome.Success("ops@x"));
|
||||
var actor = CreateActor([adapter], new NotificationOutboxOptions
|
||||
{
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
MaxParallelDeliveries = 4,
|
||||
});
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
AwaitAssert(
|
||||
() => _outboxRepository.Received(4).UpdateAsync(
|
||||
Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
|
||||
Arg.Any<CancellationToken>()),
|
||||
duration: TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(1, adapter.HighWater);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 06/IMPL-2: the EWS transport does not use MaxConcurrentConnections, so its deliveries
|
||||
/// keep the full MaxParallelDeliveries.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void EwsTransport_IsNotClampedByMaxConcurrentConnections()
|
||||
{
|
||||
SetupSmtp(maxConcurrentConnections: 1, transport: "Ews");
|
||||
SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification()).ToArray());
|
||||
var adapter = new PacedAdapter(TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("ops@x"));
|
||||
var actor = CreateActor([adapter], new NotificationOutboxOptions
|
||||
{
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
MaxParallelDeliveries = 4,
|
||||
});
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
AwaitAssert(
|
||||
() => _outboxRepository.Received(4).UpdateAsync(
|
||||
Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
|
||||
Arg.Any<CancellationToken>()),
|
||||
duration: TimeSpan.FromSeconds(10));
|
||||
Assert.True(adapter.HighWater >= 2, $"expected concurrent EWS deliveries, got {adapter.HighWater}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 06/IMPL-2: the SMTP clamp applies to Email only — SMS deliveries in the same sweep keep
|
||||
/// the full MaxParallelDeliveries.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void SmtpClamp_DoesNotThrottleSms()
|
||||
{
|
||||
SetupSmtp(maxConcurrentConnections: 1);
|
||||
SetupDueBatchOnce(Enumerable.Range(0, 4).Select(_ => MakeNotification(NotificationType.Sms)).ToArray());
|
||||
var sms = new PacedAdapter(
|
||||
TimeSpan.FromMilliseconds(400), () => DeliveryOutcome.Success("+1555"), NotificationType.Sms);
|
||||
var actor = CreateActor([sms], new NotificationOutboxOptions
|
||||
{
|
||||
DispatchInterval = TimeSpan.FromHours(1),
|
||||
MaxParallelDeliveries = 4,
|
||||
});
|
||||
|
||||
actor.Tell(InternalMessages.DispatchTick.Instance);
|
||||
|
||||
AwaitAssert(
|
||||
() => _outboxRepository.Received(4).UpdateAsync(
|
||||
Arg.Is<Notification>(n => n.Status == NotificationStatus.Delivered),
|
||||
Arg.Any<CancellationToken>()),
|
||||
duration: TimeSpan.FromSeconds(10));
|
||||
Assert.True(sms.HighWater >= 2, $"expected concurrent SMS deliveries, got {sms.HighWater}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user