feat(notification-outbox): add dispatcher loop to NotificationOutboxActor

This commit is contained in:
Joseph Doherty
2026-05-19 01:42:28 -04:00
parent 4dc9f9e159
commit c41f43c87f
4 changed files with 489 additions and 7 deletions

View File

@@ -26,4 +26,30 @@ internal static class InternalMessages
IActorRef Sender,
bool Succeeded,
string? Error);
/// <summary>
/// Periodic tick that triggers a dispatch sweep. Started as a periodic timer in
/// <c>PreStart</c> at the configured <c>DispatchInterval</c>. A singleton instance is
/// reused so the timer carries no per-tick state.
/// </summary>
internal sealed class DispatchTick
{
/// <summary>The shared singleton tick instance scheduled by the dispatch timer.</summary>
internal static readonly DispatchTick Instance = new();
private DispatchTick() { }
}
/// <summary>
/// Completion signal for an asynchronous dispatch sweep, piped back to the actor so the
/// in-flight guard is cleared on the actor thread. Sent on both success and failure of
/// the sweep — the actor only needs to know the sweep has finished.
/// </summary>
internal sealed class DispatchComplete
{
/// <summary>The shared singleton completion instance.</summary>
internal static readonly DispatchComplete Instance = new();
private DispatchComplete() { }
}
}

View File

@@ -5,33 +5,67 @@ using ScadaLink.Commons.Entities.Notifications;
using ScadaLink.Commons.Interfaces.Repositories;
using ScadaLink.Commons.Messages.Notification;
using ScadaLink.Commons.Types.Enums;
using ScadaLink.NotificationOutbox.Delivery;
using ScadaLink.NotificationOutbox.Messages;
namespace ScadaLink.NotificationOutbox;
/// <summary>
/// Central-side actor that owns the notification outbox. This task implements the ingest
/// path only: it accepts <see cref="NotificationSubmit"/> messages forwarded from sites,
/// persists each as a <see cref="Notification"/> row, and acks the submitting site.
/// Dispatch, query, and purge are added by later tasks.
/// Central-side actor that owns the notification outbox. It accepts
/// <see cref="NotificationSubmit"/> messages forwarded from sites and persists each as a
/// <see cref="Notification"/> row (the ingest path), and runs a periodic dispatch loop
/// that claims due notifications, delivers them through the matching channel adapter, and
/// applies the resulting status transition. Query and purge are added by later tasks.
/// </summary>
public class NotificationOutboxActor : ReceiveActor
public class NotificationOutboxActor : ReceiveActor, IWithTimers
{
private const string DispatchTimerKey = "dispatch";
/// <summary>Retry policy fallback used when no SMTP configuration row is present.</summary>
private const int FallbackMaxRetries = 10;
private static readonly TimeSpan FallbackRetryDelay = TimeSpan.FromMinutes(1);
private readonly IServiceProvider _serviceProvider;
private readonly NotificationOutboxOptions _options;
private readonly ILogger<NotificationOutboxActor> _logger;
private readonly IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> _adapters;
/// <summary>
/// In-flight guard for the dispatch loop. Set true at the start of a sweep and cleared
/// when the sweep's <see cref="InternalMessages.DispatchComplete"/> arrives. While true,
/// further <see cref="InternalMessages.DispatchTick"/>s are dropped so sweeps never overlap.
/// </summary>
private bool _dispatching;
/// <summary>Akka timer scheduler, assigned by the actor system via <see cref="IWithTimers"/>.</summary>
public ITimerScheduler Timers { get; set; } = null!;
public NotificationOutboxActor(
IServiceProvider serviceProvider,
NotificationOutboxOptions options,
ILogger<NotificationOutboxActor> logger)
ILogger<NotificationOutboxActor> logger,
IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> adapters)
{
_serviceProvider = serviceProvider;
_options = options;
_logger = logger;
_adapters = adapters;
Receive<NotificationSubmit>(HandleSubmit);
Receive<InternalMessages.IngestPersisted>(HandleIngestPersisted);
Receive<InternalMessages.DispatchTick>(_ => HandleDispatchTick());
Receive<InternalMessages.DispatchComplete>(_ => _dispatching = false);
}
/// <summary>
/// Starts the periodic dispatch timer once the actor is running. The tick cadence is
/// <see cref="NotificationOutboxOptions.DispatchInterval"/>.
/// </summary>
protected override void PreStart()
{
base.PreStart();
Timers.StartPeriodicTimer(
DispatchTimerKey, InternalMessages.DispatchTick.Instance, _options.DispatchInterval);
}
/// <summary>
@@ -89,6 +123,144 @@ public class NotificationOutboxActor : ReceiveActor
}
}
/// <summary>
/// Handles a dispatch tick. If a sweep is already in flight the tick is dropped so
/// sweeps never overlap; otherwise the guard is raised and an asynchronous sweep is
/// launched, with a <see cref="InternalMessages.DispatchComplete"/> piped back to
/// <see cref="Self"/> to lower the guard on the actor thread.
/// </summary>
private void HandleDispatchTick()
{
if (_dispatching)
{
return;
}
_dispatching = true;
var now = DateTimeOffset.UtcNow;
// RunDispatchPass swallows its own errors; the completion message is sent for both
// success and failure so the guard is always cleared.
RunDispatchPass(now).PipeTo(Self, success: () => InternalMessages.DispatchComplete.Instance);
}
/// <summary>
/// Runs a single dispatch sweep: claims the due batch, resolves the retry policy, and
/// delivers each notification sequentially. Per-notification failures are caught and
/// logged so one bad row never aborts the rest of the batch.
/// </summary>
private async Task RunDispatchPass(DateTimeOffset now)
{
using var scope = _serviceProvider.CreateScope();
var outboxRepository = scope.ServiceProvider.GetRequiredService<INotificationOutboxRepository>();
var notificationRepository = scope.ServiceProvider.GetRequiredService<INotificationRepository>();
IReadOnlyList<Notification> due;
try
{
due = await outboxRepository.GetDueAsync(now, _options.DispatchBatchSize);
}
catch (Exception ex)
{
_logger.LogError(ex, "Dispatch sweep failed to claim due notifications.");
return;
}
if (due.Count == 0)
{
return;
}
var (maxRetries, retryDelay) = await ResolveRetryPolicyAsync(notificationRepository);
foreach (var notification in due)
{
try
{
await DeliverOneAsync(notification, now, maxRetries, retryDelay, outboxRepository);
}
catch (Exception ex)
{
// Isolate per-notification failures so the remainder of the batch still runs.
_logger.LogError(
ex, "Dispatch failed for notification {NotificationId}.", notification.NotificationId);
}
}
}
/// <summary>
/// Resolves the retry policy from the first SMTP configuration row. When no SMTP
/// configuration exists, falls back to a conservative default — delivery itself will
/// permanently fail in that case, so the policy only acts as a guard.
/// </summary>
private async Task<(int MaxRetries, TimeSpan RetryDelay)> ResolveRetryPolicyAsync(
INotificationRepository notificationRepository)
{
var configurations = await notificationRepository.GetAllSmtpConfigurationsAsync();
var configuration = configurations.Count > 0 ? configurations[0] : null;
return configuration is null
? (FallbackMaxRetries, FallbackRetryDelay)
: (configuration.MaxRetries, configuration.RetryDelay);
}
/// <summary>
/// Delivers a single notification through its channel adapter and applies the resulting
/// status transition. A missing adapter parks the notification; otherwise the
/// <see cref="DeliveryOutcome"/> drives the transition. The updated row is always persisted.
/// </summary>
private async Task DeliverOneAsync(
Notification notification,
DateTimeOffset now,
int maxRetries,
TimeSpan retryDelay,
INotificationOutboxRepository outboxRepository)
{
if (!_adapters.TryGetValue(notification.Type, out var adapter))
{
notification.Status = NotificationStatus.Parked;
notification.LastError = $"no delivery adapter for type {notification.Type}";
notification.LastAttemptAt = now;
await outboxRepository.UpdateAsync(notification);
return;
}
var outcome = await adapter.DeliverAsync(notification);
switch (outcome.Result)
{
case DeliveryResult.Success:
notification.Status = NotificationStatus.Delivered;
notification.DeliveredAt = now;
notification.LastAttemptAt = now;
notification.ResolvedTargets = outcome.ResolvedTargets;
notification.LastError = null;
break;
case DeliveryResult.TransientFailure:
notification.LastAttemptAt = now;
notification.RetryCount++;
notification.LastError = outcome.Error;
if (notification.RetryCount >= maxRetries)
{
notification.Status = NotificationStatus.Parked;
}
else
{
notification.Status = NotificationStatus.Retrying;
notification.NextAttemptAt = now + retryDelay;
}
break;
case DeliveryResult.PermanentFailure:
notification.Status = NotificationStatus.Parked;
notification.LastAttemptAt = now;
notification.LastError = outcome.Error;
break;
}
await outboxRepository.UpdateAsync(notification);
}
private static Notification BuildNotification(NotificationSubmit msg)
{
// All current notifications are email; NotificationType has only the Email member.