feat(notification-outbox): add dispatcher loop to NotificationOutboxActor

src/ScadaLink.NotificationOutbox/Messages/InternalMessages.cs

@@ -26,4 +26,30 @@ internal static class InternalMessages
         IActorRef Sender,
         bool Succeeded,
         string? Error);
+
+    /// <summary>
+    /// Periodic tick that triggers a dispatch sweep. Started as a periodic timer in
+    /// <c>PreStart</c> at the configured <c>DispatchInterval</c>. A singleton instance is
+    /// reused so the timer carries no per-tick state.
+    /// </summary>
+    internal sealed class DispatchTick
+    {
+        /// <summary>The shared singleton tick instance scheduled by the dispatch timer.</summary>
+        internal static readonly DispatchTick Instance = new();
+
+        private DispatchTick() { }
+    }
+
+    /// <summary>
+    /// Completion signal for an asynchronous dispatch sweep, piped back to the actor so the
+    /// in-flight guard is cleared on the actor thread. Sent on both success and failure of
+    /// the sweep — the actor only needs to know the sweep has finished.
+    /// </summary>
+    internal sealed class DispatchComplete
+    {
+        /// <summary>The shared singleton completion instance.</summary>
+        internal static readonly DispatchComplete Instance = new();
+
+        private DispatchComplete() { }
+    }
 }

src/ScadaLink.NotificationOutbox/NotificationOutboxActor.cs

@@ -5,33 +5,67 @@ using ScadaLink.Commons.Entities.Notifications;
 using ScadaLink.Commons.Interfaces.Repositories;
 using ScadaLink.Commons.Messages.Notification;
 using ScadaLink.Commons.Types.Enums;
+using ScadaLink.NotificationOutbox.Delivery;
 using ScadaLink.NotificationOutbox.Messages;
 
 namespace ScadaLink.NotificationOutbox;
 
 /// <summary>
-/// Central-side actor that owns the notification outbox. This task implements the ingest
+/// Central-side actor that owns the notification outbox. It accepts
-/// path only: it accepts <see cref="NotificationSubmit"/> messages forwarded from sites,
+/// <see cref="NotificationSubmit"/> messages forwarded from sites and persists each as a
-/// persists each as a <see cref="Notification"/> row, and acks the submitting site.
+/// <see cref="Notification"/> row (the ingest path), and runs a periodic dispatch loop
-/// Dispatch, query, and purge are added by later tasks.
+/// that claims due notifications, delivers them through the matching channel adapter, and
+/// applies the resulting status transition. Query and purge are added by later tasks.
 /// </summary>
-public class NotificationOutboxActor : ReceiveActor
+public class NotificationOutboxActor : ReceiveActor, IWithTimers
 {
+    private const string DispatchTimerKey = "dispatch";
+
+    /// <summary>Retry policy fallback used when no SMTP configuration row is present.</summary>
+    private const int FallbackMaxRetries = 10;
+    private static readonly TimeSpan FallbackRetryDelay = TimeSpan.FromMinutes(1);
+
     private readonly IServiceProvider _serviceProvider;
     private readonly NotificationOutboxOptions _options;
     private readonly ILogger<NotificationOutboxActor> _logger;
+    private readonly IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> _adapters;
+
+    /// <summary>
+    /// In-flight guard for the dispatch loop. Set true at the start of a sweep and cleared
+    /// when the sweep's <see cref="InternalMessages.DispatchComplete"/> arrives. While true,
+    /// further <see cref="InternalMessages.DispatchTick"/>s are dropped so sweeps never overlap.
+    /// </summary>
+    private bool _dispatching;
+
+    /// <summary>Akka timer scheduler, assigned by the actor system via <see cref="IWithTimers"/>.</summary>
+    public ITimerScheduler Timers { get; set; } = null!;
 
     public NotificationOutboxActor(
         IServiceProvider serviceProvider,
         NotificationOutboxOptions options,
-        ILogger<NotificationOutboxActor> logger)
+        ILogger<NotificationOutboxActor> logger,
+        IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> adapters)
     {
         _serviceProvider = serviceProvider;
         _options = options;
         _logger = logger;
+        _adapters = adapters;
 
         Receive<NotificationSubmit>(HandleSubmit);
         Receive<InternalMessages.IngestPersisted>(HandleIngestPersisted);
+        Receive<InternalMessages.DispatchTick>(_ => HandleDispatchTick());
+        Receive<InternalMessages.DispatchComplete>(_ => _dispatching = false);
+    }
+
+    /// <summary>
+    /// Starts the periodic dispatch timer once the actor is running. The tick cadence is
+    /// <see cref="NotificationOutboxOptions.DispatchInterval"/>.
+    /// </summary>
+    protected override void PreStart()
+    {
+        base.PreStart();
+        Timers.StartPeriodicTimer(
+            DispatchTimerKey, InternalMessages.DispatchTick.Instance, _options.DispatchInterval);
     }
 
     /// <summary>
@@ -89,6 +123,144 @@ public class NotificationOutboxActor : ReceiveActor
         }
     }
 
+    /// <summary>
+    /// Handles a dispatch tick. If a sweep is already in flight the tick is dropped so
+    /// sweeps never overlap; otherwise the guard is raised and an asynchronous sweep is
+    /// launched, with a <see cref="InternalMessages.DispatchComplete"/> piped back to
+    /// <see cref="Self"/> to lower the guard on the actor thread.
+    /// </summary>
+    private void HandleDispatchTick()
+    {
+        if (_dispatching)
+        {
+            return;
+        }
+
+        _dispatching = true;
+        var now = DateTimeOffset.UtcNow;
+
+        // RunDispatchPass swallows its own errors; the completion message is sent for both
+        // success and failure so the guard is always cleared.
+        RunDispatchPass(now).PipeTo(Self, success: () => InternalMessages.DispatchComplete.Instance);
+    }
+
+    /// <summary>
+    /// Runs a single dispatch sweep: claims the due batch, resolves the retry policy, and
+    /// delivers each notification sequentially. Per-notification failures are caught and
+    /// logged so one bad row never aborts the rest of the batch.
+    /// </summary>
+    private async Task RunDispatchPass(DateTimeOffset now)
+    {
+        using var scope = _serviceProvider.CreateScope();
+        var outboxRepository = scope.ServiceProvider.GetRequiredService<INotificationOutboxRepository>();
+        var notificationRepository = scope.ServiceProvider.GetRequiredService<INotificationRepository>();
+
+        IReadOnlyList<Notification> due;
+        try
+        {
+            due = await outboxRepository.GetDueAsync(now, _options.DispatchBatchSize);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Dispatch sweep failed to claim due notifications.");
+            return;
+        }
+
+        if (due.Count == 0)
+        {
+            return;
+        }
+
+        var (maxRetries, retryDelay) = await ResolveRetryPolicyAsync(notificationRepository);
+
+        foreach (var notification in due)
+        {
+            try
+            {
+                await DeliverOneAsync(notification, now, maxRetries, retryDelay, outboxRepository);
+            }
+            catch (Exception ex)
+            {
+                // Isolate per-notification failures so the remainder of the batch still runs.
+                _logger.LogError(
+                    ex, "Dispatch failed for notification {NotificationId}.", notification.NotificationId);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Resolves the retry policy from the first SMTP configuration row. When no SMTP
+    /// configuration exists, falls back to a conservative default — delivery itself will
+    /// permanently fail in that case, so the policy only acts as a guard.
+    /// </summary>
+    private async Task<(int MaxRetries, TimeSpan RetryDelay)> ResolveRetryPolicyAsync(
+        INotificationRepository notificationRepository)
+    {
+        var configurations = await notificationRepository.GetAllSmtpConfigurationsAsync();
+        var configuration = configurations.Count > 0 ? configurations[0] : null;
+        return configuration is null
+            ? (FallbackMaxRetries, FallbackRetryDelay)
+            : (configuration.MaxRetries, configuration.RetryDelay);
+    }
+
+    /// <summary>
+    /// Delivers a single notification through its channel adapter and applies the resulting
+    /// status transition. A missing adapter parks the notification; otherwise the
+    /// <see cref="DeliveryOutcome"/> drives the transition. The updated row is always persisted.
+    /// </summary>
+    private async Task DeliverOneAsync(
+        Notification notification,
+        DateTimeOffset now,
+        int maxRetries,
+        TimeSpan retryDelay,
+        INotificationOutboxRepository outboxRepository)
+    {
+        if (!_adapters.TryGetValue(notification.Type, out var adapter))
+        {
+            notification.Status = NotificationStatus.Parked;
+            notification.LastError = $"no delivery adapter for type {notification.Type}";
+            notification.LastAttemptAt = now;
+            await outboxRepository.UpdateAsync(notification);
+            return;
+        }
+
+        var outcome = await adapter.DeliverAsync(notification);
+
+        switch (outcome.Result)
+        {
+            case DeliveryResult.Success:
+                notification.Status = NotificationStatus.Delivered;
+                notification.DeliveredAt = now;
+                notification.LastAttemptAt = now;
+                notification.ResolvedTargets = outcome.ResolvedTargets;
+                notification.LastError = null;
+                break;
+
+            case DeliveryResult.TransientFailure:
+                notification.LastAttemptAt = now;
+                notification.RetryCount++;
+                notification.LastError = outcome.Error;
+                if (notification.RetryCount >= maxRetries)
+                {
+                    notification.Status = NotificationStatus.Parked;
+                }
+                else
+                {
+                    notification.Status = NotificationStatus.Retrying;
+                    notification.NextAttemptAt = now + retryDelay;
+                }
+                break;
+
+            case DeliveryResult.PermanentFailure:
+                notification.Status = NotificationStatus.Parked;
+                notification.LastAttemptAt = now;
+                notification.LastError = outcome.Error;
+                break;
+        }
+
+        await outboxRepository.UpdateAsync(notification);
+    }
+
     private static Notification BuildNotification(NotificationSubmit msg)
     {
         // All current notifications are email; NotificationType has only the Email member.

tests/ScadaLink.NotificationOutbox.Tests/NotificationOutboxActorDispatchTests.cs

@@ -0,0 +1,282 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using ScadaLink.Commons.Entities.Notifications;
+using ScadaLink.Commons.Interfaces.Repositories;
+using ScadaLink.Commons.Types.Enums;
+using ScadaLink.NotificationOutbox.Delivery;
+using ScadaLink.NotificationOutbox.Messages;
+
+namespace ScadaLink.NotificationOutbox.Tests;
+
+/// <summary>
+/// Task 14: Tests for the <see cref="NotificationOutboxActor"/> dispatcher loop — the
+/// periodic sweep that claims due notifications via
+/// <see cref="INotificationOutboxRepository.GetDueAsync"/>, delivers each through the
+/// matching <see cref="INotificationDeliveryAdapter"/>, and applies the resulting status
+/// transition with <see cref="INotificationOutboxRepository.UpdateAsync"/>.
+/// </summary>
+public class NotificationOutboxActorDispatchTests : TestKit
+{
+    private readonly INotificationOutboxRepository _outboxRepository =
+        Substitute.For<INotificationOutboxRepository>();
+
+    private readonly INotificationRepository _notificationRepository =
+        Substitute.For<INotificationRepository>();
+
+    private IServiceProvider BuildServiceProvider()
+    {
+        var services = new ServiceCollection();
+        services.AddScoped(_ => _outboxRepository);
+        services.AddScoped(_ => _notificationRepository);
+        return services.BuildServiceProvider();
+    }
+
+    /// <summary>
+    /// Stub adapter whose <see cref="DeliverAsync"/> returns a configurable outcome and
+    /// optionally blocks for a delay — used to exercise the overlapping-tick guard.
+    /// </summary>
+    private sealed class StubAdapter : INotificationDeliveryAdapter
+    {
+        private readonly Func<DeliveryOutcome> _outcome;
+        private readonly TimeSpan _delay;
+
+        public StubAdapter(Func<DeliveryOutcome> outcome, TimeSpan? delay = null)
+        {
+            _outcome = outcome;
+            _delay = delay ?? TimeSpan.Zero;
+        }
+
+        public int CallCount;
+
+        public NotificationType Type => NotificationType.Email;
+
+        public async Task<DeliveryOutcome> DeliverAsync(
+            Notification notification, CancellationToken cancellationToken = default)
+        {
+            Interlocked.Increment(ref CallCount);
+            if (_delay > TimeSpan.Zero)
+            {
+                await Task.Delay(_delay, cancellationToken);
+            }
+
+            return _outcome();
+        }
+    }
+
+    private IActorRef CreateActor(
+        IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> adapters,
+        NotificationOutboxOptions? options = null)
+    {
+        return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
+            BuildServiceProvider(),
+            options ?? new NotificationOutboxOptions { DispatchInterval = TimeSpan.FromHours(1) },
+            NullLogger<NotificationOutboxActor>.Instance,
+            adapters)));
+    }
+
+    private static Notification MakeNotification(
+        NotificationType type = NotificationType.Email, int retryCount = 0)
+    {
+        return new Notification(
+            Guid.NewGuid().ToString(), type, "ops-team", "Subject", "Body", "site-1")
+        {
+            RetryCount = retryCount,
+            CreatedAt = DateTimeOffset.UtcNow,
+        };
+    }
+
+    private void SetupSmtpRetryPolicy(int maxRetries, TimeSpan retryDelay)
+    {
+        var config = new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
+        {
+            MaxRetries = maxRetries,
+            RetryDelay = retryDelay,
+        };
+        _notificationRepository.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
+            .Returns(new[] { config });
+    }
+
+    [Fact]
+    public void DispatchTick_ClaimsDueNotifications_AndInvokesAdapter()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
+        var notification = MakeNotification();
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>
+        {
+            [NotificationType.Email] = adapter,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() =>
+        {
+            _outboxRepository.Received(1).GetDueAsync(
+                Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>());
+            Assert.Equal(1, adapter.CallCount);
+        });
+    }
+
+    [Fact]
+    public void Success_MarksNotificationDelivered_WithResolvedTargets()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
+        var notification = MakeNotification();
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        var adapter = new StubAdapter(() => DeliveryOutcome.Success("ops@example.com"));
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>
+        {
+            [NotificationType.Email] = adapter,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() =>
+        {
+            _outboxRepository.Received(1).UpdateAsync(
+                Arg.Is<Notification>(n =>
+                    n.Status == NotificationStatus.Delivered &&
+                    n.DeliveredAt != null &&
+                    n.LastAttemptAt != null &&
+                    n.ResolvedTargets == "ops@example.com" &&
+                    n.LastError == null),
+                Arg.Any<CancellationToken>());
+        });
+    }
+
+    [Fact]
+    public void TransientFailure_BelowMaxRetries_MarksRetrying_AndSchedulesNextAttempt()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(3));
+        var notification = MakeNotification(retryCount: 1);
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>
+        {
+            [NotificationType.Email] = adapter,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() =>
+        {
+            _outboxRepository.Received(1).UpdateAsync(
+                Arg.Is<Notification>(n =>
+                    n.Status == NotificationStatus.Retrying &&
+                    n.RetryCount == 2 &&
+                    n.NextAttemptAt != null &&
+                    n.LastError == "smtp timeout" &&
+                    n.LastAttemptAt != null),
+                Arg.Any<CancellationToken>());
+        });
+    }
+
+    [Fact]
+    public void TransientFailure_ReachingMaxRetries_MarksParked()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 3, retryDelay: TimeSpan.FromMinutes(1));
+        // RetryCount starts at max-1; the failed attempt increments it to max.
+        var notification = MakeNotification(retryCount: 2);
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        var adapter = new StubAdapter(() => DeliveryOutcome.Transient("smtp timeout"));
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>
+        {
+            [NotificationType.Email] = adapter,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() =>
+        {
+            _outboxRepository.Received(1).UpdateAsync(
+                Arg.Is<Notification>(n =>
+                    n.Status == NotificationStatus.Parked &&
+                    n.RetryCount == 3),
+                Arg.Any<CancellationToken>());
+        });
+    }
+
+    [Fact]
+    public void PermanentFailure_MarksParked_WithLastError()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
+        var notification = MakeNotification();
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        var adapter = new StubAdapter(() => DeliveryOutcome.Permanent("invalid recipient address"));
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>
+        {
+            [NotificationType.Email] = adapter,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() =>
+        {
+            _outboxRepository.Received(1).UpdateAsync(
+                Arg.Is<Notification>(n =>
+                    n.Status == NotificationStatus.Parked &&
+                    n.LastError == "invalid recipient address" &&
+                    n.LastAttemptAt != null),
+                Arg.Any<CancellationToken>());
+        });
+    }
+
+    [Fact]
+    public void NoAdapterForType_MarksParked_WithExplanatoryError()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
+        var notification = MakeNotification();
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        // Empty adapter dictionary: no adapter resolves for the notification's type.
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>());
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        AwaitAssert(() =>
+        {
+            _outboxRepository.Received(1).UpdateAsync(
+                Arg.Is<Notification>(n =>
+                    n.Status == NotificationStatus.Parked &&
+                    n.LastError != null &&
+                    n.LastError.Contains("no delivery adapter") &&
+                    n.LastAttemptAt != null),
+                Arg.Any<CancellationToken>());
+        });
+    }
+
+    [Fact]
+    public void OverlappingTicks_WhileDispatchInFlight_DoNotClaimConcurrently()
+    {
+        SetupSmtpRetryPolicy(maxRetries: 5, retryDelay: TimeSpan.FromMinutes(1));
+        var notification = MakeNotification();
+        _outboxRepository.GetDueAsync(Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { notification });
+        // Slow adapter keeps the first sweep in flight while the second tick arrives.
+        var adapter = new StubAdapter(
+            () => DeliveryOutcome.Success("ops@example.com"),
+            delay: TimeSpan.FromMilliseconds(800));
+        var actor = CreateActor(new Dictionary<NotificationType, INotificationDeliveryAdapter>
+        {
+            [NotificationType.Email] = adapter,
+        });
+
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+        actor.Tell(InternalMessages.DispatchTick.Instance);
+
+        // Second tick is dropped by the in-flight guard: only one sweep runs.
+        AwaitAssert(
+            () => _outboxRepository.Received(1).GetDueAsync(
+                Arg.Any<DateTimeOffset>(), Arg.Any<int>(), Arg.Any<CancellationToken>()),
+            duration: TimeSpan.FromSeconds(2));
+    }
+}

tests/ScadaLink.NotificationOutbox.Tests/NotificationOutboxActorIngestTests.cs

@@ -8,6 +8,7 @@ using ScadaLink.Commons.Entities.Notifications;
 using ScadaLink.Commons.Interfaces.Repositories;
 using ScadaLink.Commons.Messages.Notification;
 using ScadaLink.Commons.Types.Enums;
+using ScadaLink.NotificationOutbox.Delivery;
 
 namespace ScadaLink.NotificationOutbox.Tests;
 
@@ -33,7 +34,8 @@ public class NotificationOutboxActorIngestTests : TestKit
         return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
             BuildServiceProvider(),
             new NotificationOutboxOptions(),
-            NullLogger<NotificationOutboxActor>.Instance)));
+            NullLogger<NotificationOutboxActor>.Instance,
+            new Dictionary<NotificationType, INotificationDeliveryAdapter>())));
     }
 
     private static NotificationSubmit MakeSubmit(string? notificationId = null)