perf(notification-outbox): bounded parallel delivery within a dispatch sweep (default 4)

src/ZB.MOM.WW.ScadaBridge.NotificationOutbox/NotificationOutboxActor.cs

@@ -318,33 +318,26 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
 
             var (maxRetries, retryDelay) = await ResolveRetryPolicyAsync(notificationRepository);
 
+            // Deliver the claimed batch with bounded parallelism. Delivery is I/O-bound
+            // (multi-second SMTP/Twilio round trips), so a strictly sequential sweep caps
+            // throughput at ~1/attempt-latency — an alarm-storm backlog behind one slow
+            // host would drain far too slowly (arch-review 04). A SemaphoreSlim caps the
+            // number of in-flight deliveries; each delivery runs on its OWN DI scope/
+            // repository (the EF DbContext is not thread-safe and rows are distinct per
+            // notification, so concurrent deliveries never contend). The cached adapters
+            // and the lifetime-scoped audit writer are already safe for concurrent use.
+            // MaxParallelDeliveries = 1 preserves the strictly sequential behaviour.
+            using var deliveryGate = new SemaphoreSlim(
+                _options.ResolvedMaxParallelDeliveries, _options.ResolvedMaxParallelDeliveries);
+            var deliveries = new List<Task>(due.Count);
             foreach (var notification in due)
             {
-                // Between deliveries, observe shutdown so we don't kick off a fresh
-                // SMTP send when the actor is already tearing down.
-                if (cancellationToken.IsCancellationRequested)
-                {
-                    return;
-                }
-
-                try
-                {
-                    await DeliverOneAsync(
-                        notification, now, maxRetries, retryDelay, outboxRepository, adapters, cancellationToken);
-                }
-                catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
-                {
-                    // In-flight delivery interrupted by shutdown. Row remains in its
-                    // pre-attempt state; next active sweep retries.
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    // Isolate per-notification failures so the remainder of the batch still runs.
-                    _logger.LogError(
-                        ex, "Dispatch failed for notification {NotificationId}.", notification.NotificationId);
-                }
+                deliveries.Add(DeliverGatedAsync(
+                    notification, now, maxRetries, retryDelay, adapters, deliveryGate, cancellationToken));
             }
+
+            // Each DeliverGatedAsync isolates its own faults, so WhenAll never throws.
+            await Task.WhenAll(deliveries);
         }
         catch (Exception ex)
         {
@@ -354,6 +347,67 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>
+    /// Delivers one notification under the sweep's concurrency gate. Acquires a permit,
+    /// runs <see cref="DeliverOneAsync"/> on a fresh per-notification DI scope (so parallel
+    /// deliveries never share an EF <c>DbContext</c>), and always releases the permit.
+    /// Faults are isolated here — shutdown cancellation is swallowed (the row stays in its
+    /// pre-attempt state for the next active sweep) and any other exception is logged — so
+    /// one bad delivery never aborts the rest of the batch and <c>Task.WhenAll</c> never faults.
+    /// </summary>
+    private async Task DeliverGatedAsync(
+        Notification notification,
+        DateTimeOffset now,
+        int maxRetries,
+        TimeSpan retryDelay,
+        IReadOnlyDictionary<NotificationType, INotificationDeliveryAdapter> adapters,
+        SemaphoreSlim gate,
+        CancellationToken cancellationToken)
+    {
+        try
+        {
+            await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException)
+        {
+            // Shutdown cancelled before this delivery acquired a permit; the row stays
+            // Pending and the next active node's sweep re-claims it. No permit to release.
+            return;
+        }
+
+        try
+        {
+            if (cancellationToken.IsCancellationRequested)
+            {
+                return;
+            }
+
+            // Each delivery owns its DI scope + repository so parallel UpdateAsync calls
+            // never share a DbContext. The adapters map + audit writer are already safe
+            // for concurrent use.
+            using var scope = _serviceProvider.CreateScope();
+            var outboxRepository = scope.ServiceProvider.GetRequiredService<INotificationOutboxRepository>();
+            await DeliverOneAsync(
+                    notification, now, maxRetries, retryDelay, outboxRepository, adapters, cancellationToken)
+                .ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            // In-flight delivery interrupted by shutdown. Row remains in its pre-attempt
+            // state; next active sweep retries.
+        }
+        catch (Exception ex)
+        {
+            // Isolate per-notification failures so the remainder of the batch still runs.
+            _logger.LogError(
+                ex, "Dispatch failed for notification {NotificationId}.", notification.NotificationId);
+        }
+        finally
+        {
+            gate.Release();
+        }
+    }
+
     /// <summary>
     /// Resolves the retry policy from the first SMTP configuration row. When no SMTP
     /// configuration exists, falls back to a conservative default — delivery itself will

src/ZB.MOM.WW.ScadaBridge.NotificationOutbox/NotificationOutboxOptions.cs

@@ -48,4 +48,23 @@ public class NotificationOutboxOptions
     /// Default: 1 minute.
     /// </summary>
     public TimeSpan DeliveredKpiWindow { get; set; } = TimeSpan.FromMinutes(1);
+
+    /// <summary>
+    /// Maximum number of notifications delivered concurrently within a single dispatch
+    /// sweep. Delivery is I/O-bound (SMTP / Twilio round trips of seconds each), so a
+    /// purely sequential sweep caps throughput at ~1/attempt-latency regardless of batch
+    /// size — an alarm storm queued behind one slow SMTP host would drain far too slowly
+    /// (arch-review 04). Each concurrent delivery runs on its own DI scope/repository, so
+    /// there is no shared-DbContext contention. Set to <c>1</c> to preserve the strictly
+    /// sequential behaviour. Clamped to a minimum of 1 via <see cref="ResolvedMaxParallelDeliveries"/>.
+    /// Default: 4.
+    /// </summary>
+    public int MaxParallelDeliveries { get; set; } = 4;
+
+    /// <summary>
+    /// Resolves the effective delivery parallelism, clamped to at least 1 so a
+    /// misconfigured <c>0</c>/negative value cannot stall the dispatcher (a zero-count
+    /// semaphore would deadlock the sweep).
+    /// </summary>
+    public int ResolvedMaxParallelDeliveries => MaxParallelDeliveries < 1 ? 1 : MaxParallelDeliveries;
 }