fix(notification-outbox): per-attempt timestamps, Pending/Retrying split claim + sweep budget, SMTP MaxConcurrentConnections clamp (review 2026-10-21 06/COR-1, 06/PERF-5, 06/IMPL-2)

2026-10-22 09:22:10 -04:00
parent ebf4b96789
commit 811a561885
10 changed files with 547 additions and 37 deletions
@@ -146,15 +146,54 @@ VALUES
        // of notifications — each carrying an nvarchar(max) Body and TypeData —
        // paid for a full snapshot copy per row plus a DetectChanges scan of the
        // whole batch on every save.
-        return await _context.Notifications
-            .AsNoTracking()
-            .Where(n => n.Status == NotificationStatus.Pending
-                || (n.Status == NotificationStatus.Retrying
-                    && n.NextAttemptAt != null
-                    && n.NextAttemptAt <= now))
-            .OrderBy(n => n.CreatedAt)
-            .Take(batchSize)
+        //
+        // The batch is SPLIT between due Retrying rows and Pending rows (review
+        // 2026-10-21 06/PERF-5). A single oldest-first claim let a retrying backlog —
+        // always older than fresh traffic — monopolise every sweep during a partial
+        // relay outage, so a new alarm email was not even attempted until the backlog
+        // parked (hours, at MaxRetries x RetryDelay). Retrying gets floor(batch/2),
+        // Pending the rest; a share the other status cannot fill is back-filled, so a
+        // sweep still claims a full batch whenever that many rows are due. Each share is
+        // oldest-first and the merged batch is returned in CreatedAt order.
+        if (batchSize <= 0)
+        {
+            return Array.Empty<Notification>();
+        }
+
+        var retryingQuota = batchSize / 2;
+        var pending = await DuePendingQuery()
+            .Take(batchSize - retryingQuota)
            .ToListAsync(cancellationToken);
+        var retrying = await DueRetryingQuery(now)
+            .Take(batchSize - pending.Count)
+            .ToListAsync(cancellationToken);
+
+        var remaining = batchSize - pending.Count - retrying.Count;
+        if (remaining > 0 && pending.Count == batchSize - retryingQuota)
+        {
+            // Retrying could not fill its share; back-fill with newer Pending rows.
+            pending.AddRange(await DuePendingQuery()
+                .Skip(pending.Count)
+                .Take(remaining)
+                .ToListAsync(cancellationToken));
+        }
+
+        pending.AddRange(retrying);
+        return pending.OrderBy(n => n.CreatedAt).ToList();
+
+        IQueryable<Notification> DuePendingQuery() => _context.Notifications
+            .AsNoTracking()
+            .Where(n => n.Status == NotificationStatus.Pending)
+            .OrderBy(n => n.CreatedAt)
+            .ThenBy(n => n.NotificationId);
+
+        IQueryable<Notification> DueRetryingQuery(DateTimeOffset dueBy) => _context.Notifications
+            .AsNoTracking()
+            .Where(n => n.Status == NotificationStatus.Retrying
+                && n.NextAttemptAt != null
+                && n.NextAttemptAt <= dueBy)
+            .OrderBy(n => n.CreatedAt)
+            .ThenBy(n => n.NotificationId);
    }

    /// <inheritdoc />