perf(store-and-forward): parallel per-target sweep lanes (cap 4), sequential within lane

src/ZB.MOM.WW.ScadaBridge.StoreAndForward/StoreAndForwardOptions.cs

@@ -29,4 +29,10 @@ public class StoreAndForwardOptions
     /// 0 = unlimited (legacy).
     /// </summary>
     public int SweepBatchLimit { get; set; } = 500;
+
+    /// <summary>
+    /// Max <c>(category, target)</c> lanes delivered concurrently per sweep.
+    /// 1 = legacy serial. Within a lane delivery stays sequential (per-target FIFO).
+    /// </summary>
+    public int SweepTargetParallelism { get; set; } = 4;
 }

src/ZB.MOM.WW.ScadaBridge.StoreAndForward/StoreAndForwardService.cs

@@ -606,20 +606,36 @@ public class StoreAndForwardService
 
             _logger.LogDebug("Retry sweep: {Count} messages due for retry", messages.Count);
 
-            var failedTargets = new HashSet<(StoreAndForwardCategory, string)>();
-            foreach (var message in messages)
-            {
-                // One transient failure per (category, target) per sweep: the target
-                // is down — burning a full timeout per remaining message serializes
-                // the sweep into hours under backlog (arch review 02, Performance #1).
-                // Skipped rows keep their RetryCount/LastAttemptAt untouched.
-                if (failedTargets.Contains((message.Category, message.Target)))
-                    continue;
+            // Group due rows into per-(category,target) lanes. GroupBy is stable, so
+            // each lane preserves created_at-ASC (oldest-first) order. Lanes run
+            // concurrently up to SweepTargetParallelism, so one slow/dead target no
+            // longer blocks delivery to healthy ones (arch review 02, Performance #2);
+            // within a lane delivery stays strictly sequential (per-target FIFO).
+            var lanes = messages
+                .GroupBy(m => (m.Category, m.Target))
+                .Select(g => g.ToList())
+                .ToList();
 
-                var outcome = await RetryMessageAsync(message);
-                if (outcome == RetryOutcome.TransientFailure)
-                    failedTargets.Add((message.Category, message.Target));
-            }
+            using var laneCap = new SemaphoreSlim(Math.Max(1, _options.SweepTargetParallelism));
+            var laneTasks = lanes.Select(async lane =>
+            {
+                await laneCap.WaitAsync();
+                try
+                {
+                    foreach (var message in lane)
+                    {
+                        // Task 8 short-circuit, per lane: one transient failure means
+                        // the target is down — skip the rest of this lane this sweep
+                        // rather than burning a full timeout per remaining message.
+                        // Skipped rows keep their RetryCount/LastAttemptAt untouched.
+                        var outcome = await RetryMessageAsync(message);
+                        if (outcome == RetryOutcome.TransientFailure)
+                            return;
+                    }
+                }
+                finally { laneCap.Release(); }
+            }).ToList();
+            await Task.WhenAll(laneTasks);
         }
         catch (Exception ex)
         {