perf(store-and-forward): short-circuit a (category,target) lane after its first transient failure per sweep

src/ZB.MOM.WW.ScadaBridge.StoreAndForward/StoreAndForwardService.cs

@@ -606,9 +606,19 @@ public class StoreAndForwardService
 
             _logger.LogDebug("Retry sweep: {Count} messages due for retry", messages.Count);
 
+            var failedTargets = new HashSet<(StoreAndForwardCategory, string)>();
             foreach (var message in messages)
             {
-                await RetryMessageAsync(message);
+                // One transient failure per (category, target) per sweep: the target
+                // is down — burning a full timeout per remaining message serializes
+                // the sweep into hours under backlog (arch review 02, Performance #1).
+                // Skipped rows keep their RetryCount/LastAttemptAt untouched.
+                if (failedTargets.Contains((message.Category, message.Target)))
+                    continue;
+
+                var outcome = await RetryMessageAsync(message);
+                if (outcome == RetryOutcome.TransientFailure)
+                    failedTargets.Add((message.Category, message.Target));
             }
         }
         catch (Exception ex)
@@ -621,12 +631,20 @@ public class StoreAndForwardService
         }
     }
 
-    private async Task RetryMessageAsync(StoreAndForwardMessage message)
+    /// <summary>
+    /// Outcome of a single message's retry attempt, used by the sweep to
+    /// short-circuit a <c>(category, target)</c> lane after its first transient
+    /// failure. <see cref="Skipped"/> = no attempt was made (no handler, or the
+    /// row's status changed under a concurrent apply); its RetryCount is untouched.
+    /// </summary>
+    internal enum RetryOutcome { Delivered, Parked, TransientFailure, Skipped }
+
+    private async Task<RetryOutcome> RetryMessageAsync(StoreAndForwardMessage message)
     {
         if (!_deliveryHandlers.TryGetValue(message.Category, out var handler))
         {
             _logger.LogWarning("No delivery handler for category {Category}", message.Category);
-            return;
+            return RetryOutcome.Skipped;
         }
 
         // Measure per-attempt
@@ -657,7 +675,7 @@ public class StoreAndForwardService
                     httpStatus: null,
                     occurredAtUtc: attemptStartUtc,
                     durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
-                return;
+                return RetryOutcome.Delivered;
             }
 
             // Permanent failure on retry — park immediately.
@@ -671,7 +689,7 @@ public class StoreAndForwardService
                 _logger.LogDebug(
                     "Message {MessageId} changed status during delivery; sweep park skipped",
                     message.Id);
-                return;
+                return RetryOutcome.Skipped;
             }
             Interlocked.Decrement(ref _bufferedCount);
             _replication?.ReplicatePark(message);
@@ -687,6 +705,7 @@ public class StoreAndForwardService
                 httpStatus: null,
                 occurredAtUtc: attemptStartUtc,
                 durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
+            return RetryOutcome.Parked;
         }
         catch (Exception ex)
         {
@@ -706,7 +725,7 @@ public class StoreAndForwardService
                     _logger.LogDebug(
                         "Message {MessageId} changed status during delivery; sweep park skipped",
                         message.Id);
-                    return;
+                    return RetryOutcome.Skipped;
                 }
                 Interlocked.Decrement(ref _bufferedCount);
                 _replication?.ReplicatePark(message);
@@ -725,6 +744,7 @@ public class StoreAndForwardService
                     httpStatus: null,
                     occurredAtUtc: attemptStartUtc,
                     durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
+                return RetryOutcome.Parked;
             }
             else
             {
@@ -734,7 +754,7 @@ public class StoreAndForwardService
                     _logger.LogDebug(
                         "Message {MessageId} changed status during delivery; sweep retry-count update skipped",
                         message.Id);
-                    return;
+                    return RetryOutcome.Skipped;
                 }
                 RaiseActivity("Retried", message.Category,
                     $"Retry {message.RetryCount}/{message.MaxRetries} for {message.Target}: {ex.Message}");
@@ -748,6 +768,7 @@ public class StoreAndForwardService
                     httpStatus: null,
                     occurredAtUtc: attemptStartUtc,
                     durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
+                return RetryOutcome.TransientFailure;
             }
         }
     }