Resolve all 46 low-severity architecture review findings

src/ZB.MOM.WW.ScadaBridge.KpiHistory/KpiHistoryRecorderActor.cs

@@ -58,9 +58,10 @@ namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
 /// <para>
 /// <b>DI scopes.</b> <see cref="IKpiSampleSource"/>s and
 /// <see cref="IKpiHistoryRepository"/> are scoped EF Core-backed services; the
-/// recorder opens a fresh <see cref="IServiceScope"/> per tick and resolves
-/// there, asynchronously disposing it (<c>CreateAsyncScope</c>) so EF Core's
-/// async connection cleanup does not block.
+/// recorder opens a fresh <see cref="IServiceScope"/> for the tick's repository
+/// and a separate scope for each source collection. A timed-out collection owns
+/// its scope until actual completion, while later sources use separate DbContexts.
+/// Scopes are asynchronously disposed (<c>CreateAsyncScope</c>).
 /// </para>
 /// <para>
 /// <b>Singleton wiring.</b> The <c>Props</c> is built in the Host on the
@@ -87,6 +88,8 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     /// </summary>
     private static readonly TimeSpan BackfillInitialDelay = TimeSpan.FromSeconds(15);
 
+    // Shared by every recorder incarnation on this root provider.
+    private readonly KpiSourceAdmission _sourceAdmission;
     private readonly IServiceProvider _serviceProvider;
     private readonly KpiHistoryOptions _options;
     private readonly ILogger<KpiHistoryRecorderActor> _logger;
@@ -176,6 +179,7 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         ILogger<KpiHistoryRecorderActor> logger)
     {
         _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
+        _sourceAdmission = serviceProvider.GetRequiredService<KpiSourceAdmission>();
         _options = options ?? throw new ArgumentNullException(nameof(options));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
@@ -307,9 +311,8 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
     }
 
     /// <summary>
-    /// Runs a single sampling pass: opens a DI scope, enumerates every registered
-    /// <see cref="IKpiSampleSource"/>, collects each source's samples (isolating per-source
-    /// faults), and bulk-writes the combined batch. The whole body is wrapped so the
+    /// Runs a single sampling pass: enumerates every registered <see cref="IKpiSampleSource"/>,
+    /// waits a bounded time for each separately scoped collection, and bulk-writes the combined batch. The whole body is wrapped so the
     /// returned task never faults — best-effort observability must never disrupt anything.
     /// </summary>
     private async Task RunSamplePass(DateTime capturedAt, CancellationToken cancellationToken)
@@ -317,17 +320,32 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         try
         {
             await using var scope = _serviceProvider.CreateAsyncScope();
-            var sources = scope.ServiceProvider.GetServices<IKpiSampleSource>();
+            var sources = scope.ServiceProvider.GetServices<IKpiSampleSource>().ToArray();
             var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
 
             var samples = new List<KpiSample>();
-            foreach (var source in sources)
+            for (var sourceIndex = 0; sourceIndex < sources.Length; sourceIndex++)
             {
+                var sourceName = sources[sourceIndex].Source;
+                // Each call owns a separate scope, so a cancellation-ignoring source
+                // neither shares a busy DbContext with healthy sources nor loses its scope.
+                var index = sourceIndex;
+                if (!_sourceAdmission.TryStart(index,
+                        () => CollectSourceAsync(index, capturedAt, cancellationToken), out var collection))
+                    continue;
+                _ = collection.ContinueWith(t =>
+                    _logger.LogError(t.Exception, "KPI source {Source} collection failed.", sourceName),
+                    CancellationToken.None, TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
+                    TaskScheduler.Default);
                 try
                 {
-                    var collected = await source.CollectAsync(capturedAt, cancellationToken);
+                    var collected = await collection.WaitAsync(_options.SourceTimeout, cancellationToken);
                     samples.AddRange(collected);
                 }
+                catch (TimeoutException)
+                {
+                    _logger.LogWarning("KPI source {Source} exceeded its collection deadline; unfinished work remains owned and will not be restarted.", sourceName);
+                }
                 catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                 {
                     // Shutdown interrupted collection; abandon the rest of the pass. The next
@@ -337,7 +355,7 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
                 catch (Exception ex)
                 {
                     // A throwing source must NOT abort the pass or the other sources.
-                    _logger.LogError(ex, "KPI source {Source} failed to collect samples.", source.Source);
+                    _logger.LogError(ex, "KPI source {Source} failed to collect samples.", sourceName);
                 }
             }
 
@@ -367,6 +385,17 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
         }
     }
 
+    /// <summary>Owns a source's scope and cancellation source until its actual collection completes, even after the caller stops waiting.</summary>
+    private async Task<IReadOnlyList<KpiSample>> CollectSourceAsync(
+        int sourceIndex, DateTime capturedAt, CancellationToken cancellationToken)
+    {
+        await using var scope = _serviceProvider.CreateAsyncScope();
+        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        deadline.CancelAfter(_options.SourceTimeout);
+        var source = scope.ServiceProvider.GetServices<IKpiSampleSource>().ElementAt(sourceIndex);
+        return await source.CollectAsync(capturedAt, deadline.Token).ConfigureAwait(false);
+    }
+
     /// <summary>
     /// Handles a purge tick: computes both retention cut-offs on the actor thread (raw samples
     /// at <see cref="KpiHistoryOptions.RetentionDays"/>, hourly rollups at the longer