perf(central): set-based ingest, aligned partition purge, KPI query shapes, EF hygiene

src/ZB.MOM.WW.ScadaBridge.SiteCallAudit/SiteCallAuditActor.cs

@@ -158,6 +158,26 @@ public class SiteCallAuditActor : ReceiveActor
     /// </summary>
     private readonly Dictionary<string, bool> _reconciliationPinned = new();
 
+    /// <summary>
+    /// Actor-system EventStream captured on the actor thread at handler-registration
+    /// time. The reconciliation pass runs off-mailbox and publishes the pinned-state
+    /// transition from there, so it must not reach through <c>Context</c>.
+    /// </summary>
+    private Akka.Event.EventStream _eventStream = null!;
+
+    /// <summary>
+    /// Single-flight guard for the off-mailbox reconciliation pass. Raised on the
+    /// actor thread when a tick launches a pass and lowered on the actor thread
+    /// when the piped <see cref="ReconciliationComplete"/> arrives, so the
+    /// per-site cursor/pinned dictionaries the pass mutates are only ever touched
+    /// by one task at a time and the mailbox supplies the memory barrier between
+    /// consecutive passes.
+    /// </summary>
+    private bool _reconciling;
+
+    /// <summary>Single-flight guard for the off-mailbox terminal-row purge pass.</summary>
+    private bool _purging;
+
     private ICancelable? _reconciliationTimer;
     private ICancelable? _purgeTimer;
 
@@ -338,8 +358,79 @@ public class SiteCallAuditActor : ReceiveActor
         // the daily terminal-row purge. Handlers stay alive across faults via
         // their own per-site / per-tick try/catch (mirroring the ingest path);
         // the timers are only started when their collaborators are available.
-        ReceiveAsync<ReconciliationTick>(_ => OnReconciliationTickAsync());
-        ReceiveAsync<PurgeTick>(_ => OnPurgeTickAsync());
+        //
+        // OFF-MAILBOX (WP2.2). Both passes run as PipeTo-completed background
+        // tasks behind a single-flight guard rather than as ReceiveAsync bodies.
+        // A ReceiveAsync handler occupies the actor for its whole duration, and a
+        // reconciliation pass is unbounded work — every site, up to
+        // MaxReconciliationPagesPerTick network pulls each, one upsert per row.
+        // Post-outage catch-up therefore blocked telemetry ingest, UI queries and
+        // KPI Asks behind it until the drain finished, and those callers timed out
+        // rather than queued. NotificationOutboxActor's dispatch sweep is the
+        // in-repo reference for this shape.
+        Receive<ReconciliationTick>(_ => HandleReconciliationTick());
+        Receive<ReconciliationComplete>(_ => _reconciling = false);
+        Receive<PurgeTick>(_ => HandlePurgeTick());
+        Receive<PurgeComplete>(_ => _purging = false);
+
+        // Captured on the actor thread so the background passes never touch
+        // Context off-thread. EventStream itself is thread-safe.
+        _eventStream = Context.System.EventStream;
+    }
+
+    /// <summary>
+    /// Launches a reconciliation pass unless one is already in flight, dropping
+    /// the tick if so. Overlapping passes are not merely wasteful — they would
+    /// race on the per-site cursor and pinned-latch dictionaries, which the
+    /// single-flight guard keeps confined to one task at a time (the guard itself
+    /// is only ever mutated on the actor thread: raised here, lowered by the
+    /// piped completion message).
+    /// </summary>
+    private void HandleReconciliationTick()
+    {
+        if (_reconciling)
+        {
+            return;
+        }
+
+        _reconciling = true;
+
+        // OnReconciliationTickAsync swallows its own per-site errors, but the
+        // failure projection is kept as a belt-and-braces guard so even a faulted
+        // task still lowers the guard — otherwise reconciliation would wedge
+        // permanently after a single unexpected throw.
+        OnReconciliationTickAsync().PipeTo(
+            Self,
+            success: () => ReconciliationComplete.Instance,
+            failure: ex =>
+            {
+                _logger.LogError(ex, "SiteCallAudit reconciliation pass faulted unexpectedly.");
+                return ReconciliationComplete.Instance;
+            });
+    }
+
+    /// <summary>
+    /// Launches a purge pass unless one is already in flight. Same single-flight
+    /// discipline as <see cref="HandleReconciliationTick"/>: a purge that outlives
+    /// its interval (a large catch-up after an outage) must not stack.
+    /// </summary>
+    private void HandlePurgeTick()
+    {
+        if (_purging)
+        {
+            return;
+        }
+
+        _purging = true;
+
+        OnPurgeTickAsync().PipeTo(
+            Self,
+            success: () => PurgeComplete.Instance,
+            failure: ex =>
+            {
+                _logger.LogError(ex, "SiteCallAudit purge pass faulted unexpectedly.");
+                return PurgeComplete.Instance;
+            });
     }
 
     /// <inheritdoc />
@@ -743,7 +834,10 @@ public class SiteCallAuditActor : ReceiveActor
         }
 
         _reconciliationPinned[siteId] = pinned;
-        Context.System.EventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
+
+        // _eventStream, not Context.System.EventStream: this runs on the
+        // off-mailbox reconciliation pass, where Context must not be touched.
+        _eventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
     }
 
     // ── Piece B: daily terminal-row purge scheduler ──
@@ -1452,6 +1546,24 @@ public class SiteCallAuditActor : ReceiveActor
         public static readonly PurgeTick Instance = new();
         private PurgeTick() { }
     }
+
+    /// <summary>
+    /// Piped back to <c>Self</c> when an off-mailbox reconciliation pass ends
+    /// (successfully or not) so the single-flight guard is lowered on the actor
+    /// thread rather than from the background task.
+    /// </summary>
+    internal sealed class ReconciliationComplete
+    {
+        public static readonly ReconciliationComplete Instance = new();
+        private ReconciliationComplete() { }
+    }
+
+    /// <summary>Purge counterpart of <see cref="ReconciliationComplete"/>.</summary>
+    internal sealed class PurgeComplete
+    {
+        public static readonly PurgeComplete Instance = new();
+        private PurgeComplete() { }
+    }
 }
 
 /// <summary>