perf(central): set-based ingest, aligned partition purge, KPI query shapes, EF hygiene

2026-08-14 21:07:12 -04:00
parent ee193cd2bb
commit 5db2a810c0
29 changed files with 3790 additions and 266 deletions
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
 using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
 using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
 using ZB.MOM.WW.ScadaBridge.Commons.Types;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
@@ -582,4 +583,122 @@ public class SiteCallAuditReconciliationTests : TestKit
        Assert.Equal(siteId, evt.SiteId);
        Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
    }
+
+    // ---------------------------------------------------------------------
+    // 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
+    //    query and KPI messages are answered while a long post-outage
+    //    catch-up is still pulling.
+    // ---------------------------------------------------------------------
+
+    /// <summary>
+    /// Pull client whose first call blocks until released, simulating a
+    /// post-outage catch-up that takes far longer than the caller's Ask timeout.
+    /// </summary>
+    private sealed class BlockingPullClient : IPullSiteCallsClient
+    {
+        private readonly TaskCompletionSource _release =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private readonly TaskCompletionSource _entered =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        /// <summary>Completes once the drain has actually started pulling.</summary>
+        public Task Entered => _entered.Task;
+
+        public void Release() => _release.TrySetResult();
+
+        public async Task<PullSiteCallsResponse> PullAsync(
+            string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
+        {
+            _entered.TrySetResult();
+            await _release.Task.ConfigureAwait(false);
+            return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
+        }
+    }
+
+    [Fact]
+    public async Task ReconciliationDrain_InFlight_DoesNotBlockIngestUpsert()
+    {
+        // The drain used to run inside ReceiveAsync, which occupies the actor for
+        // its whole duration. A post-outage catch-up (every site, many paged
+        // network pulls, one upsert per row) therefore parked telemetry ingest,
+        // UI queries and KPI Asks behind it — and those callers timed out rather
+        // than queued, so a slow site could make central look dead. This pins the
+        // fix: with the drain off-mailbox behind a single-flight guard, an ingest
+        // Ask completes promptly while the pull is still blocked.
+        var siteId = "siteSlow";
+        var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
+        var client = new BlockingPullClient();
+        var repo = new RecordingRepo();
+
+        var actor = CreateActor(sites, client, repo, FastTickOptions());
+
+        // Wait until the drain is genuinely in flight and blocked inside PullAsync.
+        await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
+
+        // The mailbox must still be serving. A generous-but-finite budget: this
+        // fails at the pre-fix behaviour (the reply only arrives once the pull
+        // unblocks, which never happens until Release below).
+        var id = TrackedOperationId.New();
+        var reply = await actor.Ask<UpsertSiteCallReply>(
+            new UpsertSiteCallCommand(NewRow(id, sourceSite: siteId)),
+            TimeSpan.FromSeconds(3));
+
+        Assert.True(reply.Accepted);
+        Assert.Equal(id, reply.TrackedOperationId);
+
+        // Let the drain finish so the actor shuts down cleanly.
+        client.Release();
+    }
+
+    [Fact]
+    public async Task ReconciliationTicks_DoNotOverlap_WhileADrainIsInFlight()
+    {
+        // Single-flight guard: with a 100 ms tick and a drain blocked for far
+        // longer, every subsequent tick must be dropped rather than starting a
+        // second concurrent pass — overlapping passes would race on the per-site
+        // cursor and pinned-latch dictionaries the drain mutates.
+        var siteId = "siteSlow";
+        var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
+        var client = new CountingBlockingPullClient();
+        var repo = new RecordingRepo();
+
+        CreateActor(sites, client, repo, FastTickOptions());
+
+        await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
+
+        // Several tick intervals elapse while the first pass is still blocked.
+        await Task.Delay(TimeSpan.FromMilliseconds(600));
+
+        Assert.Equal(1, client.CallCount);
+
+        client.Release();
+    }
+
+    /// <summary>
+    /// <see cref="BlockingPullClient"/> that also counts invocations, so a test
+    /// can prove no second pass started while the first was blocked.
+    /// </summary>
+    private sealed class CountingBlockingPullClient : IPullSiteCallsClient
+    {
+        private readonly TaskCompletionSource _release =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private readonly TaskCompletionSource _entered =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private int _callCount;
+
+        public Task Entered => _entered.Task;
+
+        public int CallCount => Volatile.Read(ref _callCount);
+
+        public void Release() => _release.TrySetResult();
+
+        public async Task<PullSiteCallsResponse> PullAsync(
+            string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
+        {
+            Interlocked.Increment(ref _callCount);
+            _entered.TrySetResult();
+            await _release.Task.ConfigureAwait(false);
+            return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
+        }
+    }
 }