merge(agent-c): rows 91/95/96 — compile-cache tests on key-scoped delta assertions; Watch/Stop/ExpectTerminated at all 23 fast-tick actor creation sites in the three remaining AuditLog.Tests classes; per-suite xunit maxParallelThreads:4 for the four MSSQL-fixture suites (matches the row-82 semaphore slots) + Test Parallelism Policy doc

2026-08-17 05:28:51 -04:00
parent ad0100f2f9 d2d4db492b
commit 106e467dfd
15 changed files with 428 additions and 54 deletions
@@ -138,6 +138,38 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            NullLogger<AuditLogPurgeActor>.Instance)));
    }

+    /// <summary>
+    /// Stops a purge actor and blocks until it is actually dead.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Deferred-work register row 95, the same leak shape row 82 fixed in
+    /// <c>PartitionPurgeTests</c> / <c>NotifyDispatcherAuditTrailTests</c>. Every actor this
+    /// class creates ticks every <c>100 ms</c>
+    /// (<see cref="AuditLogPurgeOptions.IntervalOverride"/>) and was only ever stopped
+    /// implicitly by <see cref="TestKit"/>'s <c>Dispose</c> → <c>Shutdown</c> — which is
+    /// bounded by a timeout and does NOT fail when that timeout expires. So on a loaded
+    /// machine an actor kept ticking while LATER test classes ran: the stub-repo tests burn a
+    /// DI scope and a thread every 100 ms for the rest of the assembly run, and
+    /// <c>EndToEnd_RealPartition_…</c>'s actor keeps switching partitions out of the shared
+    /// <see cref="MsSqlMigrationFixture"/> database that other classes in this assembly seed
+    /// against.
+    /// </para>
+    /// <para>
+    /// Stopping the actor inside the test that created it removes the dependency on
+    /// ActorSystem-shutdown timing entirely.
+    /// </para>
+    /// </remarks>
+    private void StopPurgeActor(IActorRef actor)
+    {
+        Watch(actor);
+        Sys.Stop(actor);
+        // A stop is processed after the in-flight tick completes, so this also guarantees no
+        // partition switch is still running against the fixture DB.
+        ExpectTerminated(actor, TimeSpan.FromSeconds(30));
+        Unwatch(actor);
+    }
+
    private static AuditLogPurgeOptions FastTickOptions(TimeSpan? interval = null) => new()
    {
        IntervalHours = 24,
@@ -163,7 +195,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
    public void Tick_Fires_OnDailyInterval()
    {
        var repo = new RecordingRepo();
-        CreateActor(repo, FastTickOptions());
+        var actor = CreateActor(repo, FastTickOptions());

        // The first scheduled tick fires after the configured interval. We
        // assert the visible side effect (the enumerator was called) rather
@@ -173,6 +205,8 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
                $"expected >= 1 enumerator call, got {repo.ThresholdQueries.Count}"),
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopPurgeActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -192,7 +226,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            RowsPerBoundary = _ => 42L,
        };

-        CreateActor(repo, FastTickOptions());
+        var actor = CreateActor(repo, FastTickOptions());

        AwaitAssert(
            () =>
@@ -202,6 +236,8 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            },
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopPurgeActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -217,7 +253,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        // and asserting the actor switched nothing despite the tick firing.
        var repo = new RecordingRepo { Boundaries = new List<DateTime>() };

-        CreateActor(repo, FastTickOptions());
+        var actor = CreateActor(repo, FastTickOptions());

        // Wait for at least one tick (visible via the enumerator call) then
        // assert no switch happened.
@@ -226,6 +262,10 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));

+        // Stop before the negative assertion: a still-ticking actor is exactly what would
+        // make it read a moving target (register row 95).
+        StopPurgeActor(actor);
+
        Assert.Empty(repo.SwitchedBoundaries);
    }

@@ -244,9 +284,10 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        };

        var probe = SubscribePurged();
-        CreateActor(repo, FastTickOptions());
+        var actor = CreateActor(repo, FastTickOptions());

        var msg = probe.ExpectMsg<AuditLogPurgedEvent>(TimeSpan.FromSeconds(5));
+        StopPurgeActor(actor);
        Assert.Equal(boundary, msg.MonthBoundary);
        Assert.Equal(1234L, msg.RowsDeleted);
        Assert.True(msg.DurationMs >= 0,
@@ -269,7 +310,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            BoundaryException = new InvalidOperationException("simulated switch failure for poison boundary"),
        };

-        CreateActor(repo, FastTickOptions());
+        var actor = CreateActor(repo, FastTickOptions());

        AwaitAssert(
            () =>
@@ -281,6 +322,8 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            },
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopPurgeActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -349,7 +392,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        };

        var probe = SubscribePurged();
-        Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
+        var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
            sp,
            Options.Create(purgeOptions),
            Options.Create(auditOptions),
@@ -363,6 +406,11 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            isMessage: m => m.MonthBoundary == janBoundary,
            max: TimeSpan.FromSeconds(30));

+        // This is the one actor in the class that mutates the SHARED fixture database, so it
+        // must not outlive the test and start switching partitions out from under the next
+        // class's seed rows (register row 95).
+        StopPurgeActor(actor);
+
        Assert.True(matched.RowsDeleted >= 1,
            $"Expected RowsDeleted >= 1 for the Jan-2026 partition; got {matched.RowsDeleted}.");

@@ -394,7 +442,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        // or some other accidental default. We use a non-default retention
        // (30 days) so the assertion isn't satisfied by the 365 default.
        var repo = new RecordingRepo();
-        CreateActor(
+        var actor = CreateActor(
            repo,
            FastTickOptions(),
            auditOptions: new AuditLogOptions { RetentionDays = 30 });
@@ -404,6 +452,8 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));

+        StopPurgeActor(actor);
+
        var threshold = repo.ThresholdQueries[0];
        var expected = DateTime.UtcNow - TimeSpan.FromDays(30);
        // 1-minute slack covers test-thread scheduling jitter between the
@@ -437,7 +487,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        var purgeOptionsWrapped = Options.Create(purgeOptions);

        var sp = BuildScopedProvider(repo);
-        Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
+        var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
            sp,
            purgeOptionsWrapped,
            auditOptions,
@@ -448,6 +498,8 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));

+        StopPurgeActor(actor);
+
        var purge = repo.ChannelPurges.First(p => p.Channel == "ApiOutbound");
        Assert.Equal(1234, purge.BatchSize);

@@ -483,7 +535,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        var purgeOptions = Options.Create(FastTickOptions());

        var sp = BuildScopedProvider(repo);
-        Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
+        var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
            sp,
            purgeOptions,
            auditOptions,
@@ -496,6 +548,10 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));

+        // Stop before the negative assertion: a still-ticking actor is exactly what would
+        // make it read a moving target (register row 95).
+        StopPurgeActor(actor);
+
        Assert.Empty(repo.ChannelPurges);
    }

@@ -527,7 +583,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        });

        var sp = BuildScopedProvider(repo);
-        Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
+        var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
            sp,
            Options.Create(purgeOptions),
            auditOptions,
@@ -542,6 +598,8 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
            },
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopPurgeActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -572,7 +630,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
        var probe = CreateTestProbe();
        Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgeFailedEvent));

-        Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
+        var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
            sp,
            Options.Create(FastTickOptions()),
            Options.Create(new AuditLogOptions()),
@@ -588,5 +646,7 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
                $"expected PurgeFailures >= 1, got {snapshot.PurgeFailures}"),
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopPurgeActor(actor);
    }
 }
@@ -252,6 +252,37 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            NullLogger<SiteAuditReconciliationActor>.Instance)));
    }

+    /// <summary>
+    /// Stops a reconciliation actor and blocks until it is actually dead.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Deferred-work register row 95, the same leak shape row 82 fixed in
+    /// <c>PartitionPurgeTests</c> / <c>NotifyDispatcherAuditTrailTests</c>. Every actor this
+    /// class creates ticks on a ~0 s repeat interval (<see cref="FastTickOptions"/>) and was
+    /// only ever stopped implicitly by <see cref="TestKit"/>'s <c>Dispose</c> →
+    /// <c>Shutdown</c>, which is bounded by a timeout and does NOT fail when it expires. A
+    /// survivor keeps pulling and INSERTING audit rows: in
+    /// <c>Tick_Duplicates_NotDoubleInserted</c> it holds a real
+    /// <see cref="AuditLogRepository"/> over the shared <see cref="MsSqlMigrationFixture"/>
+    /// database (and over a <c>DbContext</c> the test is about to dispose), so it writes into
+    /// the DB LATER test classes read from.
+    /// </para>
+    /// <para>
+    /// Stopping the actor inside the test that created it removes the dependency on
+    /// ActorSystem-shutdown timing entirely.
+    /// </para>
+    /// </remarks>
+    private void StopReconciliationActor(IActorRef actor)
+    {
+        Watch(actor);
+        Sys.Stop(actor);
+        // A stop is processed after the in-flight tick completes, so this also guarantees no
+        // pull or insert is still running against the fixture DB.
+        ExpectTerminated(actor, TimeSpan.FromSeconds(30));
+        Unwatch(actor);
+    }
+
    /// <summary>
    /// Subscribes to the EventStream and collects every
    /// <see cref="SiteAuditTelemetryStalledChanged"/> publication into a list
@@ -278,7 +309,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        var repo = new RecordingRepo();
        var opts = FastTickOptions();

-        CreateActor(sites, client, repo, opts);
+        var actor = CreateActor(sites, client, repo, opts);

        // The first scheduled tick fires after `ReconciliationIntervalSeconds`,
        // which is 0 for the test — Akka's scheduler still respects the
@@ -289,6 +320,8 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            () => Assert.True(client.Calls.Count >= 1, $"expected >= 1 pull call, got {client.Calls.Count}"),
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopReconciliationActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -304,7 +337,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        var client = new ScriptedPullClient();
        var repo = new RecordingRepo();

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        AwaitAssert(() =>
            {
@@ -313,6 +346,8 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            },
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopReconciliationActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -329,11 +364,14 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            new PullAuditEventsResponse(new[] { e1, e2 }, MoreAvailable: false));
        var repo = new RecordingRepo();

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        AwaitAssert(() => Assert.Equal(2, repo.InsertCallCount),
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopReconciliationActor(actor);
+
        Assert.Contains(repo.Inserted, e => e.EventId == e1.EventId);
        Assert.Contains(repo.Inserted, e => e.EventId == e2.EventId);
    }
@@ -371,13 +409,18 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        await using var context = CreateContext();
        var repo = new AuditLogRepository(context);

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        // Wait for the actor to ingest both rows.
        await Task.Delay(TimeSpan.FromSeconds(1));
        AwaitAssert(() => Assert.True(client.Calls.Count >= 1),
            duration: TimeSpan.FromSeconds(3));

+        // Stop before reading: this actor holds a real repository over the SHARED fixture
+        // database (and over the DbContext this test is about to dispose), so it must not
+        // outlive the test (register row 95).
+        StopReconciliationActor(actor);
+
        // Even though the pull returned 2 events, only 1 fresh row should
        // exist in MSSQL alongside the pre-existing one — InsertIfNotExistsAsync
        // is first-write-wins on EventId.
@@ -413,7 +456,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            new PullAuditEventsResponse(new[] { e1, e2, e3 }, MoreAvailable: false));
        var repo = new RecordingRepo();

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        // Wait until we have at least two pulls — the second one must use t3
        // as its `since` argument because that was the max OccurredAtUtc in
@@ -423,6 +466,10 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            duration: TimeSpan.FromSeconds(5),
            interval: TimeSpan.FromMilliseconds(50));

+        // Stop before indexing into client.Calls: the list must not grow mid-assertion
+        // (register row 95).
+        StopReconciliationActor(actor);
+
        Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc);
        Assert.Equal(t3, client.Calls[1].SinceUtc);

@@ -451,13 +498,15 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            new PullAuditEventsResponse(new[] { low, high, mid }, MoreAvailable: true));
        var repo = new RecordingRepo();

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        AwaitAssert(() => Assert.True(client.Calls.Count >= 2,
            $"need at least 2 pulls, got {client.Calls.Count}"),
            duration: TimeSpan.FromSeconds(5),
            interval: TimeSpan.FromMilliseconds(50));

+        StopReconciliationActor(actor);
+
        Assert.Equal(t, client.Calls[1].SinceUtc);
        Assert.Equal(high.EventId.ToString(), client.Calls[1].AfterId);
    }
@@ -475,13 +524,15 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            new PullAuditEventsResponse(new[] { evt }, MoreAvailable: false));
        var repo = new AlwaysThrowingRepo();

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        AwaitAssert(() => Assert.True(client.Calls.Count >= 2,
            $"need at least 2 pulls, got {client.Calls.Count}"),
            duration: TimeSpan.FromSeconds(5),
            interval: TimeSpan.FromMilliseconds(50));

+        StopReconciliationActor(actor);
+
        Assert.Equal(DateTime.MinValue, client.Calls[1].SinceUtc);
        Assert.Null(client.Calls[1].AfterId);
    }
@@ -543,7 +594,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
                new PullAuditEventsResponse(new[] { bEvent }, MoreAvailable: false));
        var repo = new RecordingRepo();

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        AwaitAssert(() =>
            {
@@ -552,6 +603,8 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            },
            duration: TimeSpan.FromSeconds(3),
            interval: TimeSpan.FromMilliseconds(50));
+
+        StopReconciliationActor(actor);
    }

    // ---------------------------------------------------------------------
@@ -574,12 +627,13 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        var repo = new RecordingRepo();
        var (probe, _) = SubscribeStalled();

-        CreateActor(sites, client, repo, FastTickOptions(stalledAfter: 2));
+        var actor = CreateActor(sites, client, repo, FastTickOptions(stalledAfter: 2));

        // Expect Stalled=true after the second non-draining tick. The probe
        // waits with its own timeout (a few seconds gives the 0 s repeat
        // interval ample slack).
        var msg = probe.ExpectMsg<SiteAuditTelemetryStalledChanged>(TimeSpan.FromSeconds(5));
+        StopReconciliationActor(actor);
        Assert.Equal("siteA", msg.SiteId);
        Assert.True(msg.Stalled);
    }
@@ -606,7 +660,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        var repo = new RecordingRepo();
        var (probe, _) = SubscribeStalled();

-        CreateActor(sites, client, repo, FastTickOptions(stalledAfter: 2));
+        var actor = CreateActor(sites, client, repo, FastTickOptions(stalledAfter: 2));

        // First publication is the stalled=true transition; second is the
        // back-to-draining flip. The actor publishes ONLY on transitions so we
@@ -615,6 +669,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        Assert.True(first.Stalled);

        var second = probe.ExpectMsg<SiteAuditTelemetryStalledChanged>(TimeSpan.FromSeconds(5));
+        StopReconciliationActor(actor);
        Assert.False(second.Stalled);
        Assert.Equal("siteA", second.SiteId);
    }
@@ -643,7 +698,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
        var repo = new PoisonInsertRepo(
            poison.EventId, new InvalidOperationException("poison row will never insert"));

-        CreateActor(sites, client, repo, FastTickOptions());
+        var actor = CreateActor(sites, client, repo, FastTickOptions());

        // After the 5th failed attempt the actor abandons the row and writes ONE
        // synthetic ReconciliationAbandoned row (fresh EventId) whose details carry
@@ -669,5 +724,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
            () => Assert.Contains(client.Calls, c => c.SinceUtc == poisonTime),
            duration: TimeSpan.FromSeconds(5),
            interval: TimeSpan.FromMilliseconds(100));
+
+        StopReconciliationActor(actor);
    }
 }
@@ -165,6 +165,37 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
        return (sp, ingest);
    }

+    /// <summary>
+    /// Stops an actor and blocks until it is actually dead.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Deferred-work register row 95, the same leak shape row 82 fixed in
+    /// <c>PartitionPurgeTests</c> / <c>NotifyDispatcherAuditTrailTests</c>. Both tests here
+    /// already called <c>Sys.Stop</c>, but that is fire-and-forget: it returns before the
+    /// actor has processed the stop, so an in-flight 100 ms tick could still be pulling and
+    /// INSERTING into the shared <see cref="MsSqlMigrationFixture"/> database — and into the
+    /// <see cref="IServiceProvider"/> the enclosing <c>await using</c> is about to dispose.
+    /// Worse, the call sat at the very END of the block, so any earlier assertion failure
+    /// skipped it entirely and left the actor ticking for the rest of the assembly run,
+    /// stopped only by <see cref="TestKit"/>'s <c>Dispose</c> → <c>Shutdown</c>, which is
+    /// timeout-bounded and does not fail when it expires.
+    /// </para>
+    /// <para>
+    /// Watch/Stop/ExpectTerminated makes the stop synchronous, and each call site now sits
+    /// immediately after the last step that needs the actor alive rather than at the end.
+    /// </para>
+    /// </remarks>
+    private void StopActor(IActorRef actor)
+    {
+        Watch(actor);
+        Sys.Stop(actor);
+        // A stop is processed after the in-flight tick completes, so this also guarantees no
+        // pull or insert is still running against the fixture DB.
+        ExpectTerminated(actor, TimeSpan.FromSeconds(30));
+        Unwatch(actor);
+    }
+
    private static SiteAuditReconciliationOptions FastTickOptions(int batchSize = 256) => new()
    {
        ReconciliationIntervalSeconds = 300,
@@ -265,6 +296,12 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
                duration: TimeSpan.FromSeconds(10),
                interval: TimeSpan.FromMilliseconds(100));

+            // Both actors have done everything the test needs; stop them HERE, before the
+            // read-back, so nothing is still writing to the shared fixture DB while step 5
+            // counts rows and so a step-5 failure cannot leak them (register row 95).
+            StopActor(reconciliationActor);
+            StopActor(ingest);
+
            // Step 5: assert no duplicates by EventId — central must have
            // exactly the 200 rows we wrote at the site (one row per EventId).
            await using var verify = CreateContext();
@@ -277,10 +314,6 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
            // And every EventId we wrote at the site is present centrally.
            Assert.True(written.All(w => centralIds.Contains(w.EventId)),
                "every site-written EventId should be present centrally.");
-
-            // Tear the actor down before disposing the harness; the actor's
-            // PostStop cancels its scheduled timer.
-            Sys.Stop(reconciliationActor);
        }
    }

@@ -303,7 +336,7 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
            await sqliteWriter.WriteAsync(NewEvent(siteId, baseOccurred.AddMilliseconds(i)));
        }

-        var (sp, _) = BuildCentralPipeline();
+        var (sp, ingest) = BuildCentralPipeline();
        await using (sp as IAsyncDisposable ?? throw new InvalidOperationException())
        {
            var pullClient = new QueueBackedPullClient(sqliteWriter);
@@ -338,6 +371,13 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
            Assert.True(pullClient.CallCount > callsAfterDrain,
                $"expected additional pull calls after drain to validate idempotency, got {pullClient.CallCount} after {callsAfterDrain}");

+            // Both actors have done everything the test needs; stop them HERE, before the
+            // read-back, so nothing is still writing to the shared fixture DB while the
+            // duplicate check counts rows and so a failure below cannot leak them
+            // (register row 95).
+            StopActor(reconciliationActor);
+            StopActor(ingest);
+
            // Central count must still be exactly totalEvents — no duplicates
            // even though the cursor + read-Reconciled-too semantics could
            // theoretically re-fetch on the second cycle.
@@ -347,8 +387,6 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
                .ToListAsync();
            Assert.Equal(totalEvents, rows.Count);
            Assert.Equal(totalEvents, rows.Select(r => r.EventId).Distinct().Count());
-
-            Sys.Stop(reconciliationActor);
        }
    }
 }
@@ -46,6 +46,22 @@
    <Using Include="Xunit" />
  </ItemGroup>

+  <ItemGroup>
+    <!--
+      Deferred-work register row 96 — per-suite in-process parallelism cap.
+      Resource-heavy suite: MSSQL-backed classes take an MsSqlMigrationFixture
+      per test CLASS (CREATE DATABASE + the full migration chain on the ONE
+      shared dev SQL Server) and most classes stand up an Akka TestKit
+      ActorSystem with fast-ticking actors. Uncapped, xunit runs
+      Environment.ProcessorCount collections at once. The cap is threads only —
+      collections stay parallel — and 4 matches the default slot count of the
+      cross-process sp_getapplock migrate semaphore (row 82), so the in-process
+      and cross-process limits agree. Policy: docs/test_infra/test_infra.md.
+      The json must be copied to the output directory to take effect.
+    -->
+    <Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
  <ItemGroup>
    <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.AuditLog/ZB.MOM.WW.ScadaBridge.AuditLog.csproj" />
    <!--
@@ -0,0 +1,4 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "maxParallelThreads": 4
+}
@@ -42,6 +42,23 @@
    <Using Include="Xunit" />
  </ItemGroup>

+  <ItemGroup>
+    <!--
+      Deferred-work register row 96 — per-suite in-process parallelism cap.
+      This is the suite that owns MsSqlMigrationFixture: every MSSQL-backed test
+      CLASS gets its own database, i.e. a CREATE DATABASE plus the full
+      ~100-migration chain against the ONE shared dev SQL Server. Uncapped,
+      xunit runs Environment.ProcessorCount collections at once, so a 14-core
+      dev box asked the server for 14 concurrent fresh-database migrations from
+      this process alone. The cap is threads only — collections stay parallel —
+      and 4 matches the default slot count of the cross-process sp_getapplock
+      migrate semaphore (row 82), so the in-process and cross-process limits
+      agree. Policy: docs/test_infra/test_infra.md.
+      The json must be copied to the output directory to take effect.
+    -->
+    <Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
  <ItemGroup>
    <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj" />
    <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Commons/ZB.MOM.WW.ScadaBridge.Commons.csproj" />
@@ -0,0 +1,4 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "maxParallelThreads": 4
+}
@@ -28,6 +28,19 @@
    <Using Include="Xunit" />
  </ItemGroup>

+  <ItemGroup>
+    <!--
+      Deferred-work register row 96 — per-suite in-process parallelism cap.
+      Resource-heavy suite: classes boot real ASP.NET Core hosts and real Akka
+      ActorSystems — Akka.Remote binds real sockets, Kestrel binds real ports,
+      gRPC channels dial them. Uncapped that is Environment.ProcessorCount full
+      hosts standing up at once. The cap is threads only; collections stay
+      parallel. Policy: docs/test_infra/test_infra.md.
+      The json must be copied to the output directory to take effect.
+    -->
+    <Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
  <ItemGroup>
    <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Host/ZB.MOM.WW.ScadaBridge.Host.csproj" />
  </ItemGroup>
@@ -0,0 +1,4 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "maxParallelThreads": 4
+}
@@ -36,6 +36,21 @@
    <Using Include="Xunit" />
  </ItemGroup>

+  <ItemGroup>
+    <!--
+      Deferred-work register row 96 — per-suite in-process parallelism cap.
+      This suite's actor tests take an MsSqlMigrationFixture per test CLASS: a
+      CREATE DATABASE plus the full migration chain on the ONE shared dev SQL
+      Server, alongside an Akka TestKit ActorSystem. Uncapped, xunit runs
+      Environment.ProcessorCount collections at once. The cap is threads only —
+      collections stay parallel — and 4 matches the default slot count of the
+      cross-process sp_getapplock migrate semaphore (row 82), so the in-process
+      and cross-process limits agree. Policy: docs/test_infra/test_infra.md.
+      The json must be copied to the output directory to take effect.
+    -->
+    <Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
  <ItemGroup>
    <ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.SiteCallAudit/ZB.MOM.WW.ScadaBridge.SiteCallAudit.csproj" />
    <!--
@@ -0,0 +1,4 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "maxParallelThreads": 4
+}
@@ -248,15 +248,20 @@ public class DeploymentWarmThenGateTests : TestKit, IDisposable
            "BatchTwo", ConfigJson("BatchTwo", sharedCode), "d2", "h2", true);

        SiteScriptCompileCache.Clear();
-        Assert.Equal(0, SiteScriptCompileCache.Hits);
+        // Baseline rather than Assert.Equal(0, Hits): Hits is a process-global counter and test
+        // classes outside this collection compile scripts in parallel, so a foreign hit can land
+        // between the Clear() and the read (deferred-work register row 91). Everything below is
+        // asserted as a delta over this baseline.
+        var hitsBefore = SiteScriptCompileCache.Hits;

        CreateDeploymentManager();

        AwaitAssert(() =>
        {
            // One compile, then repeated hits: the second pre-warm plus both PreStarts.
-            Assert.True(SiteScriptCompileCache.Hits >= 3,
-                $"expected the pre-warmed body to be served from cache, saw {SiteScriptCompileCache.Hits} hits");
+            Assert.True(SiteScriptCompileCache.Hits - hitsBefore >= 3,
+                $"expected the pre-warmed body to be served from cache, saw " +
+                $"{SiteScriptCompileCache.Hits - hitsBefore} hits over baseline {hitsBefore}");
        }, TimeSpan.FromSeconds(20));
    }
 }
@@ -7,39 +7,98 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Scripts;
 /// <summary>
 /// Tests for the process-wide <see cref="SiteScriptCompileCache"/>. The cache is
 /// static/process-wide state, so this class shares the "SiteScriptCompileCache" xunit
-/// collection with <c>ScriptCompilationServiceTests</c> (no cross-class parallelism) and
-/// calls <see cref="SiteScriptCompileCache.Clear"/> at the top of each test.
+/// collection with <c>ScriptCompilationServiceTests</c> and <c>DeploymentWarmThenGateTests</c>
+/// (no cross-class parallelism between those three, so no concurrent
+/// <see cref="SiteScriptCompileCache.Clear"/>).
 /// </summary>
+/// <remarks>
+/// <para>
+/// <b>Deferred-work register row 91.</b> That collection does NOT stop the rest of the
+/// assembly: any test class that compiles a script — Instance/Script actor tests, deployment
+/// tests — goes through <see cref="SiteScriptCompileCache.GetOrAdd"/> too, in parallel, and
+/// bumps the same process-global <see cref="SiteScriptCompileCache.Hits"/> /
+/// <see cref="SiteScriptCompileCache.Count"/>. Absolute assertions such as
+/// <c>Assert.Equal(1, Hits)</c> after a <c>Clear()</c> therefore only held while nothing else
+/// happened to be compiling; they failed roughly 1 run in 4 of a full parallel solution run and
+/// were always green in isolation.
+/// </para>
+/// <para>
+/// The fix follows the register's row-74 pattern: assert the DELTA attributable to this test's
+/// OWN keys across its own scoped action, never an absolute global value. Two habits carry it:
+/// every test keys its cache entries on a per-test-instance unique token (<see cref="_uniq"/>),
+/// so no other class can collide with or serve them; and the global counters are only ever
+/// asserted as one-sided bounds (<c>&gt;= baseline + n</c>), since foreign traffic can only add.
+/// A collection fixture spanning every script-compiling class in the assembly would fix it too,
+/// but would serialise most of the suite for the benefit of five facts.
+/// </para>
+/// </remarks>
 [Collection("SiteScriptCompileCache")]
 public class SiteScriptCompileCacheTests
 {
+    /// <summary>
+    /// Slack allowed on <see cref="SiteScriptCompileCache.Count"/> bounds for entries inserted
+    /// by test classes outside this collection while a bound test runs (register row 91). The
+    /// exactness of the eviction claims comes from the per-key probes, not from this bound.
+    /// </summary>
+    private const int ForeignEntrySlack = 512;
+
+    /// <summary>Per-test-instance token making every cache key in this test uniquely ours.</summary>
+    private readonly string _uniq = Guid.NewGuid().ToString("N");
+
+    /// <summary>Builds a cache key body that no other test class in the assembly can produce.</summary>
+    private string Code(string body) => $"/*{_uniq}*/ {body}";
+
    private static ScriptCompilationResult Ok() =>
        ScriptCompilationResult.Succeeded(CSharpScript.Create<object?>("1", ScriptOptions.Default, typeof(ScriptGlobals)));

    [Fact]
    public void GetOrAdd_SameCodeAndGlobals_ComputesOnce()
    {
-        SiteScriptCompileCache.Clear();
+        var code = Code("return 1;");
+        var hitsBefore = SiteScriptCompileCache.Hits;
        var factoryCalls = 0;
        ScriptCompilationResult Factory() { factoryCalls++; return Ok(); }

-        var r1 = SiteScriptCompileCache.GetOrAdd("return 1;", typeof(ScriptGlobals), Factory);
-        var r2 = SiteScriptCompileCache.GetOrAdd("return 1;", typeof(ScriptGlobals), Factory);
+        var r1 = SiteScriptCompileCache.GetOrAdd(code, typeof(ScriptGlobals), Factory);
+        var r2 = SiteScriptCompileCache.GetOrAdd(code, typeof(ScriptGlobals), Factory);

+        // Scoped to our own key, so these two are exact and are the real claim: one compile,
+        // and the second lookup returned that very instance rather than a re-computation.
        Assert.Equal(1, factoryCalls);
        Assert.Same(r1, r2);
-        Assert.Equal(1, SiteScriptCompileCache.Hits);
+
+        // Hits is process-global; a parallel class's compile can only push the delta up, so the
+        // claim "the second lookup was counted as a hit" is a lower bound, not an equality.
+        Assert.True(SiteScriptCompileCache.Hits >= hitsBefore + 1,
+            $"expected the second GetOrAdd to register a hit; hits went {hitsBefore} -> {SiteScriptCompileCache.Hits}");
    }

    [Fact]
    public void GetOrAdd_SameCode_DifferentGlobals_AreSeparateEntries()
    {
-        SiteScriptCompileCache.Clear();
-        SiteScriptCompileCache.GetOrAdd("1 > 0", typeof(ScriptGlobals), Ok);
-        SiteScriptCompileCache.GetOrAdd("1 > 0", typeof(TriggerExpressionGlobals), Ok);
+        var code = Code("1 > 0");
+        var scriptCompiles = 0;
+        var triggerCompiles = 0;

-        Assert.Equal(2, SiteScriptCompileCache.Count);
-        Assert.Equal(0, SiteScriptCompileCache.Hits); // no cross-globals hit
+        var script = SiteScriptCompileCache.GetOrAdd(code, typeof(ScriptGlobals),
+            () => { scriptCompiles++; return Ok(); });
+        var trigger = SiteScriptCompileCache.GetOrAdd(code, typeof(TriggerExpressionGlobals),
+            () => { triggerCompiles++; return Ok(); });
+
+        // "No cross-globals hit" asserted where it is actually observable and ours: the second
+        // lookup had to compute, and the two globals surfaces hold distinct entries. (The old
+        // form — Count == 2 and Hits == 0 on the process-global counters — was only true while
+        // no other class was compiling.)
+        Assert.Equal(1, scriptCompiles);
+        Assert.Equal(1, triggerCompiles);
+        Assert.NotSame(script, trigger);
+
+        // Each entry is independently addressable afterwards: re-getting either must serve the
+        // matching cached result, never the other globals surface's.
+        Assert.Same(script, SiteScriptCompileCache.GetOrAdd(code, typeof(ScriptGlobals),
+            () => throw new InvalidOperationException("script entry was not cached")));
+        Assert.Same(trigger, SiteScriptCompileCache.GetOrAdd(code, typeof(TriggerExpressionGlobals),
+            () => throw new InvalidOperationException("trigger entry was not cached")));
    }

    [Fact]
@@ -47,9 +106,22 @@ public class SiteScriptCompileCacheTests
    {
        SiteScriptCompileCache.Clear();
        for (var i = 0; i <= SiteScriptCompileCache.MaxEntries; i++)
-            SiteScriptCompileCache.GetOrAdd($"return {i};", typeof(ScriptGlobals), Ok);
+            SiteScriptCompileCache.GetOrAdd(Code($"return {i};"), typeof(ScriptGlobals), Ok);

-        Assert.True(SiteScriptCompileCache.Count <= SiteScriptCompileCache.MaxEntries);
+        // MaxEntries + 1 distinct inserts must not all be resident — the sweep ran. The bound
+        // carries slack for foreign inserts (register row 91); the sharp claim is the eviction
+        // probe below, which is scoped to keys only this test can produce.
+        Assert.True(SiteScriptCompileCache.Count <= SiteScriptCompileCache.MaxEntries + ForeignEntrySlack,
+            $"cache grew past its bound: {SiteScriptCompileCache.Count} vs max {SiteScriptCompileCache.MaxEntries}");
+
+        var recomputed = 0;
+        for (var i = 0; i < 16; i++)
+        {
+            var code = Code($"return {i};");
+            SiteScriptCompileCache.GetOrAdd(code, typeof(ScriptGlobals), () => { recomputed++; return Ok(); });
+        }
+
+        Assert.True(recomputed > 0, "no oldest-batch entry was evicted — the overflow sweep did not run");
    }

    // ── WP3.1: approximate LRU replaced the wholesale-Clear overflow cliff ──────────
@@ -70,15 +142,15 @@ public class SiteScriptCompileCacheTests

        // Fill to the bound. Entries 1..N are inserted oldest-first.
        for (var i = 0; i < SiteScriptCompileCache.MaxEntries; i++)
-            SiteScriptCompileCache.GetOrAdd($"return {i};", typeof(ScriptGlobals), Ok);
+            SiteScriptCompileCache.GetOrAdd(Code($"return {i};"), typeof(ScriptGlobals), Ok);

        // Touch entry 0 so it carries the NEWEST access stamp despite being inserted first.
-        var hot = SiteScriptCompileCache.GetOrAdd("return 0;", typeof(ScriptGlobals), Ok);
+        var hot = SiteScriptCompileCache.GetOrAdd(Code("return 0;"), typeof(ScriptGlobals), Ok);
        var hitsAfterTouch = SiteScriptCompileCache.Hits;

        // Push well past the bound so at least one eviction sweep definitely runs.
        for (var i = 0; i < 200; i++)
-            SiteScriptCompileCache.GetOrAdd($"return overflow{i};", typeof(ScriptGlobals), Ok);
+            SiteScriptCompileCache.GetOrAdd(Code($"return overflow{i};"), typeof(ScriptGlobals), Ok);

        // Nothing was cleared wholesale: the cache is still most of the way full and the hit
        // counter survived (Clear() would have reset it to 0).
@@ -89,16 +161,20 @@ public class SiteScriptCompileCacheTests
        // The touched entry survived the sweep — a hit, not a recompile. This is the LRU
        // property: recency, not insertion order, decides what stays.
        var hitsBefore = SiteScriptCompileCache.Hits;
-        var again = SiteScriptCompileCache.GetOrAdd("return 0;", typeof(ScriptGlobals),
+        var again = SiteScriptCompileCache.GetOrAdd(Code("return 0;"), typeof(ScriptGlobals),
            () => throw new InvalidOperationException("hot entry was evicted — LRU is not keeping recently-used entries"));
+        // Assert.Same is the sharp claim (our own key, our own instance). The counter check is a
+        // lower bound: Hits is process-global and a parallel class's hit lands in the same window
+        // (register row 91 — this was `Assert.Equal(hitsBefore + 1, …)`).
        Assert.Same(hot, again);
-        Assert.Equal(hitsBefore + 1, SiteScriptCompileCache.Hits);
+        Assert.True(SiteScriptCompileCache.Hits >= hitsBefore + 1,
+            $"expected the surviving hot entry to register a hit; hits went {hitsBefore} -> {SiteScriptCompileCache.Hits}");

        // …and untouched old entries genuinely were evicted, so the sweep really ran.
        var recomputed = 0;
        for (var i = 1; i <= 16; i++)
        {
-            var code = $"return {i};";
+            var code = Code($"return {i};");
            SiteScriptCompileCache.GetOrAdd(code, typeof(ScriptGlobals),
                () => { recomputed++; return Ok(); });
        }
@@ -119,7 +195,7 @@ public class SiteScriptCompileCacheTests
        var tasks = Enumerable.Range(0, 8).Select(t => Task.Run(() =>
        {
            for (var i = 0; i < perTask; i++)
-                SiteScriptCompileCache.GetOrAdd($"return {t}_{i};", typeof(ScriptGlobals), Ok);
+                SiteScriptCompileCache.GetOrAdd(Code($"return {t}_{i};"), typeof(ScriptGlobals), Ok);
        }));

        await Task.WhenAll(tasks);
@@ -131,7 +207,7 @@ public class SiteScriptCompileCacheTests
        // eviction bug would show up here as unbounded growth, not as a handful of extra rows.
        // (Other test classes compile into this process-wide cache concurrently, which is the
        // other reason the bound is asserted with slack rather than exactly.)
-        Assert.True(SiteScriptCompileCache.Count <= SiteScriptCompileCache.MaxEntries + 512,
+        Assert.True(SiteScriptCompileCache.Count <= SiteScriptCompileCache.MaxEntries + ForeignEntrySlack,
            $"cache grew past its bound: {SiteScriptCompileCache.Count} vs max {SiteScriptCompileCache.MaxEntries} " +
            $"after {8 * perTask} distinct inserts");
    }