From 835fc08c3f84ddfa8e2d3708f96ef4a50884bbc0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 21 Jul 2026 10:35:35 -0400 Subject: [PATCH] fix(tests): stop the integration host from ballooning to 30 GB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-reported: the integration tests leave instances behind and take 18+ GB. Measured before changing anything — a SINGLE Host.IntegrationTests process peaked at 30,153 MB RSS across its ~200 tests. The cause was not a leak and not parallelism. Referencing the Host drags in the ASP.NET Core framework reference, which turns Server GC on by default: one heap per core, tuned for throughput and deliberately reluctant to hand memory back. Correct for a server, wrong for a test host. Workstation GC takes the same suite from 30,153 MB to 6,717 MB — 78% less — with no change in runtime (45s). Two hypotheses were measured and rejected on the way, recorded here so nobody re-runs the experiment: capping xunit maxParallelThreads to 4 cost 29% wall-clock and saved nothing (30,153 -> 31,115 MB, i.e. noise), and it turns out a single process was doing all of it. Reverted. The harness fix is real but was worth only 4% on its own: TwoNodeClusterHarness disposed each node without stopping it first, and IHost.DisposeAsync only disposes the service provider — it never invokes IHostedService.StopAsync, which is where Akka.Hosting terminates the ActorSystem. So every test left two live ActorSystems, remoting transports and dispatcher pools included, rooted for the process lifetime. The class doc-comment asserted the opposite ("DisposeAsync, which runs CoordinatedShutdown"); it was wrong, and StopNodeBAsync inherited the same bug on a path failover tests depend on. The payoff is larger than the memory number. Two failures I had classified as known-flaky baseline noise now pass consistently: RoslynVirtualTagEvaluatorTests.Evaluate_racing_ClearCompiledScripts_never_fails_with_disposed and ContinuousHistorizationRecorderTests.Retry_after_writer_failure_eventually_acks. They were never flaky — they were starved. Full solution: peak summed test-process RSS 15.2 GB, zero new failures, two fewer. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW --- .../TwoNodeClusterHarness.cs | 39 ++++++++++++++++++- ...OM.WW.OtOpcUa.Host.IntegrationTests.csproj | 13 +++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs index 9bdb9c59..15175be0 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/TwoNodeClusterHarness.cs @@ -456,6 +456,41 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable $"B up={Akka.Cluster.Cluster.Get(b).State.Members.Count(m => m.Status == MemberStatus.Up)}"); } + /// + /// Stops a node's hosted services, then disposes it. + /// + /// + /// + /// is not optional, and disposal is not a substitute for + /// it. Host.DisposeAsync only disposes the service provider; it never invokes + /// . Akka.Hosting terminates the + /// from that callback, so a disposed-but-never-stopped node + /// leaves a live ActorSystem — remoting transport, dispatcher thread pools and all — + /// rooted for the lifetime of the test process. + /// + /// + /// Every test here starts two of them, so the cost compounded across the suite: a single + /// test host was measured at ~30 GB RSS, which starved timeout-based assertions + /// into false failures that looked exactly like real regressions. + /// + /// + /// The node to stop and dispose. + private static async Task ShutDownAsync(WebApplication node) + { + // Best-effort: a node already stopped (or mid-CoordinatedShutdown from a failover test) + // throws here, and a failure to stop must never mask the test's own assertion failure. + try + { + await node.StopAsync(TimeSpan.FromSeconds(10)); + } + catch + { + // fall through to disposal + } + + await node.DisposeAsync(); + } + private static int AllocateFreePort() { using var listener = new TcpListener(System.Net.IPAddress.Parse(LoopbackHost), 0); @@ -468,8 +503,8 @@ public sealed class TwoNodeClusterHarness : IAsyncDisposable /// Asynchronously disposes both nodes and cleans up the SQL database if used. public async ValueTask DisposeAsync() { - if (NodeB is not null) await NodeB.DisposeAsync(); - if (NodeA is not null) await NodeA.DisposeAsync(); + if (NodeB is not null) await ShutDownAsync(NodeB); + if (NodeA is not null) await ShutDownAsync(NodeA); if (_sqlConnString is not null) { try { await DropSqlDatabaseAsync(_sqlConnString); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.csproj b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.csproj index c678f2e0..9ae4ff79 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.csproj +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.csproj @@ -30,4 +30,17 @@ + + + false + false + +