test: deflake load-sensitive tests (lmxopcua#769)
v2-ci / unit-tests (push) Has been cancelled

- NodeManagerLockTimeBenchmark: wall-clock budget enforced only with OTOPCUA_BENCH=1; default run keeps the state-correctness asserts and logs timings.
- OpcUaClientDriverWire host-status: signal (semaphore) wait that only accepts a non-Running state observed after the server is disposed.
- AbCip PollLatencyComparison: retry once on a fixture connection reset; p95 criterion enforced only with AB_LAT_ENFORCE=1.
- ScriptRunGuardTests: orphan scripts held by a ctx.Released signal instead of a 250 ms wall-clock spin that could finish inside the 60 ms budget under load; stuck-orphan wall-clock rule disabled in the tests that pin the strike counting.
- DependencyMuxReplayTests: split the inside-grace positive control (1 h grace) from the release test (control asserted before unregister), removing the 200 ms grace race.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 05:39:22 -05:00
parent 000d7f5591
commit 11a5c1d4e2
6 changed files with 128 additions and 59 deletions
@@ -20,8 +20,9 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
/// </list>
/// The wall-clock budgets are generous for CI hardware; the before/after numbers are in the commit
/// message. P3 is pinned by the bookkeeping's own visit counter, which does not depend on the machine.
/// Tagged <c>Category=Benchmark</c>, but no filter excludes that category (CI drops only E2E + LiveIntegration),
/// so it runs by default — on purpose:
/// lmxopcua#769: the wall-clock legs only ASSERT their budget when <c>OTOPCUA_BENCH=1</c> (opt-in, like
/// <c>OTOPCUA_SCRIPT_PERF</c>); the default run still executes every teardown and asserts the state correctness.
/// Historical note on the budget: no filter excludes <c>Category=Benchmark</c> (CI drops only E2E + LiveIntegration):
/// measured after the fix at 5–6 ms (P1) and 58–64 ms (P2) on a loaded dev machine against the 1 s budget —
/// ≥ 15× headroom, so it does not flake on slow CI. The flip side: the pre-fix code (75–150 ms / 470–680 ms) also
/// fits 1 s at these sizes, so the wall-clock legs guard against gross (order-of-magnitude) regressions only; the
@@ -35,6 +36,18 @@ public sealed class NodeManagerLockTimeBenchmarkTests(ITestOutputHelper output)
// >= 80x headroom against a CPU-starved CI box while still catching an order-of-magnitude (O(N^2)) regression.
private static readonly TimeSpan Budget = TimeSpan.FromSeconds(5);
// lmxopcua#769: even 5 s flaked in a full parallel run on a starved box, so the wall-clock legs are opt-in, like
// ScriptRunnerPerfTests (OTOPCUA_SCRIPT_PERF=1): OTOPCUA_BENCH=1 enforces the budget. The default run still executes
// every teardown and asserts the state correctness (counts, SDK handles, re-population) and logs the measured time.
private static bool EnforceBudget => Environment.GetEnvironmentVariable("OTOPCUA_BENCH") == "1";
private void AssertWithinBudget(Stopwatch sw, string what)
{
output.WriteLine($"BENCH {what}: {sw.ElapsedMilliseconds} ms (budget {Budget.TotalMilliseconds:F0} ms, enforced={EnforceBudget})");
if (EnforceBudget)
sw.Elapsed.ShouldBeLessThan(Budget, $"{what} took {sw.ElapsedMilliseconds} ms");
}
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-bench-{Guid.NewGuid():N}");
/// <summary>P1 — a rebuild that discards one device folder holding 10k tags completes within the budget.</summary>
@@ -64,8 +77,7 @@ public sealed class NodeManagerLockTimeBenchmarkTests(ITestOutputHelper output)
nm.EnsureVariable($"{device}/T0", device, "T0", "Int32", writable: false, realm: AddressSpaceRealm.Raw);
nm.TryGetVariable($"{device}/T0", AddressSpaceRealm.Raw)!.Parent.ShouldBeSameAs(nm.TryGetFolder(device, AddressSpaceRealm.Raw));
output.WriteLine($"BENCH rebuild 10k-tag folder: {sw.ElapsedMilliseconds} ms");
sw.Elapsed.ShouldBeLessThan(Budget, $"rebuild of a 10k-tag folder took {sw.ElapsedMilliseconds} ms");
AssertWithinBudget(sw, "rebuild 10k-tag folder");
}
finally { await host.DisposeAsync(); }
}
@@ -101,8 +113,7 @@ public sealed class NodeManagerLockTimeBenchmarkTests(ITestOutputHelper output)
nm.TryGetVariable("area/line/keep/S0").ShouldNotBeNull();
nm.GetManagerHandle(new NodeId("area/line/eq0/S0", nm.NamespaceIndexForRealm(AddressSpaceRealm.Uns))).ShouldBeNull();
output.WriteLine($"BENCH remove 200 equipment / 50k nodes: {sw.ElapsedMilliseconds} ms");
sw.Elapsed.ShouldBeLessThan(Budget, $"removing 200 equipment took {sw.ElapsedMilliseconds} ms");
AssertWithinBudget(sw, "remove 200 equipment / 50k nodes");
}
finally { await host.DisposeAsync(); }
}