dohertj2/lmxopcua
1
0
Fork 0
Code Issues 37 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

test: deflake load-sensitive tests (lmxopcua#769)
v2-ci / unit-tests (push) Has been cancelled
Details

Browse Source 
- NodeManagerLockTimeBenchmark: wall-clock budget enforced only with OTOPCUA_BENCH=1; default run keeps the state-correctness asserts and logs timings.
- OpcUaClientDriverWire host-status: signal (semaphore) wait that only accepts a non-Running state observed after the server is disposed.
- AbCip PollLatencyComparison: retry once on a fixture connection reset; p95 criterion enforced only with AB_LAT_ENFORCE=1.
- ScriptRunGuardTests: orphan scripts held by a ctx.Released signal instead of a 250 ms wall-clock spin that could finish inside the 60 ms budget under load; stuck-orphan wall-clock rule disabled in the tests that pin the strike counting.
- DependencyMuxReplayTests: split the inside-grace positive control (1 h grace) from the release test (control asserted before unregister), removing the 200 ms grace race.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 05:39:22 -05:00
parent 000d7f5591
commit 11a5c1d4e2
6 changed files with 128 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs
+42 -33
View File
@@ -31,11 +31,10 @@ public sealed class ScriptRunGuardTests
private const string FastScript = "return 42;";
/// <summary>Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts.</summary>
/// <summary>Outlives the 60 ms budget until the test sets <see cref="FakeScriptContext.Released"/> — a signal, not a wall-clock spin, so load cannot let it finish inside the budget.</summary>
private const string BriefSpinScript =
"""
var end = System.DateTime.UtcNow.AddMilliseconds(250);
while (System.DateTime.UtcNow < end) { }
while (!ctx.Released) { }
return 1;
""";
@@ -45,8 +44,8 @@ public sealed class ScriptRunGuardTests
ScriptEvaluator<FakeScriptContext, int>.Compile(source);
private static Task<int> Run(ScriptRunGuard guard, string key, string source,
ScriptEvaluator<FakeScriptContext, int> evaluator, TimeSpan? timeout = null, CancellationToken ct = default) =>
guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct);
ScriptEvaluator<FakeScriptContext, int> evaluator, TimeSpan? timeout = null, CancellationToken ct = default, FakeScriptContext? context = null) =>
guard.RunAsync(key, source, evaluator, context ?? new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct);
[Fact]
public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot()
@@ -96,19 +95,21 @@ public sealed class ScriptRunGuardTests
[Fact]
public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating()
{
var guard = new ScriptRunGuard();
// Stuck-orphan rule off: it is wall-clock based and not what this test pins.
var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = 0 });
using var spin = Compile(BriefSpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
var held = new FakeScriptContext();
// Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run,
// Three timeouts in a row — each orphan is released (signal) and drained so the next attempt is a real run,
// not a refusal (a refusal is not a strike; see the next test).
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct, held));
await ReleaseAndDrain(guard, held);
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct, held));
await ReleaseAndDrain(guard, held);
var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct, held));
await ReleaseAndDrain(guard, held);
quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
quarantined.Message.ShouldContain("3 consecutive");
@@ -117,7 +118,7 @@ public sealed class ScriptRunGuardTests
guard.QuarantinedCount.ShouldBe(1);
// A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run.
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct, held));
guard.StartedRunCount.ShouldBe(3);
// Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key.
@@ -135,22 +136,25 @@ public sealed class ScriptRunGuardTests
[Fact]
public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
// Stuck-orphan rule off: with it on, a loaded box could age the held orphan past 10x its budget and turn
// a refusal into a stuck-quarantine, which is a different (wall-clock) rule from the one under test.
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2, StuckOrphanQuarantineFactor = 0 });
using var spin = Compile(BriefSpinScript);
var ct = TestContext.Current.CancellationToken;
var held = new FakeScriptContext();
// timeout #1, then refusals while the orphan is still running — however many, no quarantine.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
// timeout #1, then refusals while the orphan is still running (held until released) — however many, no quarantine.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct, held));
for (var i = 0; i < 5; i++)
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct, held));
guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes");
guard.RefusedWhileRunningCount.ShouldBe(5);
await WaitForIdle(guard);
await ReleaseAndDrain(guard, held);
// …and they did not RESET the streak either: the second real timeout reaches the threshold of 2.
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct, held));
guard.IsQuarantined("k", out _).ShouldBeTrue();
await WaitForIdle(guard);
await ReleaseAndDrain(guard, held);
}
/// <summary>
@@ -388,19 +392,15 @@ public sealed class ScriptRunGuardTests
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
const string throwing = "throw new System.InvalidOperationException(\"boom\");";
using var thrower = Compile(throwing);
// A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts.
const string briefSpin =
"""
var end = System.DateTime.UtcNow.AddMilliseconds(250);
while (System.DateTime.UtcNow < end) { }
return 1;
""";
// A spin that outlives the timeout until released, so the slot frees up between attempts.
const string briefSpin = BriefSpinScript;
using var spin = Compile(briefSpin);
var ct = TestContext.Current.CancellationToken;
var held = new FakeScriptContext();
// timeout #1
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
await WaitForIdle(guard);
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct, held));
await ReleaseAndDrain(guard, held);
// A throw is a completion — the script returned control — so the consecutive counter resets…
var ex = await Should.ThrowAsync<InvalidOperationException>(() => Run(guard, "k", throwing, thrower, ct: ct));
@@ -408,9 +408,10 @@ public sealed class ScriptRunGuardTests
guard.InFlightCount.ShouldBe(0);
// …which is why this second timeout does NOT reach the threshold of 2.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
held.Released = false;
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct, held));
guard.IsQuarantined("k", out _).ShouldBeFalse();
await WaitForIdle(guard);
await ReleaseAndDrain(guard, held);
}
[Fact]
@@ -442,9 +443,17 @@ public sealed class ScriptRunGuardTests
guard.QuarantinedCount.ShouldBe(1);
}
/// <summary>Releases the held script (signal), waits for the orphan to drain, and re-arms the gate for the next attempt.</summary>
private static async Task ReleaseAndDrain(ScriptRunGuard guard, FakeScriptContext held)
{
held.Released = true;
await WaitForIdle(guard);
held.Released = false;
}
private static async Task WaitForIdle(ScriptRunGuard guard)
{
var deadline = DateTime.UtcNow.AddSeconds(5);
var deadline = DateTime.UtcNow.AddSeconds(30);
while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline)
await Task.Delay(10);
guard.InFlightCount.ShouldBe(0, "the brief spin must have finished");