using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Scripting; namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; /// /// — the per-script run quarantine shared by the virtual-tag evaluator, the /// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so /// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard /// refuses to start a run while the previous one is still executing, quarantines a script after three /// consecutive non-completions, and quarantines a script whose single run allocates past the budget. /// [Trait("Category", "Unit")] public sealed class ScriptRunGuardTests { /// /// A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool /// thread it leaves behind frees itself after the test). System.DateTime is outside the sandbox /// deny-list. /// private const string SpinScript = """ var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 1; """; private const string FastScript = "return 42;"; private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60); private static ScriptEvaluator Compile(string source) => ScriptEvaluator.Compile(source); private static Task Run(ScriptRunGuard guard, string key, string source, ScriptEvaluator evaluator, TimeSpan? timeout = null, CancellationToken ct = default) => guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct); [Fact] public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot() { var guard = new ScriptRunGuard(); using var fast = Compile(FastScript); var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken); value.ShouldBe(42); guard.InFlightCount.ShouldBe(0); guard.StartedRunCount.ShouldBe(1); guard.IsQuarantined("vt-1", out _).ShouldBeFalse(); } [Fact] public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); await Should.ThrowAsync(() => Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); // The abandoned script is still executing on its pool thread — the guard knows, the caller does not. guard.InFlightCount.ShouldBe(1); guard.StartedRunCount.ShouldBe(1); } [Fact] public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); await Should.ThrowAsync(() => Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); var ex = await Should.ThrowAsync(() => Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run"); guard.InFlightCount.ShouldBe(1); guard.RefusedWhileRunningCount.ShouldBe(1); } [Fact] public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; // 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined. await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); quarantined.Message.ShouldContain("3 consecutive"); guard.IsQuarantined("bad", out var reason).ShouldBeTrue(); reason.ShouldNotBeNull(); guard.QuarantinedCount.ShouldBe(1); // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run. await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); guard.StartedRunCount.ShouldBe(1); // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key. (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42); guard.IsQuarantined("good", out _).ShouldBeFalse(); guard.StartedRunCount.ShouldBe(2); } /// /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the /// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy /// sibling keeps evaluating throughout (the positive control an absence assertion needs). /// /// /// The load-bearing assertions are on the guard's own bookkeeping ( /// / ): moves with /// every parallel test class on a loaded machine, so it is kept as a coarse secondary check only — /// it must stay far below the 30 the leak would have produced. /// [Fact] public async Task Runaway_script_leaves_at_most_one_orphan_after_30_triggers_while_a_sibling_keeps_evaluating() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; var threadsBefore = ThreadPool.ThreadCount; var healthyResults = new List(); for (var trigger = 0; trigger < 30; trigger++) { try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); } catch (ScriptTimeoutException) { } catch (ScriptRunRefusedException) { } catch (ScriptQuarantinedException) { } healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct)); } guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); guard.IsQuarantined("runaway", out _).ShouldBeTrue(); healthyResults.ShouldAllBe(v => v == 42); healthyResults.Count.ShouldBe(30); var threadDelta = ThreadPool.ThreadCount - threadsBefore; threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads"); } [Fact] public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); guard.IsQuarantined("vt", out _).ShouldBeTrue(); // The operator fixed the script: a different source identity under the same key starts clean. The // old orphan is a sunk cost — it keeps running, but it must not block the FIXED script. (await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42); guard.IsQuarantined("vt", out _).ShouldBeFalse(); guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes"); } [Fact] public async Task Reset_and_ResetAll_clear_a_quarantine() { var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; await Should.ThrowAsync(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct)); await Should.ThrowAsync(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct)); guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true); guard.Reset("a"); guard.IsQuarantined("a", out _).ShouldBeFalse(); guard.IsQuarantined("b", out _).ShouldBeTrue(); guard.ResetAll(); guard.IsQuarantined("b", out _).ShouldBeFalse(); // The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it). (await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42); } [Fact] public async Task A_run_that_allocates_past_the_budget_is_quarantined() { var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 }); // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the // allocation as dead code (which is exactly what it does to the naive one-liner). const string hungry = "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; using var evaluator = Compile(hungry); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; var ex = await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); ex.Message.ShouldContain("allocat"); guard.IsQuarantined("hog", out var reason).ShouldBeTrue(); reason.ShouldNotBeNull(); reason.ShouldContain("allocat"); // Never dispatched again, and the slot was released (the run DID complete). await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); guard.StartedRunCount.ShouldBe(1); guard.InFlightCount.ShouldBe(0); // Positive control: a modest allocation under the budget is fine. (await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42); } [Fact] public async Task A_disabled_allocation_budget_never_quarantines() { var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 }); // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the // allocation as dead code (which is exactly what it does to the naive one-liner). const string hungry = "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; using var evaluator = Compile(hungry); (await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1)); guard.IsQuarantined("hog", out _).ShouldBeFalse(); } [Fact] public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion() { var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); const string throwing = "throw new System.InvalidOperationException(\"boom\");"; using var thrower = Compile(throwing); // A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts. const string briefSpin = """ var end = System.DateTime.UtcNow.AddMilliseconds(250); while (System.DateTime.UtcNow < end) { } return 1; """; using var spin = Compile(briefSpin); var ct = TestContext.Current.CancellationToken; // timeout #1 await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); await WaitForIdle(guard); // A throw is a completion — the script returned control — so the consecutive counter resets… var ex = await Should.ThrowAsync(() => Run(guard, "k", throwing, thrower, ct: ct)); ex.Message.ShouldBe("boom"); guard.InFlightCount.ShouldBe(0); // …which is why this second timeout does NOT reach the threshold of 2. await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); guard.IsQuarantined("k", out _).ShouldBeFalse(); await WaitForIdle(guard); } [Fact] public async Task Caller_cancellation_wins_over_the_timeout() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80)); await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token)); guard.IsQuarantined("k", out _).ShouldBeFalse(); } [Fact] public async Task Quarantine_is_observable_through_the_event_and_the_counter() { var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); using var spin = Compile(SpinScript); var seen = new List<(string Key, string Reason)>(); guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason)); await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); seen.ShouldHaveSingleItem(); seen[0].Key.ShouldBe("k"); seen[0].Reason.ShouldContain("consecutive"); guard.QuarantinedCount.ShouldBe(1); } private static async Task WaitForIdle(ScriptRunGuard guard) { var deadline = DateTime.UtcNow.AddSeconds(5); while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline) await Task.Delay(10); guard.InFlightCount.ShouldBe(0, "the brief spin must have finished"); } }