feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)
A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
+105
@@ -0,0 +1,105 @@
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ScriptedAlarmEngine"/> runs every predicate through a <see cref="ScriptRunGuard"/> keyed by
|
||||
/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation
|
||||
/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is
|
||||
/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a
|
||||
/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class ScriptedAlarmEngineRunGuardTests
|
||||
{
|
||||
private const uint StatusBadInternalError = 0x80020000u;
|
||||
|
||||
private const string SpinPredicate =
|
||||
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;";
|
||||
|
||||
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
|
||||
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
|
||||
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
|
||||
|
||||
private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null)
|
||||
{
|
||||
var logger = new LoggerConfiguration().CreateLogger();
|
||||
// 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and
|
||||
// must finish inside the budget on a loaded CI box, or the positive control itself times out. The
|
||||
// spin predicate outlives this by 12x, so the runaway alarm still cannot finish first.
|
||||
return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
|
||||
scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating()
|
||||
{
|
||||
var up = new FakeUpstream();
|
||||
up.Set("Temp", 50);
|
||||
using var eng = Build(up);
|
||||
var events = new List<ScriptedAlarmEvent>();
|
||||
eng.OnEvent += (_, e) => { lock (events) events.Add(e); };
|
||||
|
||||
// The load itself evaluates once (startup recovery) — that is the first, abandoned run.
|
||||
await eng.LoadAsync(
|
||||
[Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"),
|
||||
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")],
|
||||
TestContext.Current.CancellationToken);
|
||||
|
||||
// Two more pushes: refused (orphan still running), then quarantined.
|
||||
up.Push("Temp", 60);
|
||||
await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1);
|
||||
up.Push("Temp", 70);
|
||||
await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _));
|
||||
|
||||
eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
|
||||
eng.RunGuard.InFlightCount.ShouldBe(1);
|
||||
|
||||
// The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving.
|
||||
ScriptedAlarmEvent[] snapshot;
|
||||
lock (events) snapshot = events.ToArray();
|
||||
var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray();
|
||||
quality.ShouldNotBeEmpty();
|
||||
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
|
||||
eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented");
|
||||
|
||||
// Positive control: the healthy sibling still transitions on the same engine.
|
||||
up.Push("Temp", 150);
|
||||
await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reload_lifts_the_quarantine()
|
||||
{
|
||||
var up = new FakeUpstream();
|
||||
up.Set("Temp", 50);
|
||||
using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
|
||||
await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")],
|
||||
TestContext.Current.CancellationToken);
|
||||
eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue();
|
||||
|
||||
// The operator fixed the predicate and redeployed.
|
||||
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")],
|
||||
TestContext.Current.CancellationToken);
|
||||
eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse();
|
||||
|
||||
up.Push("Temp", 150);
|
||||
await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active);
|
||||
}
|
||||
|
||||
private static async Task WaitForAsync(Func<bool> cond, int timeoutMs = 10_000)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (cond()) return;
|
||||
await Task.Delay(25);
|
||||
}
|
||||
throw new TimeoutException("Condition did not become true in time");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ScriptRunGuard"/> — the per-script run quarantine shared by the virtual-tag evaluator, the
|
||||
/// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so
|
||||
/// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard
|
||||
/// refuses to start a run while the previous one is still executing, quarantines a script after three
|
||||
/// consecutive non-completions, and quarantines a script whose single run allocates past the budget.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class ScriptRunGuardTests
|
||||
{
|
||||
/// <summary>
|
||||
/// A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool
|
||||
/// thread it leaves behind frees itself after the test). <c>System.DateTime</c> is outside the sandbox
|
||||
/// deny-list.
|
||||
/// </summary>
|
||||
private const string SpinScript =
|
||||
"""
|
||||
var end = System.DateTime.UtcNow.AddSeconds(6);
|
||||
while (System.DateTime.UtcNow < end) { }
|
||||
return 1;
|
||||
""";
|
||||
|
||||
private const string FastScript = "return 42;";
|
||||
|
||||
private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60);
|
||||
|
||||
private static ScriptEvaluator<FakeScriptContext, int> Compile(string source) =>
|
||||
ScriptEvaluator<FakeScriptContext, int>.Compile(source);
|
||||
|
||||
private static Task<int> Run(ScriptRunGuard guard, string key, string source,
|
||||
ScriptEvaluator<FakeScriptContext, int> evaluator, TimeSpan? timeout = null, CancellationToken ct = default) =>
|
||||
guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct);
|
||||
|
||||
[Fact]
|
||||
public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var fast = Compile(FastScript);
|
||||
|
||||
var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken);
|
||||
|
||||
value.ShouldBe(42);
|
||||
guard.InFlightCount.ShouldBe(0);
|
||||
guard.StartedRunCount.ShouldBe(1);
|
||||
guard.IsQuarantined("vt-1", out _).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var spin = Compile(SpinScript);
|
||||
|
||||
await Should.ThrowAsync<ScriptTimeoutException>(() =>
|
||||
Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
|
||||
|
||||
// The abandoned script is still executing on its pool thread — the guard knows, the caller does not.
|
||||
guard.InFlightCount.ShouldBe(1);
|
||||
guard.StartedRunCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var spin = Compile(SpinScript);
|
||||
await Should.ThrowAsync<ScriptTimeoutException>(() =>
|
||||
Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
|
||||
|
||||
var ex = await Should.ThrowAsync<ScriptRunRefusedException>(() =>
|
||||
Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
|
||||
|
||||
ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run");
|
||||
guard.InFlightCount.ShouldBe(1);
|
||||
guard.RefusedWhileRunningCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var spin = Compile(SpinScript);
|
||||
using var fast = Compile(FastScript);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
// 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined.
|
||||
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
|
||||
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
|
||||
var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
|
||||
|
||||
quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
quarantined.Message.ShouldContain("3 consecutive");
|
||||
guard.IsQuarantined("bad", out var reason).ShouldBeTrue();
|
||||
reason.ShouldNotBeNull();
|
||||
guard.QuarantinedCount.ShouldBe(1);
|
||||
|
||||
// A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run.
|
||||
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
|
||||
guard.StartedRunCount.ShouldBe(1);
|
||||
|
||||
// Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key.
|
||||
(await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42);
|
||||
guard.IsQuarantined("good", out _).ShouldBeFalse();
|
||||
guard.StartedRunCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The thread-leak regression. Pre-guard, a <c>while(true)</c> script on a 1 Hz trigger leaked one
|
||||
/// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the
|
||||
/// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy
|
||||
/// sibling keeps evaluating throughout (the positive control an absence assertion needs).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The load-bearing assertions are on the guard's own bookkeeping (<see cref="ScriptRunGuard.InFlightCount"/>
|
||||
/// / <see cref="ScriptRunGuard.StartedRunCount"/>): <see cref="ThreadPool.ThreadCount"/> moves with
|
||||
/// every parallel test class on a loaded machine, so it is kept as a coarse secondary check only —
|
||||
/// it must stay far below the 30 the leak would have produced.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task Runaway_script_leaves_at_most_one_orphan_after_30_triggers_while_a_sibling_keeps_evaluating()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var spin = Compile(SpinScript);
|
||||
using var fast = Compile(FastScript);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var threadsBefore = ThreadPool.ThreadCount;
|
||||
|
||||
var healthyResults = new List<int>();
|
||||
for (var trigger = 0; trigger < 30; trigger++)
|
||||
{
|
||||
try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); }
|
||||
catch (ScriptTimeoutException) { }
|
||||
catch (ScriptRunRefusedException) { }
|
||||
catch (ScriptQuarantinedException) { }
|
||||
|
||||
healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct));
|
||||
}
|
||||
|
||||
guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing");
|
||||
guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling");
|
||||
guard.IsQuarantined("runaway", out _).ShouldBeTrue();
|
||||
healthyResults.ShouldAllBe(v => v == 42);
|
||||
healthyResults.Count.ShouldBe(30);
|
||||
|
||||
var threadDelta = ThreadPool.ThreadCount - threadsBefore;
|
||||
threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var spin = Compile(SpinScript);
|
||||
using var fast = Compile(FastScript);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
|
||||
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
|
||||
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
|
||||
guard.IsQuarantined("vt", out _).ShouldBeTrue();
|
||||
|
||||
// The operator fixed the script: a different source identity under the same key starts clean. The
|
||||
// old orphan is a sunk cost — it keeps running, but it must not block the FIXED script.
|
||||
(await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42);
|
||||
guard.IsQuarantined("vt", out _).ShouldBeFalse();
|
||||
guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reset_and_ResetAll_clear_a_quarantine()
|
||||
{
|
||||
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
using var spin = Compile(SpinScript);
|
||||
using var fast = Compile(FastScript);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct));
|
||||
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct));
|
||||
guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true);
|
||||
|
||||
guard.Reset("a");
|
||||
guard.IsQuarantined("a", out _).ShouldBeFalse();
|
||||
guard.IsQuarantined("b", out _).ShouldBeTrue();
|
||||
|
||||
guard.ResetAll();
|
||||
guard.IsQuarantined("b", out _).ShouldBeFalse();
|
||||
// The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it).
|
||||
(await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_run_that_allocates_past_the_budget_is_quarantined()
|
||||
{
|
||||
var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 });
|
||||
// Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the
|
||||
// allocation as dead code (which is exactly what it does to the naive one-liner).
|
||||
const string hungry =
|
||||
"long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;";
|
||||
using var evaluator = Compile(hungry);
|
||||
using var fast = Compile(FastScript);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
var ex = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "hog", hungry, evaluator, ct: ct));
|
||||
ex.Message.ShouldContain("allocat");
|
||||
guard.IsQuarantined("hog", out var reason).ShouldBeTrue();
|
||||
reason.ShouldNotBeNull();
|
||||
reason.ShouldContain("allocat");
|
||||
|
||||
// Never dispatched again, and the slot was released (the run DID complete).
|
||||
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "hog", hungry, evaluator, ct: ct));
|
||||
guard.StartedRunCount.ShouldBe(1);
|
||||
guard.InFlightCount.ShouldBe(0);
|
||||
|
||||
// Positive control: a modest allocation under the budget is fine.
|
||||
(await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_disabled_allocation_budget_never_quarantines()
|
||||
{
|
||||
var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 });
|
||||
// Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the
|
||||
// allocation as dead code (which is exactly what it does to the naive one-liner).
|
||||
const string hungry =
|
||||
"long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;";
|
||||
using var evaluator = Compile(hungry);
|
||||
|
||||
(await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1));
|
||||
guard.IsQuarantined("hog", out _).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion()
|
||||
{
|
||||
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
|
||||
const string throwing = "throw new System.InvalidOperationException(\"boom\");";
|
||||
using var thrower = Compile(throwing);
|
||||
// A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts.
|
||||
const string briefSpin =
|
||||
"""
|
||||
var end = System.DateTime.UtcNow.AddMilliseconds(250);
|
||||
while (System.DateTime.UtcNow < end) { }
|
||||
return 1;
|
||||
""";
|
||||
using var spin = Compile(briefSpin);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
// timeout #1
|
||||
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
|
||||
await WaitForIdle(guard);
|
||||
|
||||
// A throw is a completion — the script returned control — so the consecutive counter resets…
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(() => Run(guard, "k", throwing, thrower, ct: ct));
|
||||
ex.Message.ShouldBe("boom");
|
||||
guard.InFlightCount.ShouldBe(0);
|
||||
|
||||
// …which is why this second timeout does NOT reach the threshold of 2.
|
||||
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
|
||||
guard.IsQuarantined("k", out _).ShouldBeFalse();
|
||||
await WaitForIdle(guard);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Caller_cancellation_wins_over_the_timeout()
|
||||
{
|
||||
var guard = new ScriptRunGuard();
|
||||
using var spin = Compile(SpinScript);
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80));
|
||||
|
||||
await Should.ThrowAsync<OperationCanceledException>(() =>
|
||||
Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token));
|
||||
guard.IsQuarantined("k", out _).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Quarantine_is_observable_through_the_event_and_the_counter()
|
||||
{
|
||||
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
using var spin = Compile(SpinScript);
|
||||
var seen = new List<(string Key, string Reason)>();
|
||||
guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason));
|
||||
|
||||
await Should.ThrowAsync<ScriptQuarantinedException>(() =>
|
||||
Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
|
||||
|
||||
seen.ShouldHaveSingleItem();
|
||||
seen[0].Key.ShouldBe("k");
|
||||
seen[0].Reason.ShouldContain("consecutive");
|
||||
guard.QuarantinedCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
private static async Task WaitForIdle(ScriptRunGuard guard)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(5);
|
||||
while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline)
|
||||
await Task.Delay(10);
|
||||
guard.InFlightCount.ShouldBe(0, "the brief spin must have finished");
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="CalculationEvaluator"/> runs every compiled script through a <see cref="ScriptRunGuard"/>
|
||||
/// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every
|
||||
/// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now
|
||||
/// the next trigger is refused ("previous evaluation still running"), the third consecutive
|
||||
/// non-completion quarantines the tag, and a healthy sibling on the same evaluator keeps computing.
|
||||
/// </summary>
|
||||
public sealed class CalculationEvaluatorRunGuardTests
|
||||
{
|
||||
private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
|
||||
|
||||
/// <summary>Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself.</summary>
|
||||
private const string SpinScript =
|
||||
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
|
||||
|
||||
private static readonly Dictionary<string, object?> NoDeps = new();
|
||||
|
||||
[Fact]
|
||||
public async Task A_trigger_during_the_previous_orphan_is_refused_then_the_tag_is_quarantined_and_a_sibling_keeps_computing()
|
||||
{
|
||||
using var sut = new CalculationEvaluator(
|
||||
NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
|
||||
first.Success.ShouldBeFalse();
|
||||
first.Reason.ShouldNotBeNull();
|
||||
first.Reason.ShouldContain("timed out");
|
||||
|
||||
var second = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct);
|
||||
second.Success.ShouldBeFalse();
|
||||
second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
|
||||
var third = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct);
|
||||
third.Success.ShouldBeFalse();
|
||||
third.Reason.ShouldNotBeNull();
|
||||
third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
sut.QuarantinedScripts.ShouldBe(new[] { "calc/loop" });
|
||||
|
||||
// Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger.
|
||||
sut.RunGuard.StartedRunCount.ShouldBe(1);
|
||||
sut.RunGuard.InFlightCount.ShouldBe(1);
|
||||
|
||||
// Positive control: a sibling tag on the same evaluator still computes.
|
||||
var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct);
|
||||
sibling.Success.ShouldBeTrue(sibling.Reason);
|
||||
sibling.Value.ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does()
|
||||
{
|
||||
using var sut = new CalculationEvaluator(
|
||||
NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
|
||||
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
|
||||
r.Reason.ShouldNotBeNull();
|
||||
r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
|
||||
sut.ClearCompiledScripts();
|
||||
|
||||
sut.QuarantinedScripts.ShouldBeEmpty();
|
||||
// The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag.
|
||||
var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct);
|
||||
fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
|
||||
fixedRun.Value.ShouldBe(7);
|
||||
}
|
||||
|
||||
private static Dictionary<string, Core.Abstractions.DataValueSnapshot> Snap(Dictionary<string, object?> deps)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var d = new Dictionary<string, Core.Abstractions.DataValueSnapshot>(StringComparer.Ordinal);
|
||||
foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now);
|
||||
return d;
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Engines;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="RoslynVirtualTagEvaluator"/> runs every compiled expression through a
|
||||
/// <see cref="ScriptRunGuard"/> keyed by virtual-tag id: a runaway expression leaks one orphaned pool
|
||||
/// thread, not one per trigger, the tag is quarantined after three consecutive non-completions (surfacing
|
||||
/// as a <c>Failure</c> the actor publishes Bad for), and the quarantine lifts at the deploy boundary.
|
||||
/// </summary>
|
||||
public sealed class RoslynVirtualTagEvaluatorRunGuardTests
|
||||
{
|
||||
private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
|
||||
|
||||
private const string SpinScript =
|
||||
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
|
||||
|
||||
private static readonly Dictionary<string, object?> NoDeps = new();
|
||||
|
||||
[Fact]
|
||||
public void Runaway_expression_is_refused_then_quarantined_and_a_sibling_keeps_evaluating()
|
||||
{
|
||||
using var sut = new RoslynVirtualTagEvaluator(
|
||||
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
|
||||
|
||||
var first = sut.Evaluate("vt-loop", SpinScript, NoDeps);
|
||||
first.Success.ShouldBeFalse();
|
||||
first.Reason.ShouldNotBeNull();
|
||||
first.Reason.ShouldContain("timed out");
|
||||
|
||||
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
|
||||
var third = sut.Evaluate("vt-loop", SpinScript, NoDeps);
|
||||
third.Reason.ShouldNotBeNull();
|
||||
third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
sut.QuarantinedScripts.ShouldBe(new[] { "vt-loop" });
|
||||
sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger");
|
||||
|
||||
var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps);
|
||||
sibling.Success.ShouldBeTrue(sibling.Reason);
|
||||
sibling.Value.ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResetQuarantines_lifts_the_quarantine_as_a_redeploy_does()
|
||||
{
|
||||
using var sut = new RoslynVirtualTagEvaluator(
|
||||
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
|
||||
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
|
||||
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
sut.QuarantinedScripts.ShouldNotBeEmpty();
|
||||
|
||||
sut.ResetQuarantines();
|
||||
|
||||
sut.QuarantinedScripts.ShouldBeEmpty();
|
||||
var fixedRun = sut.Evaluate("vt-loop", "return 7;", NoDeps);
|
||||
fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
|
||||
fixedRun.Value.ShouldBe(7);
|
||||
}
|
||||
}
|
||||
+8
-2
@@ -489,12 +489,18 @@ public sealed class RoslynVirtualTagEvaluatorTests
|
||||
// most evaluations then hit the warm cache instead of recompiling. Neither the cadence nor the
|
||||
// iteration count is a correctness knob any more — the assertions below hold for any number of
|
||||
// clears landing inside a single evaluation, and for any number of evaluations completed.
|
||||
var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() =>
|
||||
// One virtual-tag id PER LOOP, sharing the expression. The race under test lives in the compile
|
||||
// cache, which is keyed by source, so twelve ids on one expression contend for exactly the entry
|
||||
// twelve copies of one id did — while twelve concurrent evaluations of ONE id is a shape production
|
||||
// never produces (one VirtualTagActor per id serialises them) and exactly the shape the per-tag
|
||||
// ScriptRunGuard refuses ("previous evaluation still running", then quarantine).
|
||||
var evalLoops = Enumerable.Range(0, loopCount).Select(loop => Task.Run(() =>
|
||||
{
|
||||
var vtagId = $"vt-race-{loop}";
|
||||
var done = 0;
|
||||
for (var i = 0; i < MaxIterationsPerLoop && Environment.TickCount64 < stopAtMs; i++)
|
||||
{
|
||||
var r = sut.Evaluate("vt-race", expr, new Dictionary<string, object?> { ["a"] = i });
|
||||
var r = sut.Evaluate(vtagId, expr, new Dictionary<string, object?> { ["a"] = i });
|
||||
done++;
|
||||
if (!r.Success)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user