dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scripting): only timeouts are quarantine strikes; alarm Bad annotation once per bucket change (fix round 1 I1, I3)

Browse Source 
- I1 ScriptRunGuard: a run refused because the previous one is still
  executing is a Bad publish plus otopcua.script.run_refused, and no longer
  a quarantine strike. Only a timeout (and an allocation breach) counts;
  a refusal neither adds to nor resets the timeout streak; a completed run
  (value or throw) resets it. On a 1 Hz trigger with a 2 s budget a single
  4 s overrun is one timeout plus two refusals and must not quarantine.
  Tests updated across the guard, the alarm engine, the Calculation
  evaluator and the VT evaluator, plus
  Refusals_neither_count_toward_nor_reset_the_timeout_streak.
- I3 ScriptedAlarmEngine: the quality bucket is tracked only after the
  predicate completes (or in the inputs-not-ready path), so the tracked
  bucket is the last one annotated. A refused or quarantined predicate
  used to re-track the input bucket Good before the attempt and then flip
  it Bad, emitting QualityChanged on every push. The run-guard tests now
  count emissions (exactly 1 across N pushes, refused and quarantined
  cases), with a healthy sibling's dispatch count as the positive control
  that every push was processed. Both were red before the fix.

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 13:26:12 -04:00
parent 909b061d3a
commit 6aa5edaf30
10 changed files with 161 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs
+54 -15
View File
@@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
/// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so
/// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard
/// refuses to start a run while the previous one is still executing, quarantines a script after three
/// consecutive non-completions, and quarantines a script whose single run allocates past the budget.
/// consecutive timeouts (a refusal is not a strike), and quarantines a script whose single run allocates past the budget.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptRunGuardTests
@@ -28,6 +28,14 @@ public sealed class ScriptRunGuardTests
private const string FastScript = "return 42;";
/// <summary>Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts.</summary>
private const string BriefSpinScript =
"""
var end = System.DateTime.UtcNow.AddMilliseconds(250);
while (System.DateTime.UtcNow < end) { }
return 1;
""";
private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60);
private static ScriptEvaluator<FakeScriptContext, int> Compile(string source) =>
@@ -83,17 +91,21 @@ public sealed class ScriptRunGuardTests
}
[Fact]
public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating()
public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating()
{
var guard = new ScriptRunGuard();
using var spin = Compile(SpinScript);
using var spin = Compile(BriefSpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
// 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
// Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run,
// not a refusal (a refusal is not a strike; see the next test).
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
quarantined.Message.ShouldContain("3 consecutive");
@@ -102,19 +114,46 @@ public sealed class ScriptRunGuardTests
guard.QuarantinedCount.ShouldBe(1);
// A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run.
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct));
guard.StartedRunCount.ShouldBe(1);
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
guard.StartedRunCount.ShouldBe(3);
// Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key.
(await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42);
guard.IsQuarantined("good", out _).ShouldBeFalse();
guard.StartedRunCount.ShouldBe(2);
guard.StartedRunCount.ShouldBe(4);
}
/// <summary>
/// A refusal is NOT a strike (review fix round 1, I1): on a 1 Hz trigger with a 2 s budget a single
/// 4 s overrun is one timeout followed by two refusals, and counting those would quarantine a script
/// that merely ran long once. A refusal is a Bad publish plus its counter, and it leaves the timeout
/// streak untouched — neither adding to it nor resetting it.
/// </summary>
[Fact]
public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
using var spin = Compile(BriefSpinScript);
var ct = TestContext.Current.CancellationToken;
// timeout #1, then refusals while the orphan is still running — however many, no quarantine.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
for (var i = 0; i < 5; i++)
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes");
guard.RefusedWhileRunningCount.ShouldBe(5);
await WaitForIdle(guard);
// …and they did not RESET the streak either: the second real timeout reaches the threshold of 2.
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("k", out _).ShouldBeTrue();
await WaitForIdle(guard);
}
/// <summary>
/// The thread-leak regression. Pre-guard, a <c>while(true)</c> script on a 1 Hz trigger leaked one
/// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the
/// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy
/// first trigger's orphan is the only one, every later trigger is refused, and a healthy
/// sibling keeps evaluating throughout (the positive control an absence assertion needs).
/// </summary>
/// <remarks>
@@ -145,7 +184,9 @@ public sealed class ScriptRunGuardTests
guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing");
guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling");
guard.IsQuarantined("runaway", out _).ShouldBeTrue();
// One timeout + 29 refusals: not quarantined (a refusal is not a strike), but never re-dispatched.
guard.IsQuarantined("runaway", out _).ShouldBeFalse();
guard.RefusedWhileRunningCount.ShouldBe(29);
healthyResults.ShouldAllBe(v => v == 42);
healthyResults.Count.ShouldBe(30);
@@ -156,13 +197,11 @@ public sealed class ScriptRunGuardTests
[Fact]
public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot()
{
var guard = new ScriptRunGuard();
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
using var spin = Compile(SpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("vt", out _).ShouldBeTrue();