fix(scripting): only timeouts are quarantine strikes; alarm Bad annotation once per bucket change (fix round 1 I1, I3)

- I1 ScriptRunGuard: a run refused because the previous one is still
  executing is a Bad publish plus otopcua.script.run_refused, and no longer
  a quarantine strike. Only a timeout (and an allocation breach) counts;
  a refusal neither adds to nor resets the timeout streak; a completed run
  (value or throw) resets it. On a 1 Hz trigger with a 2 s budget a single
  4 s overrun is one timeout plus two refusals and must not quarantine.
  Tests updated across the guard, the alarm engine, the Calculation
  evaluator and the VT evaluator, plus
  Refusals_neither_count_toward_nor_reset_the_timeout_streak.
- I3 ScriptedAlarmEngine: the quality bucket is tracked only after the
  predicate completes (or in the inputs-not-ready path), so the tracked
  bucket is the last one annotated. A refused or quarantined predicate
  used to re-track the input bucket Good before the attempt and then flip
  it Bad, emitting QualityChanged on every push. The run-guard tests now
  count emissions (exactly 1 across N pushes, refused and quarantined
  cases), with a healthy sibling's dispatch count as the positive control
  that every push was processed. Both were red before the fix.

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 13:26:12 -04:00
parent 909b061d3a
commit 6aa5edaf30
10 changed files with 161 additions and 67 deletions
@@ -9,8 +9,8 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
/// <summary>
/// <see cref="ScriptedAlarmEngine"/> runs every predicate through a <see cref="ScriptRunGuard"/> keyed by
/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation
/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is
/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a
/// spawned another abandoned run); now the orphan is one, later pushes are refused (and a
/// predicate that times out repeatedly is quarantined) — frozen at its prior state and annotated Bad quality so the operator can see it — while a
/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine.
/// </summary>
[Trait("Category", "Unit")]
@@ -36,7 +36,7 @@ public sealed class ScriptedAlarmEngineRunGuardTests
}
[Fact]
public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating()
public async Task Runaway_predicate_leaks_one_orphan_is_refused_annotated_Bad_once_and_a_sibling_keeps_evaluating()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
@@ -50,20 +50,27 @@ public sealed class ScriptedAlarmEngineRunGuardTests
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
// Two more pushes: refused (orphan still running), then quarantined.
up.Push("Temp", 60);
await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1);
up.Push("Temp", 70);
await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _));
// N more pushes while the orphan runs: every one is refused (a refusal is not a strike — I1), and
// the Bad annotation is emitted ONCE for the whole streak, not once per push (I3).
const int pushes = 4;
for (var i = 0; i < pushes; i++)
{
up.Push("Temp", 60 + i);
var expected = i + 1;
await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= expected);
}
await WaitForAsync(() => eng.RunGuard.StartedRunCount >= 1 + 1 + pushes);
eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
eng.RunGuard.IsQuarantined("runaway", out _).ShouldBeFalse("refusals are not strikes");
eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
eng.RunGuard.InFlightCount.ShouldBe(1);
// The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving.
// The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving —
// exactly once across the whole run of non-completions.
ScriptedAlarmEvent[] snapshot;
lock (events) snapshot = events.ToArray();
var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray();
quality.ShouldNotBeEmpty();
quality.Length.ShouldBe(1);
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented");
@@ -72,6 +79,44 @@ public sealed class ScriptedAlarmEngineRunGuardTests
await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active);
}
/// <summary>I3 — a QUARANTINED predicate is re-attempted on every push and refused before dispatch; the
/// Bad annotation must still be emitted once, not once per push.</summary>
[Fact]
public async Task A_quarantined_predicate_emits_QualityChanged_once_across_many_pushes()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
var events = new List<ScriptedAlarmEvent>();
eng.OnEvent += (_, e) => { lock (events) events.Add(e); };
await eng.LoadAsync(
[Alarm("q", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"),
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 1000;""")],
TestContext.Current.CancellationToken);
eng.RunGuard.IsQuarantined("q", out _).ShouldBeTrue();
// Each push re-evaluates BOTH alarms (same input). The healthy sibling's dispatch count is the
// positive control proving every push was actually processed — so the quarantined alarm was
// re-attempted N times, not coalesced into one.
const int pushes = 5;
for (var i = 0; i < pushes; i++)
{
var before = eng.RunGuard.StartedRunCount;
up.Push("Temp", 60 + i);
await WaitForAsync(() => eng.RunGuard.StartedRunCount > before);
}
eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one runaway dispatch, then healthy-only dispatches");
await Task.Delay(300); // absence window for trailing emissions
ScriptedAlarmEvent[] snapshot;
lock (events) snapshot = events.ToArray();
var quality = snapshot.Where(e => e.AlarmId == "q" && e.Emission == EmissionKind.QualityChanged).ToArray();
quality.Length.ShouldBe(1, "one Bad annotation for the quarantine, not one per push");
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
snapshot.ShouldNotContain(e => e.AlarmId == "healthy" && e.Emission == EmissionKind.QualityChanged);
}
[Fact]
public async Task A_reload_lifts_the_quarantine()
{