fix(scripting): only timeouts are quarantine strikes; alarm Bad annotation once per bucket change (fix round 1 I1, I3)
- I1 ScriptRunGuard: a run refused because the previous one is still executing is a Bad publish plus otopcua.script.run_refused, and no longer a quarantine strike. Only a timeout (and an allocation breach) counts; a refusal neither adds to nor resets the timeout streak; a completed run (value or throw) resets it. On a 1 Hz trigger with a 2 s budget a single 4 s overrun is one timeout plus two refusals and must not quarantine. Tests updated across the guard, the alarm engine, the Calculation evaluator and the VT evaluator, plus Refusals_neither_count_toward_nor_reset_the_timeout_streak. - I3 ScriptedAlarmEngine: the quality bucket is tracked only after the predicate completes (or in the inputs-not-ready path), so the tracked bucket is the last one annotated. A refused or quarantined predicate used to re-track the input bucket Good before the attempt and then flip it Bad, emitting QualityChanged on every push. The run-guard tests now count emissions (exactly 1 across N pushes, refused and quarantined cases), with a healthy sibling's dispatch count as the positive control that every push was processed. Both were red before the fix. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
+56
-11
@@ -9,8 +9,8 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
|
||||
/// <summary>
|
||||
/// <see cref="ScriptedAlarmEngine"/> runs every predicate through a <see cref="ScriptRunGuard"/> keyed by
|
||||
/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation
|
||||
/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is
|
||||
/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a
|
||||
/// spawned another abandoned run); now the orphan is one, later pushes are refused (and a
|
||||
/// predicate that times out repeatedly is quarantined) — frozen at its prior state and annotated Bad quality so the operator can see it — while a
|
||||
/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
@@ -36,7 +36,7 @@ public sealed class ScriptedAlarmEngineRunGuardTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating()
|
||||
public async Task Runaway_predicate_leaks_one_orphan_is_refused_annotated_Bad_once_and_a_sibling_keeps_evaluating()
|
||||
{
|
||||
var up = new FakeUpstream();
|
||||
up.Set("Temp", 50);
|
||||
@@ -50,20 +50,27 @@ public sealed class ScriptedAlarmEngineRunGuardTests
|
||||
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")],
|
||||
TestContext.Current.CancellationToken);
|
||||
|
||||
// Two more pushes: refused (orphan still running), then quarantined.
|
||||
up.Push("Temp", 60);
|
||||
await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1);
|
||||
up.Push("Temp", 70);
|
||||
await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _));
|
||||
// N more pushes while the orphan runs: every one is refused (a refusal is not a strike — I1), and
|
||||
// the Bad annotation is emitted ONCE for the whole streak, not once per push (I3).
|
||||
const int pushes = 4;
|
||||
for (var i = 0; i < pushes; i++)
|
||||
{
|
||||
up.Push("Temp", 60 + i);
|
||||
var expected = i + 1;
|
||||
await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= expected);
|
||||
}
|
||||
await WaitForAsync(() => eng.RunGuard.StartedRunCount >= 1 + 1 + pushes);
|
||||
|
||||
eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
|
||||
eng.RunGuard.IsQuarantined("runaway", out _).ShouldBeFalse("refusals are not strikes");
|
||||
eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
|
||||
eng.RunGuard.InFlightCount.ShouldBe(1);
|
||||
|
||||
// The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving.
|
||||
// The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving —
|
||||
// exactly once across the whole run of non-completions.
|
||||
ScriptedAlarmEvent[] snapshot;
|
||||
lock (events) snapshot = events.ToArray();
|
||||
var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray();
|
||||
quality.ShouldNotBeEmpty();
|
||||
quality.Length.ShouldBe(1);
|
||||
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
|
||||
eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented");
|
||||
|
||||
@@ -72,6 +79,44 @@ public sealed class ScriptedAlarmEngineRunGuardTests
|
||||
await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active);
|
||||
}
|
||||
|
||||
/// <summary>I3 — a QUARANTINED predicate is re-attempted on every push and refused before dispatch; the
|
||||
/// Bad annotation must still be emitted once, not once per push.</summary>
|
||||
[Fact]
|
||||
public async Task A_quarantined_predicate_emits_QualityChanged_once_across_many_pushes()
|
||||
{
|
||||
var up = new FakeUpstream();
|
||||
up.Set("Temp", 50);
|
||||
using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
var events = new List<ScriptedAlarmEvent>();
|
||||
eng.OnEvent += (_, e) => { lock (events) events.Add(e); };
|
||||
|
||||
await eng.LoadAsync(
|
||||
[Alarm("q", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"),
|
||||
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 1000;""")],
|
||||
TestContext.Current.CancellationToken);
|
||||
eng.RunGuard.IsQuarantined("q", out _).ShouldBeTrue();
|
||||
|
||||
// Each push re-evaluates BOTH alarms (same input). The healthy sibling's dispatch count is the
|
||||
// positive control proving every push was actually processed — so the quarantined alarm was
|
||||
// re-attempted N times, not coalesced into one.
|
||||
const int pushes = 5;
|
||||
for (var i = 0; i < pushes; i++)
|
||||
{
|
||||
var before = eng.RunGuard.StartedRunCount;
|
||||
up.Push("Temp", 60 + i);
|
||||
await WaitForAsync(() => eng.RunGuard.StartedRunCount > before);
|
||||
}
|
||||
eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one runaway dispatch, then healthy-only dispatches");
|
||||
await Task.Delay(300); // absence window for trailing emissions
|
||||
|
||||
ScriptedAlarmEvent[] snapshot;
|
||||
lock (events) snapshot = events.ToArray();
|
||||
var quality = snapshot.Where(e => e.AlarmId == "q" && e.Emission == EmissionKind.QualityChanged).ToArray();
|
||||
quality.Length.ShouldBe(1, "one Bad annotation for the quarantine, not one per push");
|
||||
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
|
||||
snapshot.ShouldNotContain(e => e.AlarmId == "healthy" && e.Emission == EmissionKind.QualityChanged);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reload_lifts_the_quarantine()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user