feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)

A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 06:39:43 -04:00
parent ad2b5c4044
commit eb9269c337
17 changed files with 1313 additions and 74 deletions
@@ -0,0 +1,105 @@
+using Serilog;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
+
+/// <summary>
+///     <see cref="ScriptedAlarmEngine"/> runs every predicate through a <see cref="ScriptRunGuard"/> keyed by
+///     alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation
+///     spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is
+///     quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a
+///     sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine.
+/// </summary>
+[Trait("Category", "Unit")]
+public sealed class ScriptedAlarmEngineRunGuardTests
+{
+    private const uint StatusBadInternalError = 0x80020000u;
+
+    private const string SpinPredicate =
+        "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;";
+
+    private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
+        new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
+            Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
+
+    private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null)
+    {
+        var logger = new LoggerConfiguration().CreateLogger();
+        // 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and
+        // must finish inside the budget on a loaded CI box, or the positive control itself times out. The
+        // spin predicate outlives this by 12x, so the runaway alarm still cannot finish first.
+        return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
+            scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting);
+    }
+
+    [Fact]
+    public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating()
+    {
+        var up = new FakeUpstream();
+        up.Set("Temp", 50);
+        using var eng = Build(up);
+        var events = new List<ScriptedAlarmEvent>();
+        eng.OnEvent += (_, e) => { lock (events) events.Add(e); };
+
+        // The load itself evaluates once (startup recovery) — that is the first, abandoned run.
+        await eng.LoadAsync(
+            [Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"),
+             Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")],
+            TestContext.Current.CancellationToken);
+
+        // Two more pushes: refused (orphan still running), then quarantined.
+        up.Push("Temp", 60);
+        await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1);
+        up.Push("Temp", 70);
+        await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _));
+
+        eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
+        eng.RunGuard.InFlightCount.ShouldBe(1);
+
+        // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving.
+        ScriptedAlarmEvent[] snapshot;
+        lock (events) snapshot = events.ToArray();
+        var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray();
+        quality.ShouldNotBeEmpty();
+        quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
+        eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented");
+
+        // Positive control: the healthy sibling still transitions on the same engine.
+        up.Push("Temp", 150);
+        await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active);
+    }
+
+    [Fact]
+    public async Task A_reload_lifts_the_quarantine()
+    {
+        var up = new FakeUpstream();
+        up.Set("Temp", 50);
+        using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
+
+        await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")],
+            TestContext.Current.CancellationToken);
+        eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue();
+
+        // The operator fixed the predicate and redeployed.
+        await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")],
+            TestContext.Current.CancellationToken);
+        eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse();
+
+        up.Push("Temp", 150);
+        await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active);
+    }
+
+    private static async Task WaitForAsync(Func<bool> cond, int timeoutMs = 10_000)
+    {
+        var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
+        while (DateTime.UtcNow < deadline)
+        {
+            if (cond()) return;
+            await Task.Delay(25);
+        }
+        throw new TimeoutException("Condition did not become true in time");
+    }
+}