feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)

A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 06:39:43 -04:00
parent ad2b5c4044
commit eb9269c337
17 changed files with 1313 additions and 74 deletions
@@ -0,0 +1,86 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Serilog;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
+
+/// <summary>
+///     <see cref="CalculationEvaluator"/> runs every compiled script through a <see cref="ScriptRunGuard"/>
+///     keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every
+///     later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now
+///     the next trigger is refused ("previous evaluation still running"), the third consecutive
+///     non-completion quarantines the tag, and a healthy sibling on the same evaluator keeps computing.
+/// </summary>
+public sealed class CalculationEvaluatorRunGuardTests
+{
+    private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
+
+    /// <summary>Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself.</summary>
+    private const string SpinScript =
+        "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
+
+    private static readonly Dictionary<string, object?> NoDeps = new();
+
+    [Fact]
+    public async Task A_trigger_during_the_previous_orphan_is_refused_then_the_tag_is_quarantined_and_a_sibling_keeps_computing()
+    {
+        using var sut = new CalculationEvaluator(
+            NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
+        var ct = TestContext.Current.CancellationToken;
+
+        var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
+        first.Success.ShouldBeFalse();
+        first.Reason.ShouldNotBeNull();
+        first.Reason.ShouldContain("timed out");
+
+        var second = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct);
+        second.Success.ShouldBeFalse();
+        second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
+
+        var third = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct);
+        third.Success.ShouldBeFalse();
+        third.Reason.ShouldNotBeNull();
+        third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
+        sut.QuarantinedScripts.ShouldBe(new[] { "calc/loop" });
+
+        // Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger.
+        sut.RunGuard.StartedRunCount.ShouldBe(1);
+        sut.RunGuard.InFlightCount.ShouldBe(1);
+
+        // Positive control: a sibling tag on the same evaluator still computes.
+        var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct);
+        sibling.Success.ShouldBeTrue(sibling.Reason);
+        sibling.Value.ShouldBe(42);
+    }
+
+    [Fact]
+    public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does()
+    {
+        using var sut = new CalculationEvaluator(
+            NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
+            new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
+        var ct = TestContext.Current.CancellationToken;
+
+        var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
+        r.Reason.ShouldNotBeNull();
+        r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
+
+        sut.ClearCompiledScripts();
+
+        sut.QuarantinedScripts.ShouldBeEmpty();
+        // The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag.
+        var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct);
+        fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
+        fixedRun.Value.ShouldBe(7);
+    }
+
+    private static Dictionary<string, Core.Abstractions.DataValueSnapshot> Snap(Dictionary<string, object?> deps)
+    {
+        var now = DateTime.UtcNow;
+        var d = new Dictionary<string, Core.Abstractions.DataValueSnapshot>(StringComparer.Ordinal);
+        foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now);
+        return d;
+    }
+}