feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)
A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
+66
@@ -0,0 +1,66 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Engines;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="RoslynVirtualTagEvaluator"/> runs every compiled expression through a
|
||||
/// <see cref="ScriptRunGuard"/> keyed by virtual-tag id: a runaway expression leaks one orphaned pool
|
||||
/// thread, not one per trigger, the tag is quarantined after three consecutive non-completions (surfacing
|
||||
/// as a <c>Failure</c> the actor publishes Bad for), and the quarantine lifts at the deploy boundary.
|
||||
/// </summary>
|
||||
public sealed class RoslynVirtualTagEvaluatorRunGuardTests
|
||||
{
|
||||
private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
|
||||
|
||||
private const string SpinScript =
|
||||
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
|
||||
|
||||
private static readonly Dictionary<string, object?> NoDeps = new();
|
||||
|
||||
[Fact]
|
||||
public void Runaway_expression_is_refused_then_quarantined_and_a_sibling_keeps_evaluating()
|
||||
{
|
||||
using var sut = new RoslynVirtualTagEvaluator(
|
||||
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
|
||||
|
||||
var first = sut.Evaluate("vt-loop", SpinScript, NoDeps);
|
||||
first.Success.ShouldBeFalse();
|
||||
first.Reason.ShouldNotBeNull();
|
||||
first.Reason.ShouldContain("timed out");
|
||||
|
||||
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
|
||||
var third = sut.Evaluate("vt-loop", SpinScript, NoDeps);
|
||||
third.Reason.ShouldNotBeNull();
|
||||
third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
sut.QuarantinedScripts.ShouldBe(new[] { "vt-loop" });
|
||||
sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger");
|
||||
|
||||
var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps);
|
||||
sibling.Success.ShouldBeTrue(sibling.Reason);
|
||||
sibling.Value.ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResetQuarantines_lifts_the_quarantine_as_a_redeploy_does()
|
||||
{
|
||||
using var sut = new RoslynVirtualTagEvaluator(
|
||||
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
|
||||
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
||||
|
||||
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
sut.QuarantinedScripts.ShouldNotBeEmpty();
|
||||
|
||||
sut.ResetQuarantines();
|
||||
|
||||
sut.QuarantinedScripts.ShouldBeEmpty();
|
||||
var fixedRun = sut.Evaluate("vt-loop", "return 7;", NoDeps);
|
||||
fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
|
||||
fixedRun.Value.ShouldBe(7);
|
||||
}
|
||||
}
|
||||
+8
-2
@@ -489,12 +489,18 @@ public sealed class RoslynVirtualTagEvaluatorTests
|
||||
// most evaluations then hit the warm cache instead of recompiling. Neither the cadence nor the
|
||||
// iteration count is a correctness knob any more — the assertions below hold for any number of
|
||||
// clears landing inside a single evaluation, and for any number of evaluations completed.
|
||||
var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() =>
|
||||
// One virtual-tag id PER LOOP, sharing the expression. The race under test lives in the compile
|
||||
// cache, which is keyed by source, so twelve ids on one expression contend for exactly the entry
|
||||
// twelve copies of one id did — while twelve concurrent evaluations of ONE id is a shape production
|
||||
// never produces (one VirtualTagActor per id serialises them) and exactly the shape the per-tag
|
||||
// ScriptRunGuard refuses ("previous evaluation still running", then quarantine).
|
||||
var evalLoops = Enumerable.Range(0, loopCount).Select(loop => Task.Run(() =>
|
||||
{
|
||||
var vtagId = $"vt-race-{loop}";
|
||||
var done = 0;
|
||||
for (var i = 0; i < MaxIterationsPerLoop && Environment.TickCount64 < stopAtMs; i++)
|
||||
{
|
||||
var r = sut.Evaluate("vt-race", expr, new Dictionary<string, object?> { ["a"] = i });
|
||||
var r = sut.Evaluate(vtagId, expr, new Dictionary<string, object?> { ["a"] = i });
|
||||
done++;
|
||||
if (!r.Success)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user