feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)
A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
@@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines;
|
||||
/// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no
|
||||
/// longer open this window at all.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Run quarantine.</b> Every compiled run goes through one <see cref="ScriptRunGuard"/> keyed by
|
||||
/// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous
|
||||
/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive
|
||||
/// non-completions or an allocation-budget breach. Both surface as <see cref="VirtualTagEvalResult.Failure"/>
|
||||
/// so <c>VirtualTagActor</c> publishes Bad exactly as it does for a timeout. A quarantine lifts when the
|
||||
/// expression changes, on <see cref="ClearCompiledScripts"/> / <see cref="ResetQuarantines"/> (the deploy
|
||||
/// boundary), never by itself.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable
|
||||
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable
|
||||
{
|
||||
// Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and
|
||||
// Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary
|
||||
@@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
private readonly ILogger<RoslynVirtualTagEvaluator> _logger;
|
||||
private readonly SerilogLogger _scriptRoot;
|
||||
private readonly TimeSpan _runTimeout;
|
||||
private readonly ScriptRunGuard _guard;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
|
||||
/// <param name="logger">Logger for recording compilation and execution errors (host diagnostics).</param>
|
||||
/// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
|
||||
/// <param name="runTimeout">Maximum execution time for each script; defaults to 2 seconds if not specified.</param>
|
||||
/// <param name="scripting">Run-guard budget + quarantine threshold (the <c>Scripting</c> section); null ⇒ defaults.</param>
|
||||
public RoslynVirtualTagEvaluator(
|
||||
ILogger<RoslynVirtualTagEvaluator> logger,
|
||||
ScriptRootLogger scriptRoot,
|
||||
TimeSpan? runTimeout = null)
|
||||
TimeSpan? runTimeout = null,
|
||||
ScriptingOptions? scripting = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
|
||||
_runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
|
||||
_guard = new ScriptRunGuard(scripting, _scriptRoot);
|
||||
}
|
||||
|
||||
/// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
|
||||
public ScriptRunGuard RunGuard => _guard;
|
||||
|
||||
// Value-only overload. Every dependency reaches the script stamped Good with an
|
||||
// evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
|
||||
// overload instead; this one remains for callers that genuinely hold only
|
||||
@@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
|
||||
try
|
||||
{
|
||||
// Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
|
||||
// interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous
|
||||
// CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when
|
||||
// the wall-clock budget fires regardless of whether the inner task completes (the orphaned
|
||||
// thread is the documented, accepted trade-off — TimedScriptEvaluator remarks).
|
||||
var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
|
||||
var raw = timed.RunAsync(context).GetAwaiter().GetResult();
|
||||
// Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw
|
||||
// CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll
|
||||
// it), so the wall-clock budget is what returns control — and the guard is what stops the
|
||||
// abandoned run from being joined by another on the next trigger (one orphan per tag, then
|
||||
// quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed.
|
||||
var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout)
|
||||
.GetAwaiter().GetResult();
|
||||
return VirtualTagEvalResult.Ok(raw);
|
||||
}
|
||||
catch (ScriptTimeoutException)
|
||||
@@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
return VirtualTagEvalResult.Failure(
|
||||
$"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s");
|
||||
}
|
||||
catch (ScriptRunRefusedException ex)
|
||||
{
|
||||
return VirtualTagEvalResult.Failure(ex.Message);
|
||||
}
|
||||
catch (ScriptQuarantinedException ex)
|
||||
{
|
||||
return VirtualTagEvalResult.Failure(ex.Message);
|
||||
}
|
||||
catch (ObjectDisposedException) when (!_disposed && attempt == 0)
|
||||
{
|
||||
// S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our
|
||||
@@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ClearCompiledScripts() => _cache.Clear();
|
||||
public void ClearCompiledScripts()
|
||||
{
|
||||
_cache.Clear();
|
||||
// The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine
|
||||
// (an edited expression already lifts its own through the guard's source identity).
|
||||
_guard.ResetAll();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ResetQuarantines() => _guard.ResetAll();
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;
|
||||
|
||||
/// <summary>Disposes the evaluator and cleans up all cached script evaluators.</summary>
|
||||
public void Dispose()
|
||||
|
||||
@@ -3,6 +3,7 @@ using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
@@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor
|
||||
(_evaluator as IScriptCacheOwner)?.ClearCompiledScripts();
|
||||
}
|
||||
|
||||
// Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the
|
||||
// operator redeployed to retry" is one of the two documented ways out of a quarantine (the other,
|
||||
// an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map).
|
||||
(_evaluator as IScriptQuarantineOwner)?.ResetQuarantines();
|
||||
|
||||
var desired = new HashSet<string>(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal);
|
||||
|
||||
// Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its
|
||||
|
||||
Reference in New Issue
Block a user