eb9269c337
A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
125 lines
6.6 KiB
C#
125 lines
6.6 KiB
C#
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
|
|
/// <summary>
|
|
/// Wraps a <see cref="ScriptEvaluator{TContext, TResult}"/> with a per-evaluation
|
|
/// wall-clock timeout. Default is 250ms per Phase 7 plan Stream A.4; configurable
|
|
/// per tag so deployments with slower backends can widen it.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// Implemented with <see cref="Task.WaitAsync(TimeSpan, CancellationToken)"/>
|
|
/// rather than a cancellation-token-only approach because Roslyn-compiled
|
|
/// scripts don't internally poll the cancellation token unless the user code
|
|
/// does async work. A CPU-bound infinite loop in a script won't honor a
|
|
/// cooperative cancel — <c>WaitAsync</c> returns control when the timeout fires
|
|
/// regardless of whether the inner task completes.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>Known limitation:</b> when a script times out, the underlying ScriptRunner
|
|
/// task continues running on a thread-pool thread until the Roslyn runtime
|
|
/// returns. In the CPU-bound-infinite-loop case that's effectively "leaked" —
|
|
/// the thread is tied up until the runtime decides to return, which it may
|
|
/// never do. This wrapper has no memory of that orphan, so re-running the same
|
|
/// script through it on every trigger leaks one more thread per trigger — which
|
|
/// is why the production seams (virtual tags, Calculation, scripted alarms) run
|
|
/// through <see cref="ScriptRunGuard"/> instead: same timeout, plus a per-script
|
|
/// "previous run still executing" refusal and a quarantine. This class remains
|
|
/// for callers that evaluate once (tests, the authoring preview).
|
|
/// </para>
|
|
/// <para>
|
|
/// Caller-supplied <see cref="CancellationToken"/> is honored — if the caller
|
|
/// cancels before the timeout fires, the caller's cancel wins and the
|
|
/// <see cref="OperationCanceledException"/> propagates (not wrapped as
|
|
/// <see cref="ScriptTimeoutException"/>). That distinction matters: the
|
|
/// virtual-tag engine's shutdown path cancels scripts on dispose; it shouldn't
|
|
/// see those as timeouts.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class TimedScriptEvaluator<TContext, TResult>
|
|
where TContext : ScriptContext
|
|
{
|
|
/// <summary>Default timeout per Phase 7 plan Stream A.4 — 250ms.</summary>
|
|
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMilliseconds(250);
|
|
|
|
private readonly ScriptEvaluator<TContext, TResult> _inner;
|
|
|
|
/// <summary>Wall-clock budget per evaluation. Script exceeding this throws <see cref="ScriptTimeoutException"/>.</summary>
|
|
public TimeSpan Timeout { get; }
|
|
|
|
/// <summary>Initializes a new instance of the TimedScriptEvaluator class with the default timeout.</summary>
|
|
/// <param name="inner">The inner script evaluator.</param>
|
|
public TimedScriptEvaluator(ScriptEvaluator<TContext, TResult> inner)
|
|
: this(inner, DefaultTimeout)
|
|
{
|
|
}
|
|
|
|
/// <summary>Initializes a new instance of the TimedScriptEvaluator class with a custom timeout.</summary>
|
|
/// <param name="inner">The inner script evaluator.</param>
|
|
/// <param name="timeout">The evaluation timeout duration.</param>
|
|
public TimedScriptEvaluator(ScriptEvaluator<TContext, TResult> inner, TimeSpan timeout)
|
|
{
|
|
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
|
|
if (timeout <= TimeSpan.Zero)
|
|
throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive.");
|
|
Timeout = timeout;
|
|
}
|
|
|
|
/// <summary>Runs the script evaluation with the configured timeout.</summary>
|
|
/// <param name="context">The script context.</param>
|
|
/// <param name="ct">The cancellation token.</param>
|
|
/// <returns>The evaluation result.</returns>
|
|
public async Task<TResult> RunAsync(TContext context, CancellationToken ct = default)
|
|
{
|
|
if (context is null) throw new ArgumentNullException(nameof(context));
|
|
|
|
// Push evaluation to a thread-pool thread so a CPU-bound script (e.g. a tight
|
|
// loop with no async work) doesn't hog the caller's thread before WaitAsync
|
|
// gets to register its timeout. Without this, Roslyn's ScriptRunner executes
|
|
// synchronously on the calling thread and returns an already-completed Task,
|
|
// so WaitAsync sees a completed task and never fires the timeout.
|
|
var runTask = Task.Run(() => _inner.RunAsync(context, ct), ct);
|
|
try
|
|
{
|
|
return await runTask.WaitAsync(Timeout, ct).ConfigureAwait(false);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
// WaitAsync's synthesized timeout — the inner task may still be running
|
|
// on its thread-pool thread (known leak documented in the class summary).
|
|
// Wrap so callers can distinguish from user-written timeout logic.
|
|
//
|
|
// The class docs guarantee "caller-supplied cancel wins over timeout".
|
|
// When both fire at nearly the same time, WaitAsync observes them in
|
|
// non-deterministic order, so a cancel that arrives a few µs after the
|
|
// timeout still reaches here as TimeoutException. Re-check the token so
|
|
// the guarantee holds regardless of race ordering.
|
|
if (ct.IsCancellationRequested)
|
|
throw new OperationCanceledException(ct);
|
|
throw new ScriptTimeoutException(Timeout);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Thrown when a script evaluation exceeds its configured timeout. The virtual-tag
|
|
/// engine (Stream B) catches this + maps the owning tag's quality to
|
|
/// <c>BadInternalError</c> per the Phase 7 plan, logging a structured
|
|
/// warning with the offending script name so operators can locate + fix it.
|
|
/// </summary>
|
|
public sealed class ScriptTimeoutException : Exception
|
|
{
|
|
/// <summary>Gets the timeout duration that was exceeded.</summary>
|
|
public TimeSpan Timeout { get; }
|
|
|
|
/// <summary>Initializes a new instance of the ScriptTimeoutException class.</summary>
|
|
/// <param name="timeout">The timeout duration that was exceeded.</param>
|
|
public ScriptTimeoutException(TimeSpan timeout)
|
|
: base($"Script evaluation exceeded the configured timeout of {timeout.TotalMilliseconds:F1} ms. " +
|
|
"The script was either CPU-bound or blocked on a slow operation; check ctx.Logger output " +
|
|
"around the timeout and consider widening the timeout per tag, simplifying the script, or " +
|
|
"moving heavy work out of the evaluation path.")
|
|
{
|
|
Timeout = timeout;
|
|
}
|
|
}
|