03686eb5d7
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
208 lines
11 KiB
C#
208 lines
11 KiB
C#
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||
|
||
/// <summary>
|
||
/// Node-wide script runtime-safety knobs (the <c>Scripting</c> appsettings section), consumed by
|
||
/// <see cref="ScriptRunGuard"/>. One instance is shared by every script seam on the node — the
|
||
/// virtual-tag evaluator, the Calculation evaluator and the scripted-alarm engine — so a single setting
|
||
/// bounds every authored script the same way.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// The in-process sandbox cannot pre-empt a script: a <c>while(true)</c> is abandoned on its pool
|
||
/// thread at the timeout, and an allocation is observed only after the run returns. These knobs bound
|
||
/// the <em>damage</em> — one orphan per script, no further runs until the source changes — not the
|
||
/// first occurrence. Pre-emptive CPU/memory budgeting needs an out-of-process script runner.
|
||
/// </para>
|
||
/// </remarks>
|
||
public sealed class ScriptingOptions
|
||
{
|
||
/// <summary>The appsettings section this binds from.</summary>
|
||
public const string SectionName = "Scripting";
|
||
|
||
/// <summary>Default <see cref="RunTimeout"/>: 2 s — the budget virtual tags and Calculation tags have always run with.</summary>
|
||
public static readonly TimeSpan DefaultRunTimeout = TimeSpan.FromSeconds(2);
|
||
|
||
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="RunTimeout"/>: 60 s.</summary>
|
||
public static readonly TimeSpan MaxRunTimeout = TimeSpan.FromSeconds(60);
|
||
|
||
/// <summary>Default <see cref="MaxAllocBytesPerRun"/>: 64 MB.</summary>
|
||
public const long DefaultMaxAllocBytesPerRun = 64L * 1024 * 1024;
|
||
|
||
/// <summary>Default <see cref="QuarantineAfterConsecutiveTimeouts"/>: 3.</summary>
|
||
public const int DefaultQuarantineAfterConsecutiveTimeouts = 3;
|
||
|
||
/// <summary>Default <see cref="StuckOrphanQuarantineFactor"/>: 10.</summary>
|
||
public const double DefaultStuckOrphanQuarantineFactor = 10;
|
||
|
||
/// <summary>
|
||
/// Wall-clock budget for one script run on the node-wide seams: virtual tags and scripted-alarm
|
||
/// predicates. Default 2 s. A Calculation driver keeps its own per-driver <c>runTimeoutMs</c> (default
|
||
/// 2 s as well). Before lmxopcua#779 the scripted-alarm engine was built without a budget and silently
|
||
/// took the 250 ms Phase-7 default of <see cref="TimedScriptEvaluator{TContext, TResult}"/>, so an alarm
|
||
/// predicate got an eighth of the budget a virtual tag reading the same inputs got — not a decision
|
||
/// anywhere in the history, just a missing argument. Also scales the stuck-orphan rule
|
||
/// (<see cref="StuckOrphanQuarantineFactor"/> × this). Zero or negative becomes the default; above
|
||
/// <see cref="MaxRunTimeout"/> becomes that maximum.
|
||
/// </summary>
|
||
public TimeSpan RunTimeout { get; set; } = DefaultRunTimeout;
|
||
|
||
/// <summary>
|
||
/// Managed bytes one script run may allocate on its own thread (sampled with
|
||
/// <see cref="GC.GetAllocatedBytesForCurrentThread"/> around the run) before the script is
|
||
/// quarantined. Zero or negative disables the budget. Default 64 MB.
|
||
/// </summary>
|
||
public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun;
|
||
|
||
/// <summary>
|
||
/// Consecutive timeouts after which the script is quarantined: no further runs are dispatched, every
|
||
/// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit
|
||
/// reset. Only a timeout is a strike — a run refused because the previous one is still executing is
|
||
/// reported Bad and counted, but neither adds to nor resets the streak; a completed run (value or
|
||
/// throw) resets it. Default 3; values below 1 are treated as 1.
|
||
/// </summary>
|
||
public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts;
|
||
|
||
/// <summary>
|
||
/// A refusal quarantines the script immediately when the abandoned run it was refused behind has been
|
||
/// executing for longer than this many times the script's timeout (20 s at the 2 s default). This is
|
||
/// what quarantines a permanently stuck script (<c>while(true)</c>): it times out once and is then
|
||
/// only ever refused, so the timeout streak alone would never reach
|
||
/// <see cref="QuarantineAfterConsecutiveTimeouts"/>. A young orphan (a short overrun) is still just a
|
||
/// refusal. Zero or negative disables the rule. Default 10.
|
||
/// </summary>
|
||
public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor;
|
||
|
||
/// <summary>
|
||
/// Where compiled scripts run (<c>Scripting:Runner</c>, lmxopcua#780). Default
|
||
/// <see cref="ScriptRunnerMode.InProcess"/> — today's path, unchanged, and the rollback for the other modes.
|
||
/// </summary>
|
||
public ScriptRunnerMode Runner { get; set; } = ScriptRunnerMode.InProcess;
|
||
|
||
/// <summary>
|
||
/// The seams <see cref="Runner"/> applies to when it is not <see cref="ScriptRunnerMode.InProcess"/>
|
||
/// (<c>Scripting:Runner:Seams</c>, e.g. <c>["Calculation"]</c>). Null or empty means VirtualTag and Calculation:
|
||
/// scripted alarms run out of process only when listed explicitly (design §14 #10, alarms last). Bound by
|
||
/// <c>ScriptingOptionsBinding</c> from the documented key, because <c>Scripting:Runner</c> is itself a value.
|
||
/// </summary>
|
||
public IReadOnlyList<ScriptSeam>? RunnerSeams { get; set; }
|
||
|
||
/// <summary>The out-of-process worker pool's knobs (<c>Scripting:Worker</c>).</summary>
|
||
public ScriptWorkerOptions Worker { get; set; } = new();
|
||
|
||
/// <summary>The shadow comparison's knobs (<c>Scripting:Shadow</c>), for <see cref="ScriptRunnerMode.Shadow"/>.</summary>
|
||
public ScriptShadowOptions Shadow { get; set; } = new();
|
||
|
||
/// <summary>Whether <paramref name="seam"/> is selected for out-of-process execution by these options.</summary>
|
||
/// <param name="seam">The seam.</param>
|
||
/// <returns><c>true</c> when <see cref="Runner"/> is <see cref="ScriptRunnerMode.OutOfProcess"/> and the seam is in
|
||
/// <see cref="RunnerSeams"/> (or that list is empty and the seam is not <see cref="ScriptSeam.ScriptedAlarm"/>).</returns>
|
||
public bool IsOutOfProcess(ScriptSeam seam)
|
||
=> Runner == ScriptRunnerMode.OutOfProcess
|
||
&& (RunnerSeams is { Count: > 0 } seams ? seams.Contains(seam) : seam != ScriptSeam.ScriptedAlarm);
|
||
|
||
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="StuckOrphanQuarantineFactor"/>.</summary>
|
||
public const double MaxStuckOrphanQuarantineFactor = 1000;
|
||
|
||
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="QuarantineAfterConsecutiveTimeouts"/>.</summary>
|
||
public const int MaxQuarantineAfterConsecutiveTimeouts = 1000;
|
||
|
||
/// <summary>
|
||
/// Lower bound <see cref="Sanitized"/> applies to a positive <see cref="MaxAllocBytesPerRun"/>: 1 MB. A
|
||
/// smaller budget would quarantine practically every script on its first run.
|
||
/// </summary>
|
||
public const long MinMaxAllocBytesPerRun = 1024L * 1024;
|
||
|
||
/// <summary>
|
||
/// Returns a copy with every numeric knob pulled into a sane range. The bind site calls it and logs
|
||
/// <paramref name="adjustments"/>. <see cref="ScriptRunGuard"/> calls it again, so options that reach the
|
||
/// guard by another route (a Calculation driver's own config, a test) are clamped too.
|
||
/// <list type="bullet">
|
||
/// <item><see cref="StuckOrphanQuarantineFactor"/>: NaN becomes the default, a negative value becomes 0
|
||
/// (disabled), and a value above <see cref="MaxStuckOrphanQuarantineFactor"/> becomes that maximum.
|
||
/// Unclamped, an absurd factor made <c>timeout * factor</c> throw <see cref="OverflowException"/> on
|
||
/// the first refused run.</item>
|
||
/// <item><see cref="QuarantineAfterConsecutiveTimeouts"/>: into [1, <see cref="MaxQuarantineAfterConsecutiveTimeouts"/>].</item>
|
||
/// <item><see cref="MaxAllocBytesPerRun"/>: a negative value becomes 0 (disabled), and a positive value
|
||
/// below <see cref="MinMaxAllocBytesPerRun"/> becomes that minimum.</item>
|
||
/// <item><see cref="RunTimeout"/>: zero or negative becomes <see cref="DefaultRunTimeout"/>, above
|
||
/// <see cref="MaxRunTimeout"/> becomes that maximum.</item>
|
||
/// </list>
|
||
/// </summary>
|
||
/// <param name="adjustments">One line per value that was out of range, naming its key. A negative value that
|
||
/// already meant "disabled" is normalised without a line.</param>
|
||
/// <returns>A new, clamped instance. This instance is not modified.</returns>
|
||
public ScriptingOptions Sanitized(out IReadOnlyList<string> adjustments)
|
||
{
|
||
var notes = new List<string>();
|
||
|
||
var factor = StuckOrphanQuarantineFactor;
|
||
if (double.IsNaN(factor))
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(StuckOrphanQuarantineFactor)} is NaN; using the default {DefaultStuckOrphanQuarantineFactor}.");
|
||
factor = DefaultStuckOrphanQuarantineFactor;
|
||
}
|
||
else if (factor < 0)
|
||
{
|
||
factor = 0;
|
||
}
|
||
else if (factor > MaxStuckOrphanQuarantineFactor)
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(StuckOrphanQuarantineFactor)} {factor} is above the maximum; clamped to {MaxStuckOrphanQuarantineFactor}.");
|
||
factor = MaxStuckOrphanQuarantineFactor;
|
||
}
|
||
|
||
var threshold = QuarantineAfterConsecutiveTimeouts;
|
||
if (threshold < 1)
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(QuarantineAfterConsecutiveTimeouts)} {threshold} is below 1; clamped to 1.");
|
||
threshold = 1;
|
||
}
|
||
else if (threshold > MaxQuarantineAfterConsecutiveTimeouts)
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(QuarantineAfterConsecutiveTimeouts)} {threshold} is above the maximum; clamped to {MaxQuarantineAfterConsecutiveTimeouts}.");
|
||
threshold = MaxQuarantineAfterConsecutiveTimeouts;
|
||
}
|
||
|
||
var alloc = MaxAllocBytesPerRun;
|
||
if (alloc < 0)
|
||
{
|
||
alloc = 0;
|
||
}
|
||
else if (alloc > 0 && alloc < MinMaxAllocBytesPerRun)
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(MaxAllocBytesPerRun)} {alloc} is below the minimum; clamped to {MinMaxAllocBytesPerRun}.");
|
||
alloc = MinMaxAllocBytesPerRun;
|
||
}
|
||
|
||
var runTimeout = RunTimeout;
|
||
if (runTimeout <= TimeSpan.Zero)
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(RunTimeout)} {runTimeout} is not positive; using the default {DefaultRunTimeout}.");
|
||
runTimeout = DefaultRunTimeout;
|
||
}
|
||
else if (runTimeout > MaxRunTimeout)
|
||
{
|
||
notes.Add($"{SectionName}:{nameof(RunTimeout)} {runTimeout} is above the maximum; clamped to {MaxRunTimeout}.");
|
||
runTimeout = MaxRunTimeout;
|
||
}
|
||
|
||
var worker = (Worker ?? new ScriptWorkerOptions()).Sanitized(out var workerNotes);
|
||
notes.AddRange(workerNotes);
|
||
var shadow = (Shadow ?? new ScriptShadowOptions()).Sanitized(out var shadowNotes);
|
||
notes.AddRange(shadowNotes);
|
||
|
||
adjustments = notes;
|
||
return new ScriptingOptions
|
||
{
|
||
RunTimeout = runTimeout,
|
||
MaxAllocBytesPerRun = alloc,
|
||
QuarantineAfterConsecutiveTimeouts = threshold,
|
||
StuckOrphanQuarantineFactor = factor,
|
||
Runner = Runner,
|
||
RunnerSeams = RunnerSeams is null ? null : [.. RunnerSeams],
|
||
Worker = worker,
|
||
Shadow = shadow,
|
||
};
|
||
}
|
||
}
|