Files
lmxopcua/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs
T
Joseph Doherty 03686eb5d7 feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-10 07:23:06 -05:00

208 lines
11 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting;
/// <summary>
/// Node-wide script runtime-safety knobs (the <c>Scripting</c> appsettings section), consumed by
/// <see cref="ScriptRunGuard"/>. One instance is shared by every script seam on the node — the
/// virtual-tag evaluator, the Calculation evaluator and the scripted-alarm engine — so a single setting
/// bounds every authored script the same way.
/// </summary>
/// <remarks>
/// <para>
/// The in-process sandbox cannot pre-empt a script: a <c>while(true)</c> is abandoned on its pool
/// thread at the timeout, and an allocation is observed only after the run returns. These knobs bound
/// the <em>damage</em> — one orphan per script, no further runs until the source changes — not the
/// first occurrence. Pre-emptive CPU/memory budgeting needs an out-of-process script runner.
/// </para>
/// </remarks>
public sealed class ScriptingOptions
{
/// <summary>The appsettings section this binds from.</summary>
public const string SectionName = "Scripting";
/// <summary>Default <see cref="RunTimeout"/>: 2 s — the budget virtual tags and Calculation tags have always run with.</summary>
public static readonly TimeSpan DefaultRunTimeout = TimeSpan.FromSeconds(2);
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="RunTimeout"/>: 60 s.</summary>
public static readonly TimeSpan MaxRunTimeout = TimeSpan.FromSeconds(60);
/// <summary>Default <see cref="MaxAllocBytesPerRun"/>: 64 MB.</summary>
public const long DefaultMaxAllocBytesPerRun = 64L * 1024 * 1024;
/// <summary>Default <see cref="QuarantineAfterConsecutiveTimeouts"/>: 3.</summary>
public const int DefaultQuarantineAfterConsecutiveTimeouts = 3;
/// <summary>Default <see cref="StuckOrphanQuarantineFactor"/>: 10.</summary>
public const double DefaultStuckOrphanQuarantineFactor = 10;
/// <summary>
/// Wall-clock budget for one script run on the node-wide seams: virtual tags and scripted-alarm
/// predicates. Default 2 s. A Calculation driver keeps its own per-driver <c>runTimeoutMs</c> (default
/// 2 s as well). Before lmxopcua#779 the scripted-alarm engine was built without a budget and silently
/// took the 250 ms Phase-7 default of <see cref="TimedScriptEvaluator{TContext, TResult}"/>, so an alarm
/// predicate got an eighth of the budget a virtual tag reading the same inputs got — not a decision
/// anywhere in the history, just a missing argument. Also scales the stuck-orphan rule
/// (<see cref="StuckOrphanQuarantineFactor"/> × this). Zero or negative becomes the default; above
/// <see cref="MaxRunTimeout"/> becomes that maximum.
/// </summary>
public TimeSpan RunTimeout { get; set; } = DefaultRunTimeout;
/// <summary>
/// Managed bytes one script run may allocate on its own thread (sampled with
/// <see cref="GC.GetAllocatedBytesForCurrentThread"/> around the run) before the script is
/// quarantined. Zero or negative disables the budget. Default 64 MB.
/// </summary>
public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun;
/// <summary>
/// Consecutive timeouts after which the script is quarantined: no further runs are dispatched, every
/// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit
/// reset. Only a timeout is a strike — a run refused because the previous one is still executing is
/// reported Bad and counted, but neither adds to nor resets the streak; a completed run (value or
/// throw) resets it. Default 3; values below 1 are treated as 1.
/// </summary>
public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts;
/// <summary>
/// A refusal quarantines the script immediately when the abandoned run it was refused behind has been
/// executing for longer than this many times the script's timeout (20 s at the 2 s default). This is
/// what quarantines a permanently stuck script (<c>while(true)</c>): it times out once and is then
/// only ever refused, so the timeout streak alone would never reach
/// <see cref="QuarantineAfterConsecutiveTimeouts"/>. A young orphan (a short overrun) is still just a
/// refusal. Zero or negative disables the rule. Default 10.
/// </summary>
public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor;
/// <summary>
/// Where compiled scripts run (<c>Scripting:Runner</c>, lmxopcua#780). Default
/// <see cref="ScriptRunnerMode.InProcess"/> — today's path, unchanged, and the rollback for the other modes.
/// </summary>
public ScriptRunnerMode Runner { get; set; } = ScriptRunnerMode.InProcess;
/// <summary>
/// The seams <see cref="Runner"/> applies to when it is not <see cref="ScriptRunnerMode.InProcess"/>
/// (<c>Scripting:Runner:Seams</c>, e.g. <c>["Calculation"]</c>). Null or empty means VirtualTag and Calculation:
/// scripted alarms run out of process only when listed explicitly (design §14 #10, alarms last). Bound by
/// <c>ScriptingOptionsBinding</c> from the documented key, because <c>Scripting:Runner</c> is itself a value.
/// </summary>
public IReadOnlyList<ScriptSeam>? RunnerSeams { get; set; }
/// <summary>The out-of-process worker pool's knobs (<c>Scripting:Worker</c>).</summary>
public ScriptWorkerOptions Worker { get; set; } = new();
/// <summary>The shadow comparison's knobs (<c>Scripting:Shadow</c>), for <see cref="ScriptRunnerMode.Shadow"/>.</summary>
public ScriptShadowOptions Shadow { get; set; } = new();
/// <summary>Whether <paramref name="seam"/> is selected for out-of-process execution by these options.</summary>
/// <param name="seam">The seam.</param>
/// <returns><c>true</c> when <see cref="Runner"/> is <see cref="ScriptRunnerMode.OutOfProcess"/> and the seam is in
/// <see cref="RunnerSeams"/> (or that list is empty and the seam is not <see cref="ScriptSeam.ScriptedAlarm"/>).</returns>
public bool IsOutOfProcess(ScriptSeam seam)
=> Runner == ScriptRunnerMode.OutOfProcess
&& (RunnerSeams is { Count: > 0 } seams ? seams.Contains(seam) : seam != ScriptSeam.ScriptedAlarm);
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="StuckOrphanQuarantineFactor"/>.</summary>
public const double MaxStuckOrphanQuarantineFactor = 1000;
/// <summary>Upper bound <see cref="Sanitized"/> applies to <see cref="QuarantineAfterConsecutiveTimeouts"/>.</summary>
public const int MaxQuarantineAfterConsecutiveTimeouts = 1000;
/// <summary>
/// Lower bound <see cref="Sanitized"/> applies to a positive <see cref="MaxAllocBytesPerRun"/>: 1 MB. A
/// smaller budget would quarantine practically every script on its first run.
/// </summary>
public const long MinMaxAllocBytesPerRun = 1024L * 1024;
/// <summary>
/// Returns a copy with every numeric knob pulled into a sane range. The bind site calls it and logs
/// <paramref name="adjustments"/>. <see cref="ScriptRunGuard"/> calls it again, so options that reach the
/// guard by another route (a Calculation driver's own config, a test) are clamped too.
/// <list type="bullet">
/// <item><see cref="StuckOrphanQuarantineFactor"/>: NaN becomes the default, a negative value becomes 0
/// (disabled), and a value above <see cref="MaxStuckOrphanQuarantineFactor"/> becomes that maximum.
/// Unclamped, an absurd factor made <c>timeout * factor</c> throw <see cref="OverflowException"/> on
/// the first refused run.</item>
/// <item><see cref="QuarantineAfterConsecutiveTimeouts"/>: into [1, <see cref="MaxQuarantineAfterConsecutiveTimeouts"/>].</item>
/// <item><see cref="MaxAllocBytesPerRun"/>: a negative value becomes 0 (disabled), and a positive value
/// below <see cref="MinMaxAllocBytesPerRun"/> becomes that minimum.</item>
/// <item><see cref="RunTimeout"/>: zero or negative becomes <see cref="DefaultRunTimeout"/>, above
/// <see cref="MaxRunTimeout"/> becomes that maximum.</item>
/// </list>
/// </summary>
/// <param name="adjustments">One line per value that was out of range, naming its key. A negative value that
/// already meant "disabled" is normalised without a line.</param>
/// <returns>A new, clamped instance. This instance is not modified.</returns>
public ScriptingOptions Sanitized(out IReadOnlyList<string> adjustments)
{
var notes = new List<string>();
var factor = StuckOrphanQuarantineFactor;
if (double.IsNaN(factor))
{
notes.Add($"{SectionName}:{nameof(StuckOrphanQuarantineFactor)} is NaN; using the default {DefaultStuckOrphanQuarantineFactor}.");
factor = DefaultStuckOrphanQuarantineFactor;
}
else if (factor < 0)
{
factor = 0;
}
else if (factor > MaxStuckOrphanQuarantineFactor)
{
notes.Add($"{SectionName}:{nameof(StuckOrphanQuarantineFactor)} {factor} is above the maximum; clamped to {MaxStuckOrphanQuarantineFactor}.");
factor = MaxStuckOrphanQuarantineFactor;
}
var threshold = QuarantineAfterConsecutiveTimeouts;
if (threshold < 1)
{
notes.Add($"{SectionName}:{nameof(QuarantineAfterConsecutiveTimeouts)} {threshold} is below 1; clamped to 1.");
threshold = 1;
}
else if (threshold > MaxQuarantineAfterConsecutiveTimeouts)
{
notes.Add($"{SectionName}:{nameof(QuarantineAfterConsecutiveTimeouts)} {threshold} is above the maximum; clamped to {MaxQuarantineAfterConsecutiveTimeouts}.");
threshold = MaxQuarantineAfterConsecutiveTimeouts;
}
var alloc = MaxAllocBytesPerRun;
if (alloc < 0)
{
alloc = 0;
}
else if (alloc > 0 && alloc < MinMaxAllocBytesPerRun)
{
notes.Add($"{SectionName}:{nameof(MaxAllocBytesPerRun)} {alloc} is below the minimum; clamped to {MinMaxAllocBytesPerRun}.");
alloc = MinMaxAllocBytesPerRun;
}
var runTimeout = RunTimeout;
if (runTimeout <= TimeSpan.Zero)
{
notes.Add($"{SectionName}:{nameof(RunTimeout)} {runTimeout} is not positive; using the default {DefaultRunTimeout}.");
runTimeout = DefaultRunTimeout;
}
else if (runTimeout > MaxRunTimeout)
{
notes.Add($"{SectionName}:{nameof(RunTimeout)} {runTimeout} is above the maximum; clamped to {MaxRunTimeout}.");
runTimeout = MaxRunTimeout;
}
var worker = (Worker ?? new ScriptWorkerOptions()).Sanitized(out var workerNotes);
notes.AddRange(workerNotes);
var shadow = (Shadow ?? new ScriptShadowOptions()).Sanitized(out var shadowNotes);
notes.AddRange(shadowNotes);
adjustments = notes;
return new ScriptingOptions
{
RunTimeout = runTimeout,
MaxAllocBytesPerRun = alloc,
QuarantineAfterConsecutiveTimeouts = threshold,
StuckOrphanQuarantineFactor = factor,
Runner = Runner,
RunnerSeams = RunnerSeams is null ? null : [.. RunnerSeams],
Worker = worker,
Shadow = shadow,
};
}
}