feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 07:18:40 -05:00
parent 0796928504
commit 03686eb5d7
17 changed files with 1477 additions and 52 deletions
@@ -10,8 +10,10 @@ public enum ScriptRunnerMode
InProcess = 0,
/// <summary>
/// Run in process AND submit to the worker, comparing the two (design P2). Not implemented yet: a node
/// configured for it runs <see cref="InProcess"/> and logs a startup Warning.
/// Run in process — that result is the one published — AND mirror each evaluation of the selected seams to the
/// worker pool, comparing the two (design P2, <see cref="ShadowScriptRunner"/>). A time-boxed diagnostic: it
/// doubles script CPU and keeps the in-process risk. Nothing on the shadow side reaches a published value or
/// readiness.
/// </summary>
Shadow = 1,
@@ -251,3 +253,54 @@ public sealed class ScriptWorkerOptions
return null;
}
}
/// <summary>
/// The shadow comparison's knobs (the <c>Scripting:Shadow</c> section, lmxopcua#780 P2). Only consulted when
/// <see cref="ScriptingOptions.Runner"/> is <see cref="ScriptRunnerMode.Shadow"/>.
/// </summary>
public sealed class ScriptShadowOptions
{
/// <summary>Default <see cref="MaxInFlight"/>: 256.</summary>
public const int DefaultMaxInFlight = 256;
/// <summary>Default <see cref="MismatchLogsPerMinute"/>: 20.</summary>
public const int DefaultMismatchLogsPerMinute = 20;
/// <summary>
/// Shadow runs outstanding at once (all seams). An evaluation that would exceed it is not mirrored and is counted
/// as dropped (<c>reason = behind</c>): the shadow is best effort and never queues without bound. Default 256;
/// 1–100 000.
/// </summary>
public int MaxInFlight { get; set; } = DefaultMaxInFlight;
/// <summary>
/// Mismatch Warning lines per minute (each script logs once per mismatch reason until its seam redeploys; this
/// caps a burst of many scripts). Default 20; 0 (counted, never logged) – 10 000.
/// </summary>
public int MismatchLogsPerMinute { get; set; } = DefaultMismatchLogsPerMinute;
/// <summary>Returns a clamped copy; one line per out-of-range key in <paramref name="adjustments"/>.</summary>
/// <param name="adjustments">The clamps applied, naming each key.</param>
/// <returns>A new, clamped instance.</returns>
public ScriptShadowOptions Sanitized(out IReadOnlyList<string> adjustments)
{
const string prefix = ScriptingOptions.SectionName + ":Shadow:";
var notes = new List<string>();
var inFlight = MaxInFlight;
if (inFlight < 1 || inFlight > 100_000)
{
var clamped = Math.Clamp(inFlight, 1, 100_000);
notes.Add($"{prefix}{nameof(MaxInFlight)} {inFlight} is outside 1–100000; clamped to {clamped}.");
inFlight = clamped;
}
var logs = MismatchLogsPerMinute;
if (logs < 0 || logs > 10_000)
{
var clamped = Math.Clamp(logs, 0, 10_000);
notes.Add($"{prefix}{nameof(MismatchLogsPerMinute)} {logs} is outside 0–10000; clamped to {clamped}.");
logs = clamped;
}
adjustments = notes;
return new ScriptShadowOptions { MaxInFlight = inFlight, MismatchLogsPerMinute = logs };
}
}