feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 07:18:40 -05:00
parent 0796928504
commit 03686eb5d7
17 changed files with 1477 additions and 52 deletions
@@ -60,6 +60,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
private readonly TimeSpan _runTimeout;
private readonly ScriptRunGuard _guard;
private readonly IScriptRunner? _runner;
private readonly IScriptShadow? _shadow;
// Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE,
// which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a
@@ -78,7 +79,9 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
/// <param name="runTimeout">Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator).</param>
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the Calculation seam runs out of
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
/// process; null (the default) keeps the in-process guard path, unchanged. Under <c>Scripting:Runner=Shadow</c> it is an
/// <see cref="IScriptShadow"/>: the in-process path stays the published one and each compiled evaluation is mirrored to
/// it for comparison (P2).</param>
public CalculationEvaluator(
ILogger logger,
ScriptRootLogger scriptRoot,
@@ -86,7 +89,8 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
ScriptingOptions? scripting = null,
IScriptRunner? scriptRunner = null)
{
_runner = scriptRunner;
_shadow = scriptRunner as IScriptShadow;
_runner = _shadow is null ? scriptRunner : null;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
_runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
@@ -202,6 +206,17 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
return outcome.ToVirtualTagEvalResult();
}
var result = await RunInProcessAsync(calcTagId, scriptSource, context, ct).ConfigureAwait(false);
// lmxopcua#780 P2 — Shadow: `result` is what gets published; the same evaluation is mirrored to the worker pool
// and only compared. Mirror never blocks and nothing it does reaches this result.
if (_shadow is not null && result.ToMirroredOutcome() is { } mirrored)
_shadow.Mirror(new ScriptInvocation(calcTagId, ScriptContextKind.Calculation, scriptSource, readCache, _runTimeout), mirrored);
return result;
}
private async Task<VirtualTagEvalResult> RunInProcessAsync(
string calcTagId, string scriptSource, VirtualTagContext context, CancellationToken ct)
{
// A source that already failed to compile fails the same way without touching Roslyn (R07:P4).
var sourceKey = HashSource(scriptSource);
if (_compileFailures.TryGetValue(sourceKey, out var knownFailure))
@@ -366,6 +381,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
// (an edited source already lifts its own through the guard's source identity).
_guard.ResetAll();
_runner?.Reset(ScriptContextKind.Calculation);
_shadow?.Reset(ScriptContextKind.Calculation);
}
/// <inheritdoc />
@@ -373,6 +389,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
{
_guard.ResetAll();
_runner?.Reset(ScriptContextKind.Calculation);
_shadow?.Reset(ScriptContextKind.Calculation);
}
/// <inheritdoc />