feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 07:18:40 -05:00
parent 0796928504
commit 03686eb5d7
17 changed files with 1477 additions and 52 deletions
@@ -9,8 +9,10 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// the <see cref="ScriptRunnerMode.InProcess"/> default every seam gets null and keeps its in-process guard path,
/// unchanged. Under <see cref="ScriptRunnerMode.OutOfProcess"/> the selected seams share one
/// <see cref="OutOfProcessScriptRunner"/>, which starts no worker until the first compiled evaluation, so a node with
/// no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the
/// workers.
/// no scripts (or an admin-only node) never launches one. Under <see cref="ScriptRunnerMode.Shadow"/> (P2) the selected
/// seams get a <see cref="ShadowScriptRunner"/> over such a runner: they keep their in-process path, which is what they
/// publish, and mirror each compiled evaluation to the worker pool for comparison. Disposed with the host's service
/// provider, which stops the workers.
/// </summary>
public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable
{
@@ -26,25 +28,43 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
public static readonly IReadOnlyList<ScriptSeam> DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
private readonly OutOfProcessScriptRunner? _runner;
private readonly ShadowScriptRunner? _shadow;
private readonly HashSet<ScriptSeam> _seams;
private ScriptRunnerSelection(OutOfProcessScriptRunner? runner, IEnumerable<ScriptSeam> seams)
private ScriptRunnerSelection(OutOfProcessScriptRunner? runner, ShadowScriptRunner? shadow, IEnumerable<ScriptSeam> seams)
{
_runner = runner;
_shadow = shadow;
_seams = [.. seams];
}
/// <summary>The out-of-process runner, when one is configured.</summary>
/// <summary>The out-of-process runner, when one is configured (under Shadow, the one shadow runs go to).</summary>
public IScriptRunner? OutOfProcessRunner => _runner;
/// <summary>The out-of-process runner's status (the readiness check), or null when every seam runs in process.</summary>
/// <summary>The shadow comparison, under <see cref="ScriptRunnerMode.Shadow"/>; otherwise null.</summary>
public ShadowScriptRunner? Shadow => _shadow;
/// <summary>
/// The out-of-process runner's status (the readiness check), or null when every seam publishes in-process
/// results — <see cref="ScriptRunnerMode.InProcess"/>, and also <see cref="ScriptRunnerMode.Shadow"/>, whose worker
/// pool must never move readiness (see <see cref="ShadowStatus"/>).
/// </summary>
/// <returns>The status, or null.</returns>
public ScriptRunnerStatus? Status() => _runner?.Status();
public ScriptRunnerStatus? Status() => _shadow is null ? _runner?.Status() : null;
/// <summary>Under <see cref="ScriptRunnerMode.Shadow"/>: the shadow worker pool's status and the comparison counters.</summary>
/// <returns>The shadow status, or null in any other mode.</returns>
public (ScriptRunnerStatus Runner, ScriptShadowStats Stats)? ShadowStatus()
=> _shadow is not null && _runner is not null ? (_runner.Status(), _shadow.Stats()) : null;
/// <summary>The runner <paramref name="seam"/> must use, or null for its in-process path.</summary>
/// <param name="seam">The seam.</param>
/// <returns>The runner, or null.</returns>
public IScriptRunner? ForSeam(ScriptSeam seam) => _runner is not null && _seams.Contains(seam) ? _runner : null;
/// <returns>The out-of-process runner; under Shadow the <see cref="ShadowScriptRunner"/>; or null.</returns>
public IScriptRunner? ForSeam(ScriptSeam seam)
{
if (_runner is null || !_seams.Contains(seam)) return null;
return _shadow ?? (IScriptRunner)_runner;
}
/// <summary>
/// Startup checks for the runner options: returns the Warnings to log, and throws when the configuration must
@@ -63,21 +83,27 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
if (options.Worker.ValidatePaths(contentRootPath) is { } violation)
throw new InvalidOperationException($"Scripting misconfiguration: {violation}");
if (options.Runner == ScriptRunnerMode.Shadow)
var mode = $"Scripting:Runner={options.Runner}";
var shadow = options.Runner == ScriptRunnerMode.Shadow;
if (shadow)
{
warnings.Add("Scripting:Runner=Shadow is not implemented yet (lmxopcua#780 P2); every seam runs InProcess.");
return warnings;
warnings.Add($"{mode}: every selected script runs twice — in process (published) and in the ScriptWorker " +
$"(compared only: {string.Join(", ", RequestedSeams(options).Where(SupportedOutOfProcessSeams.Contains))}). " +
"It doubles script CPU and keeps the in-process risk: a time-boxed diagnostic, not a production mode.");
}
foreach (var seam in RequestedSeams(options))
{
if (!SupportedOutOfProcessSeams.Contains(seam))
warnings.Add($"Scripting:Runner=OutOfProcess: the {seam} seam cannot run out of process yet (lmxopcua#780); it stays InProcess.");
warnings.Add($"{mode}: the {seam} seam cannot run out of process yet (lmxopcua#780); it stays InProcess.");
}
var workerPath = OutOfProcessScriptRunner.ResolveWorkerPath(options.Worker);
if (!File.Exists(workerPath))
warnings.Add($"Scripting:Runner=OutOfProcess: the script worker '{workerPath}' does not exist; every out-of-process " +
"script reports BadResourceUnavailable (the runner fails closed, it never falls back to in process).");
warnings.Add(shadow
? $"{mode}: the script worker '{workerPath}' does not exist; every shadow run is skipped as shadow_unavailable " +
"(published values are unaffected)."
: $"{mode}: the script worker '{workerPath}' does not exist; every out-of-process " +
"script reports BadResourceUnavailable (the runner fails closed, it never falls back to in process).");
return warnings;
}
@@ -89,17 +115,34 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
public static ScriptRunnerSelection Create(ScriptingOptions options, SerilogLogger log, SerilogLogger? scriptRoot)
{
ArgumentNullException.ThrowIfNull(options);
if (options.Runner != ScriptRunnerMode.OutOfProcess) return new ScriptRunnerSelection(null, []);
ArgumentNullException.ThrowIfNull(log);
if (options.Runner == ScriptRunnerMode.InProcess) return new ScriptRunnerSelection(null, null, []);
var seams = RequestedSeams(options).Where(SupportedOutOfProcessSeams.Contains).ToArray();
if (seams.Length == 0) return new ScriptRunnerSelection(null, []);
return new ScriptRunnerSelection(new OutOfProcessScriptRunner(options, log, scriptRoot), seams);
if (seams.Length == 0) return new ScriptRunnerSelection(null, null, []);
if (options.Runner == ScriptRunnerMode.OutOfProcess)
return new ScriptRunnerSelection(new OutOfProcessScriptRunner(options, log, scriptRoot), null, seams);
// Shadow: the worker pool's own lines (kills, quarantines, restarts) are about shadow runs, so they carry
// ScriptRunnerRole=Shadow and go to the host log, never to the operator's script log — the published script was
// not killed or quarantined. ctx.Logger lines of shadow runs are dropped (the in-process run wrote them).
var shadowLog = log.ForContext("ScriptRunnerRole", "Shadow");
var runner = new OutOfProcessScriptRunner(options, shadowLog, shadowLog);
return new ScriptRunnerSelection(runner, new ShadowScriptRunner(runner, options, log), seams);
}
/// <inheritdoc />
public ValueTask DisposeAsync() => _runner?.DisposeAsync() ?? ValueTask.CompletedTask;
public async ValueTask DisposeAsync()
{
_shadow?.Dispose();
if (_runner is not null) await _runner.DisposeAsync().ConfigureAwait(false);
}
/// <inheritdoc />
public void Dispose() => _runner?.Dispose();
public void Dispose()
{
_shadow?.Dispose();
_runner?.Dispose();
}
private static IEnumerable<ScriptSeam> RequestedSeams(ScriptingOptions options)
=> options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams;