feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ScriptRunnerSelection.cs

@@ -9,8 +9,10 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
 ///     the <see cref="ScriptRunnerMode.InProcess"/> default every seam gets null and keeps its in-process guard path,
 ///     unchanged. Under <see cref="ScriptRunnerMode.OutOfProcess"/> the selected seams share one
 ///     <see cref="OutOfProcessScriptRunner"/>, which starts no worker until the first compiled evaluation, so a node with
-///     no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the
-///     workers.
+///     no scripts (or an admin-only node) never launches one. Under <see cref="ScriptRunnerMode.Shadow"/> (P2) the selected
+///     seams get a <see cref="ShadowScriptRunner"/> over such a runner: they keep their in-process path, which is what they
+///     publish, and mirror each compiled evaluation to the worker pool for comparison. Disposed with the host's service
+///     provider, which stops the workers.
 /// </summary>
 public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable
 {
@@ -26,25 +28,43 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
     public static readonly IReadOnlyList<ScriptSeam> DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
 
     private readonly OutOfProcessScriptRunner? _runner;
+    private readonly ShadowScriptRunner? _shadow;
     private readonly HashSet<ScriptSeam> _seams;
 
-    private ScriptRunnerSelection(OutOfProcessScriptRunner? runner, IEnumerable<ScriptSeam> seams)
+    private ScriptRunnerSelection(OutOfProcessScriptRunner? runner, ShadowScriptRunner? shadow, IEnumerable<ScriptSeam> seams)
     {
         _runner = runner;
+        _shadow = shadow;
         _seams = [.. seams];
     }
 
-    /// <summary>The out-of-process runner, when one is configured.</summary>
+    /// <summary>The out-of-process runner, when one is configured (under Shadow, the one shadow runs go to).</summary>
     public IScriptRunner? OutOfProcessRunner => _runner;
 
-    /// <summary>The out-of-process runner's status (the readiness check), or null when every seam runs in process.</summary>
+    /// <summary>The shadow comparison, under <see cref="ScriptRunnerMode.Shadow"/>; otherwise null.</summary>
+    public ShadowScriptRunner? Shadow => _shadow;
+
+    /// <summary>
+    ///     The out-of-process runner's status (the readiness check), or null when every seam publishes in-process
+    ///     results — <see cref="ScriptRunnerMode.InProcess"/>, and also <see cref="ScriptRunnerMode.Shadow"/>, whose worker
+    ///     pool must never move readiness (see <see cref="ShadowStatus"/>).
+    /// </summary>
     /// <returns>The status, or null.</returns>
-    public ScriptRunnerStatus? Status() => _runner?.Status();
+    public ScriptRunnerStatus? Status() => _shadow is null ? _runner?.Status() : null;
+
+    /// <summary>Under <see cref="ScriptRunnerMode.Shadow"/>: the shadow worker pool's status and the comparison counters.</summary>
+    /// <returns>The shadow status, or null in any other mode.</returns>
+    public (ScriptRunnerStatus Runner, ScriptShadowStats Stats)? ShadowStatus()
+        => _shadow is not null && _runner is not null ? (_runner.Status(), _shadow.Stats()) : null;
 
     /// <summary>The runner <paramref name="seam"/> must use, or null for its in-process path.</summary>
     /// <param name="seam">The seam.</param>
-    /// <returns>The runner, or null.</returns>
-    public IScriptRunner? ForSeam(ScriptSeam seam) => _runner is not null && _seams.Contains(seam) ? _runner : null;
+    /// <returns>The out-of-process runner; under Shadow the <see cref="ShadowScriptRunner"/>; or null.</returns>
+    public IScriptRunner? ForSeam(ScriptSeam seam)
+    {
+        if (_runner is null || !_seams.Contains(seam)) return null;
+        return _shadow ?? (IScriptRunner)_runner;
+    }
 
     /// <summary>
     ///     Startup checks for the runner options: returns the Warnings to log, and throws when the configuration must
@@ -63,21 +83,27 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
         if (options.Worker.ValidatePaths(contentRootPath) is { } violation)
             throw new InvalidOperationException($"Scripting misconfiguration: {violation}");
 
-        if (options.Runner == ScriptRunnerMode.Shadow)
+        var mode = $"Scripting:Runner={options.Runner}";
+        var shadow = options.Runner == ScriptRunnerMode.Shadow;
+        if (shadow)
         {
-            warnings.Add("Scripting:Runner=Shadow is not implemented yet (lmxopcua#780 P2); every seam runs InProcess.");
-            return warnings;
+            warnings.Add($"{mode}: every selected script runs twice — in process (published) and in the ScriptWorker " +
+                         $"(compared only: {string.Join(", ", RequestedSeams(options).Where(SupportedOutOfProcessSeams.Contains))}). " +
+                         "It doubles script CPU and keeps the in-process risk: a time-boxed diagnostic, not a production mode.");
         }
 
         foreach (var seam in RequestedSeams(options))
         {
             if (!SupportedOutOfProcessSeams.Contains(seam))
-                warnings.Add($"Scripting:Runner=OutOfProcess: the {seam} seam cannot run out of process yet (lmxopcua#780); it stays InProcess.");
+                warnings.Add($"{mode}: the {seam} seam cannot run out of process yet (lmxopcua#780); it stays InProcess.");
         }
         var workerPath = OutOfProcessScriptRunner.ResolveWorkerPath(options.Worker);
         if (!File.Exists(workerPath))
-            warnings.Add($"Scripting:Runner=OutOfProcess: the script worker '{workerPath}' does not exist; every out-of-process " +
-                         "script reports BadResourceUnavailable (the runner fails closed, it never falls back to in process).");
+            warnings.Add(shadow
+                ? $"{mode}: the script worker '{workerPath}' does not exist; every shadow run is skipped as shadow_unavailable " +
+                  "(published values are unaffected)."
+                : $"{mode}: the script worker '{workerPath}' does not exist; every out-of-process " +
+                  "script reports BadResourceUnavailable (the runner fails closed, it never falls back to in process).");
         return warnings;
     }
 
@@ -89,17 +115,34 @@ public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposa
     public static ScriptRunnerSelection Create(ScriptingOptions options, SerilogLogger log, SerilogLogger? scriptRoot)
     {
         ArgumentNullException.ThrowIfNull(options);
-        if (options.Runner != ScriptRunnerMode.OutOfProcess) return new ScriptRunnerSelection(null, []);
+        ArgumentNullException.ThrowIfNull(log);
+        if (options.Runner == ScriptRunnerMode.InProcess) return new ScriptRunnerSelection(null, null, []);
         var seams = RequestedSeams(options).Where(SupportedOutOfProcessSeams.Contains).ToArray();
-        if (seams.Length == 0) return new ScriptRunnerSelection(null, []);
-        return new ScriptRunnerSelection(new OutOfProcessScriptRunner(options, log, scriptRoot), seams);
+        if (seams.Length == 0) return new ScriptRunnerSelection(null, null, []);
+        if (options.Runner == ScriptRunnerMode.OutOfProcess)
+            return new ScriptRunnerSelection(new OutOfProcessScriptRunner(options, log, scriptRoot), null, seams);
+
+        // Shadow: the worker pool's own lines (kills, quarantines, restarts) are about shadow runs, so they carry
+        // ScriptRunnerRole=Shadow and go to the host log, never to the operator's script log — the published script was
+        // not killed or quarantined. ctx.Logger lines of shadow runs are dropped (the in-process run wrote them).
+        var shadowLog = log.ForContext("ScriptRunnerRole", "Shadow");
+        var runner = new OutOfProcessScriptRunner(options, shadowLog, shadowLog);
+        return new ScriptRunnerSelection(runner, new ShadowScriptRunner(runner, options, log), seams);
     }
 
     /// <inheritdoc />
-    public ValueTask DisposeAsync() => _runner?.DisposeAsync() ?? ValueTask.CompletedTask;
+    public async ValueTask DisposeAsync()
+    {
+        _shadow?.Dispose();
+        if (_runner is not null) await _runner.DisposeAsync().ConfigureAwait(false);
+    }
 
     /// <inheritdoc />
-    public void Dispose() => _runner?.Dispose();
+    public void Dispose()
+    {
+        _shadow?.Dispose();
+        _runner?.Dispose();
+    }
 
     private static IEnumerable<ScriptSeam> RequestedSeams(ScriptingOptions options)
         => options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams;

src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs

@@ -54,6 +54,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     private readonly TimeSpan _runTimeout;
     private readonly ScriptRunGuard _guard;
     private readonly IScriptRunner? _runner;
+    private readonly IScriptShadow? _shadow;
     private readonly ScriptBatchCoalescer? _batches;
     private bool _disposed;
 
@@ -66,7 +67,8 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     /// <param name="scriptRunner">
     ///     lmxopcua#780 — the out-of-process runner when <c>Scripting:Runner=OutOfProcess</c> selects the VirtualTag seam;
     ///     null (the default, <c>InProcess</c>) keeps the in-process guard path below, unchanged. The passthrough fast path
-    ///     never reaches it.
+    ///     never reaches it. Under <c>Scripting:Runner=Shadow</c> it is an <see cref="IScriptShadow"/>: the in-process path
+    ///     below stays the published one and each compiled evaluation is mirrored to it for comparison (P2).
     /// </param>
     public RoslynVirtualTagEvaluator(
         ILogger<RoslynVirtualTagEvaluator> logger,
@@ -79,13 +81,17 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
         _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
         _runTimeout = runTimeout ?? (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
         _guard = new ScriptRunGuard(scripting, _scriptRoot);
-        _runner = scriptRunner;
-        _batches = scriptRunner is null ? null : new ScriptBatchCoalescer(scriptRunner);
+        _shadow = scriptRunner as IScriptShadow;
+        _runner = _shadow is null ? scriptRunner : null;
+        _batches = _runner is null ? null : new ScriptBatchCoalescer(_runner);
     }
 
     /// <summary>The out-of-process runner this evaluator delegates compiled runs to, or null (in process).</summary>
     public IScriptRunner? ScriptRunner => _runner;
 
+    /// <summary>The shadow compiled runs are mirrored to (<c>Scripting:Runner=Shadow</c>), or null.</summary>
+    public IScriptShadow? ScriptShadow => _shadow;
+
     /// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
     public ScriptRunGuard RunGuard => _guard;
 
@@ -175,6 +181,16 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
             return outcome.ToVirtualTagEvalResult();
         }
 
+        var result = await RunInProcessAsync(virtualTagId, expression, context).ConfigureAwait(false);
+        // lmxopcua#780 P2 — Shadow: `result` is what gets published; the same evaluation is mirrored to the worker pool
+        // and only compared. Mirror never blocks and nothing it does reaches this result.
+        if (_shadow is not null && result.ToMirroredOutcome() is { } mirrored)
+            _shadow.Mirror(new ScriptInvocation(virtualTagId, ScriptContextKind.VirtualTag, expression, readCache, _runTimeout), mirrored);
+        return result;
+    }
+
+    private async Task<VirtualTagEvalResult> RunInProcessAsync(string virtualTagId, string expression, VirtualTagContext context)
+    {
         // 02/S12: fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent
         // apply-boundary ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the
         // run's disposed guard) re-fetches and retries exactly once. The loop runs at most twice — the
@@ -264,6 +280,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
         // (an edited expression already lifts its own through the guard's source identity).
         _guard.ResetAll();
         _runner?.Reset(ScriptContextKind.VirtualTag);
+        _shadow?.Reset(ScriptContextKind.VirtualTag);
     }
 
     /// <inheritdoc />
@@ -271,6 +288,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     {
         _guard.ResetAll();
         _runner?.Reset(ScriptContextKind.VirtualTag);
+        _shadow?.Reset(ScriptContextKind.VirtualTag);
     }
 
     /// <inheritdoc />

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/ScriptRunnerHealthCheck.cs

@@ -1,4 +1,5 @@
 using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
 using ZB.MOM.WW.OtOpcUa.Host.Configuration;
 
@@ -16,26 +17,35 @@ public static class ScriptRunnerHealthDecision
     ///     ServiceLevel to the peer over a script problem. Degraded is still ready (200) but visible.
     /// </summary>
     /// <param name="status">The out-of-process runner's status, or null when every seam runs in process.</param>
+    /// <param name="shadow">
+    ///     Under <c>Scripting:Runner=Shadow</c>: the shadow worker pool's status and the comparison counters. Always
+    ///     <b>Healthy</b> — every published value comes from the in-process path, so nothing on the shadow side (a
+    ///     degraded pool, an open circuit, mismatches) may move readiness. It is reported as data only.
+    /// </param>
     /// <returns>The status, a description and the per-shard data.</returns>
-    public static (HealthStatus Status, string Description, IReadOnlyDictionary<string, object> Data) Evaluate(ScriptRunnerStatus? status)
+    public static (HealthStatus Status, string Description, IReadOnlyDictionary<string, object> Data) Evaluate(
+        ScriptRunnerStatus? status, (ScriptRunnerStatus Runner, ScriptShadowStats Stats)? shadow = null)
     {
+        if (shadow is { } sh)
+        {
+            var shadowData = RunnerData(sh.Runner, "shadow.");
+            shadowData["mode"] = "Shadow";
+            shadowData["shadow.compared"] = sh.Stats.Compared;
+            shadowData["shadow.mismatched"] = sh.Stats.Mismatched;
+            shadowData["shadow.unavailable"] = sh.Stats.ShadowUnavailable;
+            shadowData["shadow.dropped"] = sh.Stats.Dropped;
+            shadowData["shadow.inFlight"] = sh.Stats.InFlight;
+            return (HealthStatus.Healthy,
+                $"Scripts run in process (Scripting:Runner=Shadow); the out-of-process shadow runner is {sh.Runner.State}: " +
+                $"{sh.Stats.Compared} compared, {sh.Stats.Mismatched} mismatch(es), {sh.Stats.Dropped} dropped. " +
+                "Shadow results are never published and never affect readiness.",
+                shadowData);
+        }
+
         if (status is null)
             return (HealthStatus.Healthy, "Scripts run in process (Scripting:Runner=InProcess).", new Dictionary<string, object>());
 
-        var data = new Dictionary<string, object>(StringComparer.Ordinal)
-        {
-            ["state"] = status.State.ToString(),
-            ["suspects"] = status.SuspectCount,
-            ["quarantined"] = status.QuarantinedCount,
-            ["peCache"] = status.PeCacheEnabled,
-        };
-        foreach (var shard in status.Shards)
-        {
-            data[$"shard.{shard.Label}"] = shard.Up
-                ? $"up pid {shard.Pid}, {shard.InFlight} in flight"
-                : shard.DownFor is { } downFor ? $"down {downFor.TotalSeconds:F0} s ({shard.ConsecutiveFailures} failure(s))" : "not started";
-        }
-
+        var data = RunnerData(status, string.Empty);
         var down = status.Shards.Where(s => s.DownFor is not null).Select(s => s.Label).ToArray();
         return status.State switch
         {
@@ -48,6 +58,24 @@ public static class ScriptRunnerHealthDecision
                 data),
         };
     }
+
+    private static Dictionary<string, object> RunnerData(ScriptRunnerStatus status, string prefix)
+    {
+        var data = new Dictionary<string, object>(StringComparer.Ordinal)
+        {
+            [$"{prefix}state"] = status.State.ToString(),
+            [$"{prefix}suspects"] = status.SuspectCount,
+            [$"{prefix}quarantined"] = status.QuarantinedCount,
+            [$"{prefix}peCache"] = status.PeCacheEnabled,
+        };
+        foreach (var shard in status.Shards)
+        {
+            data[$"{prefix}shard.{shard.Label}"] = shard.Up
+                ? $"up pid {shard.Pid}, {shard.InFlight} in flight"
+                : shard.DownFor is { } downFor ? $"down {downFor.TotalSeconds:F0} s ({shard.ConsecutiveFailures} failure(s))" : "not started";
+        }
+        return data;
+    }
 }
 
 /// <summary>The script-runner readiness component: Healthy, or Degraded-but-ready (see <see cref="ScriptRunnerHealthDecision"/>).</summary>
@@ -60,7 +88,7 @@ public sealed class ScriptRunnerHealthCheck(ScriptRunnerSelection? selection) :
     /// <inheritdoc />
     public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
     {
-        var (status, description, data) = ScriptRunnerHealthDecision.Evaluate(selection?.Status());
+        var (status, description, data) = ScriptRunnerHealthDecision.Evaluate(selection?.Status(), selection?.ShadowStatus());
         return Task.FromResult(status == HealthStatus.Healthy
             ? HealthCheckResult.Healthy(description, data)
             : HealthCheckResult.Degraded(description, data: data));