feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 07:18:40 -05:00
parent 0796928504
commit 03686eb5d7
17 changed files with 1477 additions and 52 deletions
@@ -128,12 +128,52 @@ public sealed class ScriptRunnerSelectionTests
}
[Fact]
public void Shadow_is_not_implemented_yet_and_runs_in_process_with_a_warning()
public void Shadow_hands_the_selected_seams_a_shadow_runner_and_never_reports_its_pool_to_readiness()
{
// lmxopcua#780 P2. Unset Seams: VirtualTag + Calculation; alarms only when listed (same rule as OutOfProcess).
var options = new ScriptingOptions { Runner = ScriptRunnerMode.Shadow };
ScriptRunnerSelection.Validate(options, "/srv").ShouldHaveSingleItem().ShouldContain("Shadow");
using var selection = ScriptRunnerSelection.Create(options, Silent, null);
selection.OutOfProcessRunner.ShouldBeNull();
ScriptRunnerSelection.Validate(options, "/srv").ShouldContain(w => w.Contains("runs twice") && w.Contains("time-boxed diagnostic"));
using (var selection = ScriptRunnerSelection.Create(options, Silent, null))
{
var vt = selection.ForSeam(ScriptSeam.VirtualTag).ShouldBeOfType<ShadowScriptRunner>();
selection.ForSeam(ScriptSeam.Calculation).ShouldBeSameAs(vt, "one node-wide shadow");
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeNull();
vt.Shadow.ShouldBeSameAs(selection.OutOfProcessRunner);
selection.Shadow.ShouldBeSameAs(vt);
selection.Status().ShouldBeNull("readiness must not see the shadow pool");
selection.ShadowStatus().ShouldNotBeNull();
}
var withAlarms = new ScriptingOptions { Runner = ScriptRunnerMode.Shadow, RunnerSeams = [ScriptSeam.ScriptedAlarm] };
using (var selection = ScriptRunnerSelection.Create(withAlarms, Silent, null))
{
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeOfType<ShadowScriptRunner>();
selection.ForSeam(ScriptSeam.VirtualTag).ShouldBeNull();
}
// Out of process: no shadow, the pool's status IS readiness.
using (var oop = ScriptRunnerSelection.Create(new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess }, Silent, null))
{
oop.Shadow.ShouldBeNull();
oop.ShadowStatus().ShouldBeNull();
oop.Status().ShouldNotBeNull();
}
}
[Fact]
public void Scripting_Shadow_binds_and_clamps()
{
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
{
["Scripting:Runner"] = "Shadow",
["Scripting:Shadow:MaxInFlight"] = "0",
["Scripting:Shadow:MismatchLogsPerMinute"] = "5",
}).Build();
var bound = ScriptingOptionsBinding.Bind(config, out var adjustments);
bound.Runner.ShouldBe(ScriptRunnerMode.Shadow);
bound.Shadow.MaxInFlight.ShouldBe(1);
bound.Shadow.MismatchLogsPerMinute.ShouldBe(5);
adjustments.ShouldContain(a => a.Contains("Scripting:Shadow:MaxInFlight"));
}
[Fact]