Joseph Doherty
03686eb5d7
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
230 lines
11 KiB
C#
230 lines
11 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
using Serilog;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#780 — <c>Scripting:Runner</c> binds by its documented keys (design §12), defaults to InProcess (every seam
|
|
/// gets no runner, so the in-process path is untouched), and the worker options clamp and fail closed.
|
|
/// </summary>
|
|
public sealed class ScriptRunnerSelectionTests
|
|
{
|
|
private static IConfiguration Config(Dictionary<string, string?> values) =>
|
|
new ConfigurationBuilder().AddInMemoryCollection(values).Build();
|
|
|
|
private static readonly ILogger Silent = new LoggerConfiguration().CreateLogger();
|
|
|
|
[Fact]
|
|
public void The_default_is_InProcess_and_selects_no_runner_for_any_seam()
|
|
{
|
|
var bound = ScriptingOptionsBinding.Bind(Config(new()), out var adjustments);
|
|
adjustments.ShouldBeEmpty();
|
|
bound.Runner.ShouldBe(ScriptRunnerMode.InProcess);
|
|
ScriptRunnerSelection.Validate(bound, "/srv/app").ShouldBeEmpty();
|
|
using var selection = ScriptRunnerSelection.Create(bound, Silent, null);
|
|
foreach (var seam in Enum.GetValues<ScriptSeam>()) selection.ForSeam(seam).ShouldBeNull(seam.ToString());
|
|
selection.OutOfProcessRunner.ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Every_documented_runner_key_binds()
|
|
{
|
|
var bound = ScriptingOptionsBinding.Bind(Config(new()
|
|
{
|
|
["Scripting:Runner"] = "OutOfProcess",
|
|
["Scripting:Runner:Seams:0"] = "Calculation",
|
|
["Scripting:Worker:Shards"] = "3",
|
|
["Scripting:Worker:HardGrace"] = "00:00:02",
|
|
["Scripting:Worker:HeapLimitBytes"] = "268435456",
|
|
["Scripting:Worker:QuarantineAfterKills"] = "4",
|
|
["Scripting:Worker:UnavailableGrace"] = "00:00:07",
|
|
["Scripting:Worker:IsolationProbationRuns"] = "9",
|
|
["Scripting:Worker:PeCacheDir"] = "/var/cache/otopcua-pe",
|
|
["Scripting:Worker:ExecutablePath"] = "/opt/w/ZB.MOM.WW.OtOpcUa.ScriptWorker.dll",
|
|
}), out var adjustments);
|
|
|
|
adjustments.ShouldBeEmpty();
|
|
bound.Runner.ShouldBe(ScriptRunnerMode.OutOfProcess);
|
|
bound.RunnerSeams.ShouldBe([ScriptSeam.Calculation]);
|
|
bound.Worker.Shards.ShouldBe(3);
|
|
bound.Worker.HardGrace.ShouldBe(TimeSpan.FromSeconds(2));
|
|
bound.Worker.HeapLimitBytes.ShouldBe(256L * 1024 * 1024);
|
|
bound.Worker.QuarantineAfterKills.ShouldBe(4);
|
|
bound.Worker.UnavailableGrace.ShouldBe(TimeSpan.FromSeconds(7));
|
|
bound.Worker.IsolationProbationRuns.ShouldBe(9);
|
|
bound.Worker.PeCacheDir.ShouldBe("/var/cache/otopcua-pe");
|
|
bound.Worker.ExecutablePath.ShouldBe("/opt/w/ZB.MOM.WW.OtOpcUa.ScriptWorker.dll");
|
|
bound.IsOutOfProcess(ScriptSeam.Calculation).ShouldBeTrue();
|
|
bound.IsOutOfProcess(ScriptSeam.VirtualTag).ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_misspelled_runner_mode_fails_startup()
|
|
=> Should.Throw<InvalidOperationException>(() =>
|
|
ScriptingOptionsBinding.Bind(Config(new() { ["Scripting:Runner"] = "OutOfProc" }), out _));
|
|
|
|
[Fact]
|
|
public void Worker_values_out_of_range_are_clamped_and_reported()
|
|
{
|
|
var bound = ScriptingOptionsBinding.Bind(Config(new()
|
|
{
|
|
["Scripting:Worker:Shards"] = "64",
|
|
["Scripting:Worker:HardGrace"] = "00:00:00.010",
|
|
["Scripting:Worker:HeapLimitBytes"] = "1024",
|
|
["Scripting:Worker:QuarantineAfterKills"] = "0",
|
|
["Scripting:Worker:UnavailableGrace"] = "00:05:00",
|
|
["Scripting:Worker:IsolationProbationRuns"] = "0",
|
|
}), out var adjustments);
|
|
|
|
bound.Worker.Shards.ShouldBe(ScriptWorkerOptions.MaxShards);
|
|
bound.Worker.HardGrace.ShouldBe(ScriptWorkerOptions.MinHardGrace);
|
|
bound.Worker.HeapLimitBytes.ShouldBe(ScriptWorkerOptions.MinHeapLimitBytes);
|
|
bound.Worker.QuarantineAfterKills.ShouldBe(1);
|
|
bound.Worker.UnavailableGrace.ShouldBe(ScriptWorkerOptions.MaxUnavailableGrace);
|
|
bound.Worker.IsolationProbationRuns.ShouldBe(1);
|
|
adjustments.Count.ShouldBe(6);
|
|
adjustments.ShouldAllBe(a => a.StartsWith("Scripting:Worker:"));
|
|
}
|
|
|
|
[Fact]
|
|
public void The_hard_grace_defaults_to_max_of_one_second_and_the_budget()
|
|
{
|
|
var worker = new ScriptWorkerOptions();
|
|
worker.EffectiveHardGrace(TimeSpan.FromMilliseconds(250)).ShouldBe(TimeSpan.FromSeconds(1));
|
|
worker.EffectiveHardGrace(TimeSpan.FromSeconds(2)).ShouldBe(TimeSpan.FromSeconds(2));
|
|
new ScriptWorkerOptions { HardGrace = TimeSpan.FromMilliseconds(400) }
|
|
.EffectiveHardGrace(TimeSpan.FromSeconds(2)).ShouldBe(TimeSpan.FromMilliseconds(400));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("relative/cache", "must be an absolute path")]
|
|
[InlineData("{root}/pe-cache", "inside the content root")]
|
|
[InlineData("{root}", "inside the content root")]
|
|
public void A_pe_cache_dir_that_is_relative_or_inside_the_content_root_fails_startup(string dir, string why)
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), "otopcua-root-" + Guid.NewGuid().ToString("N"));
|
|
var options = new ScriptingOptions
|
|
{
|
|
Runner = ScriptRunnerMode.OutOfProcess,
|
|
Worker = new ScriptWorkerOptions { PeCacheDir = dir.Replace("{root}", root) },
|
|
};
|
|
Should.Throw<InvalidOperationException>(() => ScriptRunnerSelection.Validate(options, root)).Message.ShouldContain(why);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_pe_cache_dir_outside_the_content_root_is_accepted()
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), "otopcua-root");
|
|
var options = new ScriptingOptions
|
|
{
|
|
Runner = ScriptRunnerMode.OutOfProcess,
|
|
Worker = new ScriptWorkerOptions { PeCacheDir = Path.Combine(Path.GetTempPath(), "otopcua-root-sibling") },
|
|
};
|
|
Should.NotThrow(() => ScriptRunnerSelection.Validate(options, root));
|
|
}
|
|
|
|
[Fact]
|
|
public void Shadow_hands_the_selected_seams_a_shadow_runner_and_never_reports_its_pool_to_readiness()
|
|
{
|
|
// lmxopcua#780 P2. Unset Seams: VirtualTag + Calculation; alarms only when listed (same rule as OutOfProcess).
|
|
var options = new ScriptingOptions { Runner = ScriptRunnerMode.Shadow };
|
|
ScriptRunnerSelection.Validate(options, "/srv").ShouldContain(w => w.Contains("runs twice") && w.Contains("time-boxed diagnostic"));
|
|
using (var selection = ScriptRunnerSelection.Create(options, Silent, null))
|
|
{
|
|
var vt = selection.ForSeam(ScriptSeam.VirtualTag).ShouldBeOfType<ShadowScriptRunner>();
|
|
selection.ForSeam(ScriptSeam.Calculation).ShouldBeSameAs(vt, "one node-wide shadow");
|
|
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeNull();
|
|
vt.Shadow.ShouldBeSameAs(selection.OutOfProcessRunner);
|
|
selection.Shadow.ShouldBeSameAs(vt);
|
|
selection.Status().ShouldBeNull("readiness must not see the shadow pool");
|
|
selection.ShadowStatus().ShouldNotBeNull();
|
|
}
|
|
|
|
var withAlarms = new ScriptingOptions { Runner = ScriptRunnerMode.Shadow, RunnerSeams = [ScriptSeam.ScriptedAlarm] };
|
|
using (var selection = ScriptRunnerSelection.Create(withAlarms, Silent, null))
|
|
{
|
|
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeOfType<ShadowScriptRunner>();
|
|
selection.ForSeam(ScriptSeam.VirtualTag).ShouldBeNull();
|
|
}
|
|
|
|
// Out of process: no shadow, the pool's status IS readiness.
|
|
using (var oop = ScriptRunnerSelection.Create(new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess }, Silent, null))
|
|
{
|
|
oop.Shadow.ShouldBeNull();
|
|
oop.ShadowStatus().ShouldBeNull();
|
|
oop.Status().ShouldNotBeNull();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Scripting_Shadow_binds_and_clamps()
|
|
{
|
|
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
|
{
|
|
["Scripting:Runner"] = "Shadow",
|
|
["Scripting:Shadow:MaxInFlight"] = "0",
|
|
["Scripting:Shadow:MismatchLogsPerMinute"] = "5",
|
|
}).Build();
|
|
var bound = ScriptingOptionsBinding.Bind(config, out var adjustments);
|
|
bound.Runner.ShouldBe(ScriptRunnerMode.Shadow);
|
|
bound.Shadow.MaxInFlight.ShouldBe(1);
|
|
bound.Shadow.MismatchLogsPerMinute.ShouldBe(5);
|
|
adjustments.ShouldContain(a => a.Contains("Scripting:Shadow:MaxInFlight"));
|
|
}
|
|
|
|
[Fact]
|
|
public void OutOfProcess_selects_only_the_supported_requested_seams()
|
|
{
|
|
// Unset Seams: VirtualTag + Calculation. Scripted alarms stay in process unless listed (design §14 #10).
|
|
var unset = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess };
|
|
ScriptRunnerSelection.Validate(unset, "/srv").ShouldNotContain(w => w.Contains("cannot run out of process"));
|
|
unset.IsOutOfProcess(ScriptSeam.ScriptedAlarm).ShouldBeFalse();
|
|
unset.IsOutOfProcess(ScriptSeam.VirtualTag).ShouldBeTrue();
|
|
using (var selection = ScriptRunnerSelection.Create(unset, Silent, null))
|
|
{
|
|
selection.ForSeam(ScriptSeam.VirtualTag).ShouldNotBeNull();
|
|
selection.ForSeam(ScriptSeam.Calculation).ShouldBeSameAs(selection.ForSeam(ScriptSeam.VirtualTag), "one node-wide runner");
|
|
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeNull("alarms go out of process only when listed explicitly");
|
|
}
|
|
|
|
// Listed explicitly: the alarm seam shares the node's one runner.
|
|
var withAlarms = new ScriptingOptions
|
|
{
|
|
Runner = ScriptRunnerMode.OutOfProcess,
|
|
RunnerSeams = [ScriptSeam.VirtualTag, ScriptSeam.ScriptedAlarm],
|
|
};
|
|
withAlarms.IsOutOfProcess(ScriptSeam.ScriptedAlarm).ShouldBeTrue();
|
|
ScriptRunnerSelection.Validate(withAlarms, "/srv").ShouldNotContain(w => w.Contains("cannot run out of process"));
|
|
using (var selection = ScriptRunnerSelection.Create(withAlarms, Silent, null))
|
|
{
|
|
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldNotBeNull();
|
|
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeSameAs(selection.ForSeam(ScriptSeam.VirtualTag), "one node-wide runner");
|
|
selection.ForSeam(ScriptSeam.Calculation).ShouldBeNull();
|
|
((IScriptRunnerProvider)selection).ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeSameAs(selection.ForSeam(ScriptSeam.ScriptedAlarm));
|
|
}
|
|
|
|
var calcOnly = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess, RunnerSeams = [ScriptSeam.Calculation] };
|
|
using (var selection = ScriptRunnerSelection.Create(calcOnly, Silent, null))
|
|
{
|
|
selection.ForSeam(ScriptSeam.Calculation).ShouldNotBeNull();
|
|
selection.ForSeam(ScriptSeam.VirtualTag).ShouldBeNull();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void A_missing_worker_is_a_startup_warning_not_a_silent_in_process_fallback()
|
|
{
|
|
var options = new ScriptingOptions
|
|
{
|
|
Runner = ScriptRunnerMode.OutOfProcess,
|
|
RunnerSeams = [ScriptSeam.VirtualTag],
|
|
Worker = new ScriptWorkerOptions { ExecutablePath = Path.Combine(Path.GetTempPath(), "missing-" + Guid.NewGuid().ToString("N"), "w.dll") },
|
|
};
|
|
ScriptRunnerSelection.Validate(options, "/srv").ShouldHaveSingleItem().ShouldContain("BadResourceUnavailable");
|
|
}
|
|
}
|