feat(scripting): out-of-process pre-emptible script runner behind Scripting:Runner (lmxopcua#780 P1)
v2-ci / unit-tests (push) Waiting to run
v2-ci / unit-tests (push) Waiting to run
Opt-in (Scripting:Runner=OutOfProcess; InProcess stays the default and its code path is unchanged). Implements the P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md with the owner's §14 decisions: - Options: Scripting:Runner (InProcess | Shadow | OutOfProcess), Scripting:Runner:Seams, Scripting:Worker:* (Shards 2, HardGrace max(1 s,T), HeapLimitBytes 512 MiB, QuarantineAfterKills 2, UnavailableGrace 5 s, IsolationProbationRuns 20, PeCacheDir validated absolute + outside the content root (fail closed), ExecutablePath). Shadow is not implemented yet: runs InProcess with a startup Warning. - Core.Scripting: IScriptRunner seam + InProcessScriptRunner (today's guard, the parity baseline). - Core.Scripting.Protocol: protobuf envelopes, 4-byte LE framing (zero/oversize/truncated/garbage refused), Variant carrying the exact CLR type; unsupported types are explicit failures. - ScriptWorker app: compiles in the worker, P-thread scheduler, host-driven soft cancel, fail-fast watchdog, heartbeats, ctx.Logger forwarding, exits on pipe EOF / parent gone, forbidden-assembly self-check; Akka and Grpc.Core.Api excluded from its folder. Shipped in script-worker/ (Build + Publish). - Core.Scripting.OutOfProcess: K shards + isolation worker, soft T / hard T+G kill, attribution (deadline kill = one key; crash = suspects -> isolation; isolation crash = strike), kill-strike quarantine, backoff 250 ms -> 30 s, fail closed, cleared allow-listed env, nonce + pid handshake on a current-user pipe, Windows Job Object. - VirtualTag + Calculation seams use the runner when selected; a runner-side failure (worker restarted / runner unavailable) publishes BadResourceUnavailable (0x80040000) — ScadaBridge#112. Script faults keep BadInternalError. The scripted-alarm seam stays in process (alarms last). Tests: new ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests (Category=ScriptWorker, real processes; green on macOS and on windev/Windows), seam tests in Host/Runtime/Calculation suites; mutation-checked. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
+170
@@ -0,0 +1,170 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — <c>Scripting:Runner</c> binds by its documented keys (design §12), defaults to InProcess (every seam
|
||||
/// gets no runner, so the in-process path is untouched), and the worker options clamp and fail closed.
|
||||
/// </summary>
|
||||
public sealed class ScriptRunnerSelectionTests
|
||||
{
|
||||
private static IConfiguration Config(Dictionary<string, string?> values) =>
|
||||
new ConfigurationBuilder().AddInMemoryCollection(values).Build();
|
||||
|
||||
private static readonly ILogger Silent = new LoggerConfiguration().CreateLogger();
|
||||
|
||||
[Fact]
|
||||
public void The_default_is_InProcess_and_selects_no_runner_for_any_seam()
|
||||
{
|
||||
var bound = ScriptingOptionsBinding.Bind(Config(new()), out var adjustments);
|
||||
adjustments.ShouldBeEmpty();
|
||||
bound.Runner.ShouldBe(ScriptRunnerMode.InProcess);
|
||||
ScriptRunnerSelection.Validate(bound, "/srv/app").ShouldBeEmpty();
|
||||
using var selection = ScriptRunnerSelection.Create(bound, Silent, null);
|
||||
foreach (var seam in Enum.GetValues<ScriptSeam>()) selection.ForSeam(seam).ShouldBeNull(seam.ToString());
|
||||
selection.OutOfProcessRunner.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_documented_runner_key_binds()
|
||||
{
|
||||
var bound = ScriptingOptionsBinding.Bind(Config(new()
|
||||
{
|
||||
["Scripting:Runner"] = "OutOfProcess",
|
||||
["Scripting:Runner:Seams:0"] = "Calculation",
|
||||
["Scripting:Worker:Shards"] = "3",
|
||||
["Scripting:Worker:HardGrace"] = "00:00:02",
|
||||
["Scripting:Worker:HeapLimitBytes"] = "268435456",
|
||||
["Scripting:Worker:QuarantineAfterKills"] = "4",
|
||||
["Scripting:Worker:UnavailableGrace"] = "00:00:07",
|
||||
["Scripting:Worker:IsolationProbationRuns"] = "9",
|
||||
["Scripting:Worker:PeCacheDir"] = "/var/cache/otopcua-pe",
|
||||
["Scripting:Worker:ExecutablePath"] = "/opt/w/ZB.MOM.WW.OtOpcUa.ScriptWorker.dll",
|
||||
}), out var adjustments);
|
||||
|
||||
adjustments.ShouldBeEmpty();
|
||||
bound.Runner.ShouldBe(ScriptRunnerMode.OutOfProcess);
|
||||
bound.RunnerSeams.ShouldBe([ScriptSeam.Calculation]);
|
||||
bound.Worker.Shards.ShouldBe(3);
|
||||
bound.Worker.HardGrace.ShouldBe(TimeSpan.FromSeconds(2));
|
||||
bound.Worker.HeapLimitBytes.ShouldBe(256L * 1024 * 1024);
|
||||
bound.Worker.QuarantineAfterKills.ShouldBe(4);
|
||||
bound.Worker.UnavailableGrace.ShouldBe(TimeSpan.FromSeconds(7));
|
||||
bound.Worker.IsolationProbationRuns.ShouldBe(9);
|
||||
bound.Worker.PeCacheDir.ShouldBe("/var/cache/otopcua-pe");
|
||||
bound.Worker.ExecutablePath.ShouldBe("/opt/w/ZB.MOM.WW.OtOpcUa.ScriptWorker.dll");
|
||||
bound.IsOutOfProcess(ScriptSeam.Calculation).ShouldBeTrue();
|
||||
bound.IsOutOfProcess(ScriptSeam.VirtualTag).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_misspelled_runner_mode_fails_startup()
|
||||
=> Should.Throw<InvalidOperationException>(() =>
|
||||
ScriptingOptionsBinding.Bind(Config(new() { ["Scripting:Runner"] = "OutOfProc" }), out _));
|
||||
|
||||
[Fact]
|
||||
public void Worker_values_out_of_range_are_clamped_and_reported()
|
||||
{
|
||||
var bound = ScriptingOptionsBinding.Bind(Config(new()
|
||||
{
|
||||
["Scripting:Worker:Shards"] = "64",
|
||||
["Scripting:Worker:HardGrace"] = "00:00:00.010",
|
||||
["Scripting:Worker:HeapLimitBytes"] = "1024",
|
||||
["Scripting:Worker:QuarantineAfterKills"] = "0",
|
||||
["Scripting:Worker:UnavailableGrace"] = "00:05:00",
|
||||
["Scripting:Worker:IsolationProbationRuns"] = "0",
|
||||
}), out var adjustments);
|
||||
|
||||
bound.Worker.Shards.ShouldBe(ScriptWorkerOptions.MaxShards);
|
||||
bound.Worker.HardGrace.ShouldBe(ScriptWorkerOptions.MinHardGrace);
|
||||
bound.Worker.HeapLimitBytes.ShouldBe(ScriptWorkerOptions.MinHeapLimitBytes);
|
||||
bound.Worker.QuarantineAfterKills.ShouldBe(1);
|
||||
bound.Worker.UnavailableGrace.ShouldBe(ScriptWorkerOptions.MaxUnavailableGrace);
|
||||
bound.Worker.IsolationProbationRuns.ShouldBe(1);
|
||||
adjustments.Count.ShouldBe(6);
|
||||
adjustments.ShouldAllBe(a => a.StartsWith("Scripting:Worker:"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_hard_grace_defaults_to_max_of_one_second_and_the_budget()
|
||||
{
|
||||
var worker = new ScriptWorkerOptions();
|
||||
worker.EffectiveHardGrace(TimeSpan.FromMilliseconds(250)).ShouldBe(TimeSpan.FromSeconds(1));
|
||||
worker.EffectiveHardGrace(TimeSpan.FromSeconds(2)).ShouldBe(TimeSpan.FromSeconds(2));
|
||||
new ScriptWorkerOptions { HardGrace = TimeSpan.FromMilliseconds(400) }
|
||||
.EffectiveHardGrace(TimeSpan.FromSeconds(2)).ShouldBe(TimeSpan.FromMilliseconds(400));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("relative/cache", "must be an absolute path")]
|
||||
[InlineData("{root}/pe-cache", "inside the content root")]
|
||||
[InlineData("{root}", "inside the content root")]
|
||||
public void A_pe_cache_dir_that_is_relative_or_inside_the_content_root_fails_startup(string dir, string why)
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "otopcua-root-" + Guid.NewGuid().ToString("N"));
|
||||
var options = new ScriptingOptions
|
||||
{
|
||||
Runner = ScriptRunnerMode.OutOfProcess,
|
||||
Worker = new ScriptWorkerOptions { PeCacheDir = dir.Replace("{root}", root) },
|
||||
};
|
||||
Should.Throw<InvalidOperationException>(() => ScriptRunnerSelection.Validate(options, root)).Message.ShouldContain(why);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pe_cache_dir_outside_the_content_root_is_accepted()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "otopcua-root");
|
||||
var options = new ScriptingOptions
|
||||
{
|
||||
Runner = ScriptRunnerMode.OutOfProcess,
|
||||
Worker = new ScriptWorkerOptions { PeCacheDir = Path.Combine(Path.GetTempPath(), "otopcua-root-sibling") },
|
||||
};
|
||||
Should.NotThrow(() => ScriptRunnerSelection.Validate(options, root));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Shadow_is_not_implemented_yet_and_runs_in_process_with_a_warning()
|
||||
{
|
||||
var options = new ScriptingOptions { Runner = ScriptRunnerMode.Shadow };
|
||||
ScriptRunnerSelection.Validate(options, "/srv").ShouldHaveSingleItem().ShouldContain("Shadow");
|
||||
using var selection = ScriptRunnerSelection.Create(options, Silent, null);
|
||||
selection.OutOfProcessRunner.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OutOfProcess_selects_only_the_supported_requested_seams()
|
||||
{
|
||||
var all = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess };
|
||||
ScriptRunnerSelection.Validate(all, "/srv").ShouldContain(w => w.Contains("ScriptedAlarm seam cannot run out of process yet"));
|
||||
using (var selection = ScriptRunnerSelection.Create(all, Silent, null))
|
||||
{
|
||||
selection.ForSeam(ScriptSeam.VirtualTag).ShouldNotBeNull();
|
||||
selection.ForSeam(ScriptSeam.Calculation).ShouldBeSameAs(selection.ForSeam(ScriptSeam.VirtualTag), "one node-wide runner");
|
||||
selection.ForSeam(ScriptSeam.ScriptedAlarm).ShouldBeNull();
|
||||
}
|
||||
|
||||
var calcOnly = new ScriptingOptions { Runner = ScriptRunnerMode.OutOfProcess, RunnerSeams = [ScriptSeam.Calculation] };
|
||||
using (var selection = ScriptRunnerSelection.Create(calcOnly, Silent, null))
|
||||
{
|
||||
selection.ForSeam(ScriptSeam.Calculation).ShouldNotBeNull();
|
||||
selection.ForSeam(ScriptSeam.VirtualTag).ShouldBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_missing_worker_is_a_startup_warning_not_a_silent_in_process_fallback()
|
||||
{
|
||||
var options = new ScriptingOptions
|
||||
{
|
||||
Runner = ScriptRunnerMode.OutOfProcess,
|
||||
RunnerSeams = [ScriptSeam.VirtualTag],
|
||||
Worker = new ScriptWorkerOptions { ExecutablePath = Path.Combine(Path.GetTempPath(), "missing-" + Guid.NewGuid().ToString("N"), "w.dll") },
|
||||
};
|
||||
ScriptRunnerSelection.Validate(options, "/srv").ShouldHaveSingleItem().ShouldContain("BadResourceUnavailable");
|
||||
}
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Engines;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Engines;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — the virtual-tag seam over <see cref="IScriptRunner"/>: with a runner (OutOfProcess) the compiled run
|
||||
/// is the runner's, keyed by virtual-tag id with the evaluator's budget, and its Unavailable outcome becomes a failure
|
||||
/// carrying <see cref="OpcUaStatus.BadResourceUnavailable"/>. Without one (the InProcess default) nothing changes.
|
||||
/// </summary>
|
||||
public sealed class RoslynVirtualTagEvaluatorScriptRunnerTests
|
||||
{
|
||||
private sealed class FakeRunner : IScriptRunner
|
||||
{
|
||||
public ConcurrentQueue<ScriptRunOutcome> Outcomes { get; } = new();
|
||||
public ConcurrentQueue<ScriptInvocation> Invocations { get; } = new();
|
||||
public List<ScriptContextKind> Resets { get; } = [];
|
||||
|
||||
public Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default)
|
||||
{
|
||||
Invocations.Enqueue(invocation);
|
||||
return Task.FromResult(Outcomes.TryDequeue(out var next) ? next : ScriptRunOutcome.Ok(null));
|
||||
}
|
||||
|
||||
public void Reset(ScriptContextKind kind) => Resets.Add(kind);
|
||||
|
||||
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind) => ["vt-q"];
|
||||
}
|
||||
|
||||
private static RoslynVirtualTagEvaluator Build(IScriptRunner? runner) =>
|
||||
new(NullLogger<RoslynVirtualTagEvaluator>.Instance, new ScriptRootLogger(new LoggerConfiguration().CreateLogger()),
|
||||
runTimeout: TimeSpan.FromMilliseconds(1500), scriptRunner: runner);
|
||||
|
||||
private static Dictionary<string, DataValueSnapshot> Deps(int a) => new(StringComparer.Ordinal)
|
||||
{
|
||||
["a"] = new DataValueSnapshot(a, 0u, DateTime.UtcNow, DateTime.UtcNow),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task Compiled_runs_go_to_the_runner_and_its_outcomes_map_onto_the_seam_contract()
|
||||
{
|
||||
var runner = new FakeRunner();
|
||||
runner.Outcomes.Enqueue(ScriptRunOutcome.Ok(42));
|
||||
runner.Outcomes.Enqueue(ScriptRunOutcome.Unavailable(ScriptRunOutcome.WorkerRestartedReason));
|
||||
runner.Outcomes.Enqueue(ScriptRunOutcome.Failed("script threw: boom"));
|
||||
using var sut = Build(runner);
|
||||
const string script = "return (int)ctx.GetTag(\"a\").Value + 1;";
|
||||
|
||||
var ok = await sut.EvaluateAsync("vt-1", script, Deps(41));
|
||||
ok.Success.ShouldBeTrue();
|
||||
ok.Value.ShouldBe(42);
|
||||
ok.StatusCode.ShouldBeNull();
|
||||
|
||||
var unavailable = await sut.EvaluateAsync("vt-1", script, Deps(41));
|
||||
unavailable.Success.ShouldBeFalse();
|
||||
unavailable.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
unavailable.StatusCode.ShouldBe(OpcUaStatus.BadResourceUnavailable);
|
||||
|
||||
var failed = await sut.EvaluateAsync("vt-1", script, Deps(41));
|
||||
failed.Reason.ShouldBe("script threw: boom");
|
||||
failed.StatusCode.ShouldBeNull("a script failure keeps BadInternalError");
|
||||
|
||||
var invocation = runner.Invocations.First();
|
||||
invocation.Key.ShouldBe("vt-1");
|
||||
invocation.Kind.ShouldBe(ScriptContextKind.VirtualTag);
|
||||
invocation.Source.ShouldBe(script);
|
||||
invocation.Timeout.ShouldBe(TimeSpan.FromMilliseconds(1500));
|
||||
invocation.Inputs["a"].Value.ShouldBe(41);
|
||||
invocation.ScriptLogger.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_passthrough_fast_path_and_an_empty_expression_never_reach_the_runner()
|
||||
{
|
||||
var runner = new FakeRunner();
|
||||
using var sut = Build(runner);
|
||||
(await sut.EvaluateAsync("vt-p", "return ctx.GetTag(\"a\").Value;", Deps(7))).Value.ShouldBe(7);
|
||||
(await sut.EvaluateAsync("vt-e", " ", Deps(7))).Success.ShouldBeFalse();
|
||||
runner.Invocations.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_deploy_boundary_resets_the_runners_virtual_tag_seam_and_its_quarantines_are_reported()
|
||||
{
|
||||
var runner = new FakeRunner();
|
||||
using var sut = Build(runner);
|
||||
sut.ClearCompiledScripts();
|
||||
sut.ResetQuarantines();
|
||||
runner.Resets.ShouldBe([ScriptContextKind.VirtualTag, ScriptContextKind.VirtualTag]);
|
||||
sut.QuarantinedScripts.ShouldBe(["vt-q"]);
|
||||
}
|
||||
|
||||
/// <summary>Positive control for the default: no runner means the in-process Roslyn path actually runs the script.</summary>
|
||||
[Fact]
|
||||
public async Task Without_a_runner_the_script_runs_in_process()
|
||||
{
|
||||
using var sut = Build(runner: null);
|
||||
sut.ScriptRunner.ShouldBeNull();
|
||||
(await sut.EvaluateAsync("vt-1", "return (int)ctx.GetTag(\"a\").Value + 1;", Deps(41))).Value.ShouldBe(42);
|
||||
sut.RunGuard.StartedRunCount.ShouldBe(1);
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — when the out-of-process runner, not the script, fails (its worker restarted, no worker ready), the
|
||||
/// virtual tag publishes <see cref="OpcUaStatus.BadResourceUnavailable"/> with its last value, so a client can tell a
|
||||
/// sick node from a broken script. Every other failure keeps the long-standing <see cref="OpcUaStatus.BadInternalError"/>.
|
||||
/// </summary>
|
||||
public sealed class VirtualTagActorRunnerUnavailableTests : RuntimeActorTestBase
|
||||
{
|
||||
private sealed class QueuedEvaluator : IVirtualTagEvaluator
|
||||
{
|
||||
public ConcurrentQueue<VirtualTagEvalResult> Results { get; } = new();
|
||||
|
||||
public VirtualTagEvalResult Evaluate(string virtualTagId, string expression, IReadOnlyDictionary<string, object?> dependencies)
|
||||
=> Results.TryDequeue(out var next) ? next : VirtualTagEvalResult.NoChange;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_runner_unavailable_failure_publishes_BadResourceUnavailable_and_a_script_failure_BadInternalError()
|
||||
{
|
||||
var parent = CreateTestProbe();
|
||||
var evaluator = new QueuedEvaluator();
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Ok(5));
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Unavailable("script worker restarted"));
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Unavailable("script worker restarted"));
|
||||
evaluator.Results.Enqueue(VirtualTagEvalResult.Failure("script threw: boom"));
|
||||
var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "x", evaluator: evaluator, dependencyRefs: ["a"]));
|
||||
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
|
||||
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().StatusCode.ShouldBe(OpcUaStatus.Good);
|
||||
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow));
|
||||
var unavailable = parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget);
|
||||
unavailable.StatusCode.ShouldBe(OpcUaStatus.BadResourceUnavailable);
|
||||
unavailable.Value.ShouldBe(5, "the last known value rides out with the Bad status");
|
||||
|
||||
// Once per transition: the same code again publishes nothing.
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 3, DateTime.UtcNow));
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
|
||||
// A script failure is a different Bad and is published.
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 4, DateTime.UtcNow));
|
||||
var scriptFault = parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget);
|
||||
scriptFault.StatusCode.ShouldBe(OpcUaStatus.BadInternalError);
|
||||
scriptFault.Value.ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_failure_without_a_status_keeps_BadInternalError()
|
||||
{
|
||||
VirtualTagEvalResult.Failure("x").StatusCode.ShouldBeNull();
|
||||
VirtualTagEvalResult.Unavailable("x").StatusCode.ShouldBe(OpcUaStatus.BadResourceUnavailable);
|
||||
OpcUaStatus.BadResourceUnavailable.ShouldBe(0x80040000u, "the OPC UA Part 6 code ScadaBridge maps");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
using System.Diagnostics;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>Worker-backed tests spawn real processes and measure wall-clock bounds, so they never run in parallel.</summary>
|
||||
[CollectionDefinition(Name, DisableParallelization = true)]
|
||||
public sealed class ScriptWorkerCollection
|
||||
{
|
||||
public const string Name = "ScriptWorker";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 design §13 "Containment": each case runs against the real <c>ScriptWorker</c> process, and each has a
|
||||
/// positive control — a healthy sibling on the SAME shard (every runner here has one shard) that keeps producing
|
||||
/// correct values before and after the event.
|
||||
/// </summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class ContainmentTests
|
||||
{
|
||||
private const string Healthy = "return 7;";
|
||||
private const string Runaway = "while (true) { }";
|
||||
private const string Cooperative = "while (!ctx.CancellationToken.IsCancellationRequested) { } return 1;";
|
||||
|
||||
/// <summary>Spins for the number of milliseconds in input <c>ms</c>, so one source serves slow and fast runs.</summary>
|
||||
private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";
|
||||
|
||||
private static IReadOnlyDictionary<string, DataValueSnapshot> Ms(int ms) => Scripts.Input("ms", ms);
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
[Fact]
|
||||
public async Task A_healthy_script_runs_out_of_process_and_reads_its_inputs()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
var outcome = await runner.RunAsync(Scripts.Vt("vt", "return (int)ctx.GetTag(\"x\").Value + 1;",
|
||||
inputs: Scripts.Input("x", 41)), Ct);
|
||||
|
||||
outcome.Status.ShouldBe(ScriptRunStatus.Ok, outcome.Reason);
|
||||
outcome.Value.ShouldBe(42);
|
||||
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(Environment.ProcessId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The headline case: <c>while(true){}</c> ignores the token. In process it would spin a host core forever; here
|
||||
/// the caller gets the usual timeout at T, the worker is killed at T + G, the host burns no CPU on it, and the
|
||||
/// sibling on the same shard keeps working. Mutation check: with the hard-deadline kill disabled the worker is
|
||||
/// never killed and the Exited wait times out.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_runaway_script_is_killed_at_its_hard_deadline_and_its_sibling_keeps_working()
|
||||
{
|
||||
var softT = TimeSpan.FromMilliseconds(400);
|
||||
var grace = TimeSpan.FromMilliseconds(500);
|
||||
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 2));
|
||||
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var outcome = await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct);
|
||||
var softElapsed = sw.Elapsed;
|
||||
outcome.Status.ShouldBe(ScriptRunStatus.Failed);
|
||||
outcome.Reason!.ShouldStartWith("script timed out after ");
|
||||
softElapsed.ShouldBeLessThan(softT + TimeSpan.FromMilliseconds(400), "the caller is answered at the soft deadline");
|
||||
|
||||
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
var killElapsed = sw.Elapsed;
|
||||
exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline);
|
||||
exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "runaway"));
|
||||
killElapsed.ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50));
|
||||
// Design budget is T + G + 100 ms; the slack here is for a loaded CI box (process reap + exit notification).
|
||||
killElapsed.ShouldBeLessThan(softT + grace + TimeSpan.FromMilliseconds(1500));
|
||||
|
||||
// No orphan in the host: an in-process while(true) would burn one host core for the whole window.
|
||||
var cpuBefore = Process.GetCurrentProcess().TotalProcessorTime;
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(1500), Ct);
|
||||
var cpuUsed = Process.GetCurrentProcess().TotalProcessorTime - cpuBefore;
|
||||
cpuUsed.ShouldBeLessThan(TimeSpan.FromMilliseconds(1000), "the runaway's thread died with its worker");
|
||||
|
||||
// The sibling recovers on a fresh worker.
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(worker.Pid);
|
||||
|
||||
// A second kill quarantines (QuarantineAfterKills = 2); the third trigger is refused without running.
|
||||
var second = runner.Shards[0].Current!;
|
||||
(await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct)).Status.ShouldBe(ScriptRunStatus.Failed);
|
||||
(await second.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct)).Reason.ShouldBe(WorkerExitReason.KilledDeadline);
|
||||
// The strike is recorded by the shard's exit handler, which runs asynchronously to the Exited completion.
|
||||
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1,
|
||||
TimeSpan.FromSeconds(5), "the second kill strike");
|
||||
var refused = await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct);
|
||||
refused.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
refused.Reason.ShouldContain("hard deadline");
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"]);
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
|
||||
// Lifted by a source change (the operator fixed it)…
|
||||
(await runner.RunAsync(Scripts.Vt("runaway", "return 1;"), Ct)).Value.ShouldBe(1);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_redeploy_lifts_a_kill_quarantine_for_its_seam_only()
|
||||
{
|
||||
var softT = TimeSpan.FromMilliseconds(300);
|
||||
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(300), quarantineAfterKills: 1));
|
||||
|
||||
(await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct)).Status.ShouldBe(ScriptRunStatus.Failed);
|
||||
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(10), "the kill quarantine");
|
||||
|
||||
runner.Reset(ScriptContextKind.Calculation);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"], "another seam's redeploy leaves it alone");
|
||||
runner.Reset(ScriptContextKind.VirtualTag);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>A script that polls its token ends itself at T: no kill, no strike beyond the timeout (design §13).</summary>
|
||||
[Fact]
|
||||
public async Task A_cooperative_script_ends_at_its_soft_deadline_without_a_kill()
|
||||
{
|
||||
var softT = TimeSpan.FromMilliseconds(400);
|
||||
var grace = TimeSpan.FromMilliseconds(1500);
|
||||
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 1));
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
var outcome = await runner.RunAsync(Scripts.Vt("polite", Cooperative, softT), Ct);
|
||||
outcome.Reason!.ShouldStartWith("script timed out after ");
|
||||
|
||||
await Task.Delay(softT + grace + TimeSpan.FromMilliseconds(500), Ct);
|
||||
worker.IsAlive.ShouldBeTrue("a script that ended itself must not cost the shard a restart");
|
||||
runner.Shards[0].Current!.Pid.ShouldBe(worker.Pid);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("no kill strike (QuarantineAfterKills = 1)");
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_second_trigger_while_the_first_still_runs_is_refused_not_queued()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
|
||||
var first = runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(1500)), Ct);
|
||||
await Task.Delay(200, Ct);
|
||||
var second = await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct);
|
||||
second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
|
||||
(await first).Value.ShouldBe(1500);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>kill -9</c> of the worker: the evaluation in flight is a victim — Unavailable "script worker restarted"
|
||||
/// (BadResourceUnavailable), not a script fault — and, with no deadline crossed, a suspect that proves itself in
|
||||
/// the isolation worker and then returns to its shard.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task An_externally_killed_worker_makes_its_in_flight_script_a_suspect_that_recovers_in_isolation()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(probation: 2));
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
var inFlight = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
||||
await Task.Delay(300, Ct);
|
||||
Process.GetProcessById(worker.Pid).Kill();
|
||||
|
||||
var victim = await inFlight.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
victim.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
victim.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
(await worker.Exited).Reason.ShouldBe(WorkerExitReason.Crash);
|
||||
runner.SuspectKeys.ShouldBe([InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "spin")]);
|
||||
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
runner.Isolation.Current.ShouldNotBeNull("a suspect runs in the isolation worker");
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
runner.SuspectKeys.ShouldBeEmpty("two clean runs (probation = 2) return it to its shard");
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
|
||||
|
||||
// The sibling recovers on a fresh shard worker.
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
runner.Shards[0].Current!.Pid.ShouldNotBe(worker.Pid);
|
||||
}
|
||||
|
||||
/// <summary>The isolation worker runs one evaluation at a time, so a crash there names exactly one key.</summary>
|
||||
[Fact]
|
||||
public async Task A_crash_in_the_isolation_worker_strikes_its_one_key_and_quarantines_it()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(quarantineAfterKills: 2));
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
|
||||
async Task<ScriptRunOutcome> RunAndKill(Func<WorkerConnection?> worker)
|
||||
{
|
||||
var run = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
||||
var live = await Scripts.Eventually(worker, TimeSpan.FromSeconds(15), "a live worker");
|
||||
await Scripts.Eventually(() => live.InFlightCount == 1, TimeSpan.FromSeconds(15), "the evaluation in flight");
|
||||
await Task.Delay(200, Ct);
|
||||
Process.GetProcessById(live.Pid).Kill();
|
||||
return await run.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
}
|
||||
|
||||
(await RunAndKill(() => runner.Shards[0].Current)).Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
runner.SuspectKeys.Count.ShouldBe(1);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("an ambiguous shard crash is not a strike");
|
||||
|
||||
(await RunAndKill(() => runner.Isolation.Current)).Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("one isolation crash is one strike");
|
||||
(await RunAndKill(() => runner.Isolation.Current)).Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1,
|
||||
TimeSpan.FromSeconds(5), "the isolated-crash quarantine");
|
||||
|
||||
var refused = await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct);
|
||||
refused.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
||||
refused.Reason.ShouldContain("isolation worker");
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
}
|
||||
|
||||
/// <summary>A deadline kill blames exactly one key: the sibling in flight on the same worker is a victim, not a suspect.</summary>
|
||||
[Fact]
|
||||
public async Task A_deadline_kill_blames_only_its_culprit()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(300), quarantineAfterKills: 1));
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
(await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7);
|
||||
// Compile the runaway first, so both evaluations are on the worker at once.
|
||||
_ = runner.Shards[0].Current!;
|
||||
|
||||
var victim = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct);
|
||||
await Task.Delay(200, Ct);
|
||||
var culprit = await runner.RunAsync(Scripts.Vt("runaway", Runaway, TimeSpan.FromMilliseconds(300)), Ct);
|
||||
culprit.Reason!.ShouldStartWith("script timed out after ");
|
||||
|
||||
var victimOutcome = await victim.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
||||
victimOutcome.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
victimOutcome.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(5), "the culprit's quarantine");
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"]);
|
||||
runner.SuspectKeys.ShouldBeEmpty("a deadline kill is unambiguous: nobody else is suspected");
|
||||
(await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>An allocation bomb dies in its worker (heap hard limit); the host's own memory does not move.</summary>
|
||||
[Fact]
|
||||
public async Task An_allocation_bomb_is_contained_in_its_worker()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(heapLimit: 128L * 1024 * 1024));
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
|
||||
GC.Collect();
|
||||
var hostBefore = GC.GetTotalMemory(forceFullCollection: true);
|
||||
var outcome = await runner.RunAsync(Scripts.Vt("bomb",
|
||||
"var hoard = new System.Collections.Generic.List<byte[]>(); while (true) { hoard.Add(new byte[1 << 20]); }",
|
||||
TimeSpan.FromSeconds(10)), Ct);
|
||||
var hostAfter = GC.GetTotalMemory(forceFullCollection: true);
|
||||
|
||||
outcome.Status.ShouldNotBe(ScriptRunStatus.Ok);
|
||||
(hostAfter - hostBefore).ShouldBeLessThan(32L * 1024 * 1024, "the hoard lived in the worker's heap, not the host's");
|
||||
await Scripts.Eventually(() => runner.Shards[0].Current, TimeSpan.FromSeconds(10), "a worker");
|
||||
(await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7);
|
||||
}
|
||||
|
||||
/// <summary>Fail closed (design §14 #4): no worker means Bad, never an in-process run.</summary>
|
||||
[Fact]
|
||||
public async Task A_worker_that_cannot_start_reports_unavailable_and_never_runs_in_process()
|
||||
{
|
||||
var missing = Path.Combine(Path.GetTempPath(), "no-such-dir-" + Guid.NewGuid().ToString("N"), OutOfProcessScriptRunner.WorkerAssemblyFileName);
|
||||
await using var runner = Scripts.Runner(Scripts.Options(workerPath: missing, unavailableGrace: TimeSpan.FromSeconds(1)));
|
||||
|
||||
var outcome = await runner.RunAsync(Scripts.Vt("vt", "return 1;"), Ct);
|
||||
outcome.Status.ShouldBe(ScriptRunStatus.Unavailable);
|
||||
outcome.Reason.ShouldBe(ScriptRunOutcome.RunnerUnavailableReason);
|
||||
outcome.ToVirtualTagEvalResult().StatusCode.ShouldBe(ZB.MOM.WW.OtOpcUa.Commons.OpcUa.OpcUaStatus.BadResourceUnavailable);
|
||||
runner.Shards[0].ConsecutiveFailures.ShouldBeGreaterThan(0, "the failed start backs off");
|
||||
}
|
||||
|
||||
/// <summary>A worker whose pipe reaches end of stream exits by itself (the Linux "host died" path, design §3.3).</summary>
|
||||
[Fact]
|
||||
public async Task A_worker_exits_by_itself_when_its_pipe_closes()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
using var process = Process.GetProcessById(worker.Pid);
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
worker.SeverPipeForTest();
|
||||
await process.WaitForExitAsync(Ct).WaitAsync(TimeSpan.FromSeconds(5), Ct);
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(2));
|
||||
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(5), Ct);
|
||||
exit.ExitCode.ShouldBe(ScriptWorkerEnvironment.ExitPipeClosed, "it noticed the EOF; the host did not have to kill it");
|
||||
}
|
||||
|
||||
/// <summary>Design §6: the environment is cleared and repopulated from an allow-list. Positive control first.</summary>
|
||||
[Fact]
|
||||
public async Task The_worker_environment_carries_no_host_secrets()
|
||||
{
|
||||
const string apiKey = "ServerHistorian__ApiKey";
|
||||
const string configDb = "ConnectionStrings__ConfigDb";
|
||||
Environment.SetEnvironmentVariable(apiKey, "sekrit");
|
||||
Environment.SetEnvironmentVariable(configDb, "Server=x;Password=y");
|
||||
try
|
||||
{
|
||||
Environment.GetEnvironmentVariable(apiKey).ShouldBe("sekrit", "positive control: the host has it");
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7);
|
||||
|
||||
var diag = await runner.Shards[0].Current!.DiagnosticsAsync(TimeSpan.FromSeconds(10));
|
||||
diag.EnvironmentNames.ShouldNotContain(apiKey);
|
||||
diag.EnvironmentNames.ShouldNotContain(configDb);
|
||||
diag.EnvironmentNames.ShouldContain(ScriptWorkerEnvironment.Nonce, "the allow-listed launch variables do arrive");
|
||||
diag.EnvironmentNames.ShouldContain("DOTNET_GCHeapHardLimit");
|
||||
diag.WorkingDirectory.ShouldNotBe(Environment.CurrentDirectory);
|
||||
// An empty per-worker directory under the temp root (macOS reports /var as /private/var, so match the leaf).
|
||||
diag.WorkingDirectory.ShouldContain(Path.Combine("otopcua-scriptworker", "otopcua-sw-"));
|
||||
Directory.EnumerateFileSystemEntries(diag.WorkingDirectory).ShouldBeEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Environment.SetEnvironmentVariable(apiKey, null);
|
||||
Environment.SetEnvironmentVariable(configDb, null);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Design §6: the host's cluster, database, OPC UA and secrets stacks are not in the worker.</summary>
|
||||
[Fact]
|
||||
public async Task The_worker_loads_none_of_the_host_stacks()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("vt", "return System.Math.Abs(-3);"), Ct)).Value.ShouldBe(3);
|
||||
var diag = await runner.Shards[0].Current!.DiagnosticsAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
diag.LoadedAssemblies.ShouldContain("ZB.MOM.WW.OtOpcUa.Core.Scripting", "positive control: the compile ran here");
|
||||
diag.LoadedAssemblies.ShouldContain("Microsoft.CodeAnalysis.CSharp");
|
||||
foreach (var prefix in ScriptWorkerEnvironment.ForbiddenAssemblyPrefixes)
|
||||
diag.LoadedAssemblies.ShouldNotContain(a => a.StartsWith(prefix, StringComparison.Ordinal), prefix);
|
||||
|
||||
// Not merely unloaded: Akka is not even deployable beside the worker.
|
||||
var folder = Path.GetDirectoryName(runner.WorkerPath)!;
|
||||
File.Exists(Path.Combine(folder, "ZB.MOM.WW.OtOpcUa.ScriptWorker.dll")).ShouldBeTrue("positive control");
|
||||
File.Exists(Path.Combine(folder, "Akka.dll")).ShouldBeFalse();
|
||||
File.Exists(Path.Combine(folder, "Grpc.Core.Api.dll")).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Script_log_lines_come_back_through_the_invocations_logger()
|
||||
{
|
||||
var sink = new CapturingSink();
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
(await runner.RunAsync(Scripts.Vt("vt", "ctx.Logger.Information(\"hello {N}\", 42); return 1;", logger: sink.CreateLogger()), Ct))
|
||||
.Value.ShouldBe(1);
|
||||
await Scripts.Eventually(() => sink.Rendered.Contains("hello 42"), TimeSpan.FromSeconds(5), "the forwarded log line");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_alarm_predicate_runs_out_of_process()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
var outcome = await runner.RunAsync(new ScriptInvocation("alarm-1", ScriptContextKind.AlarmPredicate,
|
||||
"return (double)ctx.GetTag(\"t\").Value > 50;", Scripts.Input("t", 75.0), TimeSpan.FromSeconds(5)), Ct);
|
||||
outcome.Status.ShouldBe(ScriptRunStatus.Ok, outcome.Reason);
|
||||
outcome.Value.ShouldBe(true);
|
||||
}
|
||||
|
||||
/// <summary>Design §14 #11: an unsupported type is an explicit failure, never a silent conversion.</summary>
|
||||
[Fact]
|
||||
public async Task Unsupported_input_and_result_types_are_explicit_failures()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
var result = await runner.RunAsync(Scripts.Vt("list", "return new System.Collections.Generic.List<int> { 1 };"), Ct);
|
||||
result.Status.ShouldBe(ScriptRunStatus.Failed);
|
||||
result.Reason!.ShouldStartWith("unsupported type: script returned a value of unsupported type System.Collections.Generic.List");
|
||||
|
||||
var input = await runner.RunAsync(Scripts.Vt("guid", "return 1;", inputs: Scripts.Input("g", Guid.NewGuid())), Ct);
|
||||
input.Status.ShouldBe(ScriptRunStatus.Failed);
|
||||
input.Reason!.ShouldStartWith("unsupported type: input 'g' has a value of unsupported type System.Guid");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using System.IO.Pipes;
|
||||
using System.Security.Cryptography;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>lmxopcua#780 design §6 "Pipe hardening": only the launched process, echoing its nonce, is accepted.</summary>
|
||||
public sealed class HandshakeTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static async Task<Exception?> HandshakeWith(Func<Envelope, Envelope> reply, string expectedNonce = "right", int? expectedPid = 1234)
|
||||
{
|
||||
var name = "otopcua-sw-test-" + Convert.ToHexString(RandomNumberGenerator.GetBytes(6));
|
||||
await using var server = new NamedPipeServerStream(name, PipeDirection.InOut, 1, PipeTransmissionMode.Byte,
|
||||
PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
|
||||
await using var client = new NamedPipeClientStream(".", name, PipeDirection.InOut, PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
|
||||
var accept = server.WaitForConnectionAsync(Ct);
|
||||
await client.ConnectAsync(TimeSpan.FromSeconds(5), Ct);
|
||||
await accept;
|
||||
|
||||
var rogue = Task.Run(async () =>
|
||||
{
|
||||
var hello = await ScriptWorkerFrameCodec.ReadAsync(client, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, Ct);
|
||||
await ScriptWorkerFrameCodec.WriteAsync(client, reply(hello!), ScriptWorkerFrameCodec.DefaultMaxFrameBytes, Ct);
|
||||
await client.FlushAsync(Ct);
|
||||
}, Ct);
|
||||
|
||||
try
|
||||
{
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(Ct);
|
||||
timeout.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
await WorkerConnection.HandshakeAsync(server, new Hello { Nonce = expectedNonce }, expectedNonce, expectedPid, timeout.Token);
|
||||
await rogue;
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return ex;
|
||||
}
|
||||
}
|
||||
|
||||
private static Envelope Ready(string nonce, int pid) => new()
|
||||
{
|
||||
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
||||
Ready = new Ready { Nonce = nonce, Pid = pid },
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task The_launched_process_echoing_its_nonce_is_accepted()
|
||||
=> (await HandshakeWith(_ => Ready("right", 1234))).ShouldBeNull("positive control");
|
||||
|
||||
[Fact]
|
||||
public async Task A_wrong_nonce_is_refused()
|
||||
=> (await HandshakeWith(_ => Ready("wrong", 1234))).ShouldBeOfType<ScriptWorkerProtocolException>()
|
||||
.Message.ShouldContain("nonce");
|
||||
|
||||
[Fact]
|
||||
public async Task A_client_that_is_not_the_launched_process_is_refused()
|
||||
=> (await HandshakeWith(_ => Ready("right", 999))).ShouldBeOfType<ScriptWorkerProtocolException>()
|
||||
.Message.ShouldContain("pid");
|
||||
|
||||
[Fact]
|
||||
public async Task Anything_but_ready_first_is_refused()
|
||||
=> (await HandshakeWith(_ => new Envelope
|
||||
{
|
||||
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
||||
Heartbeat = new Heartbeat(),
|
||||
})).ShouldBeOfType<ScriptWorkerProtocolException>();
|
||||
|
||||
[Fact]
|
||||
public async Task The_hello_carries_the_nonce_to_echo()
|
||||
{
|
||||
string? seen = null;
|
||||
(await HandshakeWith(hello =>
|
||||
{
|
||||
seen = hello.Hello.Nonce;
|
||||
return Ready(hello.Hello.Nonce, 1234);
|
||||
})).ShouldBeNull();
|
||||
seen.ShouldBe("right");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 design §13 "Parity": the same script, through <see cref="InProcessScriptRunner"/> (today's guard)
|
||||
/// and through the worker, gives the same status, reason, value and exact CLR value type.
|
||||
/// </summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class ParityTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
public static TheoryData<string, string> Cases() => new()
|
||||
{
|
||||
{ "arithmetic", "return 1 + 2;" },
|
||||
{ "string", "return \"a\" + ctx.GetTag(\"x\").Value;" },
|
||||
{ "throw", "throw new System.InvalidOperationException(\"boom\");" },
|
||||
{ "compile error", "return undefinedThing;" },
|
||||
{ "sandbox violation", "return System.IO.File.Exists(\"x\");" },
|
||||
{ "missing input status", "return ctx.GetTag(\"missing\").StatusCode;" },
|
||||
{ "input status", "return ctx.GetTag(\"x\").StatusCode;" },
|
||||
{ "source timestamp", "return ctx.GetTag(\"x\").SourceTimestampUtc;" },
|
||||
{ "clock", "return ctx.Now > new System.DateTime(2000, 1, 1);" },
|
||||
{ "null", "return null;" },
|
||||
{ "array", "return new double[] { 1.5, 2.5 };" },
|
||||
{ "cast of wrong type", "return (short)ctx.GetTag(\"x\").Value;" },
|
||||
};
|
||||
|
||||
private static readonly IReadOnlyDictionary<string, DataValueSnapshot> Inputs =
|
||||
new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal)
|
||||
{
|
||||
["x"] = new DataValueSnapshot(5, 0x40000000u, new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), DateTime.UtcNow),
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Cases))]
|
||||
public async Task In_and_out_of_process_agree(string name, string source)
|
||||
{
|
||||
using var inProcess = new InProcessScriptRunner();
|
||||
await using var outOfProcess = Scripts.Runner(Scripts.Options());
|
||||
var invocation = Scripts.Vt(name, source, inputs: Inputs);
|
||||
|
||||
var expected = await inProcess.RunAsync(invocation, Ct);
|
||||
var actual = await outOfProcess.RunAsync(invocation, Ct);
|
||||
|
||||
actual.Status.ShouldBe(expected.Status, $"{name}: {actual.Reason} vs {expected.Reason}");
|
||||
actual.Reason.ShouldBe(expected.Reason);
|
||||
Same(actual.Value, expected.Value, name);
|
||||
}
|
||||
|
||||
public static TheoryData<string, object> Typed() => new()
|
||||
{
|
||||
{ "bool", true }, { "sbyte", (sbyte)-2 }, { "byte", (byte)2 }, { "short", (short)-3 }, { "ushort", (ushort)3 },
|
||||
{ "int", -4 }, { "uint", 4u }, { "long", -5L }, { "ulong", 5UL }, { "float", 1.25f }, { "double", 2.5 },
|
||||
{ "string", "s" }, { "System.DateTime", new DateTime(2026, 11, 9, 0, 0, 0, DateTimeKind.Utc) },
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Unboxing parity (pins design assumption A-7): <c>(T)ctx.GetTag("x").Value</c> works out of process for every
|
||||
/// tag type, which needs the exact boxed CLR type to survive the wire.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[MemberData(nameof(Typed))]
|
||||
public async Task Unboxing_to_the_exact_type_matches(string typeName, object value)
|
||||
{
|
||||
using var inProcess = new InProcessScriptRunner();
|
||||
await using var outOfProcess = Scripts.Runner(Scripts.Options());
|
||||
var invocation = Scripts.Vt("unbox-" + typeName, $"var v = ({typeName})ctx.GetTag(\"x\").Value; return v;",
|
||||
inputs: Scripts.Input("x", value));
|
||||
|
||||
var expected = await inProcess.RunAsync(invocation, Ct);
|
||||
var actual = await outOfProcess.RunAsync(invocation, Ct);
|
||||
expected.Status.ShouldBe(ScriptRunStatus.Ok, expected.Reason);
|
||||
actual.Status.ShouldBe(ScriptRunStatus.Ok, actual.Reason);
|
||||
Same(actual.Value, expected.Value, typeName);
|
||||
}
|
||||
|
||||
private static void Same(object? actual, object? expected, string because)
|
||||
{
|
||||
if (expected is null)
|
||||
{
|
||||
actual.ShouldBeNull(because);
|
||||
return;
|
||||
}
|
||||
actual.ShouldNotBeNull(because);
|
||||
actual.GetType().ShouldBe(expected.GetType(), because);
|
||||
if (expected is Array array) ((Array)actual).Cast<object?>().ShouldBe(array.Cast<object?>(), because);
|
||||
else actual.ShouldBe(expected, because);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
using System.Buffers.Binary;
|
||||
using Google.Protobuf;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>lmxopcua#780 design §13 — frame codec and the CLR ↔ Variant mapping.</summary>
|
||||
public sealed class ProtocolTests
|
||||
{
|
||||
public static IEnumerable<object?[]> Values()
|
||||
{
|
||||
object?[] values =
|
||||
[
|
||||
null, true, false, (sbyte)-5, (byte)250, (short)-3, (ushort)7, -42, 42u, long.MinValue, ulong.MaxValue,
|
||||
1.5f, double.NaN, 3.25, "text", string.Empty, new DateTime(2026, 11, 9, 1, 2, 3, DateTimeKind.Utc),
|
||||
new[] { 1, 2, 3 }, new short[] { -1, 1 }, new[] { "a", "b" }, new[] { true, false }, Array.Empty<double>(),
|
||||
new[] { new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc) }, (short)0, 0, 0L, 0.0,
|
||||
];
|
||||
return values.Select(v => new[] { v });
|
||||
}
|
||||
|
||||
/// <summary>The exact boxed CLR type survives (design §4.4): <c>(short)ctx.GetTag(x).Value</c> must keep working.</summary>
|
||||
[Theory]
|
||||
[MemberData(nameof(Values))]
|
||||
public void Every_type_tag_round_trips_with_its_exact_clr_type(object? value)
|
||||
{
|
||||
VariantCodec.TryEncode(value, out var variant, out var unsupported).ShouldBeTrue();
|
||||
unsupported.ShouldBeNull();
|
||||
|
||||
// Through the wire, not just the object: a payload the serializer drops must still decode.
|
||||
var wire = Variant.Parser.ParseFrom(variant.ToByteArray());
|
||||
var decoded = VariantCodec.Decode(wire);
|
||||
|
||||
if (value is null)
|
||||
{
|
||||
decoded.ShouldBeNull();
|
||||
return;
|
||||
}
|
||||
decoded.ShouldNotBeNull();
|
||||
decoded.GetType().ShouldBe(value.GetType());
|
||||
if (value is Array expected)
|
||||
((Array)decoded).Cast<object?>().ShouldBe(expected.Cast<object?>());
|
||||
else if (value is double d && double.IsNaN(d))
|
||||
double.IsNaN((double)decoded).ShouldBeTrue();
|
||||
else
|
||||
decoded.ShouldBe(value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unsupported_types_are_refused_not_converted()
|
||||
{
|
||||
foreach (var value in new object[] { new List<int> { 1 }, 1.5m, Guid.NewGuid(), new { A = 1 }, new int[1, 1], new object[] { 1 } })
|
||||
{
|
||||
VariantCodec.TryEncode(value, out _, out var type).ShouldBeFalse(value.GetType().Name);
|
||||
type.ShouldNotBeNullOrEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Local_datetime_crosses_as_utc()
|
||||
{
|
||||
var local = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Local);
|
||||
VariantCodec.TryEncode(local, out var variant, out _).ShouldBeTrue();
|
||||
var decoded = (DateTime)VariantCodec.Decode(variant)!;
|
||||
decoded.Kind.ShouldBe(DateTimeKind.Utc);
|
||||
decoded.ShouldBe(local.ToUniversalTime());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_payload_that_contradicts_its_tag_is_a_protocol_fault()
|
||||
{
|
||||
Should.Throw<ScriptWorkerProtocolException>(() => VariantCodec.Decode(new Variant { Type = Variant.Types.TypeTag.Int16, S = "x" }));
|
||||
Should.Throw<ScriptWorkerProtocolException>(() => VariantCodec.Decode(new Variant { Type = (Variant.Types.TypeTag)99 }));
|
||||
Should.Throw<OverflowException>(() => VariantCodec.Decode(new Variant { Type = Variant.Types.TypeTag.Int16, I = 1 << 20 }));
|
||||
}
|
||||
|
||||
private static Envelope Sample() => new()
|
||||
{
|
||||
ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
|
||||
Seq = 7,
|
||||
Heartbeat = new Heartbeat { Inflight = 3, HeapBytes = 1234 },
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task A_frame_round_trips()
|
||||
{
|
||||
using var stream = new MemoryStream();
|
||||
await ScriptWorkerFrameCodec.WriteAsync(stream, Sample(), 1024, TestContext.Current.CancellationToken);
|
||||
stream.Position = 0;
|
||||
var read = await ScriptWorkerFrameCodec.ReadAsync(stream, 1024, TestContext.Current.CancellationToken);
|
||||
read.ShouldBe(Sample());
|
||||
(await ScriptWorkerFrameCodec.ReadAsync(stream, 1024, TestContext.Current.CancellationToken)).ShouldBeNull("a clean end of stream at a frame boundary");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Oversize_frames_are_refused_on_write_and_on_read()
|
||||
{
|
||||
using var stream = new MemoryStream();
|
||||
await Should.ThrowAsync<ScriptWorkerProtocolException>(async () =>
|
||||
await ScriptWorkerFrameCodec.WriteAsync(stream, Sample(), 4, TestContext.Current.CancellationToken));
|
||||
stream.Length.ShouldBe(0, "nothing is written for a refused frame");
|
||||
|
||||
await ScriptWorkerFrameCodec.WriteAsync(stream, Sample(), 1024, TestContext.Current.CancellationToken);
|
||||
stream.Position = 0;
|
||||
await Should.ThrowAsync<ScriptWorkerProtocolException>(async () =>
|
||||
await ScriptWorkerFrameCodec.ReadAsync(stream, 4, TestContext.Current.CancellationToken));
|
||||
}
|
||||
|
||||
public static TheoryData<string, byte[]> BadFrames()
|
||||
{
|
||||
var good = Sample().ToByteArray();
|
||||
var header = new byte[4];
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header, good.Length);
|
||||
var wrongVersion = new Envelope { ProtocolVersion = 99, Heartbeat = new Heartbeat() }.ToByteArray();
|
||||
var noBody = new Envelope { ProtocolVersion = 1, Seq = 1 }.ToByteArray();
|
||||
return new TheoryData<string, byte[]>
|
||||
{
|
||||
{ "zero length", new byte[] { 0, 0, 0, 0 } },
|
||||
{ "negative length", new byte[] { 0xFF, 0xFF, 0xFF, 0xFF } },
|
||||
{ "truncated header", new byte[] { 5, 0 } },
|
||||
{ "truncated body", [.. header, .. good[..^1]] },
|
||||
{ "garbage body", [5, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF] },
|
||||
{ "wrong protocol version", Frame(wrongVersion) },
|
||||
{ "no body", Frame(noBody) },
|
||||
};
|
||||
}
|
||||
|
||||
private static byte[] Frame(byte[] body)
|
||||
{
|
||||
var header = new byte[4];
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header, body.Length);
|
||||
return [.. header, .. body];
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(BadFrames))]
|
||||
public async Task Malformed_frames_are_protocol_faults(string why, byte[] bytes)
|
||||
{
|
||||
using var stream = new MemoryStream(bytes);
|
||||
await Should.ThrowAsync<ScriptWorkerProtocolException>(async () =>
|
||||
await ScriptWorkerFrameCodec.ReadAsync(stream, 1024, TestContext.Current.CancellationToken), why);
|
||||
}
|
||||
|
||||
/// <summary>Random bytes never escape as anything but a protocol fault (or a clean end of stream).</summary>
|
||||
[Fact]
|
||||
public async Task Fuzzed_input_only_ever_yields_a_protocol_fault()
|
||||
{
|
||||
var random = new Random(780);
|
||||
for (var i = 0; i < 3000; i++)
|
||||
{
|
||||
var bytes = new byte[random.Next(0, 64)];
|
||||
random.NextBytes(bytes);
|
||||
if (bytes.Length >= 4 && random.Next(2) == 0)
|
||||
BinaryPrimitives.WriteInt32LittleEndian(bytes, random.Next(1, bytes.Length));
|
||||
using var stream = new MemoryStream(bytes);
|
||||
try
|
||||
{
|
||||
while (await ScriptWorkerFrameCodec.ReadAsync(stream, 1024, TestContext.Current.CancellationToken) is { } envelope)
|
||||
{
|
||||
// A frame that parses must still decode its variants without an unexpected exception type.
|
||||
if (envelope.BodyCase == Envelope.BodyOneofCase.Results)
|
||||
foreach (var r in envelope.Results.Items)
|
||||
try { VariantCodec.Decode(r.Value); } catch (Exception ex) when (ex is ScriptWorkerProtocolException or OverflowException) { }
|
||||
}
|
||||
}
|
||||
catch (ScriptWorkerProtocolException)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backoff_doubles_from_250ms_to_a_30s_cap_with_20_percent_jitter()
|
||||
{
|
||||
WorkerShard.NextBackoff(1, 0.5).ShouldBe(TimeSpan.FromMilliseconds(250));
|
||||
WorkerShard.NextBackoff(2, 0.5).ShouldBe(TimeSpan.FromMilliseconds(500));
|
||||
WorkerShard.NextBackoff(3, 0.5).ShouldBe(TimeSpan.FromSeconds(1));
|
||||
WorkerShard.NextBackoff(20, 0.5).ShouldBe(TimeSpan.FromSeconds(30));
|
||||
WorkerShard.NextBackoff(1, 0).ShouldBe(TimeSpan.FromMilliseconds(200));
|
||||
WorkerShard.NextBackoff(1, 0.9999999).TotalMilliseconds.ShouldBe(300, tolerance: 0.01);
|
||||
WorkerShard.NextBackoff(40, 1).ShouldBe(TimeSpan.FromSeconds(36));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Shard_placement_is_stable_and_in_range()
|
||||
{
|
||||
// FNV-1a, not string.GetHashCode (randomised per process): a script stays on its shard across restarts.
|
||||
OutOfProcessScriptRunner.ShardIndex("1:vt-1", 2).ShouldBe(OutOfProcessScriptRunner.ShardIndex("1:vt-1", 2));
|
||||
var spread = Enumerable.Range(0, 400).Select(i => OutOfProcessScriptRunner.ShardIndex($"1:vt-{i}", 4)).ToArray();
|
||||
spread.ShouldAllBe(i => i >= 0 && i < 4);
|
||||
spread.Distinct().Count().ShouldBe(4);
|
||||
OutOfProcessScriptRunner.ShardIndex("abc", 1000).ShouldBe((int)(0x1A47E90Bu % 1000));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — the whole path a client sees: a real Calculation driver whose seam runs out of process on the real
|
||||
/// worker. A script failure publishes <c>BadInternalError</c> as today; the worker dying under a script publishes
|
||||
/// <c>BadResourceUnavailable</c> (0x80040000), the ScadaBridge-visible code; and the tag recovers Good afterwards.
|
||||
/// </summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class SeamEndToEndTests
|
||||
{
|
||||
private const uint StatusGood = 0u;
|
||||
private const uint StatusBadInternalError = 0x80020000u;
|
||||
private const uint StatusBadResourceUnavailable = 0x80040000u;
|
||||
private const string Tag = "calc/e/spin";
|
||||
|
||||
/// <summary>Spins for <c>src/ms</c> milliseconds, throws when it is negative, and returns it otherwise.</summary>
|
||||
private const string Script =
|
||||
"var ms = (double)ctx.GetTag(\"src/ms\").Value; if (ms < 0) throw new System.InvalidOperationException(\"negative\"); " +
|
||||
"var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static async Task Settle(CalculationDriver driver)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
|
||||
while (DateTime.UtcNow < deadline && !driver.IsEvaluationIdle) await Task.Delay(10);
|
||||
driver.IsEvaluationIdle.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_calculation_tag_publishes_BadResourceUnavailable_only_when_the_worker_died_under_it()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options());
|
||||
var json = $"{{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(Script)}}}";
|
||||
var driver = new CalculationDriver(
|
||||
new CalculationDriverOptions { RawTags = [new RawTagEntry(Tag, json, WriteIdempotent: false)], RunTimeout = TimeSpan.FromSeconds(20), ScriptRunner = runner },
|
||||
"calc-drv", new ScriptRootLogger(new LoggerConfiguration().CreateLogger()));
|
||||
var published = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => published.Enqueue(e);
|
||||
await driver.InitializeAsync("{}", Ct);
|
||||
|
||||
driver.OnDependencyValue("src/ms", 0.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.TryDequeue(out var good).ShouldBeTrue();
|
||||
good!.Snapshot.StatusCode.ShouldBe(StatusGood);
|
||||
good.Snapshot.Value.ShouldBe(0.0);
|
||||
|
||||
// A script fault: BadInternalError, exactly as in process.
|
||||
driver.OnDependencyValue("src/ms", -1.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.TryDequeue(out var threw).ShouldBeTrue();
|
||||
threw!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError);
|
||||
|
||||
driver.OnDependencyValue("src/ms", 1.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.TryDequeue(out var back).ShouldBeTrue();
|
||||
back!.Snapshot.StatusCode.ShouldBe(StatusGood);
|
||||
|
||||
// The worker dies under a run: the runner, not the script, failed.
|
||||
driver.OnDependencyValue("src/ms", 5000.0, StatusGood, DateTime.UtcNow);
|
||||
var worker = await Scripts.Eventually(() => runner.Shards[0].Current is { InFlightCount: 1 } w ? w : null,
|
||||
TimeSpan.FromSeconds(10), "the run in flight");
|
||||
await Task.Delay(200, Ct);
|
||||
Process.GetProcessById(worker.Pid).Kill();
|
||||
await Settle(driver);
|
||||
published.TryDequeue(out var unavailable).ShouldBeTrue();
|
||||
unavailable!.Snapshot.StatusCode.ShouldBe(StatusBadResourceUnavailable);
|
||||
unavailable.Snapshot.Value.ShouldBe(1.0, "the last good value rides out with the Bad status");
|
||||
|
||||
// Recovery: the next trigger runs (in the isolation worker, as a suspect) and publishes Good.
|
||||
driver.OnDependencyValue("src/ms", 2.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.TryDequeue(out var recovered).ShouldBeTrue();
|
||||
recovered!.Snapshot.StatusCode.ShouldBe(StatusGood);
|
||||
recovered.Snapshot.Value.ShouldBe(2.0);
|
||||
await driver.ShutdownAsync(Ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Serilog;
|
||||
using Serilog.Core;
|
||||
using Serilog.Events;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>Collects Serilog events in memory.</summary>
|
||||
internal sealed class CapturingSink : ILogEventSink
|
||||
{
|
||||
public ConcurrentQueue<LogEvent> Events { get; } = new();
|
||||
|
||||
public void Emit(LogEvent logEvent) => Events.Enqueue(logEvent);
|
||||
|
||||
public ILogger CreateLogger() => new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(this).CreateLogger();
|
||||
|
||||
public IEnumerable<string> Rendered => Events.Select(e => e.RenderMessage());
|
||||
}
|
||||
|
||||
internal static class Scripts
|
||||
{
|
||||
public static readonly IReadOnlyDictionary<string, DataValueSnapshot> NoInputs =
|
||||
new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
||||
|
||||
public static IReadOnlyDictionary<string, DataValueSnapshot> Input(string path, object? value, uint status = 0)
|
||||
=> new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal)
|
||||
{
|
||||
[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow),
|
||||
};
|
||||
|
||||
public static ScriptInvocation Vt(string key, string source, TimeSpan? timeout = null,
|
||||
IReadOnlyDictionary<string, DataValueSnapshot>? inputs = null, ILogger? logger = null)
|
||||
=> new(key, ScriptContextKind.VirtualTag, source, inputs ?? NoInputs, timeout ?? TimeSpan.FromSeconds(5), logger);
|
||||
|
||||
/// <summary>Options for a single-shard runner with short budgets, so the containment tests run in seconds.</summary>
|
||||
public static ScriptingOptions Options(int shards = 1, TimeSpan? hardGrace = null, int quarantineAfterKills = 2,
|
||||
int probation = 20, TimeSpan? unavailableGrace = null, string? workerPath = null, long heapLimit = 256L * 1024 * 1024)
|
||||
=> new()
|
||||
{
|
||||
Runner = ScriptRunnerMode.OutOfProcess,
|
||||
QuarantineAfterConsecutiveTimeouts = 100,
|
||||
Worker = new ScriptWorkerOptions
|
||||
{
|
||||
Shards = shards,
|
||||
HardGrace = hardGrace ?? TimeSpan.FromMilliseconds(500),
|
||||
QuarantineAfterKills = quarantineAfterKills,
|
||||
IsolationProbationRuns = probation,
|
||||
UnavailableGrace = unavailableGrace ?? TimeSpan.FromSeconds(30),
|
||||
ExecutablePath = workerPath,
|
||||
HeapLimitBytes = heapLimit,
|
||||
},
|
||||
};
|
||||
|
||||
public static OutOfProcessScriptRunner Runner(ScriptingOptions options, ILogger? log = null)
|
||||
=> new(options, log ?? new LoggerConfiguration().CreateLogger(), null,
|
||||
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60));
|
||||
|
||||
public static async Task<T> Eventually<T>(Func<T?> probe, TimeSpan timeout, string what) where T : class
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (probe() is { } value) return value;
|
||||
await Task.Delay(20);
|
||||
}
|
||||
throw new TimeoutException($"timed out waiting for {what}");
|
||||
}
|
||||
|
||||
public static async Task Eventually(Func<bool> probe, TimeSpan timeout, string what)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (probe()) return;
|
||||
await Task.Delay(20);
|
||||
}
|
||||
throw new TimeoutException($"timed out waiting for {what}");
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
<RootNamespace>ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="xunit.v3"/>
|
||||
<PackageReference Include="Shouldly"/>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.Scripting\ZB.MOM.WW.OtOpcUa.Core.Scripting.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol\ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess\ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess.csproj"/>
|
||||
<!-- End-to-end seam test: a real Calculation driver over the real worker. -->
|
||||
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Calculation\ZB.MOM.WW.OtOpcUa.Driver.Calculation.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<!-- The real worker, built and placed in script-worker/ beside the tests, exactly as beside the Host. -->
|
||||
<Import Project="..\..\..\src\Server\ZB.MOM.WW.OtOpcUa.ScriptWorker\build\ScriptWorker.Deploy.targets"/>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user