using System.Diagnostics; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// Worker-backed tests spawn real processes and measure wall-clock bounds, so they never run in parallel. [CollectionDefinition(Name, DisableParallelization = true)] public sealed class ScriptWorkerCollection { public const string Name = "ScriptWorker"; } /// /// lmxopcua#780 design §13 "Containment": each case runs against the real ScriptWorker process, and each has a /// positive control — a healthy sibling on the SAME shard (every runner here has one shard) that keeps producing /// correct values before and after the event. /// [Collection(ScriptWorkerCollection.Name)] [Trait("Category", "ScriptWorker")] public sealed class ContainmentTests { private const string Healthy = "return 7;"; private const string Runaway = "while (true) { }"; private const string Cooperative = "while (!ctx.CancellationToken.IsCancellationRequested) { } return 1;"; /// Spins for the number of milliseconds in input ms, so one source serves slow and fast runs. private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;"; private static IReadOnlyDictionary Ms(int ms) => Scripts.Input("ms", ms); private static CancellationToken Ct => TestContext.Current.CancellationToken; [Fact] public async Task A_healthy_script_runs_out_of_process_and_reads_its_inputs() { await using var runner = Scripts.Runner(Scripts.Options()); var outcome = await runner.RunAsync(Scripts.Vt("vt", "return (int)ctx.GetTag(\"x\").Value + 1;", inputs: Scripts.Input("x", 41)), Ct); outcome.Status.ShouldBe(ScriptRunStatus.Ok, outcome.Reason); outcome.Value.ShouldBe(42); runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(Environment.ProcessId); } /// /// The headline case: while(true){} ignores the token. In process it would spin a host core forever; here /// the caller gets the usual timeout at T, the worker is killed at T + G, the host burns no CPU on it, and the /// sibling on the same shard keeps working. Mutation check: with the hard-deadline kill disabled the worker is /// never killed and the Exited wait times out. /// [Fact] public async Task A_runaway_script_is_killed_at_its_hard_deadline_and_its_sibling_keeps_working() { var softT = TimeSpan.FromMilliseconds(400); var grace = TimeSpan.FromMilliseconds(500); await using var runner = Scripts.Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 2)); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); var worker = runner.Shards[0].Current.ShouldNotBeNull(); var sw = Stopwatch.StartNew(); var outcome = await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct); var softElapsed = sw.Elapsed; outcome.Status.ShouldBe(ScriptRunStatus.Failed); outcome.Reason!.ShouldStartWith("script timed out after "); softElapsed.ShouldBeLessThan(softT + TimeSpan.FromMilliseconds(400), "the caller is answered at the soft deadline"); var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct); var killElapsed = sw.Elapsed; exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline); exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "runaway")); killElapsed.ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50)); // Design budget is T + G + 100 ms; the slack here is for a loaded CI box (process reap + exit notification). killElapsed.ShouldBeLessThan(softT + grace + TimeSpan.FromMilliseconds(1500)); // No orphan in the host: an in-process while(true) would burn one host core for the whole window. var cpuBefore = Process.GetCurrentProcess().TotalProcessorTime; await Task.Delay(TimeSpan.FromMilliseconds(1500), Ct); var cpuUsed = Process.GetCurrentProcess().TotalProcessorTime - cpuBefore; cpuUsed.ShouldBeLessThan(TimeSpan.FromMilliseconds(1000), "the runaway's thread died with its worker"); // The sibling recovers on a fresh worker. (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(worker.Pid); // A second kill quarantines (QuarantineAfterKills = 2); the third trigger is refused without running. var second = runner.Shards[0].Current!; (await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct)).Status.ShouldBe(ScriptRunStatus.Failed); (await second.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct)).Reason.ShouldBe(WorkerExitReason.KilledDeadline); // The strike is recorded by the shard's exit handler, which runs asynchronously to the Exited completion. await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(5), "the second kill strike"); var refused = await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct); refused.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); refused.Reason.ShouldContain("hard deadline"); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"]); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); // Lifted by a source change (the operator fixed it)… (await runner.RunAsync(Scripts.Vt("runaway", "return 1;"), Ct)).Value.ShouldBe(1); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty(); } [Fact] public async Task A_redeploy_lifts_a_kill_quarantine_for_its_seam_only() { var softT = TimeSpan.FromMilliseconds(300); await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(300), quarantineAfterKills: 1)); (await runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct)).Status.ShouldBe(ScriptRunStatus.Failed); await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(10), "the kill quarantine"); runner.Reset(ScriptContextKind.Calculation); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"], "another seam's redeploy leaves it alone"); runner.Reset(ScriptContextKind.VirtualTag); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty(); } /// A script that polls its token ends itself at T: no kill, no strike beyond the timeout (design §13). [Fact] public async Task A_cooperative_script_ends_at_its_soft_deadline_without_a_kill() { var softT = TimeSpan.FromMilliseconds(400); var grace = TimeSpan.FromMilliseconds(1500); await using var runner = Scripts.Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 1)); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); var worker = runner.Shards[0].Current.ShouldNotBeNull(); var outcome = await runner.RunAsync(Scripts.Vt("polite", Cooperative, softT), Ct); outcome.Reason!.ShouldStartWith("script timed out after "); await Task.Delay(softT + grace + TimeSpan.FromMilliseconds(500), Ct); worker.IsAlive.ShouldBeTrue("a script that ended itself must not cost the shard a restart"); runner.Shards[0].Current!.Pid.ShouldBe(worker.Pid); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("no kill strike (QuarantineAfterKills = 1)"); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); } [Fact] public async Task A_second_trigger_while_the_first_still_runs_is_refused_not_queued() { await using var runner = Scripts.Runner(Scripts.Options()); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); var first = runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(1500)), Ct); await Task.Delay(200, Ct); var second = await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct); second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); (await first).Value.ShouldBe(1500); } /// /// kill -9 of the worker: the evaluation in flight is a victim — Unavailable "script worker restarted" /// (BadResourceUnavailable), not a script fault — and, with no deadline crossed, a suspect that proves itself in /// the isolation worker and then returns to its shard. /// [Fact] public async Task An_externally_killed_worker_makes_its_in_flight_script_a_suspect_that_recovers_in_isolation() { await using var runner = Scripts.Runner(Scripts.Options(probation: 2)); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); var worker = runner.Shards[0].Current.ShouldNotBeNull(); var inFlight = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct); await Task.Delay(300, Ct); Process.GetProcessById(worker.Pid).Kill(); var victim = await inFlight.WaitAsync(TimeSpan.FromSeconds(10), Ct); victim.Status.ShouldBe(ScriptRunStatus.Unavailable); victim.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); (await worker.Exited).Reason.ShouldBe(WorkerExitReason.Crash); runner.SuspectKeys.ShouldBe([InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "spin")]); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); runner.Isolation.Current.ShouldNotBeNull("a suspect runs in the isolation worker"); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); runner.SuspectKeys.ShouldBeEmpty("two clean runs (probation = 2) return it to its shard"); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty(); // The sibling recovers on a fresh shard worker. (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); runner.Shards[0].Current!.Pid.ShouldNotBe(worker.Pid); } /// The isolation worker runs one evaluation at a time, so a crash there names exactly one key. [Fact] public async Task A_crash_in_the_isolation_worker_strikes_its_one_key_and_quarantines_it() { await using var runner = Scripts.Runner(Scripts.Options(quarantineAfterKills: 2)); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); async Task RunAndKill(Func worker) { var run = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct); var live = await Scripts.Eventually(worker, TimeSpan.FromSeconds(15), "a live worker"); await Scripts.Eventually(() => live.InFlightCount == 1, TimeSpan.FromSeconds(15), "the evaluation in flight"); await Task.Delay(200, Ct); Process.GetProcessById(live.Pid).Kill(); return await run.WaitAsync(TimeSpan.FromSeconds(10), Ct); } (await RunAndKill(() => runner.Shards[0].Current)).Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); runner.SuspectKeys.Count.ShouldBe(1); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("an ambiguous shard crash is not a strike"); (await RunAndKill(() => runner.Isolation.Current)).Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty("one isolation crash is one strike"); (await RunAndKill(() => runner.Isolation.Current)).Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(5), "the isolated-crash quarantine"); var refused = await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct); refused.Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); refused.Reason.ShouldContain("isolation worker"); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); } /// A deadline kill blames exactly one key: the sibling in flight on the same worker is a victim, not a suspect. [Fact] public async Task A_deadline_kill_blames_only_its_culprit() { await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(300), quarantineAfterKills: 1)); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); (await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7); // Compile the runaway first, so both evaluations are on the worker at once. _ = runner.Shards[0].Current!; var victim = runner.RunAsync(Scripts.Vt("spin", Spin, TimeSpan.FromSeconds(20), Ms(5000)), Ct); await Task.Delay(200, Ct); var culprit = await runner.RunAsync(Scripts.Vt("runaway", Runaway, TimeSpan.FromMilliseconds(300)), Ct); culprit.Reason!.ShouldStartWith("script timed out after "); var victimOutcome = await victim.WaitAsync(TimeSpan.FromSeconds(15), Ct); victimOutcome.Status.ShouldBe(ScriptRunStatus.Unavailable); victimOutcome.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(5), "the culprit's quarantine"); runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"]); runner.SuspectKeys.ShouldBeEmpty("a deadline kill is unambiguous: nobody else is suspected"); (await runner.RunAsync(Scripts.Vt("spin", Spin, inputs: Ms(0)), Ct)).Value.ShouldBe(0); } /// An allocation bomb dies in its worker (heap hard limit); the host's own memory does not move. [Fact] public async Task An_allocation_bomb_is_contained_in_its_worker() { await using var runner = Scripts.Runner(Scripts.Options(heapLimit: 128L * 1024 * 1024)); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); GC.Collect(); var hostBefore = GC.GetTotalMemory(forceFullCollection: true); var outcome = await runner.RunAsync(Scripts.Vt("bomb", "var hoard = new System.Collections.Generic.List(); while (true) { hoard.Add(new byte[1 << 20]); }", TimeSpan.FromSeconds(10)), Ct); var hostAfter = GC.GetTotalMemory(forceFullCollection: true); outcome.Status.ShouldNotBe(ScriptRunStatus.Ok); (hostAfter - hostBefore).ShouldBeLessThan(32L * 1024 * 1024, "the hoard lived in the worker's heap, not the host's"); await Scripts.Eventually(() => runner.Shards[0].Current, TimeSpan.FromSeconds(10), "a worker"); (await runner.RunAsync(Scripts.Vt("sibling", Healthy), Ct)).Value.ShouldBe(7); } /// Fail closed (design §14 #4): no worker means Bad, never an in-process run. [Fact] public async Task A_worker_that_cannot_start_reports_unavailable_and_never_runs_in_process() { var missing = Path.Combine(Path.GetTempPath(), "no-such-dir-" + Guid.NewGuid().ToString("N"), OutOfProcessScriptRunner.WorkerAssemblyFileName); await using var runner = Scripts.Runner(Scripts.Options(workerPath: missing, unavailableGrace: TimeSpan.FromSeconds(1))); var outcome = await runner.RunAsync(Scripts.Vt("vt", "return 1;"), Ct); outcome.Status.ShouldBe(ScriptRunStatus.Unavailable); outcome.Reason.ShouldBe(ScriptRunOutcome.RunnerUnavailableReason); outcome.ToVirtualTagEvalResult().StatusCode.ShouldBe(ZB.MOM.WW.OtOpcUa.Commons.OpcUa.OpcUaStatus.BadResourceUnavailable); runner.Shards[0].ConsecutiveFailures.ShouldBeGreaterThan(0, "the failed start backs off"); } /// A worker whose pipe reaches end of stream exits by itself (the Linux "host died" path, design §3.3). [Fact] public async Task A_worker_exits_by_itself_when_its_pipe_closes() { await using var runner = Scripts.Runner(Scripts.Options()); (await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7); var worker = runner.Shards[0].Current.ShouldNotBeNull(); using var process = Process.GetProcessById(worker.Pid); var sw = Stopwatch.StartNew(); worker.SeverPipeForTest(); await process.WaitForExitAsync(Ct).WaitAsync(TimeSpan.FromSeconds(5), Ct); sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(2)); var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(5), Ct); exit.ExitCode.ShouldBe(ScriptWorkerEnvironment.ExitPipeClosed, "it noticed the EOF; the host did not have to kill it"); } /// Design §6: the environment is cleared and repopulated from an allow-list. Positive control first. [Fact] public async Task The_worker_environment_carries_no_host_secrets() { const string apiKey = "ServerHistorian__ApiKey"; const string configDb = "ConnectionStrings__ConfigDb"; Environment.SetEnvironmentVariable(apiKey, "sekrit"); Environment.SetEnvironmentVariable(configDb, "Server=x;Password=y"); try { Environment.GetEnvironmentVariable(apiKey).ShouldBe("sekrit", "positive control: the host has it"); await using var runner = Scripts.Runner(Scripts.Options()); (await runner.RunAsync(Scripts.Vt("vt", Healthy), Ct)).Value.ShouldBe(7); var diag = await runner.Shards[0].Current!.DiagnosticsAsync(TimeSpan.FromSeconds(10)); diag.EnvironmentNames.ShouldNotContain(apiKey); diag.EnvironmentNames.ShouldNotContain(configDb); diag.EnvironmentNames.ShouldContain(ScriptWorkerEnvironment.Nonce, "the allow-listed launch variables do arrive"); diag.EnvironmentNames.ShouldContain("DOTNET_GCHeapHardLimit"); diag.WorkingDirectory.ShouldNotBe(Environment.CurrentDirectory); // An empty per-worker directory under the temp root (macOS reports /var as /private/var, so match the leaf). diag.WorkingDirectory.ShouldContain(Path.Combine("otopcua-scriptworker", "otopcua-sw-")); Directory.EnumerateFileSystemEntries(diag.WorkingDirectory).ShouldBeEmpty(); } finally { Environment.SetEnvironmentVariable(apiKey, null); Environment.SetEnvironmentVariable(configDb, null); } } /// Design §6: the host's cluster, database, OPC UA and secrets stacks are not in the worker. [Fact] public async Task The_worker_loads_none_of_the_host_stacks() { await using var runner = Scripts.Runner(Scripts.Options()); (await runner.RunAsync(Scripts.Vt("vt", "return System.Math.Abs(-3);"), Ct)).Value.ShouldBe(3); var diag = await runner.Shards[0].Current!.DiagnosticsAsync(TimeSpan.FromSeconds(10)); diag.LoadedAssemblies.ShouldContain("ZB.MOM.WW.OtOpcUa.Core.Scripting", "positive control: the compile ran here"); diag.LoadedAssemblies.ShouldContain("Microsoft.CodeAnalysis.CSharp"); foreach (var prefix in ScriptWorkerEnvironment.ForbiddenAssemblyPrefixes) diag.LoadedAssemblies.ShouldNotContain(a => a.StartsWith(prefix, StringComparison.Ordinal), prefix); // Not merely unloaded: Akka is not even deployable beside the worker. var folder = Path.GetDirectoryName(runner.WorkerPath)!; File.Exists(Path.Combine(folder, "ZB.MOM.WW.OtOpcUa.ScriptWorker.dll")).ShouldBeTrue("positive control"); File.Exists(Path.Combine(folder, "Akka.dll")).ShouldBeFalse(); File.Exists(Path.Combine(folder, "Grpc.Core.Api.dll")).ShouldBeFalse(); } [Fact] public async Task Script_log_lines_come_back_through_the_invocations_logger() { var sink = new CapturingSink(); await using var runner = Scripts.Runner(Scripts.Options()); (await runner.RunAsync(Scripts.Vt("vt", "ctx.Logger.Information(\"hello {N}\", 42); return 1;", logger: sink.CreateLogger()), Ct)) .Value.ShouldBe(1); await Scripts.Eventually(() => sink.Rendered.Contains("hello 42"), TimeSpan.FromSeconds(5), "the forwarded log line"); } [Fact] public async Task An_alarm_predicate_runs_out_of_process() { await using var runner = Scripts.Runner(Scripts.Options()); var outcome = await runner.RunAsync(new ScriptInvocation("alarm-1", ScriptContextKind.AlarmPredicate, "return (double)ctx.GetTag(\"t\").Value > 50;", Scripts.Input("t", 75.0), TimeSpan.FromSeconds(5)), Ct); outcome.Status.ShouldBe(ScriptRunStatus.Ok, outcome.Reason); outcome.Value.ShouldBe(true); } /// Design §14 #11: an unsupported type is an explicit failure, never a silent conversion. [Fact] public async Task Unsupported_input_and_result_types_are_explicit_failures() { await using var runner = Scripts.Runner(Scripts.Options()); var result = await runner.RunAsync(Scripts.Vt("list", "return new System.Collections.Generic.List { 1 };"), Ct); result.Status.ShouldBe(ScriptRunStatus.Failed); result.Reason!.ShouldStartWith("unsupported type: script returned a value of unsupported type System.Collections.Generic.List"); var input = await runner.RunAsync(Scripts.Vt("guid", "return 1;", inputs: Scripts.Input("g", Guid.NewGuid())), Ct); input.Status.ShouldBe(ScriptRunStatus.Failed); input.Reason!.ShouldStartWith("unsupported type: input 'g' has a value of unsupported type System.Guid"); } }