using System.Collections.Concurrent; using System.Diagnostics; using Microsoft.Extensions.Logging.Abstractions; using Serilog; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Driver.Calculation; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// /// lmxopcua#780 P2 — Scripting:Runner=Shadow over the REAL worker: the seams publish their in-process results /// and mirror each compiled evaluation to the worker pool. Ordinary scripts compare equal; a deliberately divergent /// one (it reads which process it runs in) and an unsupported return type (design §14 #11) are counted as mismatches /// for exactly those scripts; and a runaway that only runs away in the worker is killed and quarantined THERE while /// every published value stays the in-process one. Alarms: the transition sequence is the in-process engine's, a /// killed shadow worker never annotates or clears a condition. /// [Collection(ScriptWorkerCollection.Name)] [Trait("Category", "ScriptWorker")] public sealed class ShadowModeTests { /// /// True only inside the ScriptWorker: a test-only divergence that needs no sandbox-forbidden API. The worker runs /// with InvariantGlobalization (ordinal string comparison) while the test host does linguistic (ICU/NLS) /// comparison, so a culture-sensitive string.Compare("a", "B") is negative in the host and positive in the /// worker. pins the premise. /// private const string InWorker = "(string.Compare(\"a\", \"B\") > 0)"; private static CancellationToken Ct => TestContext.Current.CancellationToken; private static ScriptingOptions ShadowOptions(TimeSpan? hardGrace = null) { var options = Scripts.Options(hardGrace: hardGrace ?? TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2); options.Runner = ScriptRunnerMode.Shadow; return options; } private static CalculationEvaluator Calc(IScriptRunner? runner, TimeSpan? budget = null) => new(NullLogger.Instance, new ScriptRootLogger(new LoggerConfiguration().CreateLogger()), budget ?? TimeSpan.FromSeconds(5), scriptRunner: runner); private static Dictionary Inputs(int i) { var now = DateTime.UtcNow; DataValueSnapshot S(object v) => new(v, 0u, now, now); return new Dictionary(StringComparer.Ordinal) { ["s"] = S((short)(5 + i)), ["d"] = S(1.25 * i), ["str"] = S("abc" + i), ["i"] = S(i), ["t"] = S(new DateTime(2026, 11, 10, 12, 0, 0, DateTimeKind.Utc).AddSeconds(i)), ["arr"] = S(new[] { 1.5, 2.5 * i }), }; } private static bool Same(VirtualTagEvalResult a, VirtualTagEvalResult b) => a.Success == b.Success && a.Reason == b.Reason && a.StatusCode == b.StatusCode && System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(a.Value, b.Value) && a.Value?.GetType() == b.Value?.GetType(); [Fact] public async Task Culture_sensitive_comparison_really_diverges_between_host_and_worker() { // Premise of InWorker: the worker is a different process whose globalization mode differs from the host's. string.Compare("a", "B").ShouldBeLessThan(0, "the test host compares linguistically"); await using var pool = Scripts.Runner(Scripts.Options()); var r = await pool.RunAsync(Scripts.Vt("probe", $"return {InWorker} ? 2 : 1;"), Ct); r.Status.ShouldBe(ScriptRunStatus.Ok, r.Reason); r.Value.ShouldBe(2, "the worker (InvariantGlobalization) compares ordinally"); } [Fact] public async Task Ordinary_scripts_match_and_a_divergent_and_an_unsupported_one_are_counted_against_exactly_those_scripts() { var log = new CapturingSink(); await using var pool = Scripts.Runner(ShadowOptions()); using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), log.CreateLogger()); using var shadowed = Calc(shadow); using var reference = Calc(null); shadowed.SubmitsBatches.ShouldBeFalse("Shadow keeps the in-process one-at-a-time pump, not the runner path"); var scripts = new Dictionary(StringComparer.Ordinal) { ["short"] = "return (short)ctx.GetTag(\"s\").Value + 1;", ["double"] = "return (double)ctx.GetTag(\"d\").Value * 2.5;", ["string"] = "return \"v=\" + ctx.GetTag(\"str\").Value;", ["intArray"] = "return new[] { (int)ctx.GetTag(\"i\").Value, 2 };", ["doubleArray"] = "var a = (double[])ctx.GetTag(\"arr\").Value; return new[] { a[1], a[0] };", ["datetime"] = "return ((DateTime)ctx.GetTag(\"t\").Value).AddSeconds(1);", ["bool"] = "return (int)ctx.GetTag(\"i\").Value > 1;", ["quality"] = "return ctx.GetTag(\"i\").StatusCode;", ["throws"] = "if ((int)ctx.GetTag(\"i\").Value >= 0) throw new InvalidOperationException(\"boom \" + ctx.GetTag(\"i\").Value); return 0;", ["divergent"] = $"return {InWorker} ? 2 : 1;", ["decimal"] = "return 1.5m;", }; for (var i = 0; i < 3; i++) { foreach (var (key, source) in scripts) { var published = await shadowed.EvaluateAsync(key, source, Inputs(i), Ct); var expected = await reference.EvaluateAsync(key, source, Inputs(i), Ct); Same(published, expected).ShouldBeTrue($"{key}: Shadow publishes exactly what InProcess publishes ({published} vs {expected})"); } (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(60))).ShouldBeTrue(); } var stats = shadow.Stats(); stats.Dropped.ShouldBe(0); stats.ShadowUnavailable.ShouldBe(0); stats.Compared.ShouldBe(scripts.Count * 3); stats.Mismatched.ShouldBe(6, string.Join(Environment.NewLine, shadow.Mismatches.Select(m => $"{m.Key}: {m.LastReason} — {m.LastSummary}"))); var mismatches = shadow.Mismatches.ToDictionary(m => m.Key); mismatches.Keys.ShouldBe(["decimal", "divergent"], ignoreOrder: true); mismatches["divergent"].LastReason.ShouldBe("value"); mismatches["divergent"].LastSummary.ShouldBe("published Ok Int32 1; shadow Ok Int32 2"); mismatches["divergent"].Count.ShouldBe(3); mismatches["decimal"].LastReason.ShouldBe("unsupported_type"); mismatches["decimal"].LastSummary.ShouldContain("System.Decimal"); var lines = log.Events.Where(e => e.MessageTemplate.Text.StartsWith("Script shadow mismatch", StringComparison.Ordinal)).ToArray(); lines.Length.ShouldBe(2, "one line per script and reason"); lines.ShouldAllBe(e => e.Level == Serilog.Events.LogEventLevel.Warning); } [Fact] public async Task A_runaway_that_runs_away_only_in_the_worker_is_killed_and_quarantined_there_and_never_reaches_a_published_value() { var log = new CapturingSink(); await using var pool = Scripts.Runner(ShadowOptions(TimeSpan.FromMilliseconds(400)), log.CreateLogger()); using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), log.CreateLogger()); using var shadowed = Calc(shadow, TimeSpan.FromMilliseconds(300)); const string runaway = $"if ({InWorker}) {{ while (true) {{ }} }} return (int)ctx.GetTag(\"i\").Value * 2;"; const string sibling = "return (int)ctx.GetTag(\"i\").Value + 100;"; var slowest = TimeSpan.Zero; var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(90); var i = 0; for (; !pool.QuarantinedKeys(ScriptContextKind.Calculation).Contains("runaway"); i++) { if (DateTime.UtcNow > deadline) throw new TimeoutException("the shadow never quarantined the runaway:" + Environment.NewLine + string.Join(Environment.NewLine, log.Rendered)); var sw = Stopwatch.StartNew(); var a = await shadowed.EvaluateAsync("runaway", runaway, Inputs(i), Ct); var b = await shadowed.EvaluateAsync("sibling", sibling, Inputs(i), Ct); if (sw.Elapsed > slowest) slowest = sw.Elapsed; a.ShouldBe(VirtualTagEvalResult.Ok(i * 2), "published from the in-process run"); b.ShouldBe(VirtualTagEvalResult.Ok(i + 100)); await Task.Delay(100, Ct); } // Still publishing correctly after the shadow quarantined it, and never quarantined on the published side. for (var j = i; j < i + 5; j++) (await shadowed.EvaluateAsync("runaway", runaway, Inputs(j), Ct)).ShouldBe(VirtualTagEvalResult.Ok(j * 2)); shadowed.QuarantinedScripts.ShouldBeEmpty(); slowest.ShouldBeLessThan(TimeSpan.FromMilliseconds(250), "a published evaluation never waits for its shadow"); log.Rendered.ShouldContain(l => l.Contains("exceeded its hard deadline")); (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue(); var record = shadow.Mismatches.ShouldHaveSingleItem("the sibling on the killed shard is a victim (not compared), never a mismatch"); record.Key.ShouldBe("runaway"); record.LastReason.ShouldBe("status"); record.LastSummary.ShouldContain("shadow Failed [budget]"); shadow.Stats().DroppedKeyBusy.ShouldBeGreaterThan(0, "triggers while its shadow run was still going were dropped, not queued"); } private static ScriptedAlarmDefinition Alarm(string id, string predicate) => new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner? runner) { var logger = new LoggerConfiguration().CreateLogger(); return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, scriptTimeout: TimeSpan.FromSeconds(2), scriptRunner: runner); } [Fact] public async Task Alarms_in_shadow_transition_exactly_as_in_process_and_a_killed_shadow_worker_never_touches_a_condition() { await using var pool = Scripts.Runner(ShadowOptions()); using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), new LoggerConfiguration().CreateLogger()); var up = new TestUpstream(); var refUp = new TestUpstream(); up.Set("x", 0); refUp.Set("x", 0); using var engine = Engine(up, shadow); using var reference = Engine(refUp, null); engine.ScriptRunner.ShouldBeNull("Shadow keeps the in-process engine path"); var events = new ConcurrentQueue(); var refEvents = new ConcurrentQueue(); engine.OnEvent += (_, e) => events.Enqueue(e); reference.OnEvent += (_, e) => refEvents.Enqueue(e); ScriptedAlarmDefinition[] defs = [ Alarm("high", "return (int)ctx.GetTag(\"x\").Value > 5;"), Alarm("divergent", $"return {InWorker};"), ]; await engine.LoadAsync(defs, Ct); await reference.LoadAsync(defs, Ct); async Task Push(int x) { up.Push("x", x); refUp.Push("x", x); await Scripts.Eventually(() => events.Count == refEvents.Count, TimeSpan.FromSeconds(10), "both engines' emissions"); await Task.Delay(100, Ct); } await Push(10); await Push(2); (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue(); // kill -9 the shadow's worker: the published engine neither annotates nor clears anything because of it. var worker = pool.Shards[0].Current.ShouldNotBeNull(); Process.GetProcessById(worker.Pid).Kill(); await Push(12); await Push(0); (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue(); await Push(9); (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue(); events.Select(e => (e.AlarmId, e.Emission, e.WorstInputStatusCode)) .ShouldBe(refEvents.Select(e => (e.AlarmId, e.Emission, e.WorstInputStatusCode)), "the in-process engine's sequence, exactly"); events.Count(e => e is { AlarmId: "high", Emission: EmissionKind.Activated }).ShouldBe(3); events.ShouldNotContain(e => e.WorstInputStatusCode != 0u, "no shadow-side failure annotates a published condition"); engine.GetState("divergent")!.Active.ShouldBe(AlarmActiveState.Inactive, "the in-process predicate (false) drives it"); var record = shadow.Mismatches.ShouldHaveSingleItem(); record.Key.ShouldBe("divergent"); record.Kind.ShouldBe(ScriptContextKind.AlarmPredicate); record.LastSummary.ShouldBe("published Ok Boolean False; shadow Ok Boolean True"); shadow.Stats().Matched.ShouldBeGreaterThan(0); } }