dohertj2/lmxopcua
1
0
Fork 0
Code Issues 24 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

Browse Source 
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 07:18:40 -05:00
parent 0796928504
commit 03686eb5d7
17 changed files with 1477 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ShadowModeTests.cs
+226
View File
@@ -0,0 +1,226 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Microsoft.Extensions.Logging.Abstractions;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// <summary>
/// lmxopcua#780 P2 — <c>Scripting:Runner=Shadow</c> over the REAL worker: the seams publish their in-process results
/// and mirror each compiled evaluation to the worker pool. Ordinary scripts compare equal; a deliberately divergent
/// one (it reads which process it runs in) and an unsupported return type (design §14 #11) are counted as mismatches
/// for exactly those scripts; and a runaway that only runs away in the worker is killed and quarantined THERE while
/// every published value stays the in-process one. Alarms: the transition sequence is the in-process engine's, a
/// killed shadow worker never annotates or clears a condition.
/// </summary>
[Collection(ScriptWorkerCollection.Name)]
[Trait("Category", "ScriptWorker")]
public sealed class ShadowModeTests
{
/// <summary>True only inside the ScriptWorker (its base directory is <c>script-worker/</c>): a test-only divergence.</summary>
private const string InWorker = "System.AppContext.BaseDirectory.Contains(\"script-worker\")";
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static ScriptingOptions ShadowOptions(TimeSpan? hardGrace = null)
{
var options = Scripts.Options(hardGrace: hardGrace ?? TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2);
options.Runner = ScriptRunnerMode.Shadow;
return options;
}
private static CalculationEvaluator Calc(IScriptRunner? runner, TimeSpan? budget = null)
=> new(NullLogger.Instance, new ScriptRootLogger(new LoggerConfiguration().CreateLogger()),
budget ?? TimeSpan.FromSeconds(5), scriptRunner: runner);
private static Dictionary<string, DataValueSnapshot> Inputs(int i)
{
var now = DateTime.UtcNow;
DataValueSnapshot S(object v) => new(v, 0u, now, now);
return new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal)
{
["s"] = S((short)(5 + i)),
["d"] = S(1.25 * i),
["str"] = S("abc" + i),
["i"] = S(i),
["t"] = S(new DateTime(2026, 11, 10, 12, 0, 0, DateTimeKind.Utc).AddSeconds(i)),
["arr"] = S(new[] { 1.5, 2.5 * i }),
};
}
private static bool Same(VirtualTagEvalResult a, VirtualTagEvalResult b)
=> a.Success == b.Success && a.Reason == b.Reason && a.StatusCode == b.StatusCode
&& System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(a.Value, b.Value)
&& a.Value?.GetType() == b.Value?.GetType();
[Fact]
public async Task Ordinary_scripts_match_and_a_divergent_and_an_unsupported_one_are_counted_against_exactly_those_scripts()
{
var log = new CapturingSink();
await using var pool = Scripts.Runner(ShadowOptions());
using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), log.CreateLogger());
using var shadowed = Calc(shadow);
using var reference = Calc(null);
shadowed.SubmitsBatches.ShouldBeFalse("Shadow keeps the in-process one-at-a-time pump, not the runner path");
var scripts = new Dictionary<string, string>(StringComparer.Ordinal)
{
["short"] = "return (short)ctx.GetTag(\"s\").Value + 1;",
["double"] = "return (double)ctx.GetTag(\"d\").Value * 2.5;",
["string"] = "return \"v=\" + ctx.GetTag(\"str\").Value;",
["intArray"] = "return new[] { (int)ctx.GetTag(\"i\").Value, 2 };",
["doubleArray"] = "var a = (double[])ctx.GetTag(\"arr\").Value; return new[] { a[1], a[0] };",
["datetime"] = "return ((DateTime)ctx.GetTag(\"t\").Value).AddSeconds(1);",
["bool"] = "return (int)ctx.GetTag(\"i\").Value > 1;",
["quality"] = "return ctx.GetTag(\"i\").StatusCode;",
["throws"] = "if ((int)ctx.GetTag(\"i\").Value >= 0) throw new InvalidOperationException(\"boom \" + ctx.GetTag(\"i\").Value); return 0;",
["divergent"] = $"return {InWorker} ? 2 : 1;",
["decimal"] = "return 1.5m;",
};
for (var i = 0; i < 3; i++)
{
foreach (var (key, source) in scripts)
{
var published = await shadowed.EvaluateAsync(key, source, Inputs(i), Ct);
var expected = await reference.EvaluateAsync(key, source, Inputs(i), Ct);
Same(published, expected).ShouldBeTrue($"{key}: Shadow publishes exactly what InProcess publishes ({published} vs {expected})");
}
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(60))).ShouldBeTrue();
}
var stats = shadow.Stats();
stats.Dropped.ShouldBe(0);
stats.ShadowUnavailable.ShouldBe(0);
stats.Compared.ShouldBe(scripts.Count * 3);
stats.Mismatched.ShouldBe(6, string.Join(Environment.NewLine, shadow.Mismatches.Select(m => $"{m.Key}: {m.LastReason} — {m.LastSummary}")));
var mismatches = shadow.Mismatches.ToDictionary(m => m.Key);
mismatches.Keys.ShouldBe(["decimal", "divergent"], ignoreOrder: true);
mismatches["divergent"].LastReason.ShouldBe("value");
mismatches["divergent"].LastSummary.ShouldBe("published Ok Int32 1; shadow Ok Int32 2");
mismatches["divergent"].Count.ShouldBe(3);
mismatches["decimal"].LastReason.ShouldBe("unsupported_type");
mismatches["decimal"].LastSummary.ShouldContain("System.Decimal");
var lines = log.Events.Where(e => e.MessageTemplate.Text.StartsWith("Script shadow mismatch", StringComparison.Ordinal)).ToArray();
lines.Length.ShouldBe(2, "one line per script and reason");
lines.ShouldAllBe(e => e.Level == Serilog.Events.LogEventLevel.Warning);
}
[Fact]
public async Task A_runaway_that_runs_away_only_in_the_worker_is_killed_and_quarantined_there_and_never_reaches_a_published_value()
{
var log = new CapturingSink();
await using var pool = Scripts.Runner(ShadowOptions(TimeSpan.FromMilliseconds(400)), log.CreateLogger());
using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), log.CreateLogger());
using var shadowed = Calc(shadow, TimeSpan.FromMilliseconds(300));
const string runaway = $"if ({InWorker}) {{ while (true) {{ }} }} return (int)ctx.GetTag(\"i\").Value * 2;";
const string sibling = "return (int)ctx.GetTag(\"i\").Value + 100;";
var slowest = TimeSpan.Zero;
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(90);
var i = 0;
for (; !pool.QuarantinedKeys(ScriptContextKind.Calculation).Contains("runaway"); i++)
{
if (DateTime.UtcNow > deadline)
throw new TimeoutException("the shadow never quarantined the runaway:" + Environment.NewLine + string.Join(Environment.NewLine, log.Rendered));
var sw = Stopwatch.StartNew();
var a = await shadowed.EvaluateAsync("runaway", runaway, Inputs(i), Ct);
var b = await shadowed.EvaluateAsync("sibling", sibling, Inputs(i), Ct);
if (sw.Elapsed > slowest) slowest = sw.Elapsed;
a.ShouldBe(VirtualTagEvalResult.Ok(i * 2), "published from the in-process run");
b.ShouldBe(VirtualTagEvalResult.Ok(i + 100));
await Task.Delay(100, Ct);
}
// Still publishing correctly after the shadow quarantined it, and never quarantined on the published side.
for (var j = i; j < i + 5; j++)
(await shadowed.EvaluateAsync("runaway", runaway, Inputs(j), Ct)).ShouldBe(VirtualTagEvalResult.Ok(j * 2));
shadowed.QuarantinedScripts.ShouldBeEmpty();
slowest.ShouldBeLessThan(TimeSpan.FromMilliseconds(250), "a published evaluation never waits for its shadow");
log.Rendered.ShouldContain(l => l.Contains("exceeded its hard deadline"));
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
var record = shadow.Mismatches.ShouldHaveSingleItem("the sibling on the killed shard is a victim (not compared), never a mismatch");
record.Key.ShouldBe("runaway");
record.LastReason.ShouldBe("status");
record.LastSummary.ShouldContain("shadow Failed [budget]");
shadow.Stats().DroppedKeyBusy.ShouldBeGreaterThan(0, "triggers while its shadow run was still going were dropped, not queued");
}
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner? runner)
{
var logger = new LoggerConfiguration().CreateLogger();
return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
scriptTimeout: TimeSpan.FromSeconds(2), scriptRunner: runner);
}
[Fact]
public async Task Alarms_in_shadow_transition_exactly_as_in_process_and_a_killed_shadow_worker_never_touches_a_condition()
{
await using var pool = Scripts.Runner(ShadowOptions());
using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), new LoggerConfiguration().CreateLogger());
var up = new TestUpstream();
var refUp = new TestUpstream();
up.Set("x", 0);
refUp.Set("x", 0);
using var engine = Engine(up, shadow);
using var reference = Engine(refUp, null);
engine.ScriptRunner.ShouldBeNull("Shadow keeps the in-process engine path");
var events = new ConcurrentQueue<ScriptedAlarmEvent>();
var refEvents = new ConcurrentQueue<ScriptedAlarmEvent>();
engine.OnEvent += (_, e) => events.Enqueue(e);
reference.OnEvent += (_, e) => refEvents.Enqueue(e);
ScriptedAlarmDefinition[] defs =
[
Alarm("high", "return (int)ctx.GetTag(\"x\").Value > 5;"),
Alarm("divergent", $"return {InWorker};"),
];
await engine.LoadAsync(defs, Ct);
await reference.LoadAsync(defs, Ct);
async Task Push(int x)
{
up.Push("x", x);
refUp.Push("x", x);
await Scripts.Eventually(() => events.Count == refEvents.Count, TimeSpan.FromSeconds(10), "both engines' emissions");
await Task.Delay(100, Ct);
}
await Push(10);
await Push(2);
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
// kill -9 the shadow's worker: the published engine neither annotates nor clears anything because of it.
var worker = pool.Shards[0].Current.ShouldNotBeNull();
Process.GetProcessById(worker.Pid).Kill();
await Push(12);
await Push(0);
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
await Push(9);
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
events.Select(e => (e.AlarmId, e.Emission, e.WorstInputStatusCode))
.ShouldBe(refEvents.Select(e => (e.AlarmId, e.Emission, e.WorstInputStatusCode)), "the in-process engine's sequence, exactly");
events.Count(e => e is { AlarmId: "high", Emission: EmissionKind.Activated }).ShouldBe(3);
events.ShouldNotContain(e => e.WorstInputStatusCode != 0u, "no shadow-side failure annotates a published condition");
engine.GetState("divergent")!.Active.ShouldBe(AlarmActiveState.Inactive, "the in-process predicate (false) drives it");
var record = shadow.Mismatches.ShouldHaveSingleItem();
record.Key.ShouldBe("divergent");
record.Kind.ShouldBe(ScriptContextKind.AlarmPredicate);
record.LastSummary.ShouldBe("published Ok Boolean False; shadow Ok Boolean True");
shadow.Stats().Matched.ShouldBeGreaterThan(0);
}
}