feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run

- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 06:37:36 -05:00
parent 11a5c1d4e2
commit 0796928504
33 changed files with 2168 additions and 117 deletions
@@ -0,0 +1,152 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// <summary>A minimal <see cref="ITagUpstreamSource"/>: set or push a value; push notifies subscribers synchronously.</summary>
internal sealed class TestUpstream : ITagUpstreamSource
{
private readonly ConcurrentDictionary<string, DataValueSnapshot> _values = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, List<Action<string, DataValueSnapshot>>> _subs = new(StringComparer.Ordinal);
public void Set(string path, object? value, uint status = 0u)
=> _values[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow);
public void Push(string path, object? value, uint status = 0u)
{
Set(path, value, status);
if (!_subs.TryGetValue(path, out var list)) return;
Action<string, DataValueSnapshot>[] observers;
lock (list) observers = [.. list];
foreach (var observer in observers) observer(path, _values[path]);
}
public DataValueSnapshot ReadTag(string path)
=> _values.TryGetValue(path, out var v) ? v : new DataValueSnapshot(null, 0x80320000u, null, DateTime.UtcNow);
public IDisposable SubscribeTag(string path, Action<string, DataValueSnapshot> observer)
{
var list = _subs.GetOrAdd(path, _ => []);
lock (list) list.Add(observer);
return new Unsubscribe(() => { lock (list) list.Remove(observer); });
}
private sealed class Unsubscribe(Action dispose) : IDisposable
{
public void Dispose() => dispose();
}
}
/// <summary>
/// lmxopcua#780 — the scripted-alarm seam over the REAL worker (design §9, alarm column): a runaway predicate is killed
/// at T + G and quarantined, its condition frozen with a <c>BadInternalError</c> annotation and never transitioned; a
/// sibling alarm on the same shard keeps evaluating; and a worker that dies under an in-flight predicate freezes its
/// condition <c>BadResourceUnavailable</c> — it never clears it — and the alarm is re-evaluated by itself on return.
/// </summary>
[Collection(ScriptWorkerCollection.Name)]
[Trait("Category", "ScriptWorker")]
public sealed class AlarmSeamTests
{
private const uint StatusBadInternalError = 0x80020000u;
private const uint StatusBadResourceUnavailable = 0x80040000u;
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner runner, TimeSpan budget)
{
var logger = new LoggerConfiguration().CreateLogger();
return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
scriptTimeout: budget, scriptRunner: runner);
}
[Fact]
public async Task A_runaway_predicate_is_killed_and_quarantined_its_alarm_frozen_Bad_and_a_sibling_keeps_evaluating()
{
var budget = TimeSpan.FromMilliseconds(300);
var log = new CapturingSink();
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2), log.CreateLogger());
var up = new TestUpstream();
up.Set("x", 0);
using var engine = Engine(up, runner, budget);
var events = new ConcurrentQueue<ScriptedAlarmEvent>();
engine.OnEvent += (_, e) => events.Enqueue(e);
await engine.LoadAsync(
[
Alarm("runaway", "if ((int)ctx.GetTag(\"x\").Value > 5) { while (true) { } } return false;"),
Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;"),
], Ct);
var first = runner.Shards[0].Current.ShouldNotBeNull();
up.Push("x", 10);
await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Activated }), TimeSpan.FromSeconds(10), "the sibling's activation");
await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "runaway", Emission: EmissionKind.QualityChanged }), TimeSpan.FromSeconds(10), "the runaway's annotation");
events.Single(e => e.AlarmId == "runaway").WorstInputStatusCode.ShouldBe(StatusBadInternalError);
(await first.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct)).CulpritKey
.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.AlarmPredicate, "runaway"));
// The second kill quarantines it; the sibling (same shard, worker restarted twice) keeps evaluating. Re-pushed:
// Exited completes before the dead worker's evaluations are released, so the first push after it can find the
// runaway's slot still held and be refused (not a strike, exactly as in process) — the next push runs it.
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
for (var x = 11; !engine.QuarantinedAlarmIds.Contains("runaway"); x++)
{
if (DateTime.UtcNow > deadline)
throw new TimeoutException("no kill quarantine; events: " +
string.Join(", ", events.Select(e => $"{e.AlarmId}:{e.Emission}:{e.WorstInputStatusCode:X8}")) +
Environment.NewLine + string.Join(Environment.NewLine, log.Rendered));
up.Push("x", x);
await Task.Delay(TimeSpan.FromSeconds(2), Ct);
}
engine.QuarantinedAlarmIds.ShouldBe(["runaway"]);
up.Push("x", 0);
await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Cleared }), TimeSpan.FromSeconds(60), "the sibling's clear");
engine.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "a predicate that never returned never transitions");
events.Where(e => e.AlarmId == "runaway").ShouldAllBe(e => e.Emission == EmissionKind.QualityChanged);
// A redeploy (reload) lifts the quarantine, as in process.
await engine.LoadAsync([Alarm("runaway", "return (int)ctx.GetTag(\"x\").Value > 5;"), Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;")], Ct);
engine.QuarantinedAlarmIds.ShouldBeEmpty();
}
[Fact]
public async Task A_worker_that_dies_under_a_predicate_freezes_it_BadResourceUnavailable_never_clears_it_and_it_is_re_evaluated_on_return()
{
await using var runner = Scripts.Runner(Scripts.Options(probation: 1));
var up = new TestUpstream();
up.Set("ms", 1);
using var engine = Engine(up, runner, TimeSpan.FromSeconds(5));
var events = new ConcurrentQueue<ScriptedAlarmEvent>();
engine.OnEvent += (_, e) => events.Enqueue(e);
await engine.LoadAsync([Alarm("slow",
"var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms > 0;")], Ct);
engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active);
var worker = runner.Shards[0].Current.ShouldNotBeNull();
events.Clear();
up.Push("ms", 1500);
await Task.Delay(300, Ct); // the predicate is spinning in the worker
Process.GetProcessById(worker.Pid).Kill();
await Scripts.Eventually(() => events.Any(e => e.Emission == EmissionKind.QualityChanged), TimeSpan.FromSeconds(10), "the runner-failure annotation");
events.First().WorstInputStatusCode.ShouldBe(StatusBadResourceUnavailable);
engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active, "a dead worker never clears an alarm");
// No further input change: the runner's recovery re-evaluates it (in the isolation worker) and the quality returns.
await Scripts.Eventually(() => events.Any(e => e is { Emission: EmissionKind.QualityChanged, WorstInputStatusCode: 0u }),
TimeSpan.FromSeconds(60), "the re-evaluation on return");
engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active);
events.ShouldNotContain(e => e.Emission == EmissionKind.Cleared);
}
}
@@ -0,0 +1,172 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
/// <summary>
/// lmxopcua#780 batch submit (design §4.5) against the real worker: <see cref="IScriptRunner.RunBatch"/> sends one frame
/// per worker and answers per item, and every item keeps its own deadlines and blame — in particular an item that has to
/// wait in the worker behind a runaway sibling is neither timed out, nor struck, nor blamed for the kill.
/// </summary>
[Collection(ScriptWorkerCollection.Name)]
[Trait("Category", "ScriptWorker")]
public sealed class BatchSubmitTests
{
/// <summary>Runs away when input <c>spin</c> is true, otherwise returns <c>n + 1</c>.</summary>
private const string MaybeRunaway = "if ((bool)ctx.GetTag(\"spin\").Value) { while (true) { } } return (int)ctx.GetTag(\"n\").Value + 1;";
/// <summary>Spins for input <c>ms</c> milliseconds, then returns it.</summary>
private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static IReadOnlyDictionary<string, DataValueSnapshot> Inputs(params (string Path, object Value)[] values)
{
var map = new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
foreach (var (path, value) in values) map[path] = new DataValueSnapshot(value, 0, DateTime.UtcNow, DateTime.UtcNow);
return map;
}
private static ScriptInvocation Item(string key, bool spin, int n, TimeSpan? timeout = null)
=> Scripts.Vt(key, MaybeRunaway, timeout, Inputs(("spin", spin), ("n", n)));
private static OutOfProcessScriptRunner Runner(ScriptingOptions options, int evalThreads)
=> new(options, new LoggerConfiguration().CreateLogger(), null,
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60), evalThreads: evalThreads);
[Fact]
public async Task A_batch_of_loaded_scripts_goes_out_as_one_frame_and_every_item_gets_its_own_result()
{
await using var runner = Runner(Scripts.Options(), evalThreads: 4);
// First evaluations load (compile) each key; those take the full path one by one.
var warm = await Task.WhenAll(runner.RunBatch(Enumerable.Range(0, 40).Select(i => Item($"k{i}", false, i)).ToArray(), Ct));
warm.ShouldAllBe(o => o.Status == ScriptRunStatus.Ok);
var frames = new ConcurrentQueue<int>();
runner.BatchSubmittedForTest = frames.Enqueue;
var outcomes = await Task.WhenAll(runner.RunBatch(Enumerable.Range(0, 40).Select(i => Item($"k{i}", false, i * 10)).ToArray(), Ct));
frames.ToArray().ShouldBe([40], "every loaded item rides one EvalBatch frame");
for (var i = 0; i < 40; i++)
{
outcomes[i].Status.ShouldBe(ScriptRunStatus.Ok, outcomes[i].Reason);
outcomes[i].Value.ShouldBe(i * 10 + 1, $"item {i} got its own result");
}
}
[Fact]
public async Task A_batch_answers_exactly_what_the_same_evaluations_answer_one_by_one()
{
// Parity: Ok, a throw, a compile error, a duplicate key (refused) and a fast item, batched vs RunAsync.
ScriptInvocation[] Items() =>
[
Scripts.Vt("ok", "return (int)ctx.GetTag(\"n\").Value * 2;", inputs: Inputs(("n", 21))),
Scripts.Vt("throws", "throw new System.InvalidOperationException(\"boom\");"),
Scripts.Vt("broken", "return undefinedThing;"),
Scripts.Vt("fast", "return 5;"),
];
await using var runner = Runner(Scripts.Options(), evalThreads: 2);
var single = new List<ScriptRunOutcome>();
foreach (var item in Items()) single.Add(await runner.RunAsync(item, Ct));
var batched = await Task.WhenAll(runner.RunBatch(Items(), Ct));
batched.Select(o => (o.Status, o.Value, o.Reason)).ShouldBe(single.Select(o => (o.Status, o.Value, o.Reason)));
single[0].Value.ShouldBe(42);
single[1].Reason!.ShouldStartWith("script threw: ");
single[2].Reason!.ShouldStartWith("compile error: ");
// In process, the default RunBatch is RunAsync per item: same answers.
using var inProcess = new InProcessScriptRunner();
var inProcessBatched = await Task.WhenAll(((IScriptRunner)inProcess).RunBatch(Items(), Ct));
inProcessBatched.Select(o => (o.Status, o.Value)).ShouldBe(single.Select(o => (o.Status, o.Value)));
// A key twice in one batch: the second is refused, exactly like two concurrent RunAsync calls.
var twice = await Task.WhenAll(runner.RunBatch([Scripts.Vt("dup", Spin, inputs: Inputs(("ms", 200))), Scripts.Vt("dup", Spin, inputs: Inputs(("ms", 1)))], Ct));
twice.Count(o => o.Reason == ScriptRunGuard.PreviousRunStillRunningReason).ShouldBe(1);
}
/// <summary>
/// The blame rule (design §5.3.1) under batching: with ONE evaluation thread, a runaway at the head of a batch holds
/// the thread and its siblings wait in the worker. The siblings are the kill's victims — Unavailable "worker
/// restarted", no timeout, no strike — and only the runaway is killed and quarantined. Mutation check: with the
/// host ignoring the worker's deferral notice, the siblings' clocks run from submit, they time out at T with the
/// runaway, and at T + G any of the four can be the one blamed.
/// </summary>
[Fact]
public async Task A_runaway_in_a_batch_is_blamed_alone_and_siblings_queued_behind_it_are_not_timed_out_or_struck()
{
var softT = TimeSpan.FromMilliseconds(400);
var grace = TimeSpan.FromMilliseconds(500);
await using var runner = Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 1), evalThreads: 1);
var keys = new[] { "runaway", "sib1", "sib2", "sib3" };
foreach (var key in keys) (await runner.RunAsync(Item(key, false, 1, softT), Ct)).Value.ShouldBe(2);
var worker = runner.Shards[0].Current.ShouldNotBeNull();
var frames = new ConcurrentQueue<int>();
runner.BatchSubmittedForTest = frames.Enqueue;
var sw = Stopwatch.StartNew();
var tasks = runner.RunBatch([Item("runaway", true, 0, softT), Item("sib1", false, 10, softT), Item("sib2", false, 20, softT), Item("sib3", false, 30, softT)], Ct);
frames.ToArray().ShouldBe([4]);
var runaway = await tasks[0];
runaway.Status.ShouldBe(ScriptRunStatus.Failed);
runaway.Reason!.ShouldStartWith("script timed out after ");
// Nothing has answered the queued siblings at the runaway's soft deadline.
tasks.Skip(1).ShouldAllBe(t => !t.IsCompleted, "a run waiting behind a busy thread is not timed out at its submit-relative T");
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline);
exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "runaway"));
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50));
var siblings = await Task.WhenAll(tasks.Skip(1));
foreach (var sibling in siblings)
{
sibling.Status.ShouldBe(ScriptRunStatus.Unavailable, "a victim of another key's kill, not a script failure");
sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
}
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count > 0, TimeSpan.FromSeconds(5), "the kill quarantine");
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"], "the siblings took no strike");
// The siblings run again on the restarted worker, with their own inputs.
var again = await Task.WhenAll(runner.RunBatch([Item("sib1", false, 10, softT), Item("sib2", false, 20, softT), Item("sib3", false, 30, softT)], Ct));
again.Select(o => o.Value).ShouldBe([11, 21, 31]);
}
/// <summary>
/// The positive control for the deferral: a run that waited behind a sibling still gets its deadlines — the full T,
/// then G — counted from when it STARTED. Mutation check: a start notice that does not re-arm the clock leaves the
/// deferred runaway with no deadline at all; it is never answered or killed.
/// </summary>
[Fact]
public async Task A_deferred_run_is_timed_and_killed_from_when_it_starts()
{
var softT = TimeSpan.FromMilliseconds(400);
var grace = TimeSpan.FromMilliseconds(500);
var head = TimeSpan.FromMilliseconds(300);
await using var runner = Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 5), evalThreads: 1);
(await runner.RunAsync(Scripts.Vt("head", Spin, softT, Inputs(("ms", 1))), Ct)).Value.ShouldBe(1);
(await runner.RunAsync(Item("tail", false, 1, softT), Ct)).Value.ShouldBe(2);
var worker = runner.Shards[0].Current.ShouldNotBeNull();
var sw = Stopwatch.StartNew();
var tasks = runner.RunBatch([Scripts.Vt("head", Spin, softT, Inputs(("ms", (int)head.TotalMilliseconds))), Item("tail", true, 0, softT)], Ct);
(await tasks[0]).Value.ShouldBe((int)head.TotalMilliseconds, "the head ran inside its budget");
var tail = await tasks[1];
var answered = sw.Elapsed;
tail.Reason!.ShouldStartWith("script timed out after ");
answered.ShouldBeGreaterThanOrEqualTo(head + softT - TimeSpan.FromMilliseconds(50), "T counts from the run's start, after the head");
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "tail"));
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(head + softT + grace - TimeSpan.FromMilliseconds(50), "G counts from the run's start too");
}
}
@@ -3,6 +3,7 @@ using System.Text;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
@@ -261,6 +262,217 @@ public sealed class ScriptRunnerPerfTests
k2Closed.EvalsPerSecond.ShouldBeGreaterThanOrEqualTo(AcceptedGateEvalsPerSecond, "the accepted 5k evals/s/node gate at K=2");
}
/// <summary>
/// lmxopcua#780 batch submit (design §4.5): a closed loop of one batch at a time, <paramref name="batchSize"/>
/// distinct loaded keys per batch, either through <see cref="IScriptRunner.RunBatch"/> or as the same evaluations
/// submitted as concurrent <see cref="IScriptRunner.RunAsync"/> calls. Latency is per batch (submit → last result).
/// </summary>
private static async Task<PhaseResult> BatchLoopAsync(IScriptRunner runner, int batchSize, bool viaBatch)
{
var deadline = Stopwatch.GetTimestamp() + (long)(Phase.TotalSeconds * Stopwatch.Frequency);
var samples = new List<double>(100_000);
long ok = 0, notOk = 0;
string? firstFailure = null;
var offset = 0;
var x = 0;
var cpu0 = Process.GetCurrentProcess().TotalProcessorTime;
var wall = Stopwatch.StartNew();
while (Stopwatch.GetTimestamp() < deadline)
{
var invocations = new ScriptInvocation[batchSize];
var keys = new int[batchSize];
for (var i = 0; i < batchSize; i++)
{
keys[i] = (offset + i) % Keys;
invocations[i] = Invocation(keys[i], x);
}
offset = (offset + batchSize) % Keys;
var t0 = Stopwatch.GetTimestamp();
var outcomes = await Task.WhenAll(viaBatch
? runner.RunBatch(invocations, Ct)
: invocations.Select(inv => runner.RunAsync(inv, Ct)).ToArray());
samples.Add(Stopwatch.GetElapsedTime(t0).TotalMicroseconds);
for (var i = 0; i < batchSize; i++)
{
var o = outcomes[i];
if (o.Status == ScriptRunStatus.Ok && Equals(o.Value, x + keys[i] % DistinctSources)) ok++;
else
{
notOk++;
firstFailure ??= $"{o.Status}: {o.Reason} (value {o.Value})";
}
}
x++;
}
var elapsed = wall.Elapsed;
var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / elapsed.TotalMilliseconds * 100;
samples.Sort();
return new PhaseResult((ok + notOk) / elapsed.TotalSeconds, Percentile(samples, 0.50), Percentile(samples, 0.99), cpu, ok, notOk, firstFailure);
}
/// <summary>
/// A runner that turns a batch back into one evaluation after another — the serial pass the alarm engine and the
/// Calc pump made before batch submit, which design §8.2 warns costs N round trips (the comparison row).
/// </summary>
private sealed class SerialRunner(IScriptRunner inner) : IScriptRunner
{
public Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default) => inner.RunAsync(invocation, ct);
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
{
var results = new Task<ScriptRunOutcome>[invocations.Count];
Task previous = Task.CompletedTask;
for (var i = 0; i < results.Length; i++) previous = results[i] = After(previous, invocations[i], ct);
return results;
}
private async Task<ScriptRunOutcome> After(Task previous, ScriptInvocation invocation, CancellationToken ct)
{
await previous.ConfigureAwait(false);
return await inner.RunAsync(invocation, ct).ConfigureAwait(false);
}
public void Reset(ScriptContextKind kind) => inner.Reset(kind);
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind) => inner.QuarantinedKeys(kind);
}
/// <summary>
/// Scripted-alarm pass latency: <paramref name="alarms"/> alarms on one input, every push flips all of them, and a
/// sample is the time from the push to the last of the N transitions the pass emits (the whole dirty set through
/// the engine's gate). The design's alarm budget is 250 ms (§1.1; the host now passes <c>Scripting:RunTimeout</c>,
/// 2 s by default, lmxopcua#779), so the pass is judged against the stricter 250 ms.
/// </summary>
private static async Task<(double P50Ms, double P99Ms, double MaxMs, double CpuPercent, int NotCompleted)> AlarmPassLatencyAsync(
IScriptRunner? runner, int alarms, int pushes)
{
var up = new TestUpstream();
up.Set("x", 0);
var logger = new Serilog.LoggerConfiguration().CreateLogger();
using var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
scriptTimeout: TimeSpan.FromMilliseconds(250), scriptRunner: runner);
var transitions = 0;
var notCompleted = 0;
var target = int.MaxValue;
var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
engine.OnEvent += (_, e) =>
{
// A Bad annotation means a predicate did not complete in that pass (timed out, refused): counted, reported.
if (e.Emission == EmissionKind.QualityChanged && e.WorstInputStatusCode != 0) Interlocked.Increment(ref notCompleted);
if (e.Emission is not (EmissionKind.Activated or EmissionKind.Cleared)) return;
if (Interlocked.Increment(ref transitions) == Volatile.Read(ref target)) done.TrySetResult();
};
await engine.LoadAsync(Enumerable.Range(0, alarms).Select(i => new ScriptedAlarmDefinition(
$"alarm-{i}", "Plant/Perf", $"alarm-{i}", AlarmKind.AlarmCondition, AlarmSeverity.High, "perf",
$"return (int)ctx.GetTag(\"x\").Value > {-(i % DistinctSources)};")).ToArray(), Ct);
var samples = new List<double>(pushes);
var cpu0 = Process.GetCurrentProcess().TotalProcessorTime;
var wall = Stopwatch.StartNew();
for (var p = 0; p < pushes + 2; p++)
{
// Alternate x = -1000 (every predicate false) and x = 1000 (every predicate true). The pass is expected to
// transition exactly the alarms not already in the target state — counted from the engine, so an alarm
// that did not complete an earlier pass cannot leave the count short. The first two passes are warm-up.
var activate = p % 2 == 1;
var expected = engine.GetAllStates().Count(c => (c.Active == AlarmActiveState.Active) != activate);
Interlocked.Exchange(ref transitions, 0);
done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Volatile.Write(ref target, expected);
var t0 = Stopwatch.GetTimestamp();
up.Push("x", activate ? 1000 : -1000);
if (expected == 0) continue;
try
{
await done.Task.WaitAsync(TimeSpan.FromSeconds(60), Ct);
}
catch (TimeoutException)
{
throw new TimeoutException($"alarm pass {p} ({(runner is null ? "in process" : runner.GetType().Name)}, {alarms} alarms): " +
$"{Volatile.Read(ref transitions)} of {expected} transitions in 60 s, " +
$"{Volatile.Read(ref notCompleted)} predicate(s) did not complete so far");
}
if (p >= 2) samples.Add(Stopwatch.GetElapsedTime(t0).TotalMilliseconds);
}
var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / wall.Elapsed.TotalMilliseconds * 100;
samples.Sort();
return (Percentile(samples, 0.50), Percentile(samples, 0.99), samples[^1], cpu, notCompleted);
}
/// <summary>
/// Batch throughput (RunBatch vs the same evaluations as concurrent RunAsync calls, K = 2) and scripted-alarm pass
/// latency (in process, out of process batched, and out of process serial). Report-only, except that every
/// result must be correct.
/// </summary>
[Fact]
public async Task Batch_throughput_and_alarm_pass_latency()
{
RequireOptIn();
var report = new StringBuilder()
.AppendLine($"### Batch submit throughput ({Env()})")
.AppendLine()
.AppendLine("| Runner | batch | submitted as | evals/s | batch p50 µs | batch p99 µs | host CPU | ok / not ok |")
.AppendLine("|---|---:|---|---:|---:|---:|---:|---|");
foreach (var (name, create) in new (string, Func<IScriptRunner>)[]
{
("InProcess", () => new InProcessScriptRunner(new ScriptingOptions())),
("OutOfProcess K=2", () => Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger())),
})
{
var runner = create();
try
{
await WarmAsync(runner);
foreach (var size in new[] { 1, 10, 100, 500 })
{
foreach (var viaBatch in new[] { true, false })
{
var r = await BatchLoopAsync(runner, size, viaBatch);
r.NotOk.ShouldBe(0, $"{name} batch {size}: {r.FirstFailure}{Environment.NewLine}{RunnerLog()}");
report.AppendLine($"| {name} | {size} | {(viaBatch ? "RunBatch" : "RunAsync ×n")} | {r.EvalsPerSecond,9:N0} | " +
$"{r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | {r.Ok:N0} / {r.NotOk:N0} |");
}
}
}
finally
{
switch (runner)
{
case IAsyncDisposable a: await a.DisposeAsync(); break;
case IDisposable d: d.Dispose(); break;
}
}
}
report.AppendLine()
.AppendLine($"### Scripted-alarm pass latency, push → last transition ({Env()})")
.AppendLine()
.AppendLine("| Engine | alarms | pass p50 ms | pass p99 ms | max ms | per alarm (p50) µs | host CPU | not completed | 250 ms |")
.AppendLine("|---|---:|---:|---:|---:|---:|---:|---:|---|");
const int pushes = 40;
await using (var shared = Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger()))
{
foreach (var alarms in new[] { 100, 500, 1000 })
{
foreach (var (name, runner) in new (string, IScriptRunner?)[]
{
("InProcess (serial)", null),
("OutOfProcess K=2, batched", shared),
("OutOfProcess K=2, serial", new SerialRunner(shared)),
})
{
var (p50, p99, max, cpu, notCompleted) = await AlarmPassLatencyAsync(runner, alarms, pushes);
report.AppendLine($"| {name} | {alarms:N0} | {p50,7:N1} | {p99,7:N1} | {max,7:N1} | {p50 * 1000 / alarms,6:N0} | " +
$"{cpu,5:N0} % | {notCompleted:N0} | {(p99 <= 250 ? "PASS" : "MISS")} |");
}
}
}
report.AppendLine()
.AppendLine("Each push flips every alarm; a pass is the engine's whole dirty set under its gate. \"serial\" replays the " +
"pre-batch one-predicate-at-a-time pass over the same runner.");
Report(report);
}
private static ScriptingOptions PerfOptions(int shards, string? peCacheDir = null)
{
var options = Scripts.Options(shards: shards, hardGrace: TimeSpan.FromSeconds(2), peCacheDir: peCacheDir,
@@ -22,6 +22,8 @@
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess\ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess.csproj"/>
<!-- End-to-end seam test: a real Calculation driver over the real worker. -->
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Calculation\ZB.MOM.WW.OtOpcUa.Driver.Calculation.csproj"/>
<!-- The scripted-alarm seam over the real worker (lmxopcua#780). -->
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms\ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj"/>
</ItemGroup>
<!-- The real worker, built and placed in script-worker/ beside the tests, exactly as beside the Host. -->