feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run
v2-ci / unit-tests (push) Waiting to run
- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results. The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged. - Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill. - Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider carries the runner to DriverHostActor. - Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial); design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms; batched p99 <= 53 ms at 1,000. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>A minimal <see cref="ITagUpstreamSource"/>: set or push a value; push notifies subscribers synchronously.</summary>
|
||||
internal sealed class TestUpstream : ITagUpstreamSource
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, DataValueSnapshot> _values = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, List<Action<string, DataValueSnapshot>>> _subs = new(StringComparer.Ordinal);
|
||||
|
||||
public void Set(string path, object? value, uint status = 0u)
|
||||
=> _values[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow);
|
||||
|
||||
public void Push(string path, object? value, uint status = 0u)
|
||||
{
|
||||
Set(path, value, status);
|
||||
if (!_subs.TryGetValue(path, out var list)) return;
|
||||
Action<string, DataValueSnapshot>[] observers;
|
||||
lock (list) observers = [.. list];
|
||||
foreach (var observer in observers) observer(path, _values[path]);
|
||||
}
|
||||
|
||||
public DataValueSnapshot ReadTag(string path)
|
||||
=> _values.TryGetValue(path, out var v) ? v : new DataValueSnapshot(null, 0x80320000u, null, DateTime.UtcNow);
|
||||
|
||||
public IDisposable SubscribeTag(string path, Action<string, DataValueSnapshot> observer)
|
||||
{
|
||||
var list = _subs.GetOrAdd(path, _ => []);
|
||||
lock (list) list.Add(observer);
|
||||
return new Unsubscribe(() => { lock (list) list.Remove(observer); });
|
||||
}
|
||||
|
||||
private sealed class Unsubscribe(Action dispose) : IDisposable
|
||||
{
|
||||
public void Dispose() => dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 — the scripted-alarm seam over the REAL worker (design §9, alarm column): a runaway predicate is killed
|
||||
/// at T + G and quarantined, its condition frozen with a <c>BadInternalError</c> annotation and never transitioned; a
|
||||
/// sibling alarm on the same shard keeps evaluating; and a worker that dies under an in-flight predicate freezes its
|
||||
/// condition <c>BadResourceUnavailable</c> — it never clears it — and the alarm is re-evaluated by itself on return.
|
||||
/// </summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class AlarmSeamTests
|
||||
{
|
||||
private const uint StatusBadInternalError = 0x80020000u;
|
||||
private const uint StatusBadResourceUnavailable = 0x80040000u;
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
|
||||
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
|
||||
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
|
||||
|
||||
private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner runner, TimeSpan budget)
|
||||
{
|
||||
var logger = new LoggerConfiguration().CreateLogger();
|
||||
return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
|
||||
scriptTimeout: budget, scriptRunner: runner);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_runaway_predicate_is_killed_and_quarantined_its_alarm_frozen_Bad_and_a_sibling_keeps_evaluating()
|
||||
{
|
||||
var budget = TimeSpan.FromMilliseconds(300);
|
||||
var log = new CapturingSink();
|
||||
await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2), log.CreateLogger());
|
||||
var up = new TestUpstream();
|
||||
up.Set("x", 0);
|
||||
using var engine = Engine(up, runner, budget);
|
||||
var events = new ConcurrentQueue<ScriptedAlarmEvent>();
|
||||
engine.OnEvent += (_, e) => events.Enqueue(e);
|
||||
await engine.LoadAsync(
|
||||
[
|
||||
Alarm("runaway", "if ((int)ctx.GetTag(\"x\").Value > 5) { while (true) { } } return false;"),
|
||||
Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;"),
|
||||
], Ct);
|
||||
var first = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
up.Push("x", 10);
|
||||
await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Activated }), TimeSpan.FromSeconds(10), "the sibling's activation");
|
||||
await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "runaway", Emission: EmissionKind.QualityChanged }), TimeSpan.FromSeconds(10), "the runaway's annotation");
|
||||
events.Single(e => e.AlarmId == "runaway").WorstInputStatusCode.ShouldBe(StatusBadInternalError);
|
||||
(await first.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct)).CulpritKey
|
||||
.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.AlarmPredicate, "runaway"));
|
||||
|
||||
// The second kill quarantines it; the sibling (same shard, worker restarted twice) keeps evaluating. Re-pushed:
|
||||
// Exited completes before the dead worker's evaluations are released, so the first push after it can find the
|
||||
// runaway's slot still held and be refused (not a strike, exactly as in process) — the next push runs it.
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
|
||||
for (var x = 11; !engine.QuarantinedAlarmIds.Contains("runaway"); x++)
|
||||
{
|
||||
if (DateTime.UtcNow > deadline)
|
||||
throw new TimeoutException("no kill quarantine; events: " +
|
||||
string.Join(", ", events.Select(e => $"{e.AlarmId}:{e.Emission}:{e.WorstInputStatusCode:X8}")) +
|
||||
Environment.NewLine + string.Join(Environment.NewLine, log.Rendered));
|
||||
up.Push("x", x);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), Ct);
|
||||
}
|
||||
engine.QuarantinedAlarmIds.ShouldBe(["runaway"]);
|
||||
up.Push("x", 0);
|
||||
await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Cleared }), TimeSpan.FromSeconds(60), "the sibling's clear");
|
||||
|
||||
engine.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "a predicate that never returned never transitions");
|
||||
events.Where(e => e.AlarmId == "runaway").ShouldAllBe(e => e.Emission == EmissionKind.QualityChanged);
|
||||
|
||||
// A redeploy (reload) lifts the quarantine, as in process.
|
||||
await engine.LoadAsync([Alarm("runaway", "return (int)ctx.GetTag(\"x\").Value > 5;"), Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;")], Ct);
|
||||
engine.QuarantinedAlarmIds.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_worker_that_dies_under_a_predicate_freezes_it_BadResourceUnavailable_never_clears_it_and_it_is_re_evaluated_on_return()
|
||||
{
|
||||
await using var runner = Scripts.Runner(Scripts.Options(probation: 1));
|
||||
var up = new TestUpstream();
|
||||
up.Set("ms", 1);
|
||||
using var engine = Engine(up, runner, TimeSpan.FromSeconds(5));
|
||||
var events = new ConcurrentQueue<ScriptedAlarmEvent>();
|
||||
engine.OnEvent += (_, e) => events.Enqueue(e);
|
||||
await engine.LoadAsync([Alarm("slow",
|
||||
"var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms > 0;")], Ct);
|
||||
engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
events.Clear();
|
||||
|
||||
up.Push("ms", 1500);
|
||||
await Task.Delay(300, Ct); // the predicate is spinning in the worker
|
||||
Process.GetProcessById(worker.Pid).Kill();
|
||||
|
||||
await Scripts.Eventually(() => events.Any(e => e.Emission == EmissionKind.QualityChanged), TimeSpan.FromSeconds(10), "the runner-failure annotation");
|
||||
events.First().WorstInputStatusCode.ShouldBe(StatusBadResourceUnavailable);
|
||||
engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active, "a dead worker never clears an alarm");
|
||||
|
||||
// No further input change: the runner's recovery re-evaluates it (in the isolation worker) and the quality returns.
|
||||
await Scripts.Eventually(() => events.Any(e => e is { Emission: EmissionKind.QualityChanged, WorstInputStatusCode: 0u }),
|
||||
TimeSpan.FromSeconds(60), "the re-evaluation on return");
|
||||
engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active);
|
||||
events.ShouldNotContain(e => e.Emission == EmissionKind.Cleared);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 batch submit (design §4.5) against the real worker: <see cref="IScriptRunner.RunBatch"/> sends one frame
|
||||
/// per worker and answers per item, and every item keeps its own deadlines and blame — in particular an item that has to
|
||||
/// wait in the worker behind a runaway sibling is neither timed out, nor struck, nor blamed for the kill.
|
||||
/// </summary>
|
||||
[Collection(ScriptWorkerCollection.Name)]
|
||||
[Trait("Category", "ScriptWorker")]
|
||||
public sealed class BatchSubmitTests
|
||||
{
|
||||
/// <summary>Runs away when input <c>spin</c> is true, otherwise returns <c>n + 1</c>.</summary>
|
||||
private const string MaybeRunaway = "if ((bool)ctx.GetTag(\"spin\").Value) { while (true) { } } return (int)ctx.GetTag(\"n\").Value + 1;";
|
||||
|
||||
/// <summary>Spins for input <c>ms</c> milliseconds, then returns it.</summary>
|
||||
private const string Spin = "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static IReadOnlyDictionary<string, DataValueSnapshot> Inputs(params (string Path, object Value)[] values)
|
||||
{
|
||||
var map = new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
|
||||
foreach (var (path, value) in values) map[path] = new DataValueSnapshot(value, 0, DateTime.UtcNow, DateTime.UtcNow);
|
||||
return map;
|
||||
}
|
||||
|
||||
private static ScriptInvocation Item(string key, bool spin, int n, TimeSpan? timeout = null)
|
||||
=> Scripts.Vt(key, MaybeRunaway, timeout, Inputs(("spin", spin), ("n", n)));
|
||||
|
||||
private static OutOfProcessScriptRunner Runner(ScriptingOptions options, int evalThreads)
|
||||
=> new(options, new LoggerConfiguration().CreateLogger(), null,
|
||||
heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(60), evalThreads: evalThreads);
|
||||
|
||||
[Fact]
|
||||
public async Task A_batch_of_loaded_scripts_goes_out_as_one_frame_and_every_item_gets_its_own_result()
|
||||
{
|
||||
await using var runner = Runner(Scripts.Options(), evalThreads: 4);
|
||||
// First evaluations load (compile) each key; those take the full path one by one.
|
||||
var warm = await Task.WhenAll(runner.RunBatch(Enumerable.Range(0, 40).Select(i => Item($"k{i}", false, i)).ToArray(), Ct));
|
||||
warm.ShouldAllBe(o => o.Status == ScriptRunStatus.Ok);
|
||||
|
||||
var frames = new ConcurrentQueue<int>();
|
||||
runner.BatchSubmittedForTest = frames.Enqueue;
|
||||
var outcomes = await Task.WhenAll(runner.RunBatch(Enumerable.Range(0, 40).Select(i => Item($"k{i}", false, i * 10)).ToArray(), Ct));
|
||||
|
||||
frames.ToArray().ShouldBe([40], "every loaded item rides one EvalBatch frame");
|
||||
for (var i = 0; i < 40; i++)
|
||||
{
|
||||
outcomes[i].Status.ShouldBe(ScriptRunStatus.Ok, outcomes[i].Reason);
|
||||
outcomes[i].Value.ShouldBe(i * 10 + 1, $"item {i} got its own result");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_batch_answers_exactly_what_the_same_evaluations_answer_one_by_one()
|
||||
{
|
||||
// Parity: Ok, a throw, a compile error, a duplicate key (refused) and a fast item, batched vs RunAsync.
|
||||
ScriptInvocation[] Items() =>
|
||||
[
|
||||
Scripts.Vt("ok", "return (int)ctx.GetTag(\"n\").Value * 2;", inputs: Inputs(("n", 21))),
|
||||
Scripts.Vt("throws", "throw new System.InvalidOperationException(\"boom\");"),
|
||||
Scripts.Vt("broken", "return undefinedThing;"),
|
||||
Scripts.Vt("fast", "return 5;"),
|
||||
];
|
||||
|
||||
await using var runner = Runner(Scripts.Options(), evalThreads: 2);
|
||||
var single = new List<ScriptRunOutcome>();
|
||||
foreach (var item in Items()) single.Add(await runner.RunAsync(item, Ct));
|
||||
var batched = await Task.WhenAll(runner.RunBatch(Items(), Ct));
|
||||
|
||||
batched.Select(o => (o.Status, o.Value, o.Reason)).ShouldBe(single.Select(o => (o.Status, o.Value, o.Reason)));
|
||||
single[0].Value.ShouldBe(42);
|
||||
single[1].Reason!.ShouldStartWith("script threw: ");
|
||||
single[2].Reason!.ShouldStartWith("compile error: ");
|
||||
|
||||
// In process, the default RunBatch is RunAsync per item: same answers.
|
||||
using var inProcess = new InProcessScriptRunner();
|
||||
var inProcessBatched = await Task.WhenAll(((IScriptRunner)inProcess).RunBatch(Items(), Ct));
|
||||
inProcessBatched.Select(o => (o.Status, o.Value)).ShouldBe(single.Select(o => (o.Status, o.Value)));
|
||||
|
||||
// A key twice in one batch: the second is refused, exactly like two concurrent RunAsync calls.
|
||||
var twice = await Task.WhenAll(runner.RunBatch([Scripts.Vt("dup", Spin, inputs: Inputs(("ms", 200))), Scripts.Vt("dup", Spin, inputs: Inputs(("ms", 1)))], Ct));
|
||||
twice.Count(o => o.Reason == ScriptRunGuard.PreviousRunStillRunningReason).ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The blame rule (design §5.3.1) under batching: with ONE evaluation thread, a runaway at the head of a batch holds
|
||||
/// the thread and its siblings wait in the worker. The siblings are the kill's victims — Unavailable "worker
|
||||
/// restarted", no timeout, no strike — and only the runaway is killed and quarantined. Mutation check: with the
|
||||
/// host ignoring the worker's deferral notice, the siblings' clocks run from submit, they time out at T with the
|
||||
/// runaway, and at T + G any of the four can be the one blamed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_runaway_in_a_batch_is_blamed_alone_and_siblings_queued_behind_it_are_not_timed_out_or_struck()
|
||||
{
|
||||
var softT = TimeSpan.FromMilliseconds(400);
|
||||
var grace = TimeSpan.FromMilliseconds(500);
|
||||
await using var runner = Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 1), evalThreads: 1);
|
||||
var keys = new[] { "runaway", "sib1", "sib2", "sib3" };
|
||||
foreach (var key in keys) (await runner.RunAsync(Item(key, false, 1, softT), Ct)).Value.ShouldBe(2);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
var frames = new ConcurrentQueue<int>();
|
||||
runner.BatchSubmittedForTest = frames.Enqueue;
|
||||
var sw = Stopwatch.StartNew();
|
||||
var tasks = runner.RunBatch([Item("runaway", true, 0, softT), Item("sib1", false, 10, softT), Item("sib2", false, 20, softT), Item("sib3", false, 30, softT)], Ct);
|
||||
frames.ToArray().ShouldBe([4]);
|
||||
|
||||
var runaway = await tasks[0];
|
||||
runaway.Status.ShouldBe(ScriptRunStatus.Failed);
|
||||
runaway.Reason!.ShouldStartWith("script timed out after ");
|
||||
// Nothing has answered the queued siblings at the runaway's soft deadline.
|
||||
tasks.Skip(1).ShouldAllBe(t => !t.IsCompleted, "a run waiting behind a busy thread is not timed out at its submit-relative T");
|
||||
|
||||
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline);
|
||||
exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "runaway"));
|
||||
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50));
|
||||
|
||||
var siblings = await Task.WhenAll(tasks.Skip(1));
|
||||
foreach (var sibling in siblings)
|
||||
{
|
||||
sibling.Status.ShouldBe(ScriptRunStatus.Unavailable, "a victim of another key's kill, not a script failure");
|
||||
sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason);
|
||||
}
|
||||
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count > 0, TimeSpan.FromSeconds(5), "the kill quarantine");
|
||||
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["runaway"], "the siblings took no strike");
|
||||
|
||||
// The siblings run again on the restarted worker, with their own inputs.
|
||||
var again = await Task.WhenAll(runner.RunBatch([Item("sib1", false, 10, softT), Item("sib2", false, 20, softT), Item("sib3", false, 30, softT)], Ct));
|
||||
again.Select(o => o.Value).ShouldBe([11, 21, 31]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The positive control for the deferral: a run that waited behind a sibling still gets its deadlines — the full T,
|
||||
/// then G — counted from when it STARTED. Mutation check: a start notice that does not re-arm the clock leaves the
|
||||
/// deferred runaway with no deadline at all; it is never answered or killed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_deferred_run_is_timed_and_killed_from_when_it_starts()
|
||||
{
|
||||
var softT = TimeSpan.FromMilliseconds(400);
|
||||
var grace = TimeSpan.FromMilliseconds(500);
|
||||
var head = TimeSpan.FromMilliseconds(300);
|
||||
await using var runner = Runner(Scripts.Options(hardGrace: grace, quarantineAfterKills: 5), evalThreads: 1);
|
||||
(await runner.RunAsync(Scripts.Vt("head", Spin, softT, Inputs(("ms", 1))), Ct)).Value.ShouldBe(1);
|
||||
(await runner.RunAsync(Item("tail", false, 1, softT), Ct)).Value.ShouldBe(2);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var tasks = runner.RunBatch([Scripts.Vt("head", Spin, softT, Inputs(("ms", (int)head.TotalMilliseconds))), Item("tail", true, 0, softT)], Ct);
|
||||
|
||||
(await tasks[0]).Value.ShouldBe((int)head.TotalMilliseconds, "the head ran inside its budget");
|
||||
var tail = await tasks[1];
|
||||
var answered = sw.Elapsed;
|
||||
tail.Reason!.ShouldStartWith("script timed out after ");
|
||||
answered.ShouldBeGreaterThanOrEqualTo(head + softT - TimeSpan.FromMilliseconds(50), "T counts from the run's start, after the head");
|
||||
|
||||
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "tail"));
|
||||
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(head + softT + grace - TimeSpan.FromMilliseconds(50), "G counts from the run's start too");
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System.Text;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
||||
|
||||
@@ -261,6 +262,217 @@ public sealed class ScriptRunnerPerfTests
|
||||
k2Closed.EvalsPerSecond.ShouldBeGreaterThanOrEqualTo(AcceptedGateEvalsPerSecond, "the accepted 5k evals/s/node gate at K=2");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#780 batch submit (design §4.5): a closed loop of one batch at a time, <paramref name="batchSize"/>
|
||||
/// distinct loaded keys per batch, either through <see cref="IScriptRunner.RunBatch"/> or as the same evaluations
|
||||
/// submitted as concurrent <see cref="IScriptRunner.RunAsync"/> calls. Latency is per batch (submit → last result).
|
||||
/// </summary>
|
||||
private static async Task<PhaseResult> BatchLoopAsync(IScriptRunner runner, int batchSize, bool viaBatch)
|
||||
{
|
||||
var deadline = Stopwatch.GetTimestamp() + (long)(Phase.TotalSeconds * Stopwatch.Frequency);
|
||||
var samples = new List<double>(100_000);
|
||||
long ok = 0, notOk = 0;
|
||||
string? firstFailure = null;
|
||||
var offset = 0;
|
||||
var x = 0;
|
||||
var cpu0 = Process.GetCurrentProcess().TotalProcessorTime;
|
||||
var wall = Stopwatch.StartNew();
|
||||
while (Stopwatch.GetTimestamp() < deadline)
|
||||
{
|
||||
var invocations = new ScriptInvocation[batchSize];
|
||||
var keys = new int[batchSize];
|
||||
for (var i = 0; i < batchSize; i++)
|
||||
{
|
||||
keys[i] = (offset + i) % Keys;
|
||||
invocations[i] = Invocation(keys[i], x);
|
||||
}
|
||||
offset = (offset + batchSize) % Keys;
|
||||
var t0 = Stopwatch.GetTimestamp();
|
||||
var outcomes = await Task.WhenAll(viaBatch
|
||||
? runner.RunBatch(invocations, Ct)
|
||||
: invocations.Select(inv => runner.RunAsync(inv, Ct)).ToArray());
|
||||
samples.Add(Stopwatch.GetElapsedTime(t0).TotalMicroseconds);
|
||||
for (var i = 0; i < batchSize; i++)
|
||||
{
|
||||
var o = outcomes[i];
|
||||
if (o.Status == ScriptRunStatus.Ok && Equals(o.Value, x + keys[i] % DistinctSources)) ok++;
|
||||
else
|
||||
{
|
||||
notOk++;
|
||||
firstFailure ??= $"{o.Status}: {o.Reason} (value {o.Value})";
|
||||
}
|
||||
}
|
||||
x++;
|
||||
}
|
||||
var elapsed = wall.Elapsed;
|
||||
var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / elapsed.TotalMilliseconds * 100;
|
||||
samples.Sort();
|
||||
return new PhaseResult((ok + notOk) / elapsed.TotalSeconds, Percentile(samples, 0.50), Percentile(samples, 0.99), cpu, ok, notOk, firstFailure);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A runner that turns a batch back into one evaluation after another — the serial pass the alarm engine and the
|
||||
/// Calc pump made before batch submit, which design §8.2 warns costs N round trips (the comparison row).
|
||||
/// </summary>
|
||||
private sealed class SerialRunner(IScriptRunner inner) : IScriptRunner
|
||||
{
|
||||
public Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default) => inner.RunAsync(invocation, ct);
|
||||
|
||||
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
|
||||
{
|
||||
var results = new Task<ScriptRunOutcome>[invocations.Count];
|
||||
Task previous = Task.CompletedTask;
|
||||
for (var i = 0; i < results.Length; i++) previous = results[i] = After(previous, invocations[i], ct);
|
||||
return results;
|
||||
}
|
||||
|
||||
private async Task<ScriptRunOutcome> After(Task previous, ScriptInvocation invocation, CancellationToken ct)
|
||||
{
|
||||
await previous.ConfigureAwait(false);
|
||||
return await inner.RunAsync(invocation, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public void Reset(ScriptContextKind kind) => inner.Reset(kind);
|
||||
|
||||
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind) => inner.QuarantinedKeys(kind);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scripted-alarm pass latency: <paramref name="alarms"/> alarms on one input, every push flips all of them, and a
|
||||
/// sample is the time from the push to the last of the N transitions the pass emits (the whole dirty set through
|
||||
/// the engine's gate). The design's alarm budget is 250 ms (§1.1; the host now passes <c>Scripting:RunTimeout</c>,
|
||||
/// 2 s by default, lmxopcua#779), so the pass is judged against the stricter 250 ms.
|
||||
/// </summary>
|
||||
private static async Task<(double P50Ms, double P99Ms, double MaxMs, double CpuPercent, int NotCompleted)> AlarmPassLatencyAsync(
|
||||
IScriptRunner? runner, int alarms, int pushes)
|
||||
{
|
||||
var up = new TestUpstream();
|
||||
up.Set("x", 0);
|
||||
var logger = new Serilog.LoggerConfiguration().CreateLogger();
|
||||
using var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
|
||||
scriptTimeout: TimeSpan.FromMilliseconds(250), scriptRunner: runner);
|
||||
var transitions = 0;
|
||||
var notCompleted = 0;
|
||||
var target = int.MaxValue;
|
||||
var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
engine.OnEvent += (_, e) =>
|
||||
{
|
||||
// A Bad annotation means a predicate did not complete in that pass (timed out, refused): counted, reported.
|
||||
if (e.Emission == EmissionKind.QualityChanged && e.WorstInputStatusCode != 0) Interlocked.Increment(ref notCompleted);
|
||||
if (e.Emission is not (EmissionKind.Activated or EmissionKind.Cleared)) return;
|
||||
if (Interlocked.Increment(ref transitions) == Volatile.Read(ref target)) done.TrySetResult();
|
||||
};
|
||||
await engine.LoadAsync(Enumerable.Range(0, alarms).Select(i => new ScriptedAlarmDefinition(
|
||||
$"alarm-{i}", "Plant/Perf", $"alarm-{i}", AlarmKind.AlarmCondition, AlarmSeverity.High, "perf",
|
||||
$"return (int)ctx.GetTag(\"x\").Value > {-(i % DistinctSources)};")).ToArray(), Ct);
|
||||
var samples = new List<double>(pushes);
|
||||
var cpu0 = Process.GetCurrentProcess().TotalProcessorTime;
|
||||
var wall = Stopwatch.StartNew();
|
||||
for (var p = 0; p < pushes + 2; p++)
|
||||
{
|
||||
// Alternate x = -1000 (every predicate false) and x = 1000 (every predicate true). The pass is expected to
|
||||
// transition exactly the alarms not already in the target state — counted from the engine, so an alarm
|
||||
// that did not complete an earlier pass cannot leave the count short. The first two passes are warm-up.
|
||||
var activate = p % 2 == 1;
|
||||
var expected = engine.GetAllStates().Count(c => (c.Active == AlarmActiveState.Active) != activate);
|
||||
Interlocked.Exchange(ref transitions, 0);
|
||||
done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
Volatile.Write(ref target, expected);
|
||||
var t0 = Stopwatch.GetTimestamp();
|
||||
up.Push("x", activate ? 1000 : -1000);
|
||||
if (expected == 0) continue;
|
||||
try
|
||||
{
|
||||
await done.Task.WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
throw new TimeoutException($"alarm pass {p} ({(runner is null ? "in process" : runner.GetType().Name)}, {alarms} alarms): " +
|
||||
$"{Volatile.Read(ref transitions)} of {expected} transitions in 60 s, " +
|
||||
$"{Volatile.Read(ref notCompleted)} predicate(s) did not complete so far");
|
||||
}
|
||||
if (p >= 2) samples.Add(Stopwatch.GetElapsedTime(t0).TotalMilliseconds);
|
||||
}
|
||||
var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / wall.Elapsed.TotalMilliseconds * 100;
|
||||
samples.Sort();
|
||||
return (Percentile(samples, 0.50), Percentile(samples, 0.99), samples[^1], cpu, notCompleted);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Batch throughput (RunBatch vs the same evaluations as concurrent RunAsync calls, K = 2) and scripted-alarm pass
|
||||
/// latency (in process, out of process batched, and out of process serial). Report-only, except that every
|
||||
/// result must be correct.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Batch_throughput_and_alarm_pass_latency()
|
||||
{
|
||||
RequireOptIn();
|
||||
var report = new StringBuilder()
|
||||
.AppendLine($"### Batch submit throughput ({Env()})")
|
||||
.AppendLine()
|
||||
.AppendLine("| Runner | batch | submitted as | evals/s | batch p50 µs | batch p99 µs | host CPU | ok / not ok |")
|
||||
.AppendLine("|---|---:|---|---:|---:|---:|---:|---|");
|
||||
|
||||
foreach (var (name, create) in new (string, Func<IScriptRunner>)[]
|
||||
{
|
||||
("InProcess", () => new InProcessScriptRunner(new ScriptingOptions())),
|
||||
("OutOfProcess K=2", () => Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger())),
|
||||
})
|
||||
{
|
||||
var runner = create();
|
||||
try
|
||||
{
|
||||
await WarmAsync(runner);
|
||||
foreach (var size in new[] { 1, 10, 100, 500 })
|
||||
{
|
||||
foreach (var viaBatch in new[] { true, false })
|
||||
{
|
||||
var r = await BatchLoopAsync(runner, size, viaBatch);
|
||||
r.NotOk.ShouldBe(0, $"{name} batch {size}: {r.FirstFailure}{Environment.NewLine}{RunnerLog()}");
|
||||
report.AppendLine($"| {name} | {size} | {(viaBatch ? "RunBatch" : "RunAsync ×n")} | {r.EvalsPerSecond,9:N0} | " +
|
||||
$"{r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | {r.Ok:N0} / {r.NotOk:N0} |");
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
switch (runner)
|
||||
{
|
||||
case IAsyncDisposable a: await a.DisposeAsync(); break;
|
||||
case IDisposable d: d.Dispose(); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.AppendLine()
|
||||
.AppendLine($"### Scripted-alarm pass latency, push → last transition ({Env()})")
|
||||
.AppendLine()
|
||||
.AppendLine("| Engine | alarms | pass p50 ms | pass p99 ms | max ms | per alarm (p50) µs | host CPU | not completed | 250 ms |")
|
||||
.AppendLine("|---|---:|---:|---:|---:|---:|---:|---:|---|");
|
||||
const int pushes = 40;
|
||||
await using (var shared = Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger()))
|
||||
{
|
||||
foreach (var alarms in new[] { 100, 500, 1000 })
|
||||
{
|
||||
foreach (var (name, runner) in new (string, IScriptRunner?)[]
|
||||
{
|
||||
("InProcess (serial)", null),
|
||||
("OutOfProcess K=2, batched", shared),
|
||||
("OutOfProcess K=2, serial", new SerialRunner(shared)),
|
||||
})
|
||||
{
|
||||
var (p50, p99, max, cpu, notCompleted) = await AlarmPassLatencyAsync(runner, alarms, pushes);
|
||||
report.AppendLine($"| {name} | {alarms:N0} | {p50,7:N1} | {p99,7:N1} | {max,7:N1} | {p50 * 1000 / alarms,6:N0} | " +
|
||||
$"{cpu,5:N0} % | {notCompleted:N0} | {(p99 <= 250 ? "PASS" : "MISS")} |");
|
||||
}
|
||||
}
|
||||
}
|
||||
report.AppendLine()
|
||||
.AppendLine("Each push flips every alarm; a pass is the engine's whole dirty set under its gate. \"serial\" replays the " +
|
||||
"pre-batch one-predicate-at-a-time pass over the same runner.");
|
||||
Report(report);
|
||||
}
|
||||
|
||||
private static ScriptingOptions PerfOptions(int shards, string? peCacheDir = null)
|
||||
{
|
||||
var options = Scripts.Options(shards: shards, hardGrace: TimeSpan.FromSeconds(2), peCacheDir: peCacheDir,
|
||||
|
||||
+2
@@ -22,6 +22,8 @@
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess\ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess.csproj"/>
|
||||
<!-- End-to-end seam test: a real Calculation driver over the real worker. -->
|
||||
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Calculation\ZB.MOM.WW.OtOpcUa.Driver.Calculation.csproj"/>
|
||||
<!-- The scripted-alarm seam over the real worker (lmxopcua#780). -->
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms\ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<!-- The real worker, built and placed in script-worker/ beside the tests, exactly as beside the Host. -->
|
||||
|
||||
Reference in New Issue
Block a user