dohertj2/lmxopcua
1
0
Fork 0
Code Issues 25 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run
Details

Browse Source 
- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 06:37:36 -05:00
parent 11a5c1d4e2
commit 0796928504
33 changed files with 2168 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs
+212
View File
@@ -3,6 +3,7 @@ using System.Text;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
@@ -261,6 +262,217 @@ public sealed class ScriptRunnerPerfTests
k2Closed.EvalsPerSecond.ShouldBeGreaterThanOrEqualTo(AcceptedGateEvalsPerSecond, "the accepted 5k evals/s/node gate at K=2");
}
/// <summary>
/// lmxopcua#780 batch submit (design §4.5): a closed loop of one batch at a time, <paramref name="batchSize"/>
/// distinct loaded keys per batch, either through <see cref="IScriptRunner.RunBatch"/> or as the same evaluations
/// submitted as concurrent <see cref="IScriptRunner.RunAsync"/> calls. Latency is per batch (submit → last result).
/// </summary>
private static async Task<PhaseResult> BatchLoopAsync(IScriptRunner runner, int batchSize, bool viaBatch)
{
var deadline = Stopwatch.GetTimestamp() + (long)(Phase.TotalSeconds * Stopwatch.Frequency);
var samples = new List<double>(100_000);
long ok = 0, notOk = 0;
string? firstFailure = null;
var offset = 0;
var x = 0;
var cpu0 = Process.GetCurrentProcess().TotalProcessorTime;
var wall = Stopwatch.StartNew();
while (Stopwatch.GetTimestamp() < deadline)
{
var invocations = new ScriptInvocation[batchSize];
var keys = new int[batchSize];
for (var i = 0; i < batchSize; i++)
{
keys[i] = (offset + i) % Keys;
invocations[i] = Invocation(keys[i], x);
}
offset = (offset + batchSize) % Keys;
var t0 = Stopwatch.GetTimestamp();
var outcomes = await Task.WhenAll(viaBatch
? runner.RunBatch(invocations, Ct)
: invocations.Select(inv => runner.RunAsync(inv, Ct)).ToArray());
samples.Add(Stopwatch.GetElapsedTime(t0).TotalMicroseconds);
for (var i = 0; i < batchSize; i++)
{
var o = outcomes[i];
if (o.Status == ScriptRunStatus.Ok && Equals(o.Value, x + keys[i] % DistinctSources)) ok++;
else
{
notOk++;
firstFailure ??= $"{o.Status}: {o.Reason} (value {o.Value})";
}
}
x++;
}
var elapsed = wall.Elapsed;
var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / elapsed.TotalMilliseconds * 100;
samples.Sort();
return new PhaseResult((ok + notOk) / elapsed.TotalSeconds, Percentile(samples, 0.50), Percentile(samples, 0.99), cpu, ok, notOk, firstFailure);
}
/// <summary>
/// A runner that turns a batch back into one evaluation after another — the serial pass the alarm engine and the
/// Calc pump made before batch submit, which design §8.2 warns costs N round trips (the comparison row).
/// </summary>
private sealed class SerialRunner(IScriptRunner inner) : IScriptRunner
{
public Task<ScriptRunOutcome> RunAsync(ScriptInvocation invocation, CancellationToken ct = default) => inner.RunAsync(invocation, ct);
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
{
var results = new Task<ScriptRunOutcome>[invocations.Count];
Task previous = Task.CompletedTask;
for (var i = 0; i < results.Length; i++) previous = results[i] = After(previous, invocations[i], ct);
return results;
}
private async Task<ScriptRunOutcome> After(Task previous, ScriptInvocation invocation, CancellationToken ct)
{
await previous.ConfigureAwait(false);
return await inner.RunAsync(invocation, ct).ConfigureAwait(false);
}
public void Reset(ScriptContextKind kind) => inner.Reset(kind);
public IReadOnlyCollection<string> QuarantinedKeys(ScriptContextKind kind) => inner.QuarantinedKeys(kind);
}
/// <summary>
/// Scripted-alarm pass latency: <paramref name="alarms"/> alarms on one input, every push flips all of them, and a
/// sample is the time from the push to the last of the N transitions the pass emits (the whole dirty set through
/// the engine's gate). The design's alarm budget is 250 ms (§1.1; the host now passes <c>Scripting:RunTimeout</c>,
/// 2 s by default, lmxopcua#779), so the pass is judged against the stricter 250 ms.
/// </summary>
private static async Task<(double P50Ms, double P99Ms, double MaxMs, double CpuPercent, int NotCompleted)> AlarmPassLatencyAsync(
IScriptRunner? runner, int alarms, int pushes)
{
var up = new TestUpstream();
up.Set("x", 0);
var logger = new Serilog.LoggerConfiguration().CreateLogger();
using var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
scriptTimeout: TimeSpan.FromMilliseconds(250), scriptRunner: runner);
var transitions = 0;
var notCompleted = 0;
var target = int.MaxValue;
var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
engine.OnEvent += (_, e) =>
{
// A Bad annotation means a predicate did not complete in that pass (timed out, refused): counted, reported.
if (e.Emission == EmissionKind.QualityChanged && e.WorstInputStatusCode != 0) Interlocked.Increment(ref notCompleted);
if (e.Emission is not (EmissionKind.Activated or EmissionKind.Cleared)) return;
if (Interlocked.Increment(ref transitions) == Volatile.Read(ref target)) done.TrySetResult();
};
await engine.LoadAsync(Enumerable.Range(0, alarms).Select(i => new ScriptedAlarmDefinition(
$"alarm-{i}", "Plant/Perf", $"alarm-{i}", AlarmKind.AlarmCondition, AlarmSeverity.High, "perf",
$"return (int)ctx.GetTag(\"x\").Value > {-(i % DistinctSources)};")).ToArray(), Ct);
var samples = new List<double>(pushes);
var cpu0 = Process.GetCurrentProcess().TotalProcessorTime;
var wall = Stopwatch.StartNew();
for (var p = 0; p < pushes + 2; p++)
{
// Alternate x = -1000 (every predicate false) and x = 1000 (every predicate true). The pass is expected to
// transition exactly the alarms not already in the target state — counted from the engine, so an alarm
// that did not complete an earlier pass cannot leave the count short. The first two passes are warm-up.
var activate = p % 2 == 1;
var expected = engine.GetAllStates().Count(c => (c.Active == AlarmActiveState.Active) != activate);
Interlocked.Exchange(ref transitions, 0);
done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Volatile.Write(ref target, expected);
var t0 = Stopwatch.GetTimestamp();
up.Push("x", activate ? 1000 : -1000);
if (expected == 0) continue;
try
{
await done.Task.WaitAsync(TimeSpan.FromSeconds(60), Ct);
}
catch (TimeoutException)
{
throw new TimeoutException($"alarm pass {p} ({(runner is null ? "in process" : runner.GetType().Name)}, {alarms} alarms): " +
$"{Volatile.Read(ref transitions)} of {expected} transitions in 60 s, " +
$"{Volatile.Read(ref notCompleted)} predicate(s) did not complete so far");
}
if (p >= 2) samples.Add(Stopwatch.GetElapsedTime(t0).TotalMilliseconds);
}
var cpu = (Process.GetCurrentProcess().TotalProcessorTime - cpu0).TotalMilliseconds / wall.Elapsed.TotalMilliseconds * 100;
samples.Sort();
return (Percentile(samples, 0.50), Percentile(samples, 0.99), samples[^1], cpu, notCompleted);
}
/// <summary>
/// Batch throughput (RunBatch vs the same evaluations as concurrent RunAsync calls, K = 2) and scripted-alarm pass
/// latency (in process, out of process batched, and out of process serial). Report-only, except that every
/// result must be correct.
/// </summary>
[Fact]
public async Task Batch_throughput_and_alarm_pass_latency()
{
RequireOptIn();
var report = new StringBuilder()
.AppendLine($"### Batch submit throughput ({Env()})")
.AppendLine()
.AppendLine("| Runner | batch | submitted as | evals/s | batch p50 µs | batch p99 µs | host CPU | ok / not ok |")
.AppendLine("|---|---:|---|---:|---:|---:|---:|---|");
foreach (var (name, create) in new (string, Func<IScriptRunner>)[]
{
("InProcess", () => new InProcessScriptRunner(new ScriptingOptions())),
("OutOfProcess K=2", () => Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger())),
})
{
var runner = create();
try
{
await WarmAsync(runner);
foreach (var size in new[] { 1, 10, 100, 500 })
{
foreach (var viaBatch in new[] { true, false })
{
var r = await BatchLoopAsync(runner, size, viaBatch);
r.NotOk.ShouldBe(0, $"{name} batch {size}: {r.FirstFailure}{Environment.NewLine}{RunnerLog()}");
report.AppendLine($"| {name} | {size} | {(viaBatch ? "RunBatch" : "RunAsync ×n")} | {r.EvalsPerSecond,9:N0} | " +
$"{r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | {r.Ok:N0} / {r.NotOk:N0} |");
}
}
}
finally
{
switch (runner)
{
case IAsyncDisposable a: await a.DisposeAsync(); break;
case IDisposable d: d.Dispose(); break;
}
}
}
report.AppendLine()
.AppendLine($"### Scripted-alarm pass latency, push → last transition ({Env()})")
.AppendLine()
.AppendLine("| Engine | alarms | pass p50 ms | pass p99 ms | max ms | per alarm (p50) µs | host CPU | not completed | 250 ms |")
.AppendLine("|---|---:|---:|---:|---:|---:|---:|---:|---|");
const int pushes = 40;
await using (var shared = Scripts.Runner(PerfOptions(2), _runnerLog.CreateLogger()))
{
foreach (var alarms in new[] { 100, 500, 1000 })
{
foreach (var (name, runner) in new (string, IScriptRunner?)[]
{
("InProcess (serial)", null),
("OutOfProcess K=2, batched", shared),
("OutOfProcess K=2, serial", new SerialRunner(shared)),
})
{
var (p50, p99, max, cpu, notCompleted) = await AlarmPassLatencyAsync(runner, alarms, pushes);
report.AppendLine($"| {name} | {alarms:N0} | {p50,7:N1} | {p99,7:N1} | {max,7:N1} | {p50 * 1000 / alarms,6:N0} | " +
$"{cpu,5:N0} % | {notCompleted:N0} | {(p99 <= 250 ? "PASS" : "MISS")} |");
}
}
}
report.AppendLine()
.AppendLine("Each push flips every alarm; a pass is the engine's whole dirty set under its gate. \"serial\" replays the " +
"pre-batch one-predicate-at-a-time pass over the same runner.");
Report(report);
}
private static ScriptingOptions PerfOptions(int shards, string? peCacheDir = null)
{
var options = Scripts.Options(shards: shards, hardGrace: TimeSpan.FromSeconds(2), peCacheDir: peCacheDir,