feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)
v2-ci / unit-tests (push) Waiting to run

- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 06:37:36 -05:00
parent 11a5c1d4e2
commit 0796928504
33 changed files with 2168 additions and 117 deletions
@@ -155,7 +155,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
int compileThreads = 0)
int compileThreads = 0, int evalThreads = 0)
{
_compileThreads = compileThreads;
ArgumentNullException.ThrowIfNull(options);
@@ -192,7 +192,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log);
if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept);
var evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
// Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the
// queue-behind-a-busy-thread case deterministic).
if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
_shards = new WorkerShard[_worker.Shards];
for (var i = 0; i < _shards.Length; i++)
{
@@ -243,6 +245,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
/// <summary>Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does.</summary>
internal Action<string>? SlotReleasingForTest { get; set; }
/// <summary>Test hook: runs once per batch frame <see cref="RunBatch"/> submits, with the number of evaluations in it.</summary>
internal Action<int>? BatchSubmittedForTest { get; set; }
/// <summary>Scoped keys currently treated as suspects (tests / diagnostics).</summary>
internal IReadOnlyCollection<string> SuspectKeys =>
_keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray();
@@ -414,21 +419,147 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
// next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
// was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
// run by then.
var heldGate = gate;
var released = 0;
void ReleaseSlot()
{
if (Interlocked.Exchange(ref released, 1) != 0) return;
SlotReleasingForTest?.Invoke(key);
Interlocked.Exchange(ref state.InFlight, 0);
heldGate?.Release();
}
var submitted = Stopwatch.GetTimestamp();
var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
_worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
_worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate));
handedOff = true;
return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return ScriptRunOutcome.Failed("evaluation cancelled");
}
finally
{
if (!handedOff)
{
Interlocked.Exchange(ref state.InFlight, 0);
gate?.Release();
}
}
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
// shard has no live worker (or for the isolation gate).
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
}
/// <inheritdoc />
/// <remarks>
/// Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script
/// already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE
/// <c>EvalBatch</c> frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect
/// bound for the one-at-a-time isolation worker) takes the full <see cref="RunAsync"/> path on its own, so a batch
/// never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker
/// has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway
/// sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported
/// <see cref="ScriptRunOutcome.WorkerRestartedReason"/>, design §9).
/// </remarks>
public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(invocations);
var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
Dictionary<WorkerConnection, List<(int Index, BatchItem Item)>>? groups = null;
for (var i = 0; i < invocations.Count; i++)
{
var invocation = invocations[i];
ArgumentNullException.ThrowIfNull(invocation);
switch (TryPrepareBatchItem(invocation, out var item, out var answered))
{
case BatchPreparation.Submit:
groups ??= [];
if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = [];
group.Add((i, item));
break;
case BatchPreparation.Answered:
outcomes[i] = Task.FromResult(answered!);
break;
default:
outcomes[i] = RunAsync(invocation, ct);
break;
}
}
if (groups is null) return outcomes;
foreach (var (connection, group) in groups)
{
var submissions = new BatchSubmission[group.Count];
for (var j = 0; j < group.Count; j++)
{
var item = group[j].Item;
submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation),
item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null));
}
var submitted = Stopwatch.GetTimestamp();
BatchSubmittedForTest?.Invoke(submissions.Length);
var pendings = connection.SubmitBatch(submissions);
for (var j = 0; j < group.Count; j++)
{
var (index, item) = group[j];
outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct);
}
}
return outcomes;
}
/// <summary>
/// The synchronous half of a batch item: everything <see cref="RunAsync"/> decides before it submits, for the case
/// where nothing has to be waited for. Takes the key's single-flight slot when it returns
/// <see cref="BatchPreparation.Submit"/>; never when it returns anything else.
/// </summary>
private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered)
{
item = null;
answered = null;
if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full;
var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
var state = Admit(key, invocation.Kind, invocation.Source);
if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full;
var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full;
if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full;
if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
{
ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
return BatchPreparation.Answered;
}
state.ScriptLogger = invocation.ScriptLogger;
NoteLoad(entry);
item = new BatchItem(key, state, invocation, shard, connection, entry);
return BatchPreparation.Submit;
}
/// <summary>
/// The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation
/// gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller
/// stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation
/// worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a
/// caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found
/// 32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike,
/// quarantine, suspect) has already run by then.
/// </summary>
private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate)
{
var released = 0;
return () =>
{
if (Interlocked.Exchange(ref released, 1) != 0) return;
SlotReleasingForTest?.Invoke(key);
Interlocked.Exchange(ref state.InFlight, 0);
heldGate?.Release();
};
}
/// <summary>Waits for one submitted evaluation and turns its result into the caller's outcome.</summary>
private async Task<ScriptRunOutcome> AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation,
WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct)
{
try
{
var result = ct.CanBeCanceled
? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
: await pending.Outcome.ConfigureAwait(false);
@@ -452,19 +583,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
{
return ScriptRunOutcome.Failed("evaluation cancelled");
}
finally
{
if (!handedOff)
{
Interlocked.Exchange(ref state.InFlight, 0);
gate?.Release();
}
}
// Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
// still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
// shard has no live worker (or for the isolation gate).
TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
}
/// <summary>
@@ -816,6 +934,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
=> LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
.Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
private enum BatchPreparation
{
/// <summary>Run it through the full <see cref="RunAsync"/> path on its own.</summary>
Full,
/// <summary>Answered without running (refused).</summary>
Answered,
/// <summary>Submit it with the batch; its slot is taken.</summary>
Submit,
}
private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard,
WorkerConnection Connection, WorkerConnection.LoadEntry Entry);
private sealed class KeyState(string source, ScriptContextKind kind)
{
private byte[]? _sha;
@@ -53,6 +53,14 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
}
}
/// <summary>One evaluation of a <see cref="WorkerConnection.SubmitBatch"/>.</summary>
/// <param name="Key">The scoped key.</param>
/// <param name="Request">The request (its id is assigned on submit).</param>
/// <param name="SoftTimeout">T.</param>
/// <param name="HardGrace">G.</param>
/// <param name="OnFinished">Runs once when the worker has answered or ended, before the caller resumes (the slot release).</param>
internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan SoftTimeout, TimeSpan HardGrace, Action? OnFinished);
/// <summary>What the host needs to launch one worker.</summary>
/// <param name="Shard">The shard label (<c>0</c>, <c>1</c>, … or <c>iso</c>).</param>
/// <param name="WorkerPath">The worker entry assembly (<c>.dll</c>, run through the dotnet muxer) or apphost.</param>
@@ -522,6 +530,51 @@ internal sealed class WorkerConnection : IAsyncDisposable
return pending;
}
/// <summary>
/// Submits several evaluations as ONE <see cref="EvalBatch"/> frame (lmxopcua#780 batch submit, design §4.5). Each
/// item gets its own deadlines and slot release exactly as through <see cref="Submit(string, EvalRequest, TimeSpan?, TimeSpan, Action?)"/>;
/// an item the worker has to queue behind busy evaluation threads has its clock stopped until it starts.
/// </summary>
/// <param name="items">The evaluations (their ids are assigned here).</param>
/// <returns>One pending evaluation per item, in order.</returns>
public PendingEval[] SubmitBatch(IReadOnlyList<BatchSubmission> items)
{
ArgumentNullException.ThrowIfNull(items);
var pendings = new PendingEval[items.Count];
var batch = new EvalBatch();
Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
for (var i = 0; i < items.Count; i++)
{
var item = items[i];
item.Request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
var pending = new PendingEval(item.Key, item.Request.EvalId, item.OnFinished, _dispatcher);
pending.Arm(item.SoftTimeout, item.HardGrace, OnHardDeadline);
_pending[item.Request.EvalId] = pending;
batch.Items.Add(item.Request);
pendings[i] = pending;
}
if (items.Count > 0 && (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted))
{
foreach (var pending in pendings)
if (_pending.TryRemove(pending.EvalId, out _)) pending.Complete(null);
}
return pendings;
}
/// <summary>
/// The compiled form of <paramref name="key"/> in this worker when it is loaded at <paramref name="sourceHashHex"/>
/// with <paramref name="inputPaths"/> and compiled cleanly — the batch fast path, which never waits for a load.
/// </summary>
/// <param name="key">The scoped key.</param>
/// <param name="sourceHashHex">The source hash (hex).</param>
/// <param name="inputPaths">The slot order.</param>
/// <returns>The entry, or null when the key needs a (re)load or its load failed.</returns>
public LoadEntry? TryGetLoaded(string key, string sourceHashHex, IReadOnlyList<string> inputPaths)
=> _loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths)
&& existing.Ready.IsCompleted && existing.Outcome is { Ok: true } && !existing.TimedOut
? existing
: null;
private void OnHardDeadline(PendingEval pending)
{
_log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
@@ -677,8 +730,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
// An eval id with no outstanding request is a protocol violation (design §6 "what the host
// trusts"), except for one that was already completed by a kill race — ignored then.
if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
else CheckIssued(result.EvalId, "result");
}
// Batch submit (lmxopcua#780): a run that waited behind busy evaluation threads has its deadlines
// restarted when it actually starts, so it is never timed out, struck or killed for a sibling's time.
// The two notices are unordered with respect to each other (see PendingEval.Start).
foreach (var id in frame.Results.StartedEvalIds)
{
if (_pending.TryGetValue(id, out var started)) started.Start();
else CheckIssued(id, "start notice");
}
foreach (var id in frame.Results.DeferredEvalIds)
{
if (_pending.TryGetValue(id, out var deferred)) deferred.Defer();
else CheckIssued(id, "deferral notice");
}
break;
case Envelope.BodyOneofCase.LoadStarted:
@@ -710,6 +775,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
}
}
/// <summary>An id the host never issued is a protocol violation; one it issued and already completed is a benign race.</summary>
private void CheckIssued(ulong evalId, string what)
{
if (evalId == 0 || evalId > (ulong)Interlocked.Read(ref _nextEvalId))
throw new ScriptWorkerProtocolException($"{what} for unknown eval id {evalId}");
}
private async Task PipeEndedAsync(WorkerExitReason? fault)
{
// The pipe is gone: the worker is dead or dying. Make sure, then report.
@@ -917,6 +989,11 @@ internal sealed class WorkerConnection : IAsyncDisposable
/// <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
/// measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
/// </summary>
/// <remarks>
/// The deadlines are absolute timestamps the timer callback re-checks, so a deferral (the worker queued the run behind
/// busy evaluation threads, lmxopcua#780 batch submit) can stop the clock and a start notice can restart it — the full
/// T, then G, from the start — without a stale tick firing early.
/// </remarks>
internal sealed class PendingEval
{
/// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
@@ -926,9 +1003,15 @@ internal sealed class PendingEval
private readonly TaskCompletionSource<EvalResult?> _outcome;
private readonly ResultDispatcher? _dispatcher;
private readonly Action? _onFinished;
private readonly object _clock = new();
private Timer? _timer;
private Action<PendingEval>? _onHardDeadline;
private TimeSpan _soft;
private TimeSpan _grace;
private long _softDue;
private long _hardDue;
private bool _deferred;
private bool _started;
private int _phase;
private int _done;
@@ -954,6 +1037,12 @@ internal sealed class PendingEval
/// <summary>T + G, for the kill log line.</summary>
public TimeSpan HardDeadline { get; private set; }
/// <summary>Whether the worker reported this run as queued and has not reported it started (tests / diagnostics).</summary>
internal bool IsDeferred
{
get { lock (_clock) return _deferred; }
}
/// <summary>The worker's result, or null when the worker ended first.</summary>
public Task<EvalResult?> Completion => _completion.Task;
@@ -966,27 +1055,96 @@ internal sealed class PendingEval
/// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
{
_soft = soft;
_grace = grace;
_onHardDeadline = onHardDeadline;
HardDeadline = soft + grace;
var now = Stopwatch.GetTimestamp();
_softDue = now + Ticks(soft);
_hardDue = _softDue + Ticks(grace);
// Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
_timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
_timer.Change(soft, Timeout.InfiniteTimeSpan);
}
/// <summary>
/// The worker queued this run behind busy evaluation threads: stop the clock until it starts (it is not running,
/// so it cannot overrun). Ignored once a start notice has arrived — the two are unordered on the wire.
/// </summary>
internal void Defer()
{
lock (_clock)
{
if (_timer is null || _started || _deferred || Volatile.Read(ref _done) != 0) return;
_deferred = true;
TryChange(Timeout.InfiniteTimeSpan);
}
}
/// <summary>
/// The worker started this run: a deferred run's deadlines restart from now (T, or T + G when its soft deadline
/// had already been reported). A run that was never deferred keeps the clock it has had since submit.
/// </summary>
internal void Start()
{
lock (_clock)
{
_started = true;
if (_timer is null || !_deferred || Volatile.Read(ref _done) != 0) return;
_deferred = false;
var now = Stopwatch.GetTimestamp();
if (Volatile.Read(ref _phase) == 0)
{
_softDue = now + Ticks(_soft);
_hardDue = _softDue + Ticks(_grace);
TryChange(_soft);
}
else
{
// Its caller already has the soft timeout (a deferral notice that arrived after T): the run still gets
// the whole T + G from its start before it can be blamed for a kill.
_hardDue = now + Ticks(_soft + _grace);
TryChange(_soft + _grace);
}
}
}
private void OnTimer()
{
if (Volatile.Read(ref _done) != 0) return;
if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
bool soft = false, hard = false;
lock (_clock)
{
_outcome.TrySetResult(SoftTimedOut);
try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
catch (ObjectDisposedException) { /* answered meanwhile */ }
return;
if (_deferred || Volatile.Read(ref _done) != 0) return;
var now = Stopwatch.GetTimestamp();
if (Volatile.Read(ref _phase) == 0)
{
// A tick armed before a deferral/start can land early: re-arm for what is left.
if (now < _softDue) { TryChange(Stopwatch.GetElapsedTime(now, _softDue)); return; }
Interlocked.Exchange(ref _phase, 1);
soft = true;
TryChange(_hardDue > now ? Stopwatch.GetElapsedTime(now, _hardDue) : TimeSpan.Zero);
}
else if (Volatile.Read(ref _phase) == 1)
{
if (now < _hardDue) { TryChange(Stopwatch.GetElapsedTime(now, _hardDue)); return; }
Interlocked.Exchange(ref _phase, 2);
hard = true;
}
}
if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
// Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult.
if (soft) _outcome.TrySetResult(SoftTimedOut);
if (hard && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
}
private void TryChange(TimeSpan due)
{
try { _timer?.Change(due < TimeSpan.Zero ? TimeSpan.Zero : due, Timeout.InfiniteTimeSpan); }
catch (ObjectDisposedException) { /* answered meanwhile */ }
}
private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
/// <summary>
/// The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
/// reader, then hand the caller's resumption to the dispatcher (or the pool).