feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)

- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs

@@ -53,6 +53,14 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
     }
 }
 
+/// <summary>One evaluation of a <see cref="WorkerConnection.SubmitBatch"/>.</summary>
+/// <param name="Key">The scoped key.</param>
+/// <param name="Request">The request (its id is assigned on submit).</param>
+/// <param name="SoftTimeout">T.</param>
+/// <param name="HardGrace">G.</param>
+/// <param name="OnFinished">Runs once when the worker has answered or ended, before the caller resumes (the slot release).</param>
+internal sealed record BatchSubmission(string Key, EvalRequest Request, TimeSpan SoftTimeout, TimeSpan HardGrace, Action? OnFinished);
+
 /// <summary>What the host needs to launch one worker.</summary>
 /// <param name="Shard">The shard label (<c>0</c>, <c>1</c>, … or <c>iso</c>).</param>
 /// <param name="WorkerPath">The worker entry assembly (<c>.dll</c>, run through the dotnet muxer) or apphost.</param>
@@ -522,6 +530,51 @@ internal sealed class WorkerConnection : IAsyncDisposable
         return pending;
     }
 
+    /// <summary>
+    ///     Submits several evaluations as ONE <see cref="EvalBatch"/> frame (lmxopcua#780 batch submit, design §4.5). Each
+    ///     item gets its own deadlines and slot release exactly as through <see cref="Submit(string, EvalRequest, TimeSpan?, TimeSpan, Action?)"/>;
+    ///     an item the worker has to queue behind busy evaluation threads has its clock stopped until it starts.
+    /// </summary>
+    /// <param name="items">The evaluations (their ids are assigned here).</param>
+    /// <returns>One pending evaluation per item, in order.</returns>
+    public PendingEval[] SubmitBatch(IReadOnlyList<BatchSubmission> items)
+    {
+        ArgumentNullException.ThrowIfNull(items);
+        var pendings = new PendingEval[items.Count];
+        var batch = new EvalBatch();
+        Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
+        for (var i = 0; i < items.Count; i++)
+        {
+            var item = items[i];
+            item.Request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
+            var pending = new PendingEval(item.Key, item.Request.EvalId, item.OnFinished, _dispatcher);
+            pending.Arm(item.SoftTimeout, item.HardGrace, OnHardDeadline);
+            _pending[item.Request.EvalId] = pending;
+            batch.Items.Add(item.Request);
+            pendings[i] = pending;
+        }
+        if (items.Count > 0 && (!Send(new Envelope { Eval = batch }) || _exited.Task.IsCompleted))
+        {
+            foreach (var pending in pendings)
+                if (_pending.TryRemove(pending.EvalId, out _)) pending.Complete(null);
+        }
+        return pendings;
+    }
+
+    /// <summary>
+    ///     The compiled form of <paramref name="key"/> in this worker when it is loaded at <paramref name="sourceHashHex"/>
+    ///     with <paramref name="inputPaths"/> and compiled cleanly — the batch fast path, which never waits for a load.
+    /// </summary>
+    /// <param name="key">The scoped key.</param>
+    /// <param name="sourceHashHex">The source hash (hex).</param>
+    /// <param name="inputPaths">The slot order.</param>
+    /// <returns>The entry, or null when the key needs a (re)load or its load failed.</returns>
+    public LoadEntry? TryGetLoaded(string key, string sourceHashHex, IReadOnlyList<string> inputPaths)
+        => _loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths)
+           && existing.Ready.IsCompleted && existing.Outcome is { Ok: true } && !existing.TimedOut
+            ? existing
+            : null;
+
     private void OnHardDeadline(PendingEval pending)
     {
         _log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
@@ -677,8 +730,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
                     // An eval id with no outstanding request is a protocol violation (design §6 "what the host
                     // trusts"), except for one that was already completed by a kill race — ignored then.
                     if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
-                    else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
-                        throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
+                    else CheckIssued(result.EvalId, "result");
+                }
+                // Batch submit (lmxopcua#780): a run that waited behind busy evaluation threads has its deadlines
+                // restarted when it actually starts, so it is never timed out, struck or killed for a sibling's time.
+                // The two notices are unordered with respect to each other (see PendingEval.Start).
+                foreach (var id in frame.Results.StartedEvalIds)
+                {
+                    if (_pending.TryGetValue(id, out var started)) started.Start();
+                    else CheckIssued(id, "start notice");
+                }
+                foreach (var id in frame.Results.DeferredEvalIds)
+                {
+                    if (_pending.TryGetValue(id, out var deferred)) deferred.Defer();
+                    else CheckIssued(id, "deferral notice");
                 }
                 break;
             case Envelope.BodyOneofCase.LoadStarted:
@@ -710,6 +775,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
         }
     }
 
+    /// <summary>An id the host never issued is a protocol violation; one it issued and already completed is a benign race.</summary>
+    private void CheckIssued(ulong evalId, string what)
+    {
+        if (evalId == 0 || evalId > (ulong)Interlocked.Read(ref _nextEvalId))
+            throw new ScriptWorkerProtocolException($"{what} for unknown eval id {evalId}");
+    }
+
     private async Task PipeEndedAsync(WorkerExitReason? fault)
     {
         // The pipe is gone: the worker is dead or dying. Make sure, then report.
@@ -917,6 +989,11 @@ internal sealed class WorkerConnection : IAsyncDisposable
 ///     <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
 ///     measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
 /// </summary>
+/// <remarks>
+///     The deadlines are absolute timestamps the timer callback re-checks, so a deferral (the worker queued the run behind
+///     busy evaluation threads, lmxopcua#780 batch submit) can stop the clock and a start notice can restart it — the full
+///     T, then G, from the start — without a stale tick firing early.
+/// </remarks>
 internal sealed class PendingEval
 {
     /// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
@@ -926,9 +1003,15 @@ internal sealed class PendingEval
     private readonly TaskCompletionSource<EvalResult?> _outcome;
     private readonly ResultDispatcher? _dispatcher;
     private readonly Action? _onFinished;
+    private readonly object _clock = new();
     private Timer? _timer;
     private Action<PendingEval>? _onHardDeadline;
+    private TimeSpan _soft;
     private TimeSpan _grace;
+    private long _softDue;
+    private long _hardDue;
+    private bool _deferred;
+    private bool _started;
     private int _phase;
     private int _done;
 
@@ -954,6 +1037,12 @@ internal sealed class PendingEval
     /// <summary>T + G, for the kill log line.</summary>
     public TimeSpan HardDeadline { get; private set; }
 
+    /// <summary>Whether the worker reported this run as queued and has not reported it started (tests / diagnostics).</summary>
+    internal bool IsDeferred
+    {
+        get { lock (_clock) return _deferred; }
+    }
+
     /// <summary>The worker's result, or null when the worker ended first.</summary>
     public Task<EvalResult?> Completion => _completion.Task;
 
@@ -966,27 +1055,96 @@ internal sealed class PendingEval
     /// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
     internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
     {
+        _soft = soft;
         _grace = grace;
         _onHardDeadline = onHardDeadline;
         HardDeadline = soft + grace;
+        var now = Stopwatch.GetTimestamp();
+        _softDue = now + Ticks(soft);
+        _hardDue = _softDue + Ticks(grace);
         // Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
         _timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
         _timer.Change(soft, Timeout.InfiniteTimeSpan);
     }
 
+    /// <summary>
+    ///     The worker queued this run behind busy evaluation threads: stop the clock until it starts (it is not running,
+    ///     so it cannot overrun). Ignored once a start notice has arrived — the two are unordered on the wire.
+    /// </summary>
+    internal void Defer()
+    {
+        lock (_clock)
+        {
+            if (_timer is null || _started || _deferred || Volatile.Read(ref _done) != 0) return;
+            _deferred = true;
+            TryChange(Timeout.InfiniteTimeSpan);
+        }
+    }
+
+    /// <summary>
+    ///     The worker started this run: a deferred run's deadlines restart from now (T, or T + G when its soft deadline
+    ///     had already been reported). A run that was never deferred keeps the clock it has had since submit.
+    /// </summary>
+    internal void Start()
+    {
+        lock (_clock)
+        {
+            _started = true;
+            if (_timer is null || !_deferred || Volatile.Read(ref _done) != 0) return;
+            _deferred = false;
+            var now = Stopwatch.GetTimestamp();
+            if (Volatile.Read(ref _phase) == 0)
+            {
+                _softDue = now + Ticks(_soft);
+                _hardDue = _softDue + Ticks(_grace);
+                TryChange(_soft);
+            }
+            else
+            {
+                // Its caller already has the soft timeout (a deferral notice that arrived after T): the run still gets
+                // the whole T + G from its start before it can be blamed for a kill.
+                _hardDue = now + Ticks(_soft + _grace);
+                TryChange(_soft + _grace);
+            }
+        }
+    }
+
     private void OnTimer()
     {
         if (Volatile.Read(ref _done) != 0) return;
-        if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
+        bool soft = false, hard = false;
+        lock (_clock)
         {
-            _outcome.TrySetResult(SoftTimedOut);
-            try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
-            catch (ObjectDisposedException) { /* answered meanwhile */ }
-            return;
+            if (_deferred || Volatile.Read(ref _done) != 0) return;
+            var now = Stopwatch.GetTimestamp();
+            if (Volatile.Read(ref _phase) == 0)
+            {
+                // A tick armed before a deferral/start can land early: re-arm for what is left.
+                if (now < _softDue) { TryChange(Stopwatch.GetElapsedTime(now, _softDue)); return; }
+                Interlocked.Exchange(ref _phase, 1);
+                soft = true;
+                TryChange(_hardDue > now ? Stopwatch.GetElapsedTime(now, _hardDue) : TimeSpan.Zero);
+            }
+            else if (Volatile.Read(ref _phase) == 1)
+            {
+                if (now < _hardDue) { TryChange(Stopwatch.GetElapsedTime(now, _hardDue)); return; }
+                Interlocked.Exchange(ref _phase, 2);
+                hard = true;
+            }
         }
-        if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
+        // Outside the lock: a caller with a dispatcher resumes synchronously inside TrySetResult.
+        if (soft) _outcome.TrySetResult(SoftTimedOut);
+        if (hard && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
     }
 
+    private void TryChange(TimeSpan due)
+    {
+        try { _timer?.Change(due < TimeSpan.Zero ? TimeSpan.Zero : due, Timeout.InfiniteTimeSpan); }
+        catch (ObjectDisposedException) { /* answered meanwhile */ }
+    }
+
+    private static long Ticks(TimeSpan span) => (long)(span.TotalSeconds * Stopwatch.Frequency);
+
     /// <summary>
     ///     The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
     ///     reader, then hand the caller's resumption to the dispatcher (or the pool).