feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)

- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs

@@ -155,7 +155,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
 
     internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
         TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
-        int compileThreads = 0)
+        int compileThreads = 0, int evalThreads = 0)
     {
         _compileThreads = compileThreads;
         ArgumentNullException.ThrowIfNull(options);
@@ -192,7 +192,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         var swept = WorkerLaunchMarker.Sweep(_markerDirectory, _log);
         if (swept.Count > 0) _log.Warning("Killed {Count} orphaned ScriptWorker process(es) at startup: {Pids}", swept.Count, swept);
 
-        var evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
+        // Design §3.2 [A-4]: P = max(2, cores / K). A positive evalThreads is a test knob (a small scheduler makes the
+        // queue-behind-a-busy-thread case deterministic).
+        if (evalThreads <= 0) evalThreads = Math.Max(2, Environment.ProcessorCount / _worker.Shards);
         _shards = new WorkerShard[_worker.Shards];
         for (var i = 0; i < _shards.Length; i++)
         {
@@ -243,6 +245,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
     /// <summary>Test hook: runs inside the continuation that releases a key's single-flight slot, just before it does.</summary>
     internal Action<string>? SlotReleasingForTest { get; set; }
 
+    /// <summary>Test hook: runs once per batch frame <see cref="RunBatch"/> submits, with the number of evaluations in it.</summary>
+    internal Action<int>? BatchSubmittedForTest { get; set; }
+
     /// <summary>Scoped keys currently treated as suspects (tests / diagnostics).</summary>
     internal IReadOnlyCollection<string> SuspectKeys =>
         _keys.Where(kv => kv.Value.Suspect).Select(kv => kv.Key).ToArray();
@@ -414,21 +419,147 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
             // next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
             // was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
             // run by then.
-            var heldGate = gate;
-            var released = 0;
-            void ReleaseSlot()
-            {
-                if (Interlocked.Exchange(ref released, 1) != 0) return;
-                SlotReleasingForTest?.Invoke(key);
-                Interlocked.Exchange(ref state.InFlight, 0);
-                heldGate?.Release();
-            }
-
             var submitted = Stopwatch.GetTimestamp();
             var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
-                _worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
+                _worker.EffectiveHardGrace(invocation.Timeout), SlotRelease(key, state, gate));
             handedOff = true;
+            return await AwaitSubmittedAsync(key, state, invocation, shard, connection, pending, submitted, ct).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (ct.IsCancellationRequested)
+        {
+            return ScriptRunOutcome.Failed("evaluation cancelled");
+        }
+        finally
+        {
+            if (!handedOff)
+            {
+                Interlocked.Exchange(ref state.InFlight, 0);
+                gate?.Release();
+            }
+        }
 
+        // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
+        // still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
+        // shard has no live worker (or for the isolation gate).
+        TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
+    }
+
+    /// <inheritdoc />
+    /// <remarks>
+    ///     Every item that can go straight to a live worker — its key admitted, not quarantined, not a suspect, its script
+    ///     already loaded there at this source and slot order — is submitted with its siblings for the same worker as ONE
+    ///     <c>EvalBatch</c> frame. Anything else (a first evaluation that needs a compile, a worker that is down, a suspect
+    ///     bound for the one-at-a-time isolation worker) takes the full <see cref="RunAsync"/> path on its own, so a batch
+    ///     never waits for a load or a restart as a whole. Per-item deadlines are armed at submit and, for an item the worker
+    ///     has to queue behind busy evaluation threads, restarted when it starts — so an item waiting behind a runaway
+    ///     sibling is neither timed out, struck nor blamed for the kill (it is that kill's victim, reported
+    ///     <see cref="ScriptRunOutcome.WorkerRestartedReason"/>, design §9).
+    /// </remarks>
+    public IReadOnlyList<Task<ScriptRunOutcome>> RunBatch(IReadOnlyList<ScriptInvocation> invocations, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(invocations);
+        var outcomes = new Task<ScriptRunOutcome>[invocations.Count];
+        Dictionary<WorkerConnection, List<(int Index, BatchItem Item)>>? groups = null;
+        for (var i = 0; i < invocations.Count; i++)
+        {
+            var invocation = invocations[i];
+            ArgumentNullException.ThrowIfNull(invocation);
+            switch (TryPrepareBatchItem(invocation, out var item, out var answered))
+            {
+                case BatchPreparation.Submit:
+                    groups ??= [];
+                    if (!groups.TryGetValue(item!.Connection, out var group)) groups[item.Connection] = group = [];
+                    group.Add((i, item));
+                    break;
+                case BatchPreparation.Answered:
+                    outcomes[i] = Task.FromResult(answered!);
+                    break;
+                default:
+                    outcomes[i] = RunAsync(invocation, ct);
+                    break;
+            }
+        }
+        if (groups is null) return outcomes;
+
+        foreach (var (connection, group) in groups)
+        {
+            var submissions = new BatchSubmission[group.Count];
+            for (var j = 0; j < group.Count; j++)
+            {
+                var item = group[j].Item;
+                submissions[j] = new BatchSubmission(item.Key, BuildRequest(item.Key, item.Entry, item.Invocation),
+                    item.Invocation.Timeout, _worker.EffectiveHardGrace(item.Invocation.Timeout), SlotRelease(item.Key, item.State, null));
+            }
+            var submitted = Stopwatch.GetTimestamp();
+            BatchSubmittedForTest?.Invoke(submissions.Length);
+            var pendings = connection.SubmitBatch(submissions);
+            for (var j = 0; j < group.Count; j++)
+            {
+                var (index, item) = group[j];
+                outcomes[index] = AwaitSubmittedAsync(item.Key, item.State, item.Invocation, item.Shard, connection, pendings[j], submitted, ct);
+            }
+        }
+        return outcomes;
+    }
+
+    /// <summary>
+    ///     The synchronous half of a batch item: everything <see cref="RunAsync"/> decides before it submits, for the case
+    ///     where nothing has to be waited for. Takes the key's single-flight slot when it returns
+    ///     <see cref="BatchPreparation.Submit"/>; never when it returns anything else.
+    /// </summary>
+    private BatchPreparation TryPrepareBatchItem(ScriptInvocation invocation, out BatchItem? item, out ScriptRunOutcome? answered)
+    {
+        item = null;
+        answered = null;
+        if (Volatile.Read(ref _disposed) != 0) return BatchPreparation.Full;
+
+        var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
+        var state = Admit(key, invocation.Kind, invocation.Source);
+        if (state.QuarantineReason is not null || _compileFailures.ContainsKey(state.FailureKey)) return BatchPreparation.Full;
+
+        var shard = state.Suspect ? _isolation : _shards[ShardIndex(key, _shards.Length)];
+        if (shard.IsolationGate is not null || shard.Current is not { } connection) return BatchPreparation.Full;
+        if (connection.TryGetLoaded(key, state.ShaHex, SlotOrder(state, invocation.Inputs)) is not { } entry) return BatchPreparation.Full;
+
+        if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0)
+        {
+            ScriptRunGuard.RefusedCounter.Add(1, new KeyValuePair<string, object?>("script", key));
+            answered = ScriptRunOutcome.Failed(ScriptRunGuard.PreviousRunStillRunningReason);
+            return BatchPreparation.Answered;
+        }
+        state.ScriptLogger = invocation.ScriptLogger;
+        NoteLoad(entry);
+        item = new BatchItem(key, state, invocation, shard, connection, entry);
+        return BatchPreparation.Submit;
+    }
+
+    /// <summary>
+    ///     The slot release of one submitted evaluation: from submit on, the key's single-flight slot (and the isolation
+    ///     gate) belong to the evaluation itself and are released when the worker answers or dies — not when the caller
+    ///     stops waiting at the soft deadline. That keeps a key single-flight between T and T + G, and the isolation
+    ///     worker strictly one-at-a-time. The release runs inside the connection BEFORE the caller's await resumes, so a
+    ///     caller that awaited one run and starts the next is never refused by the run it already has (the §8 harness found
+    ///     32 of 200 refused when the release was an unordered continuation). On a worker death, attribution (strike,
+    ///     quarantine, suspect) has already run by then.
+    /// </summary>
+    private Action SlotRelease(string key, KeyState state, SemaphoreSlim? heldGate)
+    {
+        var released = 0;
+        return () =>
+        {
+            if (Interlocked.Exchange(ref released, 1) != 0) return;
+            SlotReleasingForTest?.Invoke(key);
+            Interlocked.Exchange(ref state.InFlight, 0);
+            heldGate?.Release();
+        };
+    }
+
+    /// <summary>Waits for one submitted evaluation and turns its result into the caller's outcome.</summary>
+    private async Task<ScriptRunOutcome> AwaitSubmittedAsync(string key, KeyState state, ScriptInvocation invocation,
+        WorkerShard shard, WorkerConnection connection, PendingEval pending, long submitted, CancellationToken ct)
+    {
+        try
+        {
             var result = ct.CanBeCanceled
                 ? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
                 : await pending.Outcome.ConfigureAwait(false);
@@ -452,19 +583,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         {
             return ScriptRunOutcome.Failed("evaluation cancelled");
         }
-        finally
-        {
-            if (!handedOff)
-            {
-                Interlocked.Exchange(ref state.InFlight, 0);
-                gate?.Release();
-            }
-        }
-
-        // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
-        // still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
-        // shard has no live worker (or for the isolation gate).
-        TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
     }
 
     /// <summary>
@@ -816,6 +934,21 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         => LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
             .Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
 
+    private enum BatchPreparation
+    {
+        /// <summary>Run it through the full <see cref="RunAsync"/> path on its own.</summary>
+        Full,
+
+        /// <summary>Answered without running (refused).</summary>
+        Answered,
+
+        /// <summary>Submit it with the batch; its slot is taken.</summary>
+        Submit,
+    }
+
+    private sealed record BatchItem(string Key, KeyState State, ScriptInvocation Invocation, WorkerShard Shard,
+        WorkerConnection Connection, WorkerConnection.LoadEntry Entry);
+
     private sealed class KeyState(string source, ScriptContextKind kind)
     {
         private byte[]? _sha;