feat(scripting): batch submit, exact per-item deadlines, scripted-alarm seam on the out-of-process runner (lmxopcua#780)

- IScriptRunner.RunBatch: one EvalBatch frame per worker for every evaluation ready to run; per-item, streamed results.
  The default (InProcessScriptRunner) is RunAsync per item. Used by the Calculation pump (dequeue all, split at a
  repeated tag, publish in dequeue order), the VirtualTag seam (ScriptBatchCoalescer, no timer) and the scripted-alarm
  engine (whole dirty set). In process the pump and the engine stay one at a time, unchanged.
- Exact per-item deadlines: the worker hands work to an idle evaluation thread or defers it (EvalResults
  deferred_eval_ids); the host stops that run's clock and restarts T then G when it starts (started_eval_ids). A run
  queued behind a runaway is neither timed out nor struck and is a victim, not the culprit, at the kill.
- Scripted-alarm seam behind Scripting:Runner:Seams; unset Seams now means VirtualTag + Calculation, so alarms run out of
  process only when listed. Load still compiles each predicate (to PE bytes only) so a broken one fails the load; passes
  batch and apply in the serial order under the gate. Script faults freeze the condition BadInternalError; runner faults
  freeze it BadResourceUnavailable and re-evaluate on recovery; a dead worker never clears an alarm. IScriptRunnerProvider
  carries the runner to DriverHostActor.
- Perf harness: batch throughput (RunBatch vs RunAsync x n) and alarm-pass latency (in process / batched / serial);
  design §12.3 records macOS + windev. On windev the serial out-of-process alarm pass misses 250 ms from ~500 alarms;
  batched p99 <= 53 ms at 1,000.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ScriptRunnerSelection.cs

@@ -12,10 +12,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
 ///     no scripts (or an admin-only node) never launches one. Disposed with the host's service provider, which stops the
 ///     workers.
 /// </summary>
-public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable
+public sealed class ScriptRunnerSelection : IScriptRunnerProvider, IAsyncDisposable, IDisposable
 {
-    /// <summary>Seams the out-of-process runner can serve in this build. Scripted alarms go last (design §14 #10).</summary>
-    public static readonly IReadOnlyList<ScriptSeam> SupportedOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
+    /// <summary>Seams the out-of-process runner can serve in this build.</summary>
+    public static readonly IReadOnlyList<ScriptSeam> SupportedOutOfProcessSeams =
+        [ScriptSeam.VirtualTag, ScriptSeam.Calculation, ScriptSeam.ScriptedAlarm];
+
+    /// <summary>
+    ///     The seams an unset or empty <c>Scripting:Runner:Seams</c> selects. Scripted alarms are not among them: they go
+    ///     out of process only when listed explicitly, last in the rollout (design §14 #10 — the alarm budget and the
+    ///     engine refactor).
+    /// </summary>
+    public static readonly IReadOnlyList<ScriptSeam> DefaultOutOfProcessSeams = [ScriptSeam.VirtualTag, ScriptSeam.Calculation];
 
     private readonly OutOfProcessScriptRunner? _runner;
     private readonly HashSet<ScriptSeam> _seams;
@@ -94,5 +102,5 @@ public sealed class ScriptRunnerSelection : IAsyncDisposable, IDisposable
     public void Dispose() => _runner?.Dispose();
 
     private static IEnumerable<ScriptSeam> RequestedSeams(ScriptingOptions options)
-        => options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : Enum.GetValues<ScriptSeam>();
+        => options.RunnerSeams is { Count: > 0 } seams ? seams.Distinct() : DefaultOutOfProcessSeams;
 }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs

@@ -54,6 +54,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     private readonly TimeSpan _runTimeout;
     private readonly ScriptRunGuard _guard;
     private readonly IScriptRunner? _runner;
+    private readonly ScriptBatchCoalescer? _batches;
     private bool _disposed;
 
     /// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
@@ -79,6 +80,7 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
         _runTimeout = runTimeout ?? (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
         _guard = new ScriptRunGuard(scripting, _scriptRoot);
         _runner = scriptRunner;
+        _batches = scriptRunner is null ? null : new ScriptBatchCoalescer(scriptRunner);
     }
 
     /// <summary>The out-of-process runner this evaluator delegates compiled runs to, or null (in process).</summary>
@@ -87,6 +89,9 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
     /// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
     public ScriptRunGuard RunGuard => _guard;
 
+    /// <summary>The coalescer the out-of-process path submits through, or null in process (tests).</summary>
+    internal ScriptBatchCoalescer? Batches => _batches;
+
     // Value-only overload. Every dependency reaches the script stamped Good with an
     // evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
     // overload instead; this one remains for callers that genuinely hold only
@@ -161,9 +166,11 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
         // lmxopcua#780 — OutOfProcess: the compiled run goes to the pre-emptible worker pool, which compiles in the
         // worker and owns timeouts, kills and quarantine for this seam. An Unavailable outcome (worker restarted, no
         // worker ready) publishes BadResourceUnavailable; everything else keeps today's Bad contract.
-        if (_runner is not null)
+        // Every tag has its own actor, so tags an upstream burst made ready together arrive here concurrently; the
+        // coalescer submits those as one batch (design §4.5) and an evaluation on an idle seam at once.
+        if (_batches is not null)
         {
-            var outcome = await _runner.RunAsync(new ScriptInvocation(
+            var outcome = await _batches.RunAsync(new ScriptInvocation(
                 virtualTagId, ScriptContextKind.VirtualTag, expression, readCache, _runTimeout, scriptLog)).ConfigureAwait(false);
             return outcome.ToVirtualTagEvalResult();
         }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -385,6 +385,9 @@ if (hasDriver)
     Log.Information("Scripting runner: {RunnerMode}", scriptingOptions.Runner);
     builder.Services.AddSingleton(sp => ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection.Create(
         sp.GetRequiredService<ScriptingOptions>(), Log.Logger, sp.GetService<ScriptRootLogger>()?.Logger));
+    // The Runtime's scripted-alarm host resolves its runner through this (it cannot reference the Host).
+    builder.Services.AddSingleton<IScriptRunnerProvider>(sp =>
+        sp.GetRequiredService<ZB.MOM.WW.OtOpcUa.Host.Configuration.ScriptRunnerSelection>());
 
     // lmxopcua#777: the ONE operator-write timeout (DriverWrites:Timeout) every write bound is derived from — the
     // driver child's cancel + abandon, its queue budget, DriverHostActor's Ask and the inbound-write gateway's Ask.

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -206,6 +206,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
     /// <summary>Node-wide script run-guard options for the scripted-alarm engine; null ⇒ compiled defaults.</summary>
     private readonly ScriptingOptions? _scripting;
 
+    // lmxopcua#780 — the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm; null keeps the engine's
+    // in-process guard path.
+    private readonly IScriptRunner? _scriptedAlarmRunner;
+
     /// <summary>
     ///     Driver children this host has stopped (<see cref="StopChild"/>, an operator Restart) and is watching
     ///     until they terminate. A value burst from one of them is DROPPED in <see cref="ForwardBatchToMux"/>:
@@ -771,7 +775,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         ScriptingOptions? scripting = null,
         NativeAlarmAckPreflightView? ackPreflightView = null,
         DriverWriteBounds? writeBounds = null,
-        TimeProvider? writeClock = null) =>
+        TimeProvider? writeClock = null,
+        IScriptRunner? scriptedAlarmRunner = null) =>
         // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
         // expression tree. Six IActorRef? parameters and several interface-typed ones mean a
         // mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -785,7 +790,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
             alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
             sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
             sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
-            scripting, ackPreflightView, writeBounds, writeClock));
+            scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner));
 
     /// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
     /// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -916,8 +921,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         ScriptingOptions? scripting = null,
         NativeAlarmAckPreflightView? ackPreflightView = null,
         DriverWriteBounds? writeBounds = null,
-        TimeProvider? writeClock = null)
+        TimeProvider? writeClock = null,
+        IScriptRunner? scriptedAlarmRunner = null)
     {
+        _scriptedAlarmRunner = scriptedAlarmRunner;
         _writeBounds = writeBounds ?? DriverWriteBounds.Default;
         _writeClock = writeClock ?? TimeProvider.System;
         _ackPreflightView = ackPreflightView;
@@ -1242,13 +1249,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         var store = _alarmStateStore;
         var scriptLogger = _scriptRootLogger.Logger;
         var scripting = _scripting;
+        var alarmRunner = _scriptedAlarmRunner;
         _scriptedAlarmHost = Context.ActorOf(
             ScriptedAlarmHostActor.Props(
                 _opcUaPublishActor, _dependencyMux,
                 () =>
                 {
                     var upstream = new DependencyMuxTagUpstreamSource();
-                    return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting));
+                    return (upstream, CreateScriptedAlarmEngine(upstream, store, scriptLogger, scripting, alarmRunner));
                 },
                 _localNode, telemetryHub: _telemetryHub),
             "scripted-alarm-host");
@@ -1265,15 +1273,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
     /// <param name="store">The condition-state store.</param>
     /// <param name="scriptLogger">The root script logger.</param>
     /// <param name="scripting">The node-wide <c>Scripting</c> options; null ⇒ defaults.</param>
+    /// <param name="scriptRunner">The out-of-process runner for the ScriptedAlarm seam; null ⇒ in process.</param>
     /// <returns>A new, unloaded engine.</returns>
     internal static ScriptedAlarmEngine CreateScriptedAlarmEngine(
         ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.ITagUpstreamSource upstream, IAlarmStateStore store,
-        Serilog.ILogger scriptLogger, ScriptingOptions? scripting)
+        Serilog.ILogger scriptLogger, ScriptingOptions? scripting, IScriptRunner? scriptRunner = null)
     {
         var budget = (scripting ?? new ScriptingOptions()).Sanitized(out _).RunTimeout;
         return new ScriptedAlarmEngine(
             upstream, store, new ScriptLoggerFactory(scriptLogger), scriptLogger,
-            scriptTimeout: budget, scripting: scripting);
+            scriptTimeout: budget, scripting: scripting, scriptRunner: scriptRunner);
     }
 
     private void Bootstrap()

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs

@@ -664,7 +664,11 @@ public static class ServiceCollectionExtensions
                     // The one configured write timeout (DriverWrites:Timeout, bound + validated in Program.cs) the
                     // children's write bounds AND this host's write Ask are derived from (lmxopcua#777). Unbound in
                     // test graphs ⇒ null ⇒ DriverWriteBounds.Default.
-                    writeBounds: resolver.GetService<DriverWriteBounds>()),
+                    writeBounds: resolver.GetService<DriverWriteBounds>(),
+                    // lmxopcua#780: the out-of-process runner when Scripting:Runner:Seams lists ScriptedAlarm. Null under
+                    // the InProcess default, when the seam is not listed, and in test graphs that register no provider.
+                    scriptedAlarmRunner: resolver.GetService<ZB.MOM.WW.OtOpcUa.Core.Scripting.IScriptRunnerProvider>()
+                        ?.ForSeam(ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptSeam.ScriptedAlarm)),
                 DriverHostActorName);
             registry.Register<DriverHostActorKey>(driverHost);
 

src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs

@@ -34,8 +34,22 @@ internal sealed class WorkerHost
     private readonly int _parentPid;
     private readonly ConcurrentDictionary<string, LoadedScript> _scripts = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<long, RunningEval> _running = new();
+    /// <summary>
+    ///     Hand-off to an IDLE evaluation thread: an evaluation is put here only when a thread is known to be waiting
+    ///     for it (<see cref="_idle"/>), so whatever is put here starts at once. Everything else waits in
+    ///     <see cref="_backlog"/> and is reported to the host as deferred (lmxopcua#780 batch submit: per-item deadlines
+    ///     stay exact, see the proto's <c>EvalResults</c>).
+    /// </summary>
     private readonly BlockingCollection<EvalRequest> _queue = new(new ConcurrentQueue<EvalRequest>());
 
+    /// <summary>Evaluations that arrived while every evaluation thread was busy, in arrival order. Under <see cref="_schedLock"/>.</summary>
+    private readonly Queue<EvalRequest> _backlog = new();
+
+    private readonly object _schedLock = new();
+
+    /// <summary>Evaluation threads committed to taking from <see cref="_queue"/>, minus hand-offs not yet taken. Under <see cref="_schedLock"/>.</summary>
+    private int _idle;
+
     /// <summary>
     ///     Loads waiting for a compile thread. Compiles are long, synchronous and CPU-bound, so they run on
     ///     <see cref="CompileThreads"/> dedicated threads, never on the thread pool: a burst of cold loads (a node start
@@ -225,8 +239,20 @@ internal sealed class WorkerHost
                     if (_scripts.TryRemove(key, out var gone)) gone.Dispose();
                 return null;
             case Envelope.BodyOneofCase.Eval:
-                foreach (var item in frame.Eval.Items) Admit(item);
+            {
+                List<ulong>? deferred = null;
+                foreach (var item in frame.Eval.Items)
+                    if (!Admit(item)) (deferred ??= []).Add(item.EvalId);
+                // One notice per incoming frame. It can reach the host after the deferred run's own start notice (an
+                // evaluation thread may pick the run up before this is queued); the host treats the two as unordered.
+                if (deferred is not null)
+                {
+                    var notice = new EvalResults();
+                    notice.DeferredEvalIds.AddRange(deferred);
+                    Send(new Envelope { Results = notice });
+                }
                 return null;
+            }
             case Envelope.BodyOneofCase.Cancel:
                 if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
                 {
@@ -288,38 +314,72 @@ internal sealed class WorkerHost
         Send(new Envelope { LoadResult = result });
     }
 
-    private void Admit(EvalRequest request)
+    /// <summary>
+    ///     Schedules one evaluation: straight to an idle evaluation thread when there is one (it starts now, so the
+    ///     host's clock, started at submit, is right), otherwise into the backlog.
+    /// </summary>
+    /// <param name="request">The evaluation.</param>
+    /// <returns>False when it was deferred to the backlog (the host must stop its clock until it starts).</returns>
+    private bool Admit(EvalRequest request)
     {
-        if (_queue.Count >= _queueCapacity)
+        lock (_schedLock)
         {
-            SendResult(new EvalResult
+            if (_backlog.Count + _queue.Count < _queueCapacity)
             {
-                EvalId = request.EvalId,
-                Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" },
-            });
-            return;
+                if (_idle > 0 && _backlog.Count == 0)
+                {
+                    _idle--;
+                    _queue.Add(request);
+                    return true;
+                }
+                _backlog.Enqueue(request);
+                return false;
+            }
         }
-        _queue.Add(request);
+        // Full: answered at once, outside the lock (a send may write to the pipe on this thread).
+        SendResult(new EvalResult
+        {
+            EvalId = request.EvalId,
+            Error = new ScriptError { Kind = ScriptError.Types.Kind.Busy, Message = "script worker busy" },
+        });
+        return true;
     }
 
     private void EvalThreadLoop()
     {
-        foreach (var request in _queue.GetConsumingEnumerable())
+        EvalResult? finished = null;
+        while (true)
         {
-            EvalResult result;
+            // A thread that just finished a run takes the oldest deferred evaluation, and its start notice rides on
+            // the finished run's result frame. With nothing deferred it goes idle and waits for a hand-off.
+            EvalRequest? request = null;
+            lock (_schedLock)
+            {
+                if (finished is not null && _backlog.TryDequeue(out var next)) request = next;
+                else _idle++;
+            }
+            if (finished is not null)
+            {
+                var frame = new EvalResults();
+                frame.Items.Add(finished);
+                if (request is not null) frame.StartedEvalIds.Add(request.EvalId);
+                Send(new Envelope { Results = frame });
+                finished = null;
+            }
+            request ??= _queue.Take();
+
             try
             {
-                result = Evaluate(request);
+                finished = Evaluate(request);
             }
             catch (Exception ex)
             {
-                result = new EvalResult
+                finished = new EvalResult
                 {
                     EvalId = request.EvalId,
                     Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = ex.Message },
                 };
             }
-            SendResult(result);
         }
     }
 
@@ -431,7 +491,7 @@ internal sealed class WorkerHost
                 Heartbeat = new Heartbeat
                 {
                     Inflight = (uint)_running.Count,
-                    Queued = (uint)_queue.Count,
+                    Queued = (uint)QueuedCount(),
                     HeapBytes = GC.GetTotalMemory(false),
                     Gen2Count = (uint)GC.CollectionCount(2),
                 },
@@ -489,6 +549,11 @@ internal sealed class WorkerHost
         Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed);
     }
 
+    private int QueuedCount()
+    {
+        lock (_schedLock) return _backlog.Count + _queue.Count;
+    }
+
     private void SendResult(EvalResult result)
     {
         var results = new EvalResults();