perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered
frame reader, one timer per evaluation for T and T+G, slot release on the reader
before the caller resumes, a non-spinning per-connection result dispatcher, and
blocking pipe I/O on Unix (both ends) so the socket never enters the socket
engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2:
228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % ->
123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache
worker RSS, and times kill latency from the soft-timeout answer.

#790: the worker sends LoadStarted; the compile timeout is armed then, so a load
queued behind other compiles is never killed or struck.

#788: one Error with stack per down period, rate-limited Warnings with attempt
count, one Information on recovery.

Design doc §8.2 notes + §12.2 record.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs

@@ -136,6 +136,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
     private readonly string _markerDirectory;
     private readonly string? _peCacheDirectory;
     private readonly byte[]? _peCacheKey;
+    private readonly int _compileThreads;
     private long _peCacheHits;
     private long _peCacheMisses;
     private long _peCacheRejected;
@@ -153,8 +154,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
     }
 
     internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
-        TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null)
+        TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
+        int compileThreads = 0)
     {
+        _compileThreads = compileThreads;
         ArgumentNullException.ThrowIfNull(options);
         _options = options.Sanitized(out _);
         _worker = _options.Worker;
@@ -218,7 +221,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         _keys.Count(kv => kv.Value.QuarantineReason is not null),
         _peCacheDirectory is not null);
 
-    /// <summary>How long a compile in the worker may take before the worker is killed.</summary>
+    /// <summary>
+    ///     How long a compile in the worker may run before the worker is killed, counted from the moment a compile thread
+    ///     starts it — never the time it spent queued behind other compiles (lmxopcua#790).
+    /// </summary>
     internal TimeSpan LoadTimeout { get; }
 
     /// <summary>The shard workers (tests).</summary>
@@ -340,12 +346,12 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         if (Volatile.Read(ref _disposed) != 0) return ScriptRunOutcome.Unavailable(ScriptRunOutcome.RunnerUnavailableReason);
 
         var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
-        var state = Admit(key, invocation.Source);
+        var state = Admit(key, invocation.Kind, invocation.Source);
         if (state.QuarantineReason is { } quarantined) return ScriptRunOutcome.Failed(quarantined);
 
-        var sha = SHA256.HashData(Encoding.UTF8.GetBytes(invocation.Source));
-        var failureKey = InProcessScriptRunner.ScopedKey(invocation.Kind, Convert.ToHexString(sha));
-        if (_compileFailures.TryGetValue(failureKey, out var knownFailure))
+        // The source hash and its negative-cache key are per source, so they live on the key's state (computed once per
+        // source, not per evaluation, lmxopcua#789).
+        if (_compileFailures.TryGetValue(state.FailureKey, out var knownFailure))
         {
             CompileCounter.Add(1, new KeyValuePair<string, object?>("cache", "negative"));
             return ScriptRunOutcome.Failed(knownFailure);
@@ -364,37 +370,29 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         try
         {
             state.ScriptLogger = invocation.ScriptLogger;
-            // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero
-            // grace still kicks a start that is due, so the shards keep retrying at the backoff cap.
-            var grace = State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
             if (shard.IsolationGate is { } isolationGate)
             {
-                if (!await isolationGate.WaitAsync(grace, ct).ConfigureAwait(false))
+                if (!await isolationGate.WaitAsync(Grace(), ct).ConfigureAwait(false))
                     return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
                 gate = isolationGate;
             }
 
-            connection = await shard.GetReadyAsync(grace, ct).ConfigureAwait(false);
+            // Fast path: the shard's worker is up (no state computation, no async hop).
+            connection = shard.Current ?? await shard.GetReadyAsync(Grace(), ct).ConfigureAwait(false);
             if (connection is null) return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
 
-            var paths = invocation.Inputs.Keys.Order(StringComparer.Ordinal).ToArray();
-            WorkerConnection.LoadEntry? entry;
-            try
+            var paths = SlotOrder(state, invocation.Inputs);
+            var entry = await connection
+                .EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, state.Sha, state.ShaHex, paths)
+                .WaitAsync(ct).ConfigureAwait(false);
+            if (entry.TimedOut)
             {
-                entry = await connection
-                    .EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, sha, paths)
-                    .WaitAsync(LoadTimeout, ct).ConfigureAwait(false);
+                // A compile that kept a compile thread busy for LoadTimeout after it STARTED is the script's fault (a
+                // pathological source): the connection killed the worker and the exit handler struck the key. Time
+                // spent queued behind other compiles is not counted (lmxopcua#790).
+                return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {connection.LoadTimeout.TotalSeconds:F0} s");
             }
-            catch (TimeoutException)
-            {
-                // A compile that never finishes is the script's fault (a pathological source): the worker dies, the
-                // key takes a kill strike through the exit handler.
-                _log.Warning("Script {ScriptKey}: compile in ScriptWorker[{Shard}] exceeded {Timeout} s; killing the worker",
-                    key, shard.Label, LoadTimeout.TotalSeconds);
-                connection.Kill(WorkerExitReason.KilledLoadTimeout, key);
-                return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {LoadTimeout.TotalSeconds:F0} s");
-            }
-            if (entry?.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
+            if (entry.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
             NoteLoad(entry);
             if (!outcome.Ok)
             {
@@ -405,19 +403,17 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
                     _ => $"compile failure: {outcome.Error?.Message}",
                 };
                 // Remembered until the next deploy boundary, so a broken script never re-enters Roslyn (design §7).
-                _compileFailures.TryAdd(failureKey, reason);
+                _compileFailures.TryAdd(state.FailureKey, reason);
                 return ScriptRunOutcome.Failed(reason);
             }
 
-            var submitted = Stopwatch.GetTimestamp();
-            var pending = connection.Submit(key, BuildRequest(key, entry, invocation));
-            ArmHardDeadline(connection, pending, invocation.Timeout + _worker.EffectiveHardGrace(invocation.Timeout));
-
             // From here the in-flight slot (and the isolation gate) belong to the evaluation itself: they are released
             // when the worker answers or dies — not when this caller stops waiting at the soft deadline. That keeps a
-            // key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. On a worker
-            // death, attribution (strike, quarantine, suspect) has already run when this continuation fires.
-            handedOff = true;
+            // key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. The release runs
+            // inside the connection BEFORE the caller's await resumes, so a caller that awaited one run and starts the
+            // next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
+            // was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
+            // run by then.
             var heldGate = gate;
             var released = 0;
             void ReleaseSlot()
@@ -427,20 +423,16 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
                 Interlocked.Exchange(ref state.InFlight, 0);
                 heldGate?.Release();
             }
-            _ = pending.Completion.ContinueWith(_ => ReleaseSlot(),
-                CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
 
-            EvalResult? result;
-            try
-            {
-                result = await pending.Completion.WaitAsync(invocation.Timeout, ct).ConfigureAwait(false);
-                // The worker has answered (or died, with attribution already run): free the slot HERE, before this
-                // caller resumes. The continuation above is not ordered against this await, so relying on it alone
-                // let a caller that awaited one run and then started the next see "previous evaluation still running"
-                // (found by the §8 perf harness: 32 of 200 sequential same-key runs refused). Idempotent.
-                ReleaseSlot();
-            }
-            catch (TimeoutException)
+            var submitted = Stopwatch.GetTimestamp();
+            var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
+                _worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
+            handedOff = true;
+
+            var result = ct.CanBeCanceled
+                ? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
+                : await pending.Outcome.ConfigureAwait(false);
+            if (ReferenceEquals(result, PendingEval.SoftTimedOut))
             {
                 // As in process: the run token is cancelled only once the caller has its timeout, so a polling script
                 // ends itself (and frees its slot) without the worker being killed.
@@ -468,6 +460,32 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
                 gate?.Release();
             }
         }
+
+        // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
+        // still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
+        // shard has no live worker (or for the isolation gate).
+        TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
+    }
+
+    /// <summary>
+    ///     The slot order for this evaluation's inputs: the key's previous order when the input set is the same (the
+    ///     steady state — no sort, no allocation, lmxopcua#789), otherwise the ordinal sort of the new set.
+    /// </summary>
+    private static string[] SlotOrder(KeyState state, IReadOnlyDictionary<string, Core.Abstractions.DataValueSnapshot> inputs)
+    {
+        var cached = state.Paths;
+        if (cached is not null && cached.Length == inputs.Count)
+        {
+            var same = true;
+            foreach (var path in cached)
+            {
+                if (!inputs.ContainsKey(path)) { same = false; break; }
+            }
+            if (same) return cached;
+        }
+        var fresh = inputs.Keys.Order(StringComparer.Ordinal).ToArray();
+        state.Paths = fresh;
+        return fresh;
     }
 
     /// <inheritdoc />
@@ -555,7 +573,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
             HandshakeTimeout: TimeSpan.FromSeconds(15),
             PeCacheDirectory: _peCacheDirectory,
             PeCacheKey: _peCacheKey,
-            MarkerDirectory: _markerDirectory);
+            MarkerDirectory: _markerDirectory,
+            LoadTimeout: LoadTimeout,
+            CompileThreads: _compileThreads);
         return new WorkerShard(label, isolation,
             (onExiting, ct) => WorkerConnection.StartAsync(spec, _log, OnLogs, onExiting, ct),
             OnWorkerExit, _log);
@@ -591,15 +611,15 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         }
     }
 
-    private KeyState Admit(string key, string source)
+    private KeyState Admit(string key, ScriptContextKind kind, string source)
     {
         while (true)
         {
-            var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), source);
-            if (string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
+            var state = _keys.GetOrAdd(key, static (_, arg) => new KeyState(arg.source, arg.kind), (source, kind));
+            if (ReferenceEquals(state.Source, source) || string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
             // The script changed under this key: a clean slate, exactly as the in-process guard does. An evaluation
             // of the old source still in flight releases the OLD state's slot when it ends.
-            if (_keys.TryUpdate(key, new KeyState(source), state)) continue;
+            if (_keys.TryUpdate(key, new KeyState(source, kind), state)) continue;
         }
     }
 
@@ -638,23 +658,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         return request;
     }
 
-    private void ArmHardDeadline(WorkerConnection connection, PendingEval pending, TimeSpan after)
-    {
-        var cts = new CancellationTokenSource(after);
-        var registration = cts.Token.Register(() =>
-        {
-            if (pending.Completion.IsCompleted) return;
-            _log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
-                pending.Key, (long)after.TotalMilliseconds, connection.Shard, connection.Pid);
-            connection.Kill(WorkerExitReason.KilledDeadline, pending.Key);
-        });
-        _ = pending.Completion.ContinueWith(_ =>
-        {
-            registration.Dispose();
-            cts.Dispose();
-        }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
-    }
-
     private ScriptRunOutcome Complete(string key, KeyState state, EvalResult result, WorkerShard shard, WorkerConnection connection)
     {
         // The script returned control (a value or a throw): the soft-timeout streak resets, as in process.
@@ -813,9 +816,25 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         => LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
             .Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
 
-    private sealed class KeyState(string source)
+    private sealed class KeyState(string source, ScriptContextKind kind)
     {
+        private byte[]? _sha;
+        private string? _shaHex;
+        private string? _failureKey;
+
         public readonly string Source = source;
+
+        /// <summary>SHA-256 of <see cref="Source"/> (computed on first use; a benign race computes the same bytes).</summary>
+        public byte[] Sha => _sha ??= SHA256.HashData(Encoding.UTF8.GetBytes(Source));
+
+        /// <summary>Its upper-case hex form.</summary>
+        public string ShaHex => _shaHex ??= Convert.ToHexString(Sha);
+
+        /// <summary>The negative compile cache's key for this source (kind-scoped).</summary>
+        public string FailureKey => _failureKey ??= InProcessScriptRunner.ScopedKey(kind, ShaHex);
+
+        /// <summary>The slot order last used for this key (see <see cref="SlotOrder"/>).</summary>
+        public string[]? Paths;
         public readonly List<DateTime> KillStrikes = [];
         public int InFlight;
         public int ConsecutiveTimeouts;