perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered
frame reader, one timer per evaluation for T and T+G, slot release on the reader
before the caller resumes, a non-spinning per-connection result dispatcher, and
blocking pipe I/O on Unix (both ends) so the socket never enters the socket
engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2:
228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % ->
123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache
worker RSS, and times kill latency from the soft-timeout answer.

#790: the worker sends LoadStarted; the compile timeout is armed then, so a load
queued behind other compiles is never killed or struck.

#788: one Error with stack per down period, rate-limited Warnings with attempt
count, one Information on recovery.

Design doc §8.2 notes + §12.2 record.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/OutOfProcessScriptRunner.cs

@@ -136,6 +136,7 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
     private readonly string _markerDirectory;
     private readonly string? _peCacheDirectory;
     private readonly byte[]? _peCacheKey;
+    private readonly int _compileThreads;
     private long _peCacheHits;
     private long _peCacheMisses;
     private long _peCacheRejected;
@@ -153,8 +154,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
     }
 
     internal OutOfProcessScriptRunner(ScriptingOptions options, ILogger? log, ILogger? scriptRoot,
-        TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null)
+        TimeSpan heartbeatInterval, TimeSpan loadTimeout, TimeSpan? maintenanceInterval = null, string? markerDirectory = null,
+        int compileThreads = 0)
     {
+        _compileThreads = compileThreads;
         ArgumentNullException.ThrowIfNull(options);
         _options = options.Sanitized(out _);
         _worker = _options.Worker;
@@ -218,7 +221,10 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         _keys.Count(kv => kv.Value.QuarantineReason is not null),
         _peCacheDirectory is not null);
 
-    /// <summary>How long a compile in the worker may take before the worker is killed.</summary>
+    /// <summary>
+    ///     How long a compile in the worker may run before the worker is killed, counted from the moment a compile thread
+    ///     starts it — never the time it spent queued behind other compiles (lmxopcua#790).
+    /// </summary>
     internal TimeSpan LoadTimeout { get; }
 
     /// <summary>The shard workers (tests).</summary>
@@ -340,12 +346,12 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         if (Volatile.Read(ref _disposed) != 0) return ScriptRunOutcome.Unavailable(ScriptRunOutcome.RunnerUnavailableReason);
 
         var key = InProcessScriptRunner.ScopedKey(invocation.Kind, invocation.Key);
-        var state = Admit(key, invocation.Source);
+        var state = Admit(key, invocation.Kind, invocation.Source);
         if (state.QuarantineReason is { } quarantined) return ScriptRunOutcome.Failed(quarantined);
 
-        var sha = SHA256.HashData(Encoding.UTF8.GetBytes(invocation.Source));
-        var failureKey = InProcessScriptRunner.ScopedKey(invocation.Kind, Convert.ToHexString(sha));
-        if (_compileFailures.TryGetValue(failureKey, out var knownFailure))
+        // The source hash and its negative-cache key are per source, so they live on the key's state (computed once per
+        // source, not per evaluation, lmxopcua#789).
+        if (_compileFailures.TryGetValue(state.FailureKey, out var knownFailure))
         {
             CompileCounter.Add(1, new KeyValuePair<string, object?>("cache", "negative"));
             return ScriptRunOutcome.Failed(knownFailure);
@@ -364,37 +370,29 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         try
         {
             state.ScriptLogger = invocation.ScriptLogger;
-            // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero
-            // grace still kicks a start that is due, so the shards keep retrying at the backoff cap.
-            var grace = State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
             if (shard.IsolationGate is { } isolationGate)
             {
-                if (!await isolationGate.WaitAsync(grace, ct).ConfigureAwait(false))
+                if (!await isolationGate.WaitAsync(Grace(), ct).ConfigureAwait(false))
                     return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
                 gate = isolationGate;
             }
 
-            connection = await shard.GetReadyAsync(grace, ct).ConfigureAwait(false);
+            // Fast path: the shard's worker is up (no state computation, no async hop).
+            connection = shard.Current ?? await shard.GetReadyAsync(Grace(), ct).ConfigureAwait(false);
             if (connection is null) return Unavailable(ScriptRunOutcome.RunnerUnavailableReason, shard, null);
 
-            var paths = invocation.Inputs.Keys.Order(StringComparer.Ordinal).ToArray();
-            WorkerConnection.LoadEntry? entry;
-            try
+            var paths = SlotOrder(state, invocation.Inputs);
+            var entry = await connection
+                .EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, state.Sha, state.ShaHex, paths)
+                .WaitAsync(ct).ConfigureAwait(false);
+            if (entry.TimedOut)
             {
-                entry = await connection
-                    .EnsureLoadedAsync(key, WorkerShard.ToWire(invocation.Kind), invocation.Source, sha, paths)
-                    .WaitAsync(LoadTimeout, ct).ConfigureAwait(false);
+                // A compile that kept a compile thread busy for LoadTimeout after it STARTED is the script's fault (a
+                // pathological source): the connection killed the worker and the exit handler struck the key. Time
+                // spent queued behind other compiles is not counted (lmxopcua#790).
+                return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {connection.LoadTimeout.TotalSeconds:F0} s");
             }
-            catch (TimeoutException)
-            {
-                // A compile that never finishes is the script's fault (a pathological source): the worker dies, the
-                // key takes a kill strike through the exit handler.
-                _log.Warning("Script {ScriptKey}: compile in ScriptWorker[{Shard}] exceeded {Timeout} s; killing the worker",
-                    key, shard.Label, LoadTimeout.TotalSeconds);
-                connection.Kill(WorkerExitReason.KilledLoadTimeout, key);
-                return ScriptRunOutcome.Failed($"compile failure: compile did not finish within {LoadTimeout.TotalSeconds:F0} s");
-            }
-            if (entry?.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
+            if (entry.Outcome is not { } outcome) return Unavailable(ScriptRunOutcome.WorkerRestartedReason, RouteFor(key, state), connection);
             NoteLoad(entry);
             if (!outcome.Ok)
             {
@@ -405,19 +403,17 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
                     _ => $"compile failure: {outcome.Error?.Message}",
                 };
                 // Remembered until the next deploy boundary, so a broken script never re-enters Roslyn (design §7).
-                _compileFailures.TryAdd(failureKey, reason);
+                _compileFailures.TryAdd(state.FailureKey, reason);
                 return ScriptRunOutcome.Failed(reason);
             }
 
-            var submitted = Stopwatch.GetTimestamp();
-            var pending = connection.Submit(key, BuildRequest(key, entry, invocation));
-            ArmHardDeadline(connection, pending, invocation.Timeout + _worker.EffectiveHardGrace(invocation.Timeout));
-
             // From here the in-flight slot (and the isolation gate) belong to the evaluation itself: they are released
             // when the worker answers or dies — not when this caller stops waiting at the soft deadline. That keeps a
-            // key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. On a worker
-            // death, attribution (strike, quarantine, suspect) has already run when this continuation fires.
-            handedOff = true;
+            // key single-flight between T and T + G, and the isolation worker strictly one-at-a-time. The release runs
+            // inside the connection BEFORE the caller's await resumes, so a caller that awaited one run and starts the
+            // next is never refused by the run it already has (the §8 harness found 32 of 200 refused when the release
+            // was an unordered continuation). On a worker death, attribution (strike, quarantine, suspect) has already
+            // run by then.
             var heldGate = gate;
             var released = 0;
             void ReleaseSlot()
@@ -427,20 +423,16 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
                 Interlocked.Exchange(ref state.InFlight, 0);
                 heldGate?.Release();
             }
-            _ = pending.Completion.ContinueWith(_ => ReleaseSlot(),
-                CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
 
-            EvalResult? result;
-            try
-            {
-                result = await pending.Completion.WaitAsync(invocation.Timeout, ct).ConfigureAwait(false);
-                // The worker has answered (or died, with attribution already run): free the slot HERE, before this
-                // caller resumes. The continuation above is not ordered against this await, so relying on it alone
-                // let a caller that awaited one run and then started the next see "previous evaluation still running"
-                // (found by the §8 perf harness: 32 of 200 sequential same-key runs refused). Idempotent.
-                ReleaseSlot();
-            }
-            catch (TimeoutException)
+            var submitted = Stopwatch.GetTimestamp();
+            var pending = connection.Submit(key, BuildRequest(key, entry, invocation), invocation.Timeout,
+                _worker.EffectiveHardGrace(invocation.Timeout), ReleaseSlot);
+            handedOff = true;
+
+            var result = ct.CanBeCanceled
+                ? await pending.Outcome.WaitAsync(ct).ConfigureAwait(false)
+                : await pending.Outcome.ConfigureAwait(false);
+            if (ReferenceEquals(result, PendingEval.SoftTimedOut))
             {
                 // As in process: the run token is cancelled only once the caller has its timeout, so a polling script
                 // ends itself (and frees its slot) without the worker being killed.
@@ -468,6 +460,32 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
                 gate?.Release();
             }
         }
+
+        // Circuit open (design §5.3.6): answer at once instead of making every trigger wait out the grace. A zero grace
+        // still kicks a start that is due, so the shards keep retrying at the backoff cap. Only computed when the
+        // shard has no live worker (or for the isolation gate).
+        TimeSpan Grace() => State == ScriptRunnerState.Unavailable ? TimeSpan.Zero : _worker.UnavailableGrace;
+    }
+
+    /// <summary>
+    ///     The slot order for this evaluation's inputs: the key's previous order when the input set is the same (the
+    ///     steady state — no sort, no allocation, lmxopcua#789), otherwise the ordinal sort of the new set.
+    /// </summary>
+    private static string[] SlotOrder(KeyState state, IReadOnlyDictionary<string, Core.Abstractions.DataValueSnapshot> inputs)
+    {
+        var cached = state.Paths;
+        if (cached is not null && cached.Length == inputs.Count)
+        {
+            var same = true;
+            foreach (var path in cached)
+            {
+                if (!inputs.ContainsKey(path)) { same = false; break; }
+            }
+            if (same) return cached;
+        }
+        var fresh = inputs.Keys.Order(StringComparer.Ordinal).ToArray();
+        state.Paths = fresh;
+        return fresh;
     }
 
     /// <inheritdoc />
@@ -555,7 +573,9 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
             HandshakeTimeout: TimeSpan.FromSeconds(15),
             PeCacheDirectory: _peCacheDirectory,
             PeCacheKey: _peCacheKey,
-            MarkerDirectory: _markerDirectory);
+            MarkerDirectory: _markerDirectory,
+            LoadTimeout: LoadTimeout,
+            CompileThreads: _compileThreads);
         return new WorkerShard(label, isolation,
             (onExiting, ct) => WorkerConnection.StartAsync(spec, _log, OnLogs, onExiting, ct),
             OnWorkerExit, _log);
@@ -591,15 +611,15 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         }
     }
 
-    private KeyState Admit(string key, string source)
+    private KeyState Admit(string key, ScriptContextKind kind, string source)
     {
         while (true)
         {
-            var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), source);
-            if (string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
+            var state = _keys.GetOrAdd(key, static (_, arg) => new KeyState(arg.source, arg.kind), (source, kind));
+            if (ReferenceEquals(state.Source, source) || string.Equals(state.Source, source, StringComparison.Ordinal)) return state;
             // The script changed under this key: a clean slate, exactly as the in-process guard does. An evaluation
             // of the old source still in flight releases the OLD state's slot when it ends.
-            if (_keys.TryUpdate(key, new KeyState(source), state)) continue;
+            if (_keys.TryUpdate(key, new KeyState(source, kind), state)) continue;
         }
     }
 
@@ -638,23 +658,6 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         return request;
     }
 
-    private void ArmHardDeadline(WorkerConnection connection, PendingEval pending, TimeSpan after)
-    {
-        var cts = new CancellationTokenSource(after);
-        var registration = cts.Token.Register(() =>
-        {
-            if (pending.Completion.IsCompleted) return;
-            _log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
-                pending.Key, (long)after.TotalMilliseconds, connection.Shard, connection.Pid);
-            connection.Kill(WorkerExitReason.KilledDeadline, pending.Key);
-        });
-        _ = pending.Completion.ContinueWith(_ =>
-        {
-            registration.Dispose();
-            cts.Dispose();
-        }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
-    }
-
     private ScriptRunOutcome Complete(string key, KeyState state, EvalResult result, WorkerShard shard, WorkerConnection connection)
     {
         // The script returned control (a value or a throw): the soft-timeout streak resets, as in process.
@@ -813,9 +816,25 @@ public sealed class OutOfProcessScriptRunner : IScriptRunner, IAsyncDisposable,
         => LiveRunners.Keys.SelectMany(r => r.AllShards.Select(s => s.Status()))
             .Select(s => new Measurement<int>(value(s), new KeyValuePair<string, object?>("shard", s.Label)));
 
-    private sealed class KeyState(string source)
+    private sealed class KeyState(string source, ScriptContextKind kind)
     {
+        private byte[]? _sha;
+        private string? _shaHex;
+        private string? _failureKey;
+
         public readonly string Source = source;
+
+        /// <summary>SHA-256 of <see cref="Source"/> (computed on first use; a benign race computes the same bytes).</summary>
+        public byte[] Sha => _sha ??= SHA256.HashData(Encoding.UTF8.GetBytes(Source));
+
+        /// <summary>Its upper-case hex form.</summary>
+        public string ShaHex => _shaHex ??= Convert.ToHexString(Sha);
+
+        /// <summary>The negative compile cache's key for this source (kind-scoped).</summary>
+        public string FailureKey => _failureKey ??= InProcessScriptRunner.ScopedKey(kind, ShaHex);
+
+        /// <summary>The slot order last used for this key (see <see cref="SlotOrder"/>).</summary>
+        public string[]? Paths;
         public readonly List<DateTime> KillStrikes = [];
         public int InFlight;
         public int ConsecutiveTimeouts;

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/ResultDispatcher.cs

@@ -0,0 +1,90 @@
+using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
+
+/// <summary>
+///     The thread that resumes the callers of one worker's evaluations (lmxopcua#789).
+/// </summary>
+/// <remarks>
+///     <para>Why not the thread pool: a result that wakes a pool worker makes that worker spin-wait for more work when it
+///     is done, and results arrive one at a time, spread out, so every evaluation paid a full spin. The §8 harness
+///     measured that spin as most of the host's CPU at 10k evals/s (with
+///     <c>DOTNET_ThreadPool_UnfairSemaphoreSpinLimit=0</c> the out-of-process runner cost only ~10 points of a core more
+///     than in-process). This thread blocks without spinning (<see cref="Monitor.Wait(object)"/>) and runs the
+///     continuations of every result that is ready in one wake-up.</para>
+///     <para>Why not the reader thread: the continuation is the caller's code (the Calculation pump publishes to OPC UA
+///     subscribers after its await). On the reader it could hold up heartbeats and results, and a slow consumer could
+///     get a healthy worker killed for silence. Here a slow consumer delays only the resumption of other callers; the
+///     reader has already settled each result — its deadline timer cancelled, its key's slot released — before handing
+///     it over, so no deadline can fire on an evaluation that has answered.</para>
+///     <para>A caller that blocks on another evaluation's result from inside its continuation (sync over async) would
+///     wait on this very thread; no seam does, and <see cref="IScriptRunner.RunAsync"/> is async-only for that reason.</para>
+/// </remarks>
+internal sealed class ResultDispatcher
+{
+    private readonly object _gate = new();
+    private readonly Queue<(PendingEval Pending, EvalResult? Result)> _queue = new();
+    private bool _waiting;
+    private bool _stopped;
+
+    /// <summary>Starts the thread.</summary>
+    /// <param name="name">The thread name.</param>
+    public ResultDispatcher(string name)
+    {
+        new Thread(Run) { IsBackground = true, Name = name }.Start();
+    }
+
+    /// <summary>Queues a caller's resumption. After <see cref="Stop"/> it runs on the calling thread instead.</summary>
+    /// <param name="pending">The evaluation.</param>
+    /// <param name="result">Its result (null: the worker ended).</param>
+    public void Post(PendingEval pending, EvalResult? result)
+    {
+        lock (_gate)
+        {
+            if (!_stopped)
+            {
+                _queue.Enqueue((pending, result));
+                if (_waiting) Monitor.Pulse(_gate);
+                return;
+            }
+        }
+        pending.SignalOutcome(result);
+    }
+
+    /// <summary>Lets the thread drain what is queued and end.</summary>
+    public void Stop()
+    {
+        lock (_gate)
+        {
+            _stopped = true;
+            Monitor.Pulse(_gate);
+        }
+    }
+
+    private void Run()
+    {
+        var batch = new List<(PendingEval Pending, EvalResult? Result)>();
+        while (true)
+        {
+            lock (_gate)
+            {
+                while (_queue.Count == 0)
+                {
+                    if (_stopped) return;
+                    _waiting = true;
+                    Monitor.Wait(_gate);
+                    _waiting = false;
+                }
+                while (_queue.TryDequeue(out var item)) batch.Add(item);
+            }
+            foreach (var (pending, result) in batch)
+            {
+                // TrySetResult runs the caller's continuation here; an await or ContinueWith captures its own
+                // exceptions, so nothing should escape — but this thread must survive a misbehaving one regardless.
+                try { pending.SignalOutcome(result); }
+                catch (Exception) { /* the caller's own task carries its failure */ }
+            }
+            batch.Clear();
+        }
+    }
+}

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WindowsJobObject.cs

@@ -59,6 +59,14 @@ internal sealed class WindowsJobObject : IDisposable
         }
     }
 
+    /// <summary>
+    ///     Kills every process in the job at once (lmxopcua#789). The job admits exactly one process, so this is the
+    ///     worker's whole tree without <c>Process.Kill(entireProcessTree: true)</c>'s system-wide process snapshot,
+    ///     which cost about 100–200 ms of the Windows kill latency.
+    /// </summary>
+    /// <returns>False when the job could not be terminated (the caller falls back to a process kill).</returns>
+    public bool Terminate() => !_handle.IsClosed && TerminateJobObject(_handle, unchecked((uint)-1));
+
     /// <summary>Closes the job handle, which kills every process still in it.</summary>
     public void Dispose() => _handle.Dispose();
 
@@ -98,7 +106,7 @@ internal sealed class WindowsJobObject : IDisposable
         public UIntPtr PeakJobMemoryUsed;
     }
 
-#pragma warning disable SYSLIB1054 // DllImport kept: LibraryImport would need AllowUnsafeBlocks for three calls.
+#pragma warning disable SYSLIB1054 // DllImport kept: LibraryImport would need AllowUnsafeBlocks for four calls.
     [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
     private static extern SafeFileHandle CreateJobObjectW(IntPtr securityAttributes, string? name);
 
@@ -110,5 +118,9 @@ internal sealed class WindowsJobObject : IDisposable
     [DllImport("kernel32.dll", SetLastError = true)]
     [return: MarshalAs(UnmanagedType.Bool)]
     private static extern bool AssignProcessToJobObject(SafeFileHandle job, IntPtr process);
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    [return: MarshalAs(UnmanagedType.Bool)]
+    private static extern bool TerminateJobObject(SafeFileHandle job, uint exitCode);
 #pragma warning restore SYSLIB1054
 }

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs

@@ -3,7 +3,6 @@ using System.Diagnostics;
 using System.IO.Pipes;
 using System.Security.Cryptography;
 using System.Text;
-using System.Threading.Channels;
 using Serilog;
 using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
 using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
@@ -65,6 +64,11 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
 /// <param name="PeCacheDirectory">The compiled-PE cache directory handed to the worker; null = no cache.</param>
 /// <param name="PeCacheKey">The cache's HMAC key (with <paramref name="PeCacheDirectory"/>).</param>
 /// <param name="MarkerDirectory">Where the launch marker for the orphan sweep is written; null = none.</param>
+/// <param name="LoadTimeout">
+///     How long one compile may run, counted from the worker's <c>LoadStarted</c> for it (not from when the host sent
+///     it, lmxopcua#790), before the worker is killed and the key struck; null = 60 s.
+/// </param>
+/// <param name="CompileThreads">The worker's dedicated compile threads; 0 = its default.</param>
 internal sealed record WorkerLaunchSpec(
     string Shard,
     string WorkerPath,
@@ -75,7 +79,16 @@ internal sealed record WorkerLaunchSpec(
     TimeSpan HandshakeTimeout,
     string? PeCacheDirectory = null,
     byte[]? PeCacheKey = null,
-    string? MarkerDirectory = null);
+    string? MarkerDirectory = null,
+    TimeSpan? LoadTimeout = null,
+    int CompileThreads = 0)
+{
+    /// <summary>The default compile timeout.</summary>
+    public static readonly TimeSpan DefaultLoadTimeout = TimeSpan.FromSeconds(60);
+
+    /// <summary><see cref="LoadTimeout"/> or its default.</summary>
+    public TimeSpan EffectiveLoadTimeout => LoadTimeout ?? DefaultLoadTimeout;
+}
 
 /// <summary>
 ///     One live ScriptWorker process and its pipe (lmxopcua#780, design §3–§6): launch with a cleared, allow-listed
@@ -101,13 +114,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
     private readonly string _workDir;
     private readonly ILogger _log;
     private readonly Action<LogBatch> _onLogs;
-    private readonly Channel<Envelope> _outbound = Channel.CreateUnbounded<Envelope>(new UnboundedChannelOptions { SingleReader = true });
+    private readonly ScriptWorkerFrameWriter _writer;
+    private readonly ResultDispatcher _dispatcher;
     private readonly ConcurrentDictionary<ulong, PendingEval> _pending = new();
-    private readonly ConcurrentDictionary<ulong, (string Key, TaskCompletionSource<LoadResult> Completion)> _loads = new();
+    private readonly ConcurrentDictionary<ulong, LoadInFlight> _loads = new();
     private readonly ConcurrentDictionary<string, LoadEntry> _loaded = new(StringComparer.Ordinal);
     private readonly TaskCompletionSource<WorkerExit> _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
     private readonly CancellationTokenSource _stopping = new();
     private readonly TimeSpan _heartbeatInterval;
+    private TimeSpan _loadTimeout;
     private readonly Action<WorkerExit> _onExiting;
     private readonly string? _markerPath;
     private TaskCompletionSource<DiagnosticsReply>? _diag;
@@ -116,15 +131,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
     private int _finished;
     private (WorkerExitReason Reason, string? Culprit)? _killedFor;
     private long _lastFrameTimestamp;
-    private long _seq;
     private long _nextEvalId;
     private long _nextLoadId;
-    private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
     private int _disposed;
 
     private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, string workDir,
-        ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, Action<WorkerExit> onExiting, string? markerPath)
+        ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, TimeSpan loadTimeout, Action<WorkerExit> onExiting,
+        string? markerPath)
     {
+        _loadTimeout = loadTimeout;
         _onExiting = onExiting;
         _markerPath = markerPath;
         Shard = shard;
@@ -136,6 +151,10 @@ internal sealed class WorkerConnection : IAsyncDisposable
         _onLogs = onLogs;
         _heartbeatInterval = heartbeatInterval;
         Pid = process.Id;
+        // The handshake used seq 1. A failed write means the pipe is gone: the worker is dead or dying.
+        _writer = new ScriptWorkerFrameWriter(pipe, OnWriteFault, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, firstSeq: 1,
+            synchronous: ScriptWorkerFrameCodec.UseBlockingIo);
+        _dispatcher = new ResultDispatcher($"ScriptWorker[{shard}].Results");
     }
 
     /// <summary>The shard label.</summary>
@@ -156,6 +175,16 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <summary>Evaluations currently outstanding on this worker.</summary>
     public int InFlightCount => _pending.Count;
 
+    /// <summary>
+    ///     The compile timeout applied to loads that START from now on (test hook: a test warms the worker's compiler
+    ///     under the launch value, then shortens it).
+    /// </summary>
+    internal TimeSpan LoadTimeout
+    {
+        get => _loadTimeout;
+        set => _loadTimeout = value;
+    }
+
     /// <summary>Compiles currently outstanding on this worker.</summary>
     public int LoadingCount => _loads.Count;
 
@@ -234,16 +263,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
                 HardGraceMs = (uint)Math.Clamp(spec.HardGrace.TotalMilliseconds, 1, uint.MaxValue),
                 HeartbeatIntervalMs = (uint)spec.HeartbeatInterval.TotalMilliseconds,
                 MaxLogLines = 1000,
+                CompileThreads = (uint)Math.Max(0, spec.CompileThreads),
             };
             if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey)
             {
                 hello.PeCacheDir = cacheDir;
                 hello.PeCacheKey = Google.Protobuf.ByteString.CopyFrom(cacheKey);
             }
-            await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
+            if (ScriptWorkerFrameCodec.UseBlockingIo)
+                await HandshakeBlockingAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
+            else
+                await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
 
             var connection = new WorkerConnection(spec.Shard, process, pipe, job, workDir, log, onLogs, spec.HeartbeatInterval,
-                onExiting, markerPath);
+                spec.EffectiveLoadTimeout, onExiting, markerPath);
             connection.Begin();
             return connection;
         }
@@ -280,8 +313,41 @@ internal sealed class WorkerConnection : IAsyncDisposable
         }, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
         await stream.FlushAsync(ct).ConfigureAwait(false);
 
-        var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false)
-                    ?? throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
+        var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
+        return CheckReady(reply, expectedNonce, expectedPid);
+    }
+
+    /// <summary>
+    ///     As <see cref="HandshakeAsync"/> with blocking I/O only (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>):
+    ///     one asynchronous call would turn the socket under a Unix pipe non-blocking for the connection's whole life.
+    ///     The blocking read runs off the caller and is abandoned at <paramref name="ct"/>; the caller then kills the
+    ///     worker and disposes the pipe, which ends it.
+    /// </summary>
+    /// <param name="stream">The accepted pipe.</param>
+    /// <param name="hello">The Hello to send.</param>
+    /// <param name="expectedNonce">The nonce put in the worker's environment.</param>
+    /// <param name="expectedPid">The launched process id; null skips the check.</param>
+    /// <param name="ct">Cancellation (the handshake timeout).</param>
+    /// <returns>The worker's Ready.</returns>
+    internal static async Task<Ready> HandshakeBlockingAsync(Stream stream, Hello hello, string expectedNonce, int? expectedPid,
+        CancellationToken ct)
+    {
+        var reply = await Task.Run(() =>
+        {
+            ScriptWorkerFrameCodec.Write(stream, new Envelope
+            {
+                ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
+                Seq = 1,
+                Hello = hello,
+            }, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
+            return ScriptWorkerFrameCodec.Read(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
+        }, CancellationToken.None).WaitAsync(ct).ConfigureAwait(false);
+        return CheckReady(reply, expectedNonce, expectedPid);
+    }
+
+    private static Ready CheckReady(Envelope? reply, string expectedNonce, int? expectedPid)
+    {
+        if (reply is null) throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
         if (reply.BodyCase != Envelope.BodyOneofCase.Ready)
             throw new ScriptWorkerProtocolException($"expected Ready, got {reply.BodyCase}");
         var expected = Encoding.UTF8.GetBytes(expectedNonce);
@@ -361,8 +427,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
         ReadyTimestamp = Stopwatch.GetTimestamp();
         Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp);
         Interlocked.Exchange(ref _lastUsedTimestamp, ReadyTimestamp);
-        _ = Task.Run(ReadLoopAsync);
-        _ = Task.Run(WriteLoopAsync);
+        if (!ScriptWorkerFrameCodec.UseBlockingIo) _ = Task.Run(ReadLoopAsync);
+        else new Thread(ReadLoopBlocking) { IsBackground = true, Name = $"ScriptWorker[{Shard}].Reader" }.Start();
         _ = Task.Run(HeartbeatMonitorAsync);
         _ = Task.Run(WatchProcessAsync);
     }
@@ -373,39 +439,39 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <param name="source">The source.</param>
     /// <param name="sourceHash">SHA-256 of the source.</param>
     /// <param name="inputPaths">The slot order.</param>
-    /// <returns>The load outcome and the slot map; null when the worker ended first.</returns>
-    public Task<LoadEntry?> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
+    /// <returns>The entry once the load has an outcome (see <see cref="LoadEntry.Outcome"/>).</returns>
+    public Task<LoadEntry> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
+        => EnsureLoadedAsync(key, kind, source, sourceHash, Convert.ToHexString(sourceHash), inputPaths);
+
+    /// <summary>As the other overload, with the hash's hex form precomputed (the runner caches it per key, lmxopcua#789).</summary>
+    /// <param name="key">The scoped key.</param>
+    /// <param name="kind">The context kind.</param>
+    /// <param name="source">The source.</param>
+    /// <param name="sourceHash">SHA-256 of the source.</param>
+    /// <param name="sourceHashHex">Its upper-case hex form.</param>
+    /// <param name="inputPaths">The slot order.</param>
+    /// <returns>
+    ///     The entry once the load has an outcome: <see cref="LoadEntry.Outcome"/> is the worker's verdict, or null when
+    ///     the worker ended first; <see cref="LoadEntry.TimedOut"/> marks the compile this connection killed the worker
+    ///     for.
+    /// </returns>
+    public Task<LoadEntry> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, string sourceHashHex,
+        IReadOnlyList<string> inputPaths)
     {
-        var hashHex = Convert.ToHexString(sourceHash);
-        if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == hashHex && existing.SameSlots(inputPaths))
+        if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths))
             return existing.Ready;
 
-        var entry = new LoadEntry(hashHex, inputPaths);
+        var entry = new LoadEntry(sourceHashHex, inputPaths);
         _loaded[key] = entry;
         Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
         var loadId = (ulong)Interlocked.Increment(ref _nextLoadId);
-        var tcs = new TaskCompletionSource<LoadResult>(TaskCreationOptions.RunContinuationsAsynchronously);
-        _loads[loadId] = (key, tcs);
         var load = new LoadScripts { LoadId = loadId };
         var script = new ScriptSource { Key = key, Kind = kind, Source = source, SourceSha256 = Google.Protobuf.ByteString.CopyFrom(sourceHash) };
         script.InputPaths.AddRange(inputPaths);
         load.Scripts.Add(script);
-        if (!Send(new Envelope { Load = load })) { _loads.TryRemove(loadId, out _); entry.Complete(null); return entry.Ready; }
-
-        _ = tcs.Task.ContinueWith(t =>
-        {
-            if (t.IsCompletedSuccessfully)
-            {
-                var outcome = t.Result.Outcomes.FirstOrDefault(o => o.Key == key);
-                entry.Complete(outcome ?? new LoadOutcome { Key = key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } });
-                if (outcome is { Ok: false }) _loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
-            }
-            else
-            {
-                entry.Complete(null);
-                _loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
-            }
-        }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+        var inFlight = new LoadInFlight(loadId, key, entry, this);
+        _loads[loadId] = inFlight;
+        if (!Send(new Envelope { Load = load }) && _loads.TryRemove(loadId, out _)) inFlight.Finish(null);
         return entry.Ready;
     }
 
@@ -423,11 +489,29 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <param name="key">The scoped key.</param>
     /// <param name="request">The request (its id is assigned here).</param>
     /// <returns>The pending evaluation.</returns>
-    public PendingEval Submit(string key, EvalRequest request)
+    public PendingEval Submit(string key, EvalRequest request) => Submit(key, request, null, TimeSpan.Zero, null);
+
+    /// <summary>
+    ///     Submits one evaluation with its two deadlines (design §5.1) on ONE timer (lmxopcua#789): at
+    ///     <paramref name="softTimeout"/> <see cref="PendingEval.Outcome"/> yields <see cref="PendingEval.SoftTimedOut"/>,
+    ///     and <paramref name="hardGrace"/> later, if the worker still has not answered, the worker is killed with this
+    ///     key as the culprit.
+    /// </summary>
+    /// <param name="key">The scoped key.</param>
+    /// <param name="request">The request (its id is assigned here).</param>
+    /// <param name="softTimeout">T; null = no deadlines.</param>
+    /// <param name="hardGrace">G.</param>
+    /// <param name="onFinished">
+    ///     Runs once, synchronously, when the worker has answered or ended — BEFORE <see cref="PendingEval.Outcome"/>
+    ///     completes — so a caller that resumes from its await never finds its own run still holding the key's slot.
+    /// </param>
+    /// <returns>The pending evaluation.</returns>
+    public PendingEval Submit(string key, EvalRequest request, TimeSpan? softTimeout, TimeSpan hardGrace, Action? onFinished)
     {
         request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
         Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
-        var pending = new PendingEval(key, request.EvalId);
+        var pending = new PendingEval(key, request.EvalId, onFinished, _dispatcher);
+        if (softTimeout is { } soft) pending.Arm(soft, hardGrace, OnHardDeadline);
         _pending[request.EvalId] = pending;
         var batch = new EvalBatch();
         batch.Items.Add(request);
@@ -438,6 +522,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
         return pending;
     }
 
+    private void OnHardDeadline(PendingEval pending)
+    {
+        _log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
+            pending.Key, (long)pending.HardDeadline.TotalMilliseconds, Shard, Pid);
+        Kill(WorkerExitReason.KilledDeadline, pending.Key);
+    }
+
     /// <summary>Cancels a run's <c>ctx.CancellationToken</c> in the worker (sent at the soft deadline).</summary>
     /// <param name="pending">The evaluation.</param>
     public void Cancel(PendingEval pending)
@@ -467,6 +558,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
         }
         try
         {
+            // The job holds exactly this one process (ACTIVE_PROCESS = 1), so terminating it is the whole tree, at once.
+            if (OperatingSystem.IsWindows() && _job is WindowsJobObject job && job.Terminate()) return;
             if (!_process.HasExited) _process.Kill(entireProcessTree: true);
         }
         catch (InvalidOperationException) { /* already gone */ }
@@ -502,7 +595,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
             try { await _exited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
         }
         _stopping.Cancel();
-        _outbound.Writer.TryComplete();
+        _writer.Complete();
         await _pipe.DisposeAsync().ConfigureAwait(false);
         _job?.Dispose();
         _process.Dispose();
@@ -513,106 +606,112 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <summary>Test hook: closes the host end of the pipe without touching the process, as a dying host would.</summary>
     internal void SeverPipeForTest() => _pipe.Dispose();
 
-    private bool Send(Envelope envelope)
+    private bool Send(Envelope envelope) => _writer.TryWrite(envelope);
+
+    private void OnWriteFault(Exception ex)
     {
-        envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
-        return _outbound.Writer.TryWrite(envelope);
+        if (!_stopping.IsCancellationRequested)
+            _log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
+        Kill(WorkerExitReason.Crash, null);
     }
 
-    private async Task WriteLoopAsync()
+    /// <summary>
+    ///     The reader, on a dedicated thread with blocking reads of a socket that never turns non-blocking
+    ///     (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>, lmxopcua#789): a frame wakes this thread straight from
+    ///     the kernel, instead of the socket engine's epoll thread queueing a thread-pool work item per read. Not on
+    ///     Windows, where a synchronous read of an overlapped pipe is itself completed through the I/O completion port;
+    ///     the async loop is the cheaper path there.
+    /// </summary>
+    private void ReadLoopBlocking()
     {
-        var reader = _outbound.Reader;
+        WorkerExitReason? fault = null;
+        var reader = new ScriptWorkerFrameReader(_pipe);
         try
         {
-            while (await reader.WaitToReadAsync(_stopping.Token).ConfigureAwait(false))
-            {
-                // Opportunistic batching (design §4.5): every evaluation queued right now goes out as ONE EvalBatch.
-                EvalBatch? batch = null;
-                while (reader.TryRead(out var envelope))
-                {
-                    if (envelope.BodyCase == Envelope.BodyOneofCase.Eval)
-                    {
-                        (batch ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
-                        continue;
-                    }
-                    envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
-                    await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
-                }
-                if (batch is not null)
-                {
-                    await ScriptWorkerFrameCodec.WriteAsync(_pipe, new Envelope
-                    {
-                        ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
-                        Seq = (ulong)Interlocked.Increment(ref _seq),
-                        Eval = batch,
-                    }, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
-                }
-                await _pipe.FlushAsync(_stopping.Token).ConfigureAwait(false);
-            }
+            while (reader.Read() is { } frame) HandleFrame(frame);
         }
-        catch (OperationCanceledException) { }
-        catch (Exception ex) when (ex is IOException or ObjectDisposedException or ScriptWorkerProtocolException)
+        catch (ScriptWorkerProtocolException ex)
         {
-            if (!_stopping.IsCancellationRequested)
-                _log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
-            Kill(WorkerExitReason.Crash, null);
+            fault = ProtocolFault(ex);
         }
+        catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException
+                                       or System.Net.Sockets.SocketException)
+        {
+        }
+        _ = PipeEndedAsync(fault);
     }
 
     private async Task ReadLoopAsync()
     {
         WorkerExitReason? fault = null;
+        var reader = new ScriptWorkerFrameReader(_pipe);
         try
         {
-            while (true)
-            {
-                var frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
-                if (frame is null) break;
-                Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
-                switch (frame.BodyCase)
-                {
-                    case Envelope.BodyOneofCase.Results:
-                        foreach (var result in frame.Results.Items)
-                        {
-                            // An eval id with no outstanding request is a protocol violation (design §6 "what the host
-                            // trusts"), except for one that was already completed by a kill race — ignored then.
-                            if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
-                            else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
-                                throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
-                        }
-                        break;
-                    case Envelope.BodyOneofCase.LoadResult:
-                        if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Completion.TrySetResult(frame.LoadResult);
-                        else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
-                        break;
-                    case Envelope.BodyOneofCase.Logs:
-                        try { _onLogs(frame.Logs); }
-                        catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
-                        break;
-                    case Envelope.BodyOneofCase.Heartbeat:
-                        Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
-                        break;
-                    case Envelope.BodyOneofCase.DiagReply:
-                        Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
-                        break;
-                    case Envelope.BodyOneofCase.Fault:
-                        _log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
-                        break;
-                    default:
-                        throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
-                }
-            }
+            while (await reader.ReadAsync(_stopping.Token).ConfigureAwait(false) is { } frame) HandleFrame(frame);
         }
         catch (OperationCanceledException) { }
         catch (ScriptWorkerProtocolException ex)
         {
-            _log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
-            fault = WorkerExitReason.ProtocolFault;
+            fault = ProtocolFault(ex);
         }
         catch (Exception ex) when (ex is IOException or ObjectDisposedException)
         {
         }
+        await PipeEndedAsync(fault).ConfigureAwait(false);
+    }
 
+    private WorkerExitReason ProtocolFault(ScriptWorkerProtocolException ex)
+    {
+        _log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
+        return WorkerExitReason.ProtocolFault;
+    }
+
+    private void HandleFrame(Envelope frame)
+    {
+        Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
+        switch (frame.BodyCase)
+        {
+            case Envelope.BodyOneofCase.Results:
+                foreach (var result in frame.Results.Items)
+                {
+                    // An eval id with no outstanding request is a protocol violation (design §6 "what the host
+                    // trusts"), except for one that was already completed by a kill race — ignored then.
+                    if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
+                    else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
+                        throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
+                }
+                break;
+            case Envelope.BodyOneofCase.LoadStarted:
+                // The compile timeout starts now (lmxopcua#790). An unknown id is a load already completed by a
+                // kill race (ignored) or one never issued (a protocol violation).
+                if (_loads.TryGetValue(frame.LoadStarted.LoadId, out var starting)) starting.Started(_loadTimeout);
+                else if (frame.LoadStarted.LoadId == 0 || frame.LoadStarted.LoadId > (ulong)Interlocked.Read(ref _nextLoadId))
+                    throw new ScriptWorkerProtocolException($"load start for unknown load id {frame.LoadStarted.LoadId}");
+                break;
+            case Envelope.BodyOneofCase.LoadResult:
+                if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Finish(frame.LoadResult);
+                else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
+                break;
+            case Envelope.BodyOneofCase.Logs:
+                try { _onLogs(frame.Logs); }
+                catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
+                break;
+            case Envelope.BodyOneofCase.Heartbeat:
+                Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
+                break;
+            case Envelope.BodyOneofCase.DiagReply:
+                Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
+                break;
+            case Envelope.BodyOneofCase.Fault:
+                _log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
+                break;
+            default:
+                throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
+        }
+    }
+
+    private async Task PipeEndedAsync(WorkerExitReason? fault)
+    {
         // The pipe is gone: the worker is dead or dying. Make sure, then report.
         try
         {
@@ -670,7 +769,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
         if (Interlocked.Exchange(ref _finished, 1) != 0) return;
         // No new frame can be queued from here on, so every Submit / load either fails its Send (and completes itself)
         // or was registered before this point and is completed below.
-        _outbound.Writer.TryComplete();
+        _writer.Complete();
         int? code = null;
         try { if (_process.HasExited) code = _process.ExitCode; } catch (InvalidOperationException) { }
 
@@ -692,9 +791,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
         foreach (var id in _pending.Keys.ToArray())
             if (_pending.TryRemove(id, out var pending)) pending.Complete(null);
         foreach (var id in _loads.Keys.ToArray())
-            if (_loads.TryRemove(id, out var load)) load.Completion.TrySetCanceled();
+            if (_loads.TryRemove(id, out var load)) load.Finish(null);
         Volatile.Read(ref _diag)?.TrySetCanceled();
-        _outbound.Writer.TryComplete();
+        // Resumes the callers just completed above, then ends; a later Post runs inline.
+        _dispatcher.Stop();
+    }
+
+    /// <summary>The compile timer of a started load fired: kill the worker, blaming this key alone (lmxopcua#790).</summary>
+    private void OnLoadTimeout(LoadInFlight load)
+    {
+        if (!_loads.ContainsKey(load.Id)) return;
+        load.Entry.TimedOut = true;
+        _log.Warning("Script {ScriptKey}: compile in ScriptWorker[{Shard}] has run for {Timeout} s since it started; killing the worker",
+            load.Key, Shard, _loadTimeout.TotalSeconds);
+        Kill(WorkerExitReason.KilledLoadTimeout, load.Key);
     }
 
     private static string SafeExitCode(Process process)
@@ -708,10 +818,52 @@ internal sealed class WorkerConnection : IAsyncDisposable
         try { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } catch { /* temp dir; best effort */ }
     }
 
+    /// <summary>One load sent to the worker and not yet answered: its compile timer runs from <c>LoadStarted</c>.</summary>
+    private sealed class LoadInFlight(ulong id, string key, LoadEntry entry, WorkerConnection owner)
+    {
+        private Timer? _timer;
+        private int _finished;
+
+        public ulong Id { get; } = id;
+
+        public string Key { get; } = key;
+
+        public LoadEntry Entry { get; } = entry;
+
+        /// <summary>The worker has started compiling it: arm the compile timeout (once).</summary>
+        public void Started(TimeSpan timeout)
+        {
+            if (Volatile.Read(ref _finished) != 0 || _timer is not null) return;
+            var timer = new Timer(static s => { var l = (LoadInFlight)s!; l.Owner.OnLoadTimeout(l); }, this, Timeout.Infinite, Timeout.Infinite);
+            if (Interlocked.CompareExchange(ref _timer, timer, null) is not null) { timer.Dispose(); return; }
+            timer.Change(timeout, Timeout.InfiniteTimeSpan);
+            if (Volatile.Read(ref _finished) != 0) timer.Dispose();
+        }
+
+        private WorkerConnection Owner { get; } = owner;
+
+        /// <summary>The worker answered (<paramref name="result"/>) or ended (null).</summary>
+        public void Finish(LoadResult? result)
+        {
+            if (Interlocked.Exchange(ref _finished, 1) != 0) return;
+            Volatile.Read(ref _timer)?.Dispose();
+            if (result is null)
+            {
+                Owner._loaded.TryRemove(new KeyValuePair<string, LoadEntry>(Key, Entry));
+                Entry.Complete(null);
+                return;
+            }
+            var outcome = result.Outcomes.FirstOrDefault(o => o.Key == Key)
+                          ?? new LoadOutcome { Key = Key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } };
+            if (!outcome.Ok) Owner._loaded.TryRemove(new KeyValuePair<string, LoadEntry>(Key, Entry));
+            Entry.Complete(outcome);
+        }
+    }
+
     /// <summary>A key's compiled form in this worker.</summary>
     internal sealed class LoadEntry
     {
-        private readonly TaskCompletionSource<LoadEntry?> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private readonly TaskCompletionSource<LoadEntry> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
 
         /// <summary>Set once the runner has counted this load in its metrics.</summary>
         public int Counted;
@@ -732,10 +884,17 @@ internal sealed class WorkerConnection : IAsyncDisposable
         /// <summary>Path → slot in the registered order.</summary>
         public IReadOnlyDictionary<string, uint> Slots { get; }
 
-        /// <summary>The worker's verdict (null until it arrives).</summary>
+        /// <summary>The worker's verdict; null until it arrives, and null for good when the worker ended first.</summary>
         public LoadOutcome? Outcome { get; private set; }
 
-        public Task<LoadEntry?> Ready => _ready.Task;
+        /// <summary>
+        ///     Set when this connection killed the worker because THIS compile overran its timeout from its start
+        ///     (lmxopcua#790); the waiters then report a compile timeout instead of a worker restart.
+        /// </summary>
+        public volatile bool TimedOut;
+
+        /// <summary>Completes with this entry once the load has an outcome (or the worker ended).</summary>
+        public Task<LoadEntry> Ready => _ready.Task;
 
         public bool SameSlots(IReadOnlyList<string> inputPaths)
         {
@@ -748,20 +907,42 @@ internal sealed class WorkerConnection : IAsyncDisposable
         public void Complete(LoadOutcome? outcome)
         {
             Outcome = outcome;
-            _ready.TrySetResult(outcome is null ? null : this);
+            _ready.TrySetResult(this);
         }
     }
 }
 
-/// <summary>One evaluation outstanding on a worker.</summary>
+/// <summary>
+///     One evaluation outstanding on a worker. Its deadlines share ONE timer (lmxopcua#789: the per-eval
+///     <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
+///     measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
+/// </summary>
 internal sealed class PendingEval
 {
-    private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
+    /// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
+    internal static readonly EvalResult SoftTimedOut = new();
 
-    public PendingEval(string key, ulong evalId)
+    private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
+    private readonly TaskCompletionSource<EvalResult?> _outcome;
+    private readonly ResultDispatcher? _dispatcher;
+    private readonly Action? _onFinished;
+    private Timer? _timer;
+    private Action<PendingEval>? _onHardDeadline;
+    private TimeSpan _grace;
+    private int _phase;
+    private int _done;
+
+    public PendingEval(string key, ulong evalId, Action? onFinished = null, ResultDispatcher? dispatcher = null)
     {
         Key = key;
         EvalId = evalId;
+        _onFinished = onFinished;
+        _dispatcher = dispatcher;
+        // With a dispatcher the caller resumes ON the dispatcher's thread (synchronous continuation, no pool hop);
+        // without one, on the thread pool as usual.
+        _outcome = new TaskCompletionSource<EvalResult?>(dispatcher is null
+            ? TaskCreationOptions.RunContinuationsAsynchronously
+            : TaskCreationOptions.None);
     }
 
     /// <summary>The scoped key.</summary>
@@ -770,8 +951,59 @@ internal sealed class PendingEval
     /// <summary>The eval id on the wire.</summary>
     public ulong EvalId { get; }
 
+    /// <summary>T + G, for the kill log line.</summary>
+    public TimeSpan HardDeadline { get; private set; }
+
     /// <summary>The worker's result, or null when the worker ended first.</summary>
     public Task<EvalResult?> Completion => _completion.Task;
 
-    public void Complete(EvalResult? result) => _completion.TrySetResult(result);
+    /// <summary>
+    ///     What the caller waits on: the worker's result, null when the worker ended first, or
+    ///     <see cref="SoftTimedOut"/> at the soft deadline (the run itself is still outstanding then).
+    /// </summary>
+    public Task<EvalResult?> Outcome => _outcome.Task;
+
+    /// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
+    internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
+    {
+        _grace = grace;
+        _onHardDeadline = onHardDeadline;
+        HardDeadline = soft + grace;
+        // Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
+        _timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
+        _timer.Change(soft, Timeout.InfiniteTimeSpan);
+    }
+
+    private void OnTimer()
+    {
+        if (Volatile.Read(ref _done) != 0) return;
+        if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
+        {
+            _outcome.TrySetResult(SoftTimedOut);
+            try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
+            catch (ObjectDisposedException) { /* answered meanwhile */ }
+            return;
+        }
+        if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
+    }
+
+    /// <summary>
+    ///     The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
+    ///     reader, then hand the caller's resumption to the dispatcher (or the pool).
+    /// </summary>
+    public void Complete(EvalResult? result)
+    {
+        if (Interlocked.Exchange(ref _done, 1) != 0) return;
+        _timer?.Dispose();
+        try { _onFinished?.Invoke(); }
+        finally
+        {
+            _completion.TrySetResult(result);
+            if (_dispatcher is { } dispatcher) dispatcher.Post(this, result);
+            else _outcome.TrySetResult(result);
+        }
+    }
+
+    /// <summary>Resumes the caller (on the dispatcher's thread). A soft timeout may already have done so.</summary>
+    internal void SignalOutcome(EvalResult? result) => _outcome.TrySetResult(result);
 }

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerShard.cs

@@ -45,6 +45,12 @@ internal sealed class WorkerShard : IAsyncDisposable
     internal static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(30);
     internal static readonly TimeSpan HealthyResetAfter = TimeSpan.FromSeconds(60);
 
+    /// <summary>
+    ///     While a shard keeps failing to start, at most one Warning per this interval (lmxopcua#788); the other retries
+    ///     log at Debug. The first failure of a down period is the one Error (with the exception).
+    /// </summary>
+    internal static readonly TimeSpan DefaultStartFailureWarningInterval = TimeSpan.FromSeconds(60);
+
     private static readonly Counter<long> StartsCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
         "otopcua.script.worker.starts", unit: "{process}", description: "ScriptWorker processes started (tag: shard).");
 
@@ -58,6 +64,7 @@ internal sealed class WorkerShard : IAsyncDisposable
     private readonly object _gate = new();
     private readonly Random _jitter = new();
     private readonly CancellationTokenSource _disposing = new();
+    private readonly TimeSpan _startFailureWarningInterval;
     private WorkerConnection? _current;
     private Task<WorkerConnection?>? _starting;
     private TaskCompletionSource? _recovered;
@@ -67,9 +74,15 @@ internal sealed class WorkerShard : IAsyncDisposable
     private int _consecutiveFailures;
     private bool _disposed;
 
+    // The current run of failed starts (lmxopcua#788): how many, since when, and when the last Warning went out.
+    private int _startFailures;
+    private long _startFailingSince;
+    private long _lastStartFailureWarning;
+
     public WorkerShard(string label, bool isolation, Func<Action<WorkerExit>, CancellationToken, Task<WorkerConnection>> launch,
-        Action<WorkerShard, WorkerExit> onExit, ILogger log)
+        Action<WorkerShard, WorkerExit> onExit, ILogger log, TimeSpan? startFailureWarningInterval = null)
     {
+        _startFailureWarningInterval = startFailureWarningInterval ?? DefaultStartFailureWarningInterval;
         Label = label;
         IsIsolation = isolation;
         _launch = launch;
@@ -298,7 +311,21 @@ internal sealed class WorkerShard : IAsyncDisposable
         {
             connection = await _launch(Attribute, CancellationToken.None).ConfigureAwait(false);
             StartsCounter.Add(1, new KeyValuePair<string, object?>("shard", Label));
-            _log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
+            int failedBefore;
+            long failingSince;
+            lock (_gate)
+            {
+                failedBefore = _startFailures;
+                failingSince = _startFailingSince;
+                _startFailures = 0;
+                _startFailingSince = 0;
+                _lastStartFailureWarning = 0;
+            }
+            if (failedBefore == 0)
+                _log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
+            else
+                _log.Information("ScriptWorker[{Shard}] started, pid {Pid}, after {Attempts} failed start attempt(s) over {Outage}",
+                    Label, connection.Pid, failedBefore, Stopwatch.GetElapsedTime(failingSince));
             var started = connection;
             _ = started.Exited.ContinueWith(t => OnExited(started, t.Result), CancellationToken.None,
                 TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Default);
@@ -320,13 +347,31 @@ internal sealed class WorkerShard : IAsyncDisposable
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does", Label);
+            int attempt;
+            TimeSpan failingFor;
+            bool warn;
             lock (_gate)
             {
                 _starting = null;
                 MarkDownLocked();
                 ScheduleBackoffLocked();
+                attempt = ++_startFailures;
+                var now = Stopwatch.GetTimestamp();
+                if (attempt == 1) _startFailingSince = now;
+                failingFor = Stopwatch.GetElapsedTime(_startFailingSince, now);
+                warn = attempt > 1 && (_lastStartFailureWarning == 0
+                                       || Stopwatch.GetElapsedTime(_lastStartFailureWarning, now) >= _startFailureWarningInterval);
+                if (attempt == 1 || warn) _lastStartFailureWarning = now;
             }
+            // lmxopcua#788: one Error with the stack per down period; the retries (every backoff, capped at 30 s, per
+            // shard) are a rate-limited Warning without it, and the rest Debug. Recovery is one Information line above.
+            if (attempt == 1)
+                _log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does (retrying with backoff)", Label);
+            else if (warn)
+                _log.Warning("ScriptWorker[{Shard}] still failing to start: {Attempts} failed attempt(s) over {Outage}; last error: {Error}",
+                    Label, attempt, failingFor, ex.Message);
+            else
+                _log.Debug("ScriptWorker[{Shard}] start attempt {Attempts} failed: {Error}", Label, attempt, ex.Message);
             if (connection is not null) await connection.DisposeAsync().ConfigureAwait(false);
             return null;
         }