perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered
frame reader, one timer per evaluation for T and T+G, slot release on the reader
before the caller resumes, a non-spinning per-connection result dispatcher, and
blocking pipe I/O on Unix (both ends) so the socket never enters the socket
engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2:
228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % ->
123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache
worker RSS, and times kill latency from the soft-timeout answer.

#790: the worker sends LoadStarted; the compile timeout is armed then, so a load
queued behind other compiles is never killed or struck.

#788: one Error with stack per down period, rate-limited Warnings with attempt
count, one Information on recovery.

Design doc §8.2 notes + §12.2 record.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerConnection.cs

@@ -3,7 +3,6 @@ using System.Diagnostics;
 using System.IO.Pipes;
 using System.Security.Cryptography;
 using System.Text;
-using System.Threading.Channels;
 using Serilog;
 using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
 using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
@@ -65,6 +64,11 @@ internal sealed record WorkerExit(WorkerExitReason Reason, string? CulpritKey, i
 /// <param name="PeCacheDirectory">The compiled-PE cache directory handed to the worker; null = no cache.</param>
 /// <param name="PeCacheKey">The cache's HMAC key (with <paramref name="PeCacheDirectory"/>).</param>
 /// <param name="MarkerDirectory">Where the launch marker for the orphan sweep is written; null = none.</param>
+/// <param name="LoadTimeout">
+///     How long one compile may run, counted from the worker's <c>LoadStarted</c> for it (not from when the host sent
+///     it, lmxopcua#790), before the worker is killed and the key struck; null = 60 s.
+/// </param>
+/// <param name="CompileThreads">The worker's dedicated compile threads; 0 = its default.</param>
 internal sealed record WorkerLaunchSpec(
     string Shard,
     string WorkerPath,
@@ -75,7 +79,16 @@ internal sealed record WorkerLaunchSpec(
     TimeSpan HandshakeTimeout,
     string? PeCacheDirectory = null,
     byte[]? PeCacheKey = null,
-    string? MarkerDirectory = null);
+    string? MarkerDirectory = null,
+    TimeSpan? LoadTimeout = null,
+    int CompileThreads = 0)
+{
+    /// <summary>The default compile timeout.</summary>
+    public static readonly TimeSpan DefaultLoadTimeout = TimeSpan.FromSeconds(60);
+
+    /// <summary><see cref="LoadTimeout"/> or its default.</summary>
+    public TimeSpan EffectiveLoadTimeout => LoadTimeout ?? DefaultLoadTimeout;
+}
 
 /// <summary>
 ///     One live ScriptWorker process and its pipe (lmxopcua#780, design §3–§6): launch with a cleared, allow-listed
@@ -101,13 +114,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
     private readonly string _workDir;
     private readonly ILogger _log;
     private readonly Action<LogBatch> _onLogs;
-    private readonly Channel<Envelope> _outbound = Channel.CreateUnbounded<Envelope>(new UnboundedChannelOptions { SingleReader = true });
+    private readonly ScriptWorkerFrameWriter _writer;
+    private readonly ResultDispatcher _dispatcher;
     private readonly ConcurrentDictionary<ulong, PendingEval> _pending = new();
-    private readonly ConcurrentDictionary<ulong, (string Key, TaskCompletionSource<LoadResult> Completion)> _loads = new();
+    private readonly ConcurrentDictionary<ulong, LoadInFlight> _loads = new();
     private readonly ConcurrentDictionary<string, LoadEntry> _loaded = new(StringComparer.Ordinal);
     private readonly TaskCompletionSource<WorkerExit> _exited = new(TaskCreationOptions.RunContinuationsAsynchronously);
     private readonly CancellationTokenSource _stopping = new();
     private readonly TimeSpan _heartbeatInterval;
+    private TimeSpan _loadTimeout;
     private readonly Action<WorkerExit> _onExiting;
     private readonly string? _markerPath;
     private TaskCompletionSource<DiagnosticsReply>? _diag;
@@ -116,15 +131,15 @@ internal sealed class WorkerConnection : IAsyncDisposable
     private int _finished;
     private (WorkerExitReason Reason, string? Culprit)? _killedFor;
     private long _lastFrameTimestamp;
-    private long _seq;
     private long _nextEvalId;
     private long _nextLoadId;
-    private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
     private int _disposed;
 
     private WorkerConnection(string shard, Process process, NamedPipeServerStream pipe, IDisposable? job, string workDir,
-        ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, Action<WorkerExit> onExiting, string? markerPath)
+        ILogger log, Action<LogBatch> onLogs, TimeSpan heartbeatInterval, TimeSpan loadTimeout, Action<WorkerExit> onExiting,
+        string? markerPath)
     {
+        _loadTimeout = loadTimeout;
         _onExiting = onExiting;
         _markerPath = markerPath;
         Shard = shard;
@@ -136,6 +151,10 @@ internal sealed class WorkerConnection : IAsyncDisposable
         _onLogs = onLogs;
         _heartbeatInterval = heartbeatInterval;
         Pid = process.Id;
+        // The handshake used seq 1. A failed write means the pipe is gone: the worker is dead or dying.
+        _writer = new ScriptWorkerFrameWriter(pipe, OnWriteFault, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, firstSeq: 1,
+            synchronous: ScriptWorkerFrameCodec.UseBlockingIo);
+        _dispatcher = new ResultDispatcher($"ScriptWorker[{shard}].Results");
     }
 
     /// <summary>The shard label.</summary>
@@ -156,6 +175,16 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <summary>Evaluations currently outstanding on this worker.</summary>
     public int InFlightCount => _pending.Count;
 
+    /// <summary>
+    ///     The compile timeout applied to loads that START from now on (test hook: a test warms the worker's compiler
+    ///     under the launch value, then shortens it).
+    /// </summary>
+    internal TimeSpan LoadTimeout
+    {
+        get => _loadTimeout;
+        set => _loadTimeout = value;
+    }
+
     /// <summary>Compiles currently outstanding on this worker.</summary>
     public int LoadingCount => _loads.Count;
 
@@ -234,16 +263,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
                 HardGraceMs = (uint)Math.Clamp(spec.HardGrace.TotalMilliseconds, 1, uint.MaxValue),
                 HeartbeatIntervalMs = (uint)spec.HeartbeatInterval.TotalMilliseconds,
                 MaxLogLines = 1000,
+                CompileThreads = (uint)Math.Max(0, spec.CompileThreads),
             };
             if (spec.PeCacheDirectory is { } cacheDir && spec.PeCacheKey is { } cacheKey)
             {
                 hello.PeCacheDir = cacheDir;
                 hello.PeCacheKey = Google.Protobuf.ByteString.CopyFrom(cacheKey);
             }
-            await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
+            if (ScriptWorkerFrameCodec.UseBlockingIo)
+                await HandshakeBlockingAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
+            else
+                await HandshakeAsync(pipe, hello, nonce, process.Id, handshakeCts.Token).ConfigureAwait(false);
 
             var connection = new WorkerConnection(spec.Shard, process, pipe, job, workDir, log, onLogs, spec.HeartbeatInterval,
-                onExiting, markerPath);
+                spec.EffectiveLoadTimeout, onExiting, markerPath);
             connection.Begin();
             return connection;
         }
@@ -280,8 +313,41 @@ internal sealed class WorkerConnection : IAsyncDisposable
         }, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
         await stream.FlushAsync(ct).ConfigureAwait(false);
 
-        var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false)
-                    ?? throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
+        var reply = await ScriptWorkerFrameCodec.ReadAsync(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes, ct).ConfigureAwait(false);
+        return CheckReady(reply, expectedNonce, expectedPid);
+    }
+
+    /// <summary>
+    ///     As <see cref="HandshakeAsync"/> with blocking I/O only (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>):
+    ///     one asynchronous call would turn the socket under a Unix pipe non-blocking for the connection's whole life.
+    ///     The blocking read runs off the caller and is abandoned at <paramref name="ct"/>; the caller then kills the
+    ///     worker and disposes the pipe, which ends it.
+    /// </summary>
+    /// <param name="stream">The accepted pipe.</param>
+    /// <param name="hello">The Hello to send.</param>
+    /// <param name="expectedNonce">The nonce put in the worker's environment.</param>
+    /// <param name="expectedPid">The launched process id; null skips the check.</param>
+    /// <param name="ct">Cancellation (the handshake timeout).</param>
+    /// <returns>The worker's Ready.</returns>
+    internal static async Task<Ready> HandshakeBlockingAsync(Stream stream, Hello hello, string expectedNonce, int? expectedPid,
+        CancellationToken ct)
+    {
+        var reply = await Task.Run(() =>
+        {
+            ScriptWorkerFrameCodec.Write(stream, new Envelope
+            {
+                ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
+                Seq = 1,
+                Hello = hello,
+            }, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
+            return ScriptWorkerFrameCodec.Read(stream, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
+        }, CancellationToken.None).WaitAsync(ct).ConfigureAwait(false);
+        return CheckReady(reply, expectedNonce, expectedPid);
+    }
+
+    private static Ready CheckReady(Envelope? reply, string expectedNonce, int? expectedPid)
+    {
+        if (reply is null) throw new ScriptWorkerProtocolException("the worker closed the pipe during the handshake");
         if (reply.BodyCase != Envelope.BodyOneofCase.Ready)
             throw new ScriptWorkerProtocolException($"expected Ready, got {reply.BodyCase}");
         var expected = Encoding.UTF8.GetBytes(expectedNonce);
@@ -361,8 +427,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
         ReadyTimestamp = Stopwatch.GetTimestamp();
         Volatile.Write(ref _lastFrameTimestamp, ReadyTimestamp);
         Interlocked.Exchange(ref _lastUsedTimestamp, ReadyTimestamp);
-        _ = Task.Run(ReadLoopAsync);
-        _ = Task.Run(WriteLoopAsync);
+        if (!ScriptWorkerFrameCodec.UseBlockingIo) _ = Task.Run(ReadLoopAsync);
+        else new Thread(ReadLoopBlocking) { IsBackground = true, Name = $"ScriptWorker[{Shard}].Reader" }.Start();
         _ = Task.Run(HeartbeatMonitorAsync);
         _ = Task.Run(WatchProcessAsync);
     }
@@ -373,39 +439,39 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <param name="source">The source.</param>
     /// <param name="sourceHash">SHA-256 of the source.</param>
     /// <param name="inputPaths">The slot order.</param>
-    /// <returns>The load outcome and the slot map; null when the worker ended first.</returns>
-    public Task<LoadEntry?> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
+    /// <returns>The entry once the load has an outcome (see <see cref="LoadEntry.Outcome"/>).</returns>
+    public Task<LoadEntry> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, IReadOnlyList<string> inputPaths)
+        => EnsureLoadedAsync(key, kind, source, sourceHash, Convert.ToHexString(sourceHash), inputPaths);
+
+    /// <summary>As the other overload, with the hash's hex form precomputed (the runner caches it per key, lmxopcua#789).</summary>
+    /// <param name="key">The scoped key.</param>
+    /// <param name="kind">The context kind.</param>
+    /// <param name="source">The source.</param>
+    /// <param name="sourceHash">SHA-256 of the source.</param>
+    /// <param name="sourceHashHex">Its upper-case hex form.</param>
+    /// <param name="inputPaths">The slot order.</param>
+    /// <returns>
+    ///     The entry once the load has an outcome: <see cref="LoadEntry.Outcome"/> is the worker's verdict, or null when
+    ///     the worker ended first; <see cref="LoadEntry.TimedOut"/> marks the compile this connection killed the worker
+    ///     for.
+    /// </returns>
+    public Task<LoadEntry> EnsureLoadedAsync(string key, ContextKind kind, string source, byte[] sourceHash, string sourceHashHex,
+        IReadOnlyList<string> inputPaths)
     {
-        var hashHex = Convert.ToHexString(sourceHash);
-        if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == hashHex && existing.SameSlots(inputPaths))
+        if (_loaded.TryGetValue(key, out var existing) && existing.SourceHash == sourceHashHex && existing.SameSlots(inputPaths))
             return existing.Ready;
 
-        var entry = new LoadEntry(hashHex, inputPaths);
+        var entry = new LoadEntry(sourceHashHex, inputPaths);
         _loaded[key] = entry;
         Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
         var loadId = (ulong)Interlocked.Increment(ref _nextLoadId);
-        var tcs = new TaskCompletionSource<LoadResult>(TaskCreationOptions.RunContinuationsAsynchronously);
-        _loads[loadId] = (key, tcs);
         var load = new LoadScripts { LoadId = loadId };
         var script = new ScriptSource { Key = key, Kind = kind, Source = source, SourceSha256 = Google.Protobuf.ByteString.CopyFrom(sourceHash) };
         script.InputPaths.AddRange(inputPaths);
         load.Scripts.Add(script);
-        if (!Send(new Envelope { Load = load })) { _loads.TryRemove(loadId, out _); entry.Complete(null); return entry.Ready; }
-
-        _ = tcs.Task.ContinueWith(t =>
-        {
-            if (t.IsCompletedSuccessfully)
-            {
-                var outcome = t.Result.Outcomes.FirstOrDefault(o => o.Key == key);
-                entry.Complete(outcome ?? new LoadOutcome { Key = key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } });
-                if (outcome is { Ok: false }) _loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
-            }
-            else
-            {
-                entry.Complete(null);
-                _loaded.TryRemove(new KeyValuePair<string, LoadEntry>(key, entry));
-            }
-        }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+        var inFlight = new LoadInFlight(loadId, key, entry, this);
+        _loads[loadId] = inFlight;
+        if (!Send(new Envelope { Load = load }) && _loads.TryRemove(loadId, out _)) inFlight.Finish(null);
         return entry.Ready;
     }
 
@@ -423,11 +489,29 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <param name="key">The scoped key.</param>
     /// <param name="request">The request (its id is assigned here).</param>
     /// <returns>The pending evaluation.</returns>
-    public PendingEval Submit(string key, EvalRequest request)
+    public PendingEval Submit(string key, EvalRequest request) => Submit(key, request, null, TimeSpan.Zero, null);
+
+    /// <summary>
+    ///     Submits one evaluation with its two deadlines (design §5.1) on ONE timer (lmxopcua#789): at
+    ///     <paramref name="softTimeout"/> <see cref="PendingEval.Outcome"/> yields <see cref="PendingEval.SoftTimedOut"/>,
+    ///     and <paramref name="hardGrace"/> later, if the worker still has not answered, the worker is killed with this
+    ///     key as the culprit.
+    /// </summary>
+    /// <param name="key">The scoped key.</param>
+    /// <param name="request">The request (its id is assigned here).</param>
+    /// <param name="softTimeout">T; null = no deadlines.</param>
+    /// <param name="hardGrace">G.</param>
+    /// <param name="onFinished">
+    ///     Runs once, synchronously, when the worker has answered or ended — BEFORE <see cref="PendingEval.Outcome"/>
+    ///     completes — so a caller that resumes from its await never finds its own run still holding the key's slot.
+    /// </param>
+    /// <returns>The pending evaluation.</returns>
+    public PendingEval Submit(string key, EvalRequest request, TimeSpan? softTimeout, TimeSpan hardGrace, Action? onFinished)
     {
         request.EvalId = (ulong)Interlocked.Increment(ref _nextEvalId);
         Interlocked.Exchange(ref _lastUsedTimestamp, Stopwatch.GetTimestamp());
-        var pending = new PendingEval(key, request.EvalId);
+        var pending = new PendingEval(key, request.EvalId, onFinished, _dispatcher);
+        if (softTimeout is { } soft) pending.Arm(soft, hardGrace, OnHardDeadline);
         _pending[request.EvalId] = pending;
         var batch = new EvalBatch();
         batch.Items.Add(request);
@@ -438,6 +522,13 @@ internal sealed class WorkerConnection : IAsyncDisposable
         return pending;
     }
 
+    private void OnHardDeadline(PendingEval pending)
+    {
+        _log.Warning("Script {ScriptKey} exceeded its hard deadline ({Deadline} ms); killing ScriptWorker[{Shard}] pid {Pid}",
+            pending.Key, (long)pending.HardDeadline.TotalMilliseconds, Shard, Pid);
+        Kill(WorkerExitReason.KilledDeadline, pending.Key);
+    }
+
     /// <summary>Cancels a run's <c>ctx.CancellationToken</c> in the worker (sent at the soft deadline).</summary>
     /// <param name="pending">The evaluation.</param>
     public void Cancel(PendingEval pending)
@@ -467,6 +558,8 @@ internal sealed class WorkerConnection : IAsyncDisposable
         }
         try
         {
+            // The job holds exactly this one process (ACTIVE_PROCESS = 1), so terminating it is the whole tree, at once.
+            if (OperatingSystem.IsWindows() && _job is WindowsJobObject job && job.Terminate()) return;
             if (!_process.HasExited) _process.Kill(entireProcessTree: true);
         }
         catch (InvalidOperationException) { /* already gone */ }
@@ -502,7 +595,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
             try { await _exited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } catch (TimeoutException) { }
         }
         _stopping.Cancel();
-        _outbound.Writer.TryComplete();
+        _writer.Complete();
         await _pipe.DisposeAsync().ConfigureAwait(false);
         _job?.Dispose();
         _process.Dispose();
@@ -513,106 +606,112 @@ internal sealed class WorkerConnection : IAsyncDisposable
     /// <summary>Test hook: closes the host end of the pipe without touching the process, as a dying host would.</summary>
     internal void SeverPipeForTest() => _pipe.Dispose();
 
-    private bool Send(Envelope envelope)
+    private bool Send(Envelope envelope) => _writer.TryWrite(envelope);
+
+    private void OnWriteFault(Exception ex)
     {
-        envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
-        return _outbound.Writer.TryWrite(envelope);
+        if (!_stopping.IsCancellationRequested)
+            _log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
+        Kill(WorkerExitReason.Crash, null);
     }
 
-    private async Task WriteLoopAsync()
+    /// <summary>
+    ///     The reader, on a dedicated thread with blocking reads of a socket that never turns non-blocking
+    ///     (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>, lmxopcua#789): a frame wakes this thread straight from
+    ///     the kernel, instead of the socket engine's epoll thread queueing a thread-pool work item per read. Not on
+    ///     Windows, where a synchronous read of an overlapped pipe is itself completed through the I/O completion port;
+    ///     the async loop is the cheaper path there.
+    /// </summary>
+    private void ReadLoopBlocking()
     {
-        var reader = _outbound.Reader;
+        WorkerExitReason? fault = null;
+        var reader = new ScriptWorkerFrameReader(_pipe);
         try
         {
-            while (await reader.WaitToReadAsync(_stopping.Token).ConfigureAwait(false))
-            {
-                // Opportunistic batching (design §4.5): every evaluation queued right now goes out as ONE EvalBatch.
-                EvalBatch? batch = null;
-                while (reader.TryRead(out var envelope))
-                {
-                    if (envelope.BodyCase == Envelope.BodyOneofCase.Eval)
-                    {
-                        (batch ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
-                        continue;
-                    }
-                    envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
-                    await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
-                }
-                if (batch is not null)
-                {
-                    await ScriptWorkerFrameCodec.WriteAsync(_pipe, new Envelope
-                    {
-                        ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion,
-                        Seq = (ulong)Interlocked.Increment(ref _seq),
-                        Eval = batch,
-                    }, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
-                }
-                await _pipe.FlushAsync(_stopping.Token).ConfigureAwait(false);
-            }
+            while (reader.Read() is { } frame) HandleFrame(frame);
         }
-        catch (OperationCanceledException) { }
-        catch (Exception ex) when (ex is IOException or ObjectDisposedException or ScriptWorkerProtocolException)
+        catch (ScriptWorkerProtocolException ex)
         {
-            if (!_stopping.IsCancellationRequested)
-                _log.Debug(ex, "ScriptWorker[{Shard}] pipe write ended", Shard);
-            Kill(WorkerExitReason.Crash, null);
+            fault = ProtocolFault(ex);
         }
+        catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException
+                                       or System.Net.Sockets.SocketException)
+        {
+        }
+        _ = PipeEndedAsync(fault);
     }
 
     private async Task ReadLoopAsync()
     {
         WorkerExitReason? fault = null;
+        var reader = new ScriptWorkerFrameReader(_pipe);
         try
         {
-            while (true)
-            {
-                var frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes, _stopping.Token).ConfigureAwait(false);
-                if (frame is null) break;
-                Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
-                switch (frame.BodyCase)
-                {
-                    case Envelope.BodyOneofCase.Results:
-                        foreach (var result in frame.Results.Items)
-                        {
-                            // An eval id with no outstanding request is a protocol violation (design §6 "what the host
-                            // trusts"), except for one that was already completed by a kill race — ignored then.
-                            if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
-                            else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
-                                throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
-                        }
-                        break;
-                    case Envelope.BodyOneofCase.LoadResult:
-                        if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Completion.TrySetResult(frame.LoadResult);
-                        else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
-                        break;
-                    case Envelope.BodyOneofCase.Logs:
-                        try { _onLogs(frame.Logs); }
-                        catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
-                        break;
-                    case Envelope.BodyOneofCase.Heartbeat:
-                        Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
-                        break;
-                    case Envelope.BodyOneofCase.DiagReply:
-                        Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
-                        break;
-                    case Envelope.BodyOneofCase.Fault:
-                        _log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
-                        break;
-                    default:
-                        throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
-                }
-            }
+            while (await reader.ReadAsync(_stopping.Token).ConfigureAwait(false) is { } frame) HandleFrame(frame);
         }
         catch (OperationCanceledException) { }
         catch (ScriptWorkerProtocolException ex)
         {
-            _log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
-            fault = WorkerExitReason.ProtocolFault;
+            fault = ProtocolFault(ex);
         }
         catch (Exception ex) when (ex is IOException or ObjectDisposedException)
         {
         }
+        await PipeEndedAsync(fault).ConfigureAwait(false);
+    }
 
+    private WorkerExitReason ProtocolFault(ScriptWorkerProtocolException ex)
+    {
+        _log.Warning("ScriptWorker[{Shard}] protocol violation: {Message}; killing pid {Pid}", Shard, ex.Message, Pid);
+        return WorkerExitReason.ProtocolFault;
+    }
+
+    private void HandleFrame(Envelope frame)
+    {
+        Volatile.Write(ref _lastFrameTimestamp, Stopwatch.GetTimestamp());
+        switch (frame.BodyCase)
+        {
+            case Envelope.BodyOneofCase.Results:
+                foreach (var result in frame.Results.Items)
+                {
+                    // An eval id with no outstanding request is a protocol violation (design §6 "what the host
+                    // trusts"), except for one that was already completed by a kill race — ignored then.
+                    if (_pending.TryRemove(result.EvalId, out var pending)) pending.Complete(result);
+                    else if (result.EvalId == 0 || result.EvalId > (ulong)Interlocked.Read(ref _nextEvalId))
+                        throw new ScriptWorkerProtocolException($"result for unknown eval id {result.EvalId}");
+                }
+                break;
+            case Envelope.BodyOneofCase.LoadStarted:
+                // The compile timeout starts now (lmxopcua#790). An unknown id is a load already completed by a
+                // kill race (ignored) or one never issued (a protocol violation).
+                if (_loads.TryGetValue(frame.LoadStarted.LoadId, out var starting)) starting.Started(_loadTimeout);
+                else if (frame.LoadStarted.LoadId == 0 || frame.LoadStarted.LoadId > (ulong)Interlocked.Read(ref _nextLoadId))
+                    throw new ScriptWorkerProtocolException($"load start for unknown load id {frame.LoadStarted.LoadId}");
+                break;
+            case Envelope.BodyOneofCase.LoadResult:
+                if (_loads.TryRemove(frame.LoadResult.LoadId, out var load)) load.Finish(frame.LoadResult);
+                else throw new ScriptWorkerProtocolException($"load result for unknown load id {frame.LoadResult.LoadId}");
+                break;
+            case Envelope.BodyOneofCase.Logs:
+                try { _onLogs(frame.Logs); }
+                catch (Exception ex) { _log.Warning(ex, "ScriptWorker[{Shard}] log forwarding threw", Shard); }
+                break;
+            case Envelope.BodyOneofCase.Heartbeat:
+                Volatile.Write(ref _lastHeartbeat, frame.Heartbeat);
+                break;
+            case Envelope.BodyOneofCase.DiagReply:
+                Volatile.Read(ref _diag)?.TrySetResult(frame.DiagReply);
+                break;
+            case Envelope.BodyOneofCase.Fault:
+                _log.Warning("ScriptWorker[{Shard}] reported a fault: {Fault}", Shard, frame.Fault.Message);
+                break;
+            default:
+                throw new ScriptWorkerProtocolException($"unexpected frame {frame.BodyCase} from the worker");
+        }
+    }
+
+    private async Task PipeEndedAsync(WorkerExitReason? fault)
+    {
         // The pipe is gone: the worker is dead or dying. Make sure, then report.
         try
         {
@@ -670,7 +769,7 @@ internal sealed class WorkerConnection : IAsyncDisposable
         if (Interlocked.Exchange(ref _finished, 1) != 0) return;
         // No new frame can be queued from here on, so every Submit / load either fails its Send (and completes itself)
         // or was registered before this point and is completed below.
-        _outbound.Writer.TryComplete();
+        _writer.Complete();
         int? code = null;
         try { if (_process.HasExited) code = _process.ExitCode; } catch (InvalidOperationException) { }
 
@@ -692,9 +791,20 @@ internal sealed class WorkerConnection : IAsyncDisposable
         foreach (var id in _pending.Keys.ToArray())
             if (_pending.TryRemove(id, out var pending)) pending.Complete(null);
         foreach (var id in _loads.Keys.ToArray())
-            if (_loads.TryRemove(id, out var load)) load.Completion.TrySetCanceled();
+            if (_loads.TryRemove(id, out var load)) load.Finish(null);
         Volatile.Read(ref _diag)?.TrySetCanceled();
-        _outbound.Writer.TryComplete();
+        // Resumes the callers just completed above, then ends; a later Post runs inline.
+        _dispatcher.Stop();
+    }
+
+    /// <summary>The compile timer of a started load fired: kill the worker, blaming this key alone (lmxopcua#790).</summary>
+    private void OnLoadTimeout(LoadInFlight load)
+    {
+        if (!_loads.ContainsKey(load.Id)) return;
+        load.Entry.TimedOut = true;
+        _log.Warning("Script {ScriptKey}: compile in ScriptWorker[{Shard}] has run for {Timeout} s since it started; killing the worker",
+            load.Key, Shard, _loadTimeout.TotalSeconds);
+        Kill(WorkerExitReason.KilledLoadTimeout, load.Key);
     }
 
     private static string SafeExitCode(Process process)
@@ -708,10 +818,52 @@ internal sealed class WorkerConnection : IAsyncDisposable
         try { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } catch { /* temp dir; best effort */ }
     }
 
+    /// <summary>One load sent to the worker and not yet answered: its compile timer runs from <c>LoadStarted</c>.</summary>
+    private sealed class LoadInFlight(ulong id, string key, LoadEntry entry, WorkerConnection owner)
+    {
+        private Timer? _timer;
+        private int _finished;
+
+        public ulong Id { get; } = id;
+
+        public string Key { get; } = key;
+
+        public LoadEntry Entry { get; } = entry;
+
+        /// <summary>The worker has started compiling it: arm the compile timeout (once).</summary>
+        public void Started(TimeSpan timeout)
+        {
+            if (Volatile.Read(ref _finished) != 0 || _timer is not null) return;
+            var timer = new Timer(static s => { var l = (LoadInFlight)s!; l.Owner.OnLoadTimeout(l); }, this, Timeout.Infinite, Timeout.Infinite);
+            if (Interlocked.CompareExchange(ref _timer, timer, null) is not null) { timer.Dispose(); return; }
+            timer.Change(timeout, Timeout.InfiniteTimeSpan);
+            if (Volatile.Read(ref _finished) != 0) timer.Dispose();
+        }
+
+        private WorkerConnection Owner { get; } = owner;
+
+        /// <summary>The worker answered (<paramref name="result"/>) or ended (null).</summary>
+        public void Finish(LoadResult? result)
+        {
+            if (Interlocked.Exchange(ref _finished, 1) != 0) return;
+            Volatile.Read(ref _timer)?.Dispose();
+            if (result is null)
+            {
+                Owner._loaded.TryRemove(new KeyValuePair<string, LoadEntry>(Key, Entry));
+                Entry.Complete(null);
+                return;
+            }
+            var outcome = result.Outcomes.FirstOrDefault(o => o.Key == Key)
+                          ?? new LoadOutcome { Key = Key, Ok = false, Error = new ScriptError { Kind = ScriptError.Types.Kind.Threw, Message = "no load outcome" } };
+            if (!outcome.Ok) Owner._loaded.TryRemove(new KeyValuePair<string, LoadEntry>(Key, Entry));
+            Entry.Complete(outcome);
+        }
+    }
+
     /// <summary>A key's compiled form in this worker.</summary>
     internal sealed class LoadEntry
     {
-        private readonly TaskCompletionSource<LoadEntry?> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private readonly TaskCompletionSource<LoadEntry> _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
 
         /// <summary>Set once the runner has counted this load in its metrics.</summary>
         public int Counted;
@@ -732,10 +884,17 @@ internal sealed class WorkerConnection : IAsyncDisposable
         /// <summary>Path → slot in the registered order.</summary>
         public IReadOnlyDictionary<string, uint> Slots { get; }
 
-        /// <summary>The worker's verdict (null until it arrives).</summary>
+        /// <summary>The worker's verdict; null until it arrives, and null for good when the worker ended first.</summary>
         public LoadOutcome? Outcome { get; private set; }
 
-        public Task<LoadEntry?> Ready => _ready.Task;
+        /// <summary>
+        ///     Set when this connection killed the worker because THIS compile overran its timeout from its start
+        ///     (lmxopcua#790); the waiters then report a compile timeout instead of a worker restart.
+        /// </summary>
+        public volatile bool TimedOut;
+
+        /// <summary>Completes with this entry once the load has an outcome (or the worker ended).</summary>
+        public Task<LoadEntry> Ready => _ready.Task;
 
         public bool SameSlots(IReadOnlyList<string> inputPaths)
         {
@@ -748,20 +907,42 @@ internal sealed class WorkerConnection : IAsyncDisposable
         public void Complete(LoadOutcome? outcome)
         {
             Outcome = outcome;
-            _ready.TrySetResult(outcome is null ? null : this);
+            _ready.TrySetResult(this);
         }
     }
 }
 
-/// <summary>One evaluation outstanding on a worker.</summary>
+/// <summary>
+///     One evaluation outstanding on a worker. Its deadlines share ONE timer (lmxopcua#789: the per-eval
+///     <c>WaitAsync(timeout)</c> timer, the hard-deadline <c>CancellationTokenSource</c> and their continuations were a
+///     measurable share of the host's CPU): it fires at the soft deadline T and is re-armed for the grace G.
+/// </summary>
 internal sealed class PendingEval
 {
-    private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
+    /// <summary>What <see cref="Outcome"/> yields when the soft deadline passed first. Never on the wire.</summary>
+    internal static readonly EvalResult SoftTimedOut = new();
 
-    public PendingEval(string key, ulong evalId)
+    private readonly TaskCompletionSource<EvalResult?> _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
+    private readonly TaskCompletionSource<EvalResult?> _outcome;
+    private readonly ResultDispatcher? _dispatcher;
+    private readonly Action? _onFinished;
+    private Timer? _timer;
+    private Action<PendingEval>? _onHardDeadline;
+    private TimeSpan _grace;
+    private int _phase;
+    private int _done;
+
+    public PendingEval(string key, ulong evalId, Action? onFinished = null, ResultDispatcher? dispatcher = null)
     {
         Key = key;
         EvalId = evalId;
+        _onFinished = onFinished;
+        _dispatcher = dispatcher;
+        // With a dispatcher the caller resumes ON the dispatcher's thread (synchronous continuation, no pool hop);
+        // without one, on the thread pool as usual.
+        _outcome = new TaskCompletionSource<EvalResult?>(dispatcher is null
+            ? TaskCreationOptions.RunContinuationsAsynchronously
+            : TaskCreationOptions.None);
     }
 
     /// <summary>The scoped key.</summary>
@@ -770,8 +951,59 @@ internal sealed class PendingEval
     /// <summary>The eval id on the wire.</summary>
     public ulong EvalId { get; }
 
+    /// <summary>T + G, for the kill log line.</summary>
+    public TimeSpan HardDeadline { get; private set; }
+
     /// <summary>The worker's result, or null when the worker ended first.</summary>
     public Task<EvalResult?> Completion => _completion.Task;
 
-    public void Complete(EvalResult? result) => _completion.TrySetResult(result);
+    /// <summary>
+    ///     What the caller waits on: the worker's result, null when the worker ended first, or
+    ///     <see cref="SoftTimedOut"/> at the soft deadline (the run itself is still outstanding then).
+    /// </summary>
+    public Task<EvalResult?> Outcome => _outcome.Task;
+
+    /// <summary>Starts the deadline timer. Called once, before the request is sent.</summary>
+    internal void Arm(TimeSpan soft, TimeSpan grace, Action<PendingEval> onHardDeadline)
+    {
+        _grace = grace;
+        _onHardDeadline = onHardDeadline;
+        HardDeadline = soft + grace;
+        // Created disarmed and assigned before it can fire, so the soft tick can always re-arm it for the grace.
+        _timer = new Timer(static s => ((PendingEval)s!).OnTimer(), this, Timeout.Infinite, Timeout.Infinite);
+        _timer.Change(soft, Timeout.InfiniteTimeSpan);
+    }
+
+    private void OnTimer()
+    {
+        if (Volatile.Read(ref _done) != 0) return;
+        if (Interlocked.CompareExchange(ref _phase, 1, 0) == 0)
+        {
+            _outcome.TrySetResult(SoftTimedOut);
+            try { _timer?.Change(_grace, Timeout.InfiniteTimeSpan); }
+            catch (ObjectDisposedException) { /* answered meanwhile */ }
+            return;
+        }
+        if (Interlocked.CompareExchange(ref _phase, 2, 1) == 1 && Volatile.Read(ref _done) == 0) _onHardDeadline?.Invoke(this);
+    }
+
+    /// <summary>
+    ///     The worker answered (or ended, with null): cancel the deadlines and release the key's slot HERE, on the
+    ///     reader, then hand the caller's resumption to the dispatcher (or the pool).
+    /// </summary>
+    public void Complete(EvalResult? result)
+    {
+        if (Interlocked.Exchange(ref _done, 1) != 0) return;
+        _timer?.Dispose();
+        try { _onFinished?.Invoke(); }
+        finally
+        {
+            _completion.TrySetResult(result);
+            if (_dispatcher is { } dispatcher) dispatcher.Post(this, result);
+            else _outcome.TrySetResult(result);
+        }
+    }
+
+    /// <summary>Resumes the caller (on the dispatcher's thread). A soft timeout may already have done so.</summary>
+    internal void SignalOutcome(EvalResult? result) => _outcome.TrySetResult(result);
 }