perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered
frame reader, one timer per evaluation for T and T+G, slot release on the reader
before the caller resumes, a non-spinning per-connection result dispatcher, and
blocking pipe I/O on Unix (both ends) so the socket never enters the socket
engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2:
228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % ->
123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache
worker RSS, and times kill latency from the soft-timeout answer.

#790: the worker sends LoadStarted; the compile timeout is armed then, so a load
queued behind other compiles is never killed or struck.

#788: one Error with stack per down period, rate-limited Warnings with attempt
count, one Information on recovery.

Design doc §8.2 notes + §12.2 record.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/Program.cs

@@ -22,7 +22,9 @@ internal static class Program
             PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly);
         try
         {
-            await pipe.ConnectAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
+            // Synchronous on Unix so the socket under the pipe stays blocking (ScriptWorkerFrameCodec.UseBlockingIo).
+            if (ScriptWorkerFrameCodec.UseBlockingIo) pipe.Connect(TimeSpan.FromSeconds(10));
+            else await pipe.ConnectAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
         }
         catch (Exception ex)
         {

src/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker/WorkerHost.cs

@@ -2,7 +2,6 @@ using System.Collections;
 using System.Collections.Concurrent;
 using System.Diagnostics;
 using System.IO.Pipes;
-using System.Threading.Channels;
 using Serilog;
 using Serilog.Core;
 using Serilog.Events;
@@ -33,8 +32,6 @@ internal sealed class WorkerHost
     private readonly NamedPipeClientStream _pipe;
     private readonly string _nonce;
     private readonly int _parentPid;
-    private readonly Channel<Envelope> _outbound = Channel.CreateUnbounded<Envelope>(
-        new UnboundedChannelOptions { SingleReader = true });
     private readonly ConcurrentDictionary<string, LoadedScript> _scripts = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<long, RunningEval> _running = new();
     private readonly BlockingCollection<EvalRequest> _queue = new(new ConcurrentQueue<EvalRequest>());
@@ -47,8 +44,16 @@ internal sealed class WorkerHost
     /// </summary>
     private readonly BlockingCollection<LoadScripts> _loads = new(new ConcurrentQueue<LoadScripts>());
 
-    /// <summary>Dedicated compile threads: half the processors, at least 1, at most 8.</summary>
-    internal static int CompileThreads => Math.Clamp(Environment.ProcessorCount / 2, 1, 8);
+    /// <summary>Default dedicated compile threads: half the processors, at least 1, at most 8.</summary>
+    internal static int DefaultCompileThreads => Math.Clamp(Environment.ProcessorCount / 2, 1, 8);
+
+    /// <summary>
+    ///     Outbound frames (lmxopcua#789): the thread that produces a frame — an evaluation thread with its result, a
+    ///     compile thread with its load result — writes it itself when no write is in progress, instead of handing it to
+    ///     a writer task through a channel (one thread-pool wake-up and spin per result).
+    /// </summary>
+    private ScriptWorkerFrameWriter? _writer;
+    private int _compileThreads = DefaultCompileThreads;
     private readonly ForwardingLogSink _logSink;
     private readonly ILogger _scriptLogger;
     private int _maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes;
@@ -71,7 +76,11 @@ internal sealed class WorkerHost
     {
         try
         {
-            var hello = await ScriptWorkerFrameCodec.ReadAsync(_pipe, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
+            // Blocking I/O on Unix from the first byte (lmxopcua#789): one async call would turn the socket under the
+            // pipe non-blocking for good, and every later read would go through the socket engine.
+            var hello = ScriptWorkerFrameCodec.UseBlockingIo
+                ? ScriptWorkerFrameCodec.Read(_pipe, ScriptWorkerFrameCodec.DefaultMaxFrameBytes)
+                : await ScriptWorkerFrameCodec.ReadAsync(_pipe, ScriptWorkerFrameCodec.DefaultMaxFrameBytes);
             if (hello is null) return ScriptWorkerEnvironment.ExitPipeClosed;
             if (hello.BodyCase != Envelope.BodyOneofCase.Hello)
                 throw new ScriptWorkerProtocolException($"expected Hello, got {hello.BodyCase}");
@@ -79,7 +88,7 @@ internal sealed class WorkerHost
 
             // The nonce is echoed, not trusted: the host compares it with the one it put in OUR environment, which
             // proves the process on the other end of its pipe is the one it launched.
-            await ScriptWorkerFrameCodec.WriteAsync(_pipe, Wrap(new Envelope
+            var ready = Wrap(new Envelope
             {
                 Ready = new Ready
                 {
@@ -87,14 +96,16 @@ internal sealed class WorkerHost
                     Pid = Environment.ProcessId,
                     RuntimeVersion = Environment.Version.ToString(),
                 },
-            }), _maxFrameBytes);
+            });
+            if (ScriptWorkerFrameCodec.UseBlockingIo) ScriptWorkerFrameCodec.Write(_pipe, ready, _maxFrameBytes);
+            else await ScriptWorkerFrameCodec.WriteAsync(_pipe, ready, _maxFrameBytes);
 
             _logSink.Capacity = (int)Math.Max(1, hello.Hello.MaxLogLines == 0 ? 1000 : hello.Hello.MaxLogLines);
+            _writer = new ScriptWorkerFrameWriter(_pipe, OnWriteFault, _maxFrameBytes, firstSeq: Interlocked.Read(ref _seq),
+                synchronous: ScriptWorkerFrameCodec.UseBlockingIo);
             StartBackground(Math.Max(1, (int)hello.Hello.EvalThreads));
-            var writer = Task.Run(WriteLoopAsync);
-            var exit = await ReadLoopAsync();
-            _outbound.Writer.TryComplete();
-            try { await writer.WaitAsync(TimeSpan.FromSeconds(1)); } catch { /* exiting regardless */ }
+            var exit = ScriptWorkerFrameCodec.UseBlockingIo ? await ReadOnDedicatedThread() : await ReadLoopAsync();
+            _writer.Complete();
             return exit;
         }
         catch (ScriptWorkerProtocolException ex)
@@ -114,6 +125,7 @@ internal sealed class WorkerHost
         if (hello.QueueCapacity > 0) _queueCapacity = (int)hello.QueueCapacity;
         if (hello.HardGraceMs > 0) _hardGrace = TimeSpan.FromMilliseconds(hello.HardGraceMs);
         if (hello.HeartbeatIntervalMs > 0) _heartbeatInterval = TimeSpan.FromMilliseconds(hello.HeartbeatIntervalMs);
+        if (hello.CompileThreads > 0) _compileThreads = (int)Math.Min(hello.CompileThreads, 64);
         if (!string.IsNullOrEmpty(hello.PeCacheDir) && hello.PeCacheKey.Length == ScriptPeCacheStore.KeyLength)
         {
             // Design §7: the host owns the key; it reaches us only here, never through the environment or a file.
@@ -127,7 +139,7 @@ internal sealed class WorkerHost
         {
             new Thread(EvalThreadLoop) { IsBackground = true, Name = $"ScriptWorker.Eval{i}" }.Start();
         }
-        for (var i = 0; i < CompileThreads; i++)
+        for (var i = 0; i < _compileThreads; i++)
         {
             new Thread(CompileThreadLoop) { IsBackground = true, Name = $"ScriptWorker.Compile{i}" }.Start();
         }
@@ -137,14 +149,50 @@ internal sealed class WorkerHost
         if (_parentPid > 0) _ = Task.Run(ParentWatchLoopAsync);
     }
 
+    /// <summary>The blocking read loop on its own thread (Unix, lmxopcua#789); completes with the exit code.</summary>
+    private Task<int> ReadOnDedicatedThread()
+    {
+        var done = new TaskCompletionSource<int>(TaskCreationOptions.RunContinuationsAsynchronously);
+        new Thread(() =>
+        {
+            var reader = new ScriptWorkerFrameReader(_pipe, _maxFrameBytes);
+            while (true)
+            {
+                Envelope? frame;
+                try
+                {
+                    frame = reader.Read();
+                }
+                catch (ScriptWorkerProtocolException ex)
+                {
+                    Console.Error.WriteLine($"ScriptWorker: protocol fault: {ex.Message}");
+                    done.TrySetResult(ScriptWorkerEnvironment.ExitProtocolFault);
+                    return;
+                }
+                catch (Exception ex) when (ex is IOException or ObjectDisposedException or System.Net.Sockets.SocketException)
+                {
+                    done.TrySetResult(ScriptWorkerEnvironment.ExitPipeClosed);
+                    return;
+                }
+                if (HandleFrame(frame) is { } exit)
+                {
+                    done.TrySetResult(exit);
+                    return;
+                }
+            }
+        }) { IsBackground = true, Name = "ScriptWorker.Reader" }.Start();
+        return done.Task;
+    }
+
     private async Task<int> ReadLoopAsync()
     {
+        var reader = new ScriptWorkerFrameReader(_pipe, _maxFrameBytes);
         while (true)
         {
             Envelope? frame;
             try
             {
-                frame = await ScriptWorkerFrameCodec.ReadAsync(_pipe, _maxFrameBytes);
+                frame = await reader.ReadAsync();
             }
             catch (ScriptWorkerProtocolException ex)
             {
@@ -155,42 +203,47 @@ internal sealed class WorkerHost
             {
                 return ScriptWorkerEnvironment.ExitPipeClosed;
             }
-            if (frame is null) return ScriptWorkerEnvironment.ExitPipeClosed; // host gone: fatal by design
+            if (HandleFrame(frame) is { } exit) return exit;
+        }
+    }
 
-            switch (frame.BodyCase)
-            {
-                case Envelope.BodyOneofCase.Load:
+    /// <summary>Acts on one frame from the host.</summary>
+    /// <param name="frame">The frame; null = end of stream.</param>
+    /// <returns>The exit code when the worker must end, otherwise null.</returns>
+    private int? HandleFrame(Envelope? frame)
+    {
+        if (frame is null) return ScriptWorkerEnvironment.ExitPipeClosed; // host gone: fatal by design
+        switch (frame.BodyCase)
+        {
+            case Envelope.BodyOneofCase.Load:
+                // Compile off the reader, so evaluations for already-loaded keys keep flowing during a load, and off
+                // the thread pool, so the heartbeat and the writer keep flowing during a burst of them.
+                _loads.Add(frame.Load);
+                return null;
+            case Envelope.BodyOneofCase.Unload:
+                foreach (var key in frame.Unload.Keys)
+                    if (_scripts.TryRemove(key, out var gone)) gone.Dispose();
+                return null;
+            case Envelope.BodyOneofCase.Eval:
+                foreach (var item in frame.Eval.Items) Admit(item);
+                return null;
+            case Envelope.BodyOneofCase.Cancel:
+                if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
                 {
-                    // Compile off the reader, so evaluations for already-loaded keys keep flowing during a load, and
-                    // off the thread pool, so the heartbeat and the writer keep flowing during a burst of them.
-                    _loads.Add(frame.Load);
-                    break;
+                    try { toCancel.Cancellation.Cancel(); } catch (ObjectDisposedException) { /* just finished */ }
                 }
-                case Envelope.BodyOneofCase.Unload:
-                    foreach (var key in frame.Unload.Keys)
-                        if (_scripts.TryRemove(key, out var gone)) gone.Dispose();
-                    break;
-                case Envelope.BodyOneofCase.Eval:
-                    foreach (var item in frame.Eval.Items) Admit(item);
-                    break;
-                case Envelope.BodyOneofCase.Cancel:
-                    if (_running.TryGetValue((long)frame.Cancel.EvalId, out var toCancel))
-                    {
-                        try { toCancel.Cancellation.Cancel(); } catch (ObjectDisposedException) { /* just finished */ }
-                    }
-                    break;
-                case Envelope.BodyOneofCase.DiagRequest:
-                    Send(new Envelope { DiagReply = Diagnostics() });
-                    break;
-                case Envelope.BodyOneofCase.Shutdown:
-                    return 0;
-                case Envelope.BodyOneofCase.Fault:
-                    await Console.Error.WriteLineAsync($"ScriptWorker: host fault: {frame.Fault.Message}");
-                    return ScriptWorkerEnvironment.ExitProtocolFault;
-                default:
-                    await Console.Error.WriteLineAsync($"ScriptWorker: unexpected frame {frame.BodyCase}");
-                    return ScriptWorkerEnvironment.ExitProtocolFault;
-            }
+                return null;
+            case Envelope.BodyOneofCase.DiagRequest:
+                Send(new Envelope { DiagReply = Diagnostics() });
+                return null;
+            case Envelope.BodyOneofCase.Shutdown:
+                return 0;
+            case Envelope.BodyOneofCase.Fault:
+                Console.Error.WriteLine($"ScriptWorker: host fault: {frame.Fault.Message}");
+                return ScriptWorkerEnvironment.ExitProtocolFault;
+            default:
+                Console.Error.WriteLine($"ScriptWorker: unexpected frame {frame.BodyCase}");
+                return ScriptWorkerEnvironment.ExitProtocolFault;
         }
     }
 
@@ -201,6 +254,8 @@ internal sealed class WorkerHost
 
     private void Load(LoadScripts load)
     {
+        // The host's compile timeout runs from here, not from when it sent the load (lmxopcua#790).
+        Send(new Envelope { LoadStarted = new LoadStarted { LoadId = load.LoadId } });
         var result = new LoadResult { LoadId = load.LoadId };
         foreach (var script in load.Scripts)
         {
@@ -423,41 +478,15 @@ internal sealed class WorkerHost
         }
     }
 
-    private async Task WriteLoopAsync()
+    private static void OnWriteFault(Exception ex)
     {
-        var reader = _outbound.Reader;
-        var pendingResults = new EvalResults();
-        try
+        // The host is gone (or a frame broke the protocol): fatal by design.
+        if (ex is ScriptWorkerProtocolException)
         {
-            while (await reader.WaitToReadAsync())
-            {
-                // Opportunistic coalescing (design §4.5): drain whatever is queued right now, merging every completed
-                // evaluation into one EvalResults frame; no timer, so an idle pipe adds no latency.
-                while (reader.TryRead(out var envelope))
-                {
-                    if (envelope.BodyCase == Envelope.BodyOneofCase.Results)
-                    {
-                        pendingResults.Items.AddRange(envelope.Results.Items);
-                        continue;
-                    }
-                    await ScriptWorkerFrameCodec.WriteAsync(_pipe, envelope, _maxFrameBytes);
-                }
-                if (pendingResults.Items.Count > 0)
-                {
-                    await ScriptWorkerFrameCodec.WriteAsync(_pipe, Wrap(new Envelope { Results = pendingResults }), _maxFrameBytes);
-                    pendingResults = new EvalResults();
-                }
-                await _pipe.FlushAsync();
-            }
-        }
-        catch (IOException)
-        {
-            Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed);
-        }
-        catch (ObjectDisposedException)
-        {
-            Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed);
+            Console.Error.WriteLine($"ScriptWorker: protocol fault writing a frame: {ex.Message}");
+            Environment.Exit(ScriptWorkerEnvironment.ExitProtocolFault);
         }
+        Environment.Exit(ScriptWorkerEnvironment.ExitPipeClosed);
     }
 
     private void SendResult(EvalResult result)
@@ -467,7 +496,7 @@ internal sealed class WorkerHost
         Send(new Envelope { Results = results });
     }
 
-    private void Send(Envelope envelope) => _outbound.Writer.TryWrite(Wrap(envelope));
+    private void Send(Envelope envelope) => _writer?.TryWrite(envelope);
 
     private Envelope Wrap(Envelope envelope)
     {