perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered
frame reader, one timer per evaluation for T and T+G, slot release on the reader
before the caller resumes, a non-spinning per-connection result dispatcher, and
blocking pipe I/O on Unix (both ends) so the socket never enters the socket
engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2:
228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % ->
123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache
worker RSS, and times kill latency from the soft-timeout answer.

#790: the worker sends LoadStarted; the compile timeout is armed then, so a load
queued behind other compiles is never killed or struck.

#788: one Error with stack per down period, rate-limited Warnings with attempt
count, one Information on recovery.

Design doc §8.2 notes + §12.2 record.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/Protos/script_worker.proto

@@ -26,6 +26,7 @@ message Envelope {
     DiagnosticsRequest diag_request = 21;  // host -> worker (tests / ops)
     DiagnosticsReply diag_reply = 22;      // worker -> host
     Cancel cancel = 23;                    // host -> worker, at the soft deadline
+    LoadStarted load_started = 24;         // worker -> host, when a compile thread picks a load up (lmxopcua#790)
   }
 }
 
@@ -39,6 +40,7 @@ message Hello {
   uint32 max_log_lines = 7;         // outbound log buffer cap
   string pe_cache_dir = 8;          // the node's compiled-PE cache (design §7); empty = no cache
   bytes pe_cache_key = 9;           // its HMAC key; the worker loads only entries that verify under it
+  uint32 compile_threads = 10;      // dedicated compile threads; 0 = the worker's default (min(8, max(1, CPUs/2)))
 }
 
 message Ready {
@@ -81,6 +83,13 @@ enum PeCache {
   PE_CACHE_REJECTED = 3;            // an entry failed verification, was deleted, and the source was compiled
 }
 
+// A compile thread has taken the load off the worker's queue and is about to compile (or PE-load) it. The host's load
+// timeout runs from here, not from when the host sent the load: a load queued behind other compiles is not stuck and
+// must not be blamed for the queue (lmxopcua#790).
+message LoadStarted {
+  uint64 load_id = 1;
+}
+
 message LoadResult {
   uint64 load_id = 1;
   repeated LoadOutcome outcomes = 2;

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFrameCodec.cs

@@ -62,30 +62,14 @@ public static class ScriptWorkerFrameCodec
         if (got < 4) throw new ScriptWorkerProtocolException("truncated frame header");
 
         var size = BinaryPrimitives.ReadInt32LittleEndian(header);
-        if (size == 0) throw new ScriptWorkerProtocolException("zero-length frame");
-        if (size < 0 || size > maxFrameBytes)
-            throw new ScriptWorkerProtocolException($"frame length {size} is outside 1..{maxFrameBytes}");
+        CheckLength(size, maxFrameBytes);
 
         var body = ArrayPool<byte>.Shared.Rent(size);
         try
         {
             var read = await ReadFullyAsync(stream, body.AsMemory(0, size), ct).ConfigureAwait(false);
             if (read < size) throw new ScriptWorkerProtocolException($"truncated frame: {read} of {size} bytes");
-            Envelope envelope;
-            try
-            {
-                envelope = Envelope.Parser.ParseFrom(body, 0, size);
-            }
-            catch (InvalidProtocolBufferException ex)
-            {
-                throw new ScriptWorkerProtocolException($"unparseable frame: {ex.Message}", ex);
-            }
-            if (envelope.ProtocolVersion != ProtocolVersion)
-                throw new ScriptWorkerProtocolException(
-                    $"protocol version {envelope.ProtocolVersion} is not the supported {ProtocolVersion}");
-            if (envelope.BodyCase == Envelope.BodyOneofCase.None)
-                throw new ScriptWorkerProtocolException("frame carries no body");
-            return envelope;
+            return ParseBody(body.AsSpan(0, size));
         }
         finally
         {
@@ -93,6 +77,104 @@ public static class ScriptWorkerFrameCodec
         }
     }
 
+    /// <summary>
+    ///     Whether this platform's pipe should be driven with blocking I/O (lmxopcua#789). On Unix a .NET pipe is a
+    ///     socket, and ONE asynchronous operation on it switches it to non-blocking for good, after which even a
+    ///     "blocking" read goes through the socket engine (an epoll-thread hand-off and a spinning wait per frame) —
+    ///     measured as most of the host's per-evaluation CPU. Kept blocking, a read is a plain kernel wait on a
+    ///     dedicated thread. Not on Windows, where an overlapped pipe handle is needed for full duplex and its
+    ///     asynchronous path is the efficient one.
+    /// </summary>
+    public static bool UseBlockingIo => !OperatingSystem.IsWindows();
+
+    /// <summary>Writes one frame synchronously (the blocking-I/O handshake). Not thread-safe.</summary>
+    /// <param name="stream">The pipe.</param>
+    /// <param name="envelope">The envelope.</param>
+    /// <param name="maxFrameBytes">The cap.</param>
+    public static void Write(Stream stream, Envelope envelope, int maxFrameBytes)
+    {
+        ArgumentNullException.ThrowIfNull(stream);
+        ArgumentNullException.ThrowIfNull(envelope);
+        var size = envelope.CalculateSize();
+        if (size == 0) throw new ScriptWorkerProtocolException("refusing to write an empty frame");
+        if (size > maxFrameBytes)
+            throw new ScriptWorkerProtocolException($"frame of {size} bytes exceeds the {maxFrameBytes}-byte cap");
+        var buffer = ArrayPool<byte>.Shared.Rent(size + 4);
+        try
+        {
+            BinaryPrimitives.WriteInt32LittleEndian(buffer, size);
+            envelope.WriteTo(new Span<byte>(buffer, 4, size));
+            stream.Write(buffer, 0, size + 4);
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(buffer);
+        }
+    }
+
+    /// <summary>
+    ///     Reads exactly one frame synchronously, never past it (the blocking-I/O handshake: what follows belongs to the
+    ///     connection's own reader).
+    /// </summary>
+    /// <param name="stream">The pipe.</param>
+    /// <param name="maxFrameBytes">The cap.</param>
+    /// <returns>The envelope, or null on a clean end of stream at a frame boundary.</returns>
+    public static Envelope? Read(Stream stream, int maxFrameBytes)
+    {
+        ArgumentNullException.ThrowIfNull(stream);
+        Span<byte> header = stackalloc byte[4];
+        var got = stream.ReadAtLeast(header, 4, throwOnEndOfStream: false);
+        if (got == 0) return null;
+        if (got < 4) throw new ScriptWorkerProtocolException("truncated frame header");
+        var size = BinaryPrimitives.ReadInt32LittleEndian(header);
+        CheckLength(size, maxFrameBytes);
+        var body = ArrayPool<byte>.Shared.Rent(size);
+        try
+        {
+            var read = stream.ReadAtLeast(body.AsSpan(0, size), size, throwOnEndOfStream: false);
+            if (read < size) throw new ScriptWorkerProtocolException($"truncated frame: {read} of {size} bytes");
+            return ParseBody(body.AsSpan(0, size));
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(body);
+        }
+    }
+
+    /// <summary>Validates a frame's length prefix against the cap.</summary>
+    /// <param name="size">The decoded length.</param>
+    /// <param name="maxFrameBytes">The negotiated cap.</param>
+    /// <exception cref="ScriptWorkerProtocolException">A zero, negative or oversize length.</exception>
+    internal static void CheckLength(int size, int maxFrameBytes)
+    {
+        if (size == 0) throw new ScriptWorkerProtocolException("zero-length frame");
+        if (size < 0 || size > maxFrameBytes)
+            throw new ScriptWorkerProtocolException($"frame length {size} is outside 1..{maxFrameBytes}");
+    }
+
+    /// <summary>Parses and validates one frame body (everything after the length prefix).</summary>
+    /// <param name="body">The body bytes.</param>
+    /// <returns>The envelope.</returns>
+    /// <exception cref="ScriptWorkerProtocolException">Unparseable, the wrong protocol version, or no body.</exception>
+    internal static Envelope ParseBody(ReadOnlySpan<byte> body)
+    {
+        Envelope envelope;
+        try
+        {
+            envelope = Envelope.Parser.ParseFrom(body);
+        }
+        catch (InvalidProtocolBufferException ex)
+        {
+            throw new ScriptWorkerProtocolException($"unparseable frame: {ex.Message}", ex);
+        }
+        if (envelope.ProtocolVersion != ProtocolVersion)
+            throw new ScriptWorkerProtocolException(
+                $"protocol version {envelope.ProtocolVersion} is not the supported {ProtocolVersion}");
+        if (envelope.BodyCase == Envelope.BodyOneofCase.None)
+            throw new ScriptWorkerProtocolException("frame carries no body");
+        return envelope;
+    }
+
     private static async ValueTask<int> ReadFullyAsync(Stream stream, Memory<byte> buffer, CancellationToken ct)
     {
         var total = 0;

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol/ScriptWorkerFramePipe.cs

@@ -0,0 +1,262 @@
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Collections.Concurrent;
+using Google.Protobuf;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol;
+
+/// <summary>
+///     A buffered reader of <see cref="ScriptWorkerFrameCodec"/> frames (lmxopcua#789). One read from the pipe pulls in
+///     every frame the peer has written so far, and they are then parsed from the buffer without another read, so a
+///     batch of results costs one wake-up instead of two reads per frame. Same validation as
+///     <see cref="ScriptWorkerFrameCodec.ReadAsync"/>: zero-length, oversize, truncated, unparseable, wrong-version and
+///     bodiless frames are a <see cref="ScriptWorkerProtocolException"/>. Not thread-safe: one reader per stream.
+/// </summary>
+public sealed class ScriptWorkerFrameReader
+{
+    private const int InitialBuffer = 64 * 1024;
+
+    private readonly Stream _stream;
+    private byte[] _buffer = new byte[InitialBuffer];
+    private int _start;
+    private int _end;
+
+    /// <summary>Initializes the reader.</summary>
+    /// <param name="stream">The pipe.</param>
+    /// <param name="maxFrameBytes">The frame cap.</param>
+    public ScriptWorkerFrameReader(Stream stream, int maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes)
+    {
+        ArgumentNullException.ThrowIfNull(stream);
+        _stream = stream;
+        MaxFrameBytes = maxFrameBytes;
+    }
+
+    /// <summary>The negotiated frame cap.</summary>
+    public int MaxFrameBytes { get; set; }
+
+    /// <summary>Reads the next frame.</summary>
+    /// <param name="ct">Cancellation.</param>
+    /// <returns>The envelope, or null on a clean end of stream at a frame boundary.</returns>
+    public async ValueTask<Envelope?> ReadAsync(CancellationToken ct = default)
+    {
+        while (true)
+        {
+            if (TryTake(out var envelope, out var need)) return envelope;
+            Prepare(need);
+            var n = await _stream.ReadAsync(_buffer.AsMemory(_end), ct).ConfigureAwait(false);
+            if (n == 0) return EndOfStream();
+            _end += n;
+        }
+    }
+
+    /// <summary>Reads the next frame, blocking (for a dedicated reader thread).</summary>
+    /// <returns>The envelope, or null on a clean end of stream at a frame boundary.</returns>
+    public Envelope? Read()
+    {
+        while (true)
+        {
+            if (TryTake(out var envelope, out var need)) return envelope;
+            Prepare(need);
+            var n = _stream.Read(_buffer, _end, _buffer.Length - _end);
+            if (n == 0) return EndOfStream();
+            _end += n;
+        }
+    }
+
+    private bool TryTake(out Envelope? envelope, out int need)
+    {
+        envelope = null;
+        var available = _end - _start;
+        if (available < 4)
+        {
+            need = 4;
+            return false;
+        }
+        var size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.AsSpan(_start, 4));
+        ScriptWorkerFrameCodec.CheckLength(size, MaxFrameBytes);
+        need = size + 4;
+        if (available < need) return false;
+        envelope = ScriptWorkerFrameCodec.ParseBody(_buffer.AsSpan(_start + 4, size));
+        _start += need;
+        if (_start == _end) _start = _end = 0;
+        return true;
+    }
+
+    /// <summary>Makes room for a frame of <paramref name="need"/> bytes (prefix included) starting at <c>_start</c>.</summary>
+    private void Prepare(int need)
+    {
+        if (_start + need <= _buffer.Length && _end < _buffer.Length) return;
+        var pending = _end - _start;
+        if (need > _buffer.Length)
+        {
+            var bigger = new byte[Math.Max(need, _buffer.Length * 2)];
+            Buffer.BlockCopy(_buffer, _start, bigger, 0, pending);
+            _buffer = bigger;
+        }
+        else if (_start > 0)
+        {
+            Buffer.BlockCopy(_buffer, _start, _buffer, 0, pending);
+        }
+        _start = 0;
+        _end = pending;
+    }
+
+    private Envelope? EndOfStream()
+    {
+        var pending = _end - _start;
+        if (pending == 0) return null;
+        if (pending < 4) throw new ScriptWorkerProtocolException("truncated frame header");
+        var size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.AsSpan(_start, 4));
+        throw new ScriptWorkerProtocolException($"truncated frame: {pending - 4} of {size} bytes");
+    }
+}
+
+/// <summary>
+///     A combining frame writer (lmxopcua#789): the thread that queues a frame writes it itself when no write is in
+///     progress, so the common case needs no hand-off to a writer task (no thread-pool wake-up, no spin-wait), and
+///     frames queued while a write is in progress are written by that writer in the same pass. Within a pass consecutive
+///     <see cref="EvalBatch"/> items are merged into one batch and consecutive <see cref="EvalResults"/> items into one
+///     (design §4.5 opportunistic batching), every frame is serialised into one buffer, and the buffer goes out in one
+///     write. Frame order is preserved.
+/// </summary>
+/// <remarks>
+///     <para>Asynchronous mode: a write that cannot complete at once (a full pipe) continues on the I/O completion, and
+///     frames queued meanwhile wait for it. Synchronous mode (<see cref="ScriptWorkerFrameCodec.UseBlockingIo"/>, so
+///     the socket under a Unix pipe never turns non-blocking): the draining thread blocks while the pipe is full. Only
+///     one thread drains at a time — every other producer just queues and returns — and a peer that stops reading is
+///     killed by its supervisor (the host's heartbeat check), which fails the write and releases that thread.</para>
+///     <para>A failed write completes the writer and reports the exception once through <c>onFault</c>.</para>
+/// </remarks>
+public sealed class ScriptWorkerFrameWriter
+{
+    private readonly Stream _stream;
+    private readonly Action<Exception> _onFault;
+    private readonly ConcurrentQueue<Envelope> _queue = new();
+    private readonly ArrayBufferWriter<byte> _pass = new(16 * 1024);
+    private readonly bool _synchronous;
+    private int _draining;
+    private int _completed;
+    private int _faulted;
+    private long _seq;
+
+    /// <summary>Initializes the writer.</summary>
+    /// <param name="stream">The pipe.</param>
+    /// <param name="onFault">Called once when a write fails (or a frame is refused); the writer is then completed.</param>
+    /// <param name="maxFrameBytes">The frame cap.</param>
+    /// <param name="firstSeq">The last sequence number already used (the handshake's).</param>
+    /// <param name="synchronous">Write with blocking calls (see the remarks).</param>
+    public ScriptWorkerFrameWriter(Stream stream, Action<Exception> onFault,
+        int maxFrameBytes = ScriptWorkerFrameCodec.DefaultMaxFrameBytes, long firstSeq = 0, bool synchronous = false)
+    {
+        _synchronous = synchronous;
+        ArgumentNullException.ThrowIfNull(stream);
+        ArgumentNullException.ThrowIfNull(onFault);
+        _stream = stream;
+        _onFault = onFault;
+        MaxFrameBytes = maxFrameBytes;
+        _seq = firstSeq;
+    }
+
+    /// <summary>The negotiated frame cap.</summary>
+    public int MaxFrameBytes { get; set; }
+
+    /// <summary>Whether <see cref="Complete"/> has been called or a write failed.</summary>
+    public bool IsCompleted => Volatile.Read(ref _completed) != 0;
+
+    /// <summary>
+    ///     Queues a frame and, when no write is in progress, writes it on the calling thread. Returns false once the
+    ///     writer is completed; the frame is then dropped.
+    /// </summary>
+    /// <param name="envelope">The frame (its version and sequence number are set here).</param>
+    /// <returns>Whether the frame was accepted.</returns>
+    public bool TryWrite(Envelope envelope)
+    {
+        ArgumentNullException.ThrowIfNull(envelope);
+        // A full fence before the completion check, so a caller that registered state (a pending evaluation) before
+        // calling here and a completer that completes and then sweeps that state cannot both miss each other.
+        Interlocked.MemoryBarrier();
+        if (Volatile.Read(ref _completed) != 0) return false;
+        envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
+        _queue.Enqueue(envelope);
+        if (Interlocked.CompareExchange(ref _draining, 1, 0) == 0) _ = DrainAsync();
+        return true;
+    }
+
+    /// <summary>Stops accepting frames. Frames already queued are still written by a pass in progress.</summary>
+    public void Complete() => Interlocked.Exchange(ref _completed, 1);
+
+    private async Task DrainAsync()
+    {
+        try
+        {
+            while (true)
+            {
+                FillPass();
+                if (_pass.WrittenCount > 0)
+                {
+                    if (_synchronous) _stream.Write(_pass.WrittenSpan);
+                    else await _stream.WriteAsync(_pass.WrittenMemory).ConfigureAwait(false);
+                    _pass.ResetWrittenCount();
+                    // Synchronous on purpose: a pipe has no write buffer to flush, and PipeStream does not override
+                    // FlushAsync, so the base Stream.FlushAsync would queue a thread-pool work item per pass.
+                    _stream.Flush();
+                }
+
+                Volatile.Write(ref _draining, 0);
+                // A frame queued after the last TryDequeue but before the flag dropped saw _draining == 1 and left it
+                // to us: take the flag back and go round again, unless another thread already has.
+                if (_queue.IsEmpty || Interlocked.CompareExchange(ref _draining, 1, 0) != 0) return;
+            }
+        }
+        catch (Exception ex)
+        {
+            Interlocked.Exchange(ref _completed, 1);
+            _pass.ResetWrittenCount();
+            while (_queue.TryDequeue(out _)) { }
+            Volatile.Write(ref _draining, 0);
+            if (Interlocked.Exchange(ref _faulted, 1) == 0) _onFault(ex);
+        }
+    }
+
+    private void FillPass()
+    {
+        EvalBatch? evals = null;
+        EvalResults? results = null;
+        while (_queue.TryDequeue(out var envelope))
+        {
+            switch (envelope.BodyCase)
+            {
+                case Envelope.BodyOneofCase.Eval:
+                    if (results is not null) { Append(new Envelope { Results = results }); results = null; }
+                    (evals ??= new EvalBatch()).Items.AddRange(envelope.Eval.Items);
+                    break;
+                case Envelope.BodyOneofCase.Results:
+                    if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
+                    (results ??= new EvalResults()).Items.AddRange(envelope.Results.Items);
+                    break;
+                default:
+                    if (evals is not null) { Append(new Envelope { Eval = evals }); evals = null; }
+                    if (results is not null) { Append(new Envelope { Results = results }); results = null; }
+                    Append(envelope);
+                    break;
+            }
+        }
+        if (evals is not null) Append(new Envelope { Eval = evals });
+        if (results is not null) Append(new Envelope { Results = results });
+    }
+
+    private void Append(Envelope envelope)
+    {
+        envelope.ProtocolVersion = ScriptWorkerFrameCodec.ProtocolVersion;
+        envelope.Seq = (ulong)Interlocked.Increment(ref _seq);
+        var size = envelope.CalculateSize();
+        if (size == 0) throw new ScriptWorkerProtocolException("refusing to write an empty frame");
+        if (size > MaxFrameBytes)
+            throw new ScriptWorkerProtocolException($"frame of {size} bytes exceeds the {MaxFrameBytes}-byte cap");
+        var span = _pass.GetSpan(size + 4);
+        BinaryPrimitives.WriteInt32LittleEndian(span, size);
+        envelope.WriteTo(span.Slice(4, size));
+        _pass.Advance(size + 4);
+    }
+}