perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered frame reader, one timer per evaluation for T and T+G, slot release on the reader before the caller resumes, a non-spinning per-connection result dispatcher, and blocking pipe I/O on Unix (both ends) so the socket never enters the socket engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2: 228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % -> 123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache worker RSS, and times kill latency from the soft-timeout answer. #790: the worker sends LoadStarted; the compile timeout is armed then, so a load queued behind other compiles is never killed or struck. #788: one Error with stack per down period, rate-limited Warnings with attempt count, one Information on recovery. Design doc §8.2 notes + §12.2 record. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-10 04:25:08 -05:00
parent 963b0d66fc
commit 8582ce91bc
14 changed files with 1576 additions and 317 deletions
@@ -173,6 +173,157 @@ public sealed class ProtocolTests
        }
    }

+    // ---------------------------------------------------------------- buffered reader / combining writer (lmxopcua#789)
+
+    /// <summary>Hands out at most <c>chunk</c> bytes per read, so frames straddle reads at every possible offset.</summary>
+    private sealed class ChunkedStream(byte[] data, int chunk) : MemoryStream(data)
+    {
+        public override int Read(byte[] buffer, int offset, int count) => base.Read(buffer, offset, Math.Min(count, chunk));
+
+        public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
+            => base.ReadAsync(buffer[..Math.Min(buffer.Length, chunk)], cancellationToken);
+    }
+
+    private static byte[] Frames(int count, int bigEvery = 0)
+    {
+        using var stream = new MemoryStream();
+        for (var i = 0; i < count; i++)
+        {
+            var env = new Envelope { ProtocolVersion = 1, Seq = (ulong)i + 1, Heartbeat = new Heartbeat { Inflight = (uint)i } };
+            if (bigEvery > 0 && i % bigEvery == 0)
+                env = new Envelope { ProtocolVersion = 1, Seq = (ulong)i + 1, Fault = new Fault { Message = new string('x', 100_000 + i) } };
+            ScriptWorkerFrameCodec.WriteAsync(stream, env, ScriptWorkerFrameCodec.DefaultMaxFrameBytes).AsTask().GetAwaiter().GetResult();
+        }
+        return stream.ToArray();
+    }
+
+    /// <summary>
+    ///     The buffered reader yields exactly the frames the per-frame codec wrote, in order, however the bytes are
+    ///     split across reads — including frames larger than its initial 64 KiB buffer — through both its async and its
+    ///     blocking read, and ends cleanly at a frame boundary.
+    /// </summary>
+    [Theory]
+    [InlineData(1, false)]
+    [InlineData(3, true)]
+    [InlineData(7, false)]
+    [InlineData(65_536, true)]
+    [InlineData(1_000_000, false)]
+    public async Task The_buffered_reader_reassembles_frames_split_at_any_offset(int chunk, bool blocking)
+    {
+        var bytes = Frames(200, bigEvery: 37);
+        var reader = new ScriptWorkerFrameReader(new ChunkedStream(bytes, chunk));
+        for (var i = 0; i < 200; i++)
+        {
+            var env = (blocking ? reader.Read() : await reader.ReadAsync(TestContext.Current.CancellationToken)).ShouldNotBeNull($"frame {i}");
+            env.Seq.ShouldBe((ulong)i + 1);
+            if (i % 37 == 0) env.Fault.Message.Length.ShouldBe(100_000 + i);
+            else env.Heartbeat.Inflight.ShouldBe((uint)i);
+        }
+        (blocking ? reader.Read() : await reader.ReadAsync(TestContext.Current.CancellationToken)).ShouldBeNull("a clean end of stream");
+    }
+
+    /// <summary>The buffered reader rejects exactly what the codec rejects.</summary>
+    [Theory]
+    [MemberData(nameof(BadFrames))]
+    public async Task The_buffered_reader_rejects_malformed_frames_like_the_codec(string why, byte[] bytes)
+    {
+        await Should.ThrowAsync<ScriptWorkerProtocolException>(async () =>
+            await new ScriptWorkerFrameReader(new MemoryStream(bytes), 1024).ReadAsync(TestContext.Current.CancellationToken), why);
+        Should.Throw<ScriptWorkerProtocolException>(() => new ScriptWorkerFrameReader(new ChunkedStream(bytes, 1), 1024).Read(), why);
+    }
+
+    /// <summary>Random bytes never escape the buffered reader as anything but a protocol fault (or a clean end).</summary>
+    [Fact]
+    public void Fuzzed_input_only_ever_yields_a_protocol_fault_from_the_buffered_reader()
+    {
+        var random = new Random(789);
+        for (var i = 0; i < 3000; i++)
+        {
+            var bytes = new byte[random.Next(0, 64)];
+            random.NextBytes(bytes);
+            if (bytes.Length >= 4 && random.Next(2) == 0)
+                BinaryPrimitives.WriteInt32LittleEndian(bytes, random.Next(1, bytes.Length));
+            var reader = new ScriptWorkerFrameReader(new ChunkedStream(bytes, random.Next(1, 9)), 1024);
+            try { while (reader.Read() is not null) { } }
+            catch (ScriptWorkerProtocolException) { }
+        }
+    }
+
+    /// <summary>A stream whose first write waits on a gate, so frames queue up behind a write in progress.</summary>
+    private sealed class GatedStream : MemoryStream
+    {
+        public readonly TaskCompletionSource Gate = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        public int Writes;
+
+        public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
+        {
+            if (Interlocked.Increment(ref Writes) == 1) await Gate.Task;
+            await base.WriteAsync(buffer, cancellationToken);
+        }
+    }
+
+    private static List<Envelope> ReadAll(byte[] bytes)
+    {
+        var reader = new ScriptWorkerFrameReader(new MemoryStream(bytes));
+        var frames = new List<Envelope>();
+        while (reader.Read() is { } env) frames.Add(env);
+        return frames;
+    }
+
+    private static Envelope Eval(ulong id) => new() { Eval = new EvalBatch { Items = { new EvalRequest { EvalId = id, Key = "k" } } } };
+
+    /// <summary>
+    ///     The combining writer (lmxopcua#789): frames queued while a write is in progress go out in the next pass, with
+    ///     consecutive evaluations merged into one batch, every other frame kept in its place (a Cancel is never
+    ///     reordered ahead of the evaluation it cancels), and sequence numbers increasing.
+    /// </summary>
+    [Fact]
+    public async Task The_combining_writer_batches_consecutive_evaluations_and_keeps_frame_order()
+    {
+        var stream = new GatedStream();
+        Exception? fault = null;
+        var writer = new ScriptWorkerFrameWriter(stream, ex => fault = ex, firstSeq: 1);
+
+        writer.TryWrite(Eval(1)).ShouldBeTrue(); // starts a pass that blocks on the gate
+        writer.TryWrite(Eval(2)).ShouldBeTrue();
+        writer.TryWrite(Eval(3)).ShouldBeTrue();
+        writer.TryWrite(new Envelope { Cancel = new Cancel { EvalId = 2 } }).ShouldBeTrue();
+        writer.TryWrite(Eval(4)).ShouldBeTrue();
+        stream.Gate.SetResult();
+        await Scripts.Eventually(() =>
+        {
+            try { return ReadAll(stream.ToArray()).Count == 4; }
+            catch (ScriptWorkerProtocolException) { return false; } // a pass caught mid-copy
+        }, TimeSpan.FromSeconds(5), "both passes written");
+
+        fault.ShouldBeNull();
+        var frames = ReadAll(stream.ToArray());
+        frames.Select(f => f.BodyCase).ShouldBe([Envelope.BodyOneofCase.Eval, Envelope.BodyOneofCase.Eval,
+            Envelope.BodyOneofCase.Cancel, Envelope.BodyOneofCase.Eval]);
+        frames[0].Eval.Items.Select(e => e.EvalId).ShouldBe([1UL]);
+        frames[1].Eval.Items.Select(e => e.EvalId).ShouldBe([2UL, 3UL], "queued behind the write in progress: one batch");
+        frames[2].Cancel.EvalId.ShouldBe(2UL);
+        frames[3].Eval.Items.Select(e => e.EvalId).ShouldBe([4UL]);
+        frames.Select(f => f.Seq).ShouldBe([2UL, 3UL, 4UL, 5UL]);
+        frames.ShouldAllBe(f => f.ProtocolVersion == ScriptWorkerFrameCodec.ProtocolVersion);
+        stream.Writes.ShouldBe(2, "one write per pass");
+    }
+
+    /// <summary>A failed write completes the writer, reports once, and later frames are refused rather than lost silently.</summary>
+    [Fact]
+    public void A_failed_write_completes_the_writer_and_reports_once()
+    {
+        var faults = 0;
+        var stream = new MemoryStream();
+        stream.Dispose();
+        var writer = new ScriptWorkerFrameWriter(stream, _ => Interlocked.Increment(ref faults));
+        writer.TryWrite(Eval(1)).ShouldBeTrue();
+        faults.ShouldBe(1);
+        writer.IsCompleted.ShouldBeTrue();
+        writer.TryWrite(Eval(2)).ShouldBeFalse();
+        faults.ShouldBe(1);
+    }
+
    [Fact]
    public void Backoff_doubles_from_250ms_to_a_30s_cap_with_20_percent_jitter()
    {
@@ -191,8 +191,10 @@ public sealed class ScriptRunnerPerfTests
        return new PhaseResult(issued / elapsed.TotalSeconds, Percentile(samples, 0.50), Percentile(samples, 0.99), cpu, ok, notOk, firstFailure);
    }

-    private static string Row(string runner, string phase, PhaseResult r)
-        => $"| {runner} | {phase} | {r.EvalsPerSecond,9:N0} | {r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | {r.Ok:N0} / {r.NotOk:N0} |" +
+    private static string Row(string runner, string phase, PhaseResult r, PhaseResult? inProcess = null)
+        => $"| {runner} | {phase} | {r.EvalsPerSecond,9:N0} | {r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | " +
+           (inProcess is null ? "— |" : $"{r.HostCpuPercent - inProcess.HostCpuPercent,6:+0;-0;0} % |") +
+           $" {r.Ok:N0} / {r.NotOk:N0} |" +
           (r.FirstFailure is null ? "" : $" first failure: {r.FirstFailure}");

    /// <summary>
@@ -206,9 +208,12 @@ public sealed class ScriptRunnerPerfTests
        var report = new StringBuilder()
            .AppendLine($"### Script runner perf ({Env()})")
            .AppendLine()
-            .AppendLine("| Runner | Phase | evals/s | p50 µs | p99 µs | host CPU | ok / not ok |")
-            .AppendLine("|---|---|---:|---:|---:|---:|---|");
+            .AppendLine("| Runner | Phase | evals/s | p50 µs | p99 µs | host CPU | vs InProcess | ok / not ok |")
+            .AppendLine("|---|---|---:|---:|---:|---:|---:|---|");
        PhaseResult? k2Closed = null;
+        // The in-process run of each phase is the harness's own floor (its pacing loop, its continuations, the pool):
+        // the "vs InProcess" column is what the out-of-process runner adds on top — the §8.2 "host CPU for IPC".
+        var baseline = new Dictionary<string, PhaseResult>(StringComparer.Ordinal);

        var configs = new (string Name, Func<IScriptRunner> Create)[]
        {
@@ -223,11 +228,16 @@ public sealed class ScriptRunnerPerfTests
            try
            {
                await WarmAsync(runner);
-                report.AppendLine(Row(name, "idle, sequential", await IdleLatencyAsync(runner)));
-                report.AppendLine(Row(name, "paced 5,000/s", await PacedAsync(runner, 5_000)));
-                report.AppendLine(Row(name, "paced 10,000/s", await PacedAsync(runner, 10_000)));
+                void Add(string phase, PhaseResult r)
+                {
+                    if (name == "InProcess") baseline[phase] = r;
+                    report.AppendLine(Row(name, phase, r, name == "InProcess" ? null : baseline.GetValueOrDefault(phase)));
+                }
+                Add("idle, sequential", await IdleLatencyAsync(runner));
+                Add("paced 5,000/s", await PacedAsync(runner, 5_000));
+                Add("paced 10,000/s", await PacedAsync(runner, 10_000));
                var closed = await ClosedLoopAsync(runner, concurrency: 64);
-                report.AppendLine(Row(name, "closed loop ×64", closed));
+                Add("closed loop ×64", closed);
                if (name == "OutOfProcess K=2") k2Closed = closed;
            }
            finally
@@ -242,7 +252,8 @@ public sealed class ScriptRunnerPerfTests

        report.AppendLine()
            .AppendLine("§8.2 budgets: added latency p50 ≤ 150 µs (idle), p99 ≤ 2 ms at 10k/s, ≥ 10,000 evals/s at K=2, " +
-                        "host CPU ≤ 25 % of one core at 10k/s. Accepted gate (owner, §14): ≥ 5,000 evals/s/node.");
+                        "host CPU for IPC (the \"vs InProcess\" column) ≤ 25 % of one core at 10k/s. " +
+                        "Accepted gate (owner, §14): ≥ 5,000 evals/s/node.");
        Report(report);

        k2Closed.ShouldNotBeNull();
@@ -286,7 +297,7 @@ public sealed class ScriptRunnerPerfTests

        var report = new StringBuilder().AppendLine($"### Script worker restart / RSS / kill latency ({Env()})").AppendLine();
        TimeSpan cold, warmFresh, killRestart;
-        long rssIdle, rssLoaded;
+        long rssIdle, rssLoaded, rssWarmIdle, rssWarmLoaded;
        await using (var runner = Scripts.Runner(PerfOptions(2, cacheDir), _runnerLog.CreateLogger()))
        {
            (await runner.RunAsync(Inv(0), Ct)).Status.ShouldBe(ScriptRunStatus.Ok);
@@ -296,8 +307,17 @@ public sealed class ScriptRunnerPerfTests
            rssLoaded = runner.Shards.Max(s => s.Current is { } c ? Process.GetProcessById(c.Pid).WorkingSet64 : 0);
        }
        await using (var runner = Scripts.Runner(PerfOptions(2, cacheDir), _runnerLog.CreateLogger()))
+        {
+            // A worker that has only loaded verified PEs from the cache: Roslyn is never entered in it.
+            (await runner.RunAsync(Inv(0), Ct)).Status.ShouldBe(ScriptRunStatus.Ok);
+            await Task.Delay(500, Ct);
+            rssWarmIdle = runner.Shards.Max(s => s.Current is { } c ? Process.GetProcessById(c.Pid).WorkingSet64 : 0);
+            runner.PeCacheCounts.Misses.ShouldBe(0, "every load came from the cache");
+        }
+        await using (var runner = Scripts.Runner(PerfOptions(2, cacheDir), _runnerLog.CreateLogger()))
        {
            warmFresh = await AllFirstResults(runner);
+            rssWarmLoaded = runner.Shards.Max(s => s.Current is { } c ? Process.GetProcessById(c.Pid).WorkingSet64 : 0);
            // A kill of both shard workers: time from the kill to every script's next correct result (includes the
            // 250 ms first restart backoff and the cache load).
            foreach (var shard in runner.Shards) Process.GetProcessById(shard.Current!.Pid).Kill();
@@ -316,11 +336,14 @@ public sealed class ScriptRunnerPerfTests
            {
                (await runner.RunAsync(Scripts.Vt("warm", "return 1;"), Ct)).Status.ShouldBe(ScriptRunStatus.Ok);
                var worker = runner.Shards[0].Current!;
-                var sw = Stopwatch.StartNew();
+                // Timed from the soft-timeout answer (T after the submit), not from the call: the call also compiles
+                // the cold runaway key, which on the Windows VM alone is 100–200 ms (lmxopcua#789 — the first
+                // harness counted it as kill latency). Kill latency = exit − (answer + G).
                var outcome = await runner.RunAsync(Scripts.Vt("runaway-" + i, "while (true) { }", TimeSpan.FromMilliseconds(200)), Ct);
+                var sw = Stopwatch.StartNew();
                outcome.Status.ShouldBe(ScriptRunStatus.Failed);
                await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
-                killLatencies.Add((sw.Elapsed - TimeSpan.FromMilliseconds(1200)).TotalMilliseconds);
+                killLatencies.Add((sw.Elapsed - TimeSpan.FromMilliseconds(1000)).TotalMilliseconds);
            }
        }

@@ -329,8 +352,10 @@ public sealed class ScriptRunnerPerfTests
            .AppendLine($"| {scripts} scripts, empty cache, all first results (compile + write) | {cold.TotalMilliseconds:N0} ms | — |")
            .AppendLine($"| {scripts} scripts, fresh runner on a warm cache, all first results | {warmFresh.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
            .AppendLine($"| kill both shards → every script's next correct result | {killRestart.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
-            .AppendLine($"| worker RSS after first eval (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
-            .AppendLine($"| worker RSS with {scripts / 2} scripts loaded per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |")
+            .AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
+            .AppendLine($"| worker RSS with {scripts / 2} scripts compiled per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |")
+            .AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {rssWarmIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
+            .AppendLine($"| worker RSS with {scripts / 2} scripts loaded from the cache per shard (max) | {rssWarmLoaded / 1048576.0:N0} MB | — |")
            .AppendLine($"| kill latency after T+G (5 runs, ms) | {string.Join(" / ", killLatencies.Select(l => l.ToString("N0")))} | ≤ 100 ms |");
        Report(report);
    }
@@ -132,8 +132,11 @@ public sealed class SupervisionTests
    ///     Single flight must not refuse a caller's NEXT run because of the run it already awaited: the slot is freed
    ///     before the awaiting caller resumes. The §8 perf harness found the slot freed by a continuation unordered
    ///     against the caller's await, so 32 of 200 back-to-back runs of one key came back "previous evaluation still
-    ///     running" — which the VirtualTag and Calculation seams publish as Bad. Mutation check: drop the inline
-    ///     release after the await and this goes red.
+    ///     running" — which the VirtualTag and Calculation seams publish as Bad. Since lmxopcua#789 the release runs in
+    ///     the connection, on the reader, before the caller's resumption is handed to the result dispatcher; the second
+    ///     phase makes every release slow (the test hook sleeps inside it) so the ordering, not luck, is what keeps the
+    ///     caller from being refused. Mutation check: release after handing off the resumption and the second phase goes
+    ///     red.
    /// </summary>
    [Fact]
    public async Task A_caller_that_awaited_its_run_is_never_refused_by_it()
@@ -147,6 +150,15 @@ public sealed class SupervisionTests
            if (outcome.Status != ScriptRunStatus.Ok || !Equals(outcome.Value, i + 7)) refused.Add($"#{i}: {outcome.Reason}");
        }
        refused.ShouldBeEmpty();
+
+        runner.SlotReleasingForTest = _ => Thread.Sleep(5);
+        for (var i = 0; i < 50; i++)
+        {
+            var outcome = await runner.RunAsync(
+                Scripts.Vt("seq", "return (int)ctx.GetTag(\"x\").Value + 7;", inputs: Scripts.Input("x", i)), Ct);
+            if (outcome.Status != ScriptRunStatus.Ok || !Equals(outcome.Value, i + 7)) refused.Add($"slow release #{i}: {outcome.Reason}");
+        }
+        refused.ShouldBeEmpty();
    }

    /// <summary>
@@ -487,4 +499,203 @@ public sealed class SupervisionTests
        status.State.ShouldBe(ScriptRunnerState.Ready);
        status.Shards.Single(s => s.Label == "0").Pid.ShouldBe(runner.Shards[0].Current!.Pid);
    }
+
+    // ---------------------------------------------------------------- lmxopcua#789: who resumes the caller
+
+    /// <summary>
+    ///     lmxopcua#789: callers resume on the connection's result dispatcher, never on the pipe reader — so a consumer
+    ///     that blocks after its await (the Calculation pump publishes to OPC UA subscribers there) cannot stop the reader
+    ///     from taking heartbeats and get a healthy worker killed for silence. Here a 100 ms heartbeat (kill after 300 ms
+    ///     of silence) and a continuation that blocks for 1.5 s. Mutation check: resume callers inline on the reader (a
+    ///     synchronous-continuation TaskCompletionSource completed by the reader, no dispatcher) and the worker is killed
+    ///     (KilledHeartbeat).
+    /// </summary>
+    [Fact]
+    public async Task A_caller_that_blocks_after_its_result_does_not_starve_the_reader()
+    {
+        var sink = new CapturingSink();
+        await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
+            heartbeatInterval: TimeSpan.FromMilliseconds(100), loadTimeout: TimeSpan.FromSeconds(60));
+        (await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7);
+        var worker = runner.Shards[0].Current.ShouldNotBeNull();
+
+        string? resumedOn = null;
+        await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct).ContinueWith(t =>
+        {
+            // Runs wherever RunAsync's own continuation completed it: the thread that resumed the caller.
+            resumedOn = Thread.CurrentThread.Name;
+            t.Result.Value.ShouldBe(7);
+            Thread.Sleep(1_500);
+        }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+
+        // Behaviour first: the worker was not killed for silence while the caller blocked.
+        (await Task.WhenAny(worker.Exited, Task.Delay(1_000, Ct))).ShouldNotBe(worker.Exited,
+            string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal))));
+        var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)));
+        resumedOn.ShouldBe("ScriptWorker[0].Results", log);
+        (await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7);
+        runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
+    }
+
+    // ---------------------------------------------------------------- lmxopcua#788: start-failure logging
+
+    /// <summary>
+    ///     lmxopcua#788: a shard that keeps failing to start logs ONE Error with the exception for the down period; its
+    ///     retries are a Warning without a stack, carrying the attempt count, rate-limited (the rest at Debug); and the
+    ///     start that ends the outage is one Information line naming how many attempts failed. The docker-dev drill logged
+    ///     28 Error+stack entries in two minutes on one node before this. Mutation check: log every failure at Error with
+    ///     the exception again and the single-Error assertion goes red (3 Errors).
+    /// </summary>
+    [Theory]
+    [InlineData(0, 2, 0)]
+    [InlineData(3_600_000, 0, 2)]
+    public async Task A_shard_that_cannot_start_logs_one_error_then_concise_rate_limited_retries_then_its_recovery(
+        int warnEveryMs, int expectedWarnings, int expectedDebugs)
+    {
+        var sink = new CapturingSink();
+        var log = sink.CreateLogger();
+        var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
+            256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(5), HeartbeatInterval: TimeSpan.FromSeconds(1),
+            HandshakeTimeout: TimeSpan.FromSeconds(15));
+        var attempts = 0;
+        await using var shard = new WorkerShard("t", isolation: false,
+            (onExiting, ct) => Interlocked.Increment(ref attempts) <= 3
+                ? throw new InvalidOperationException("the worker exited (code 1) before connecting to its pipe")
+                : WorkerConnection.StartAsync(spec, log, _ => { }, onExiting, ct),
+            (_, _) => { }, log, startFailureWarningInterval: TimeSpan.FromMilliseconds(warnEveryMs));
+
+        // The background recovery retries at the backoff (250 ms, 500 ms, 1 s, ±20 %) until the 4th attempt starts.
+        await shard.WhenRecoveredAsync(null).WaitAsync(TimeSpan.FromSeconds(30), Ct);
+        shard.Current.ShouldNotBeNull();
+        attempts.ShouldBe(4);
+
+        var events = sink.Events.Where(e => e.MessageTemplate.Text.StartsWith("ScriptWorker[{Shard}]", StringComparison.Ordinal)).ToList();
+        string Dump() => string.Join(Environment.NewLine, events.Select(e => $"{e.Level} {e.RenderMessage()} {(e.Exception is null ? "" : "+exception")}"));
+        static int Attempts(Serilog.Events.LogEvent e) => int.Parse(e.Properties["Attempts"].ToString(), System.Globalization.CultureInfo.InvariantCulture);
+
+        var errors = events.Where(e => e.Level == Serilog.Events.LogEventLevel.Error).ToList();
+        errors.Count.ShouldBe(1, Dump());
+        errors[0].Exception.ShouldNotBeNull("the first failure of the outage carries the stack");
+
+        var warnings = events.Where(e => e.Level == Serilog.Events.LogEventLevel.Warning).ToList();
+        warnings.Count.ShouldBe(expectedWarnings, Dump());
+        warnings.ShouldAllBe(e => e.Exception == null);
+        warnings.Select(Attempts).ShouldBe(expectedWarnings == 0 ? [] : [2, 3]);
+        events.Count(e => e.Level == Serilog.Events.LogEventLevel.Debug && e.Properties.ContainsKey("Attempts")).ShouldBe(expectedDebugs, Dump());
+
+        var recovered = events.Single(e => e.Level == Serilog.Events.LogEventLevel.Information && e.Properties.ContainsKey("Attempts"));
+        Attempts(recovered).ShouldBe(3);
+        recovered.RenderMessage().ShouldContain("after 3 failed start attempt(s)");
+    }
+
+    // ---------------------------------------------------------------- lmxopcua#790: compile timeout from compile start
+
+    /// <summary>A script of <paramref name="lines"/> statements: Roslyn needs time roughly proportional to it.</summary>
+    private static string Lines(int lines, int salt)
+    {
+        var body = new StringBuilder("var a = ").Append(salt).Append(";\n");
+        for (var i = 0; i < lines; i++) body.Append("a += (int)ctx.GetTag(\"x\").Value * ").Append(i % 97).Append(";\n");
+        return body.Append("return a;").ToString();
+    }
+
+    private static int LinesValue(int lines, int salt) => salt + Enumerable.Range(0, lines).Sum(i => i % 97);
+
+    /// <summary>
+    ///     lmxopcua#790: the compile timeout runs from when a compile thread STARTS a load, so a queue of compiles that
+    ///     takes several load timeouts to drain kills nothing and strikes nobody. One compile thread; the compiler is
+    ///     warmed under a generous timeout (a cold Roslyn start alone takes seconds on the Windows VM), then the live
+    ///     connection's timeout drops to four times one measured compile (at least 3 s; a 500-statement script takes
+    ///     ~0.1 s on an M-series Mac and ~1.5 s on the Windows VM) and a burst of such scripts, sized from that and grown
+    ///     until it does, queues for over three timeouts — asserted, so the test cannot pass vacuously
+    ///     — while each compile stays far inside the timeout. Mutation check: time the load from when the host SENT it
+    ///     (the old <c>WaitAsync(LoadTimeout)</c> around the load) and the worker is killed and the queued keys are
+    ///     struck.
+    /// </summary>
+    [Fact]
+    public async Task Compiles_queued_behind_others_longer_than_the_load_timeout_are_not_killed_or_struck()
+    {
+        const int lines = 500;
+        var sink = new CapturingSink();
+        await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
+            heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(120), compileThreads: 1);
+        (await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
+        var worker = runner.Shards[0].Current.ShouldNotBeNull();
+
+        ScriptInvocation Script(string prefix, int i)
+            => Scripts.Vt($"{prefix}-{i}", Lines(lines, i), TimeSpan.FromSeconds(300), Scripts.Input("x", 1));
+
+        // Warm the compiler, then calibrate: the compile time of one such script on this box.
+        for (var i = 0; i < 3; i++) (await runner.RunAsync(Script("warm", i), Ct)).Value.ShouldBe(LinesValue(lines, i));
+        var probe = Stopwatch.StartNew();
+        for (var i = 0; i < 3; i++) (await runner.RunAsync(Script("probe", i), Ct)).Value.ShouldBe(LinesValue(lines, i));
+        var perCompile = probe.Elapsed / 3;
+        var loadTimeout = TimeSpan.FromTicks(Math.Max(TimeSpan.FromSeconds(3).Ticks, perCompile.Ticks * 4));
+        worker.LoadTimeout = loadTimeout;
+        var count = (int)Math.Clamp(Math.Ceiling(7 * loadTimeout / perCompile), 10, 300);
+
+        // A burst compiles faster per script than the sequential probe (the compiler keeps warming up), so grow the burst
+        // until it really outlasts three load timeouts; each burst must pass the same no-kill checks.
+        ScriptRunOutcome[] outcomes = [];
+        var elapsed = TimeSpan.Zero;
+        for (var round = 0; round < 4; round++)
+        {
+            runner.Reset(ScriptContextKind.VirtualTag); // unload the previous round's scripts (worker heap)
+            var sw = Stopwatch.StartNew();
+            var prefix = "queued" + round;
+            outcomes = await Task.WhenAll(Enumerable.Range(0, count).Select(i => runner.RunAsync(Script(prefix, i), Ct)));
+            elapsed = sw.Elapsed;
+            for (var i = 0; i < count; i++) outcomes[i].Value.ShouldBe(LinesValue(lines, i), $"round {round}: {outcomes[i].Status} {outcomes[i].Reason}");
+            if (elapsed > loadTimeout * 3) break;
+            count = Math.Min(1_200, count * 2);
+        }
+
+        var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)).Take(5));
+        elapsed.ShouldBeGreaterThan(loadTimeout * 3,
+            $"precondition: {count} compiles (probe {perCompile.TotalMilliseconds:F0} ms each) must queue for several load timeouts, or this proves nothing");
+        worker.Exited.IsCompleted.ShouldBeFalse(log);
+        runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
+        runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
+        runner.SuspectKeys.ShouldBeEmpty();
+        sink.Rendered.ShouldNotContain(l => l.Contains("killing the worker", StringComparison.Ordinal));
+    }
+
+    /// <summary>
+    ///     The positive control for the test above: a compile that is still running <c>LoadTimeout</c> after it STARTED is
+    ///     the script's fault — the worker is killed (<c>KilledLoadTimeout</c>), the caller gets the compile-timeout
+    ///     failure, the key takes a kill strike (the second quarantines it), and the healthy key on the same shard is
+    ///     not blamed. The healthy key comes from a primed PE cache, so a fresh worker's cold Roslyn start never counts
+    ///     against the short timeout. Mutation check: never arm the timer on <c>LoadStarted</c> and the huge compile
+    ///     simply finishes (status Ok, red).
+    /// </summary>
+    [Fact]
+    public async Task A_compile_still_running_a_load_timeout_after_it_started_is_killed_and_struck()
+    {
+        using var dir = new Scripts.TempDir();
+        var cache = Path.Combine(dir.Path, "pe-cache");
+        await using (var primer = Scripts.Runner(Scripts.Options(peCacheDir: cache)))
+            (await primer.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
+
+        var sink = new CapturingSink();
+        await using var runner = new OutOfProcessScriptRunner(Scripts.Options(quarantineAfterKills: 2, peCacheDir: cache),
+            sink.CreateLogger(), null, heartbeatInterval: TimeSpan.FromSeconds(1), loadTimeout: TimeSpan.FromSeconds(2), compileThreads: 1);
+        (await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
+
+        // 20,000 statements: about 4 s of Roslyn on an M-series Mac, longer elsewhere.
+        var huge = Scripts.Vt("huge", Lines(20_000, 0), TimeSpan.FromSeconds(30), Scripts.Input("x", 1));
+        for (var strike = 1; strike <= 2; strike++)
+        {
+            var before = runner.Shards[0].Current ?? await runner.Shards[0].GetReadyAsync(TimeSpan.FromSeconds(30), Ct);
+            var outcome = await runner.RunAsync(huge, Ct).WaitAsync(TimeSpan.FromSeconds(60), Ct);
+            outcome.Status.ShouldBe(ScriptRunStatus.Failed, $"strike {strike}");
+            outcome.Reason.ShouldNotBeNull().ShouldContain("compile did not finish");
+            var exit = await before.ShouldNotBeNull().Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
+            exit.Reason.ShouldBe(WorkerExitReason.KilledLoadTimeout);
+            exit.CulpritKey.ShouldBe(VtKey("huge"));
+        }
+
+        var third = await runner.RunAsync(huge, Ct);
+        third.Reason.ShouldNotBeNull().ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
+        runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBe(["huge"]);
+        (await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7, "the healthy key on the same shard is not blamed");
+    }
 }