perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered frame reader, one timer per evaluation for T and T+G, slot release on the reader before the caller resumes, a non-spinning per-connection result dispatcher, and blocking pipe I/O on Unix (both ends) so the socket never enters the socket engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2: 228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % -> 123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache worker RSS, and times kill latency from the soft-timeout answer. #790: the worker sends LoadStarted; the compile timeout is armed then, so a load queued behind other compiles is never killed or struck. #788: one Error with stack per down period, rate-limited Warnings with attempt count, one Information on recovery. Design doc §8.2 notes + §12.2 record. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-10 04:25:08 -05:00
parent 963b0d66fc
commit 8582ce91bc
14 changed files with 1576 additions and 317 deletions
@@ -191,8 +191,10 @@ public sealed class ScriptRunnerPerfTests
        return new PhaseResult(issued / elapsed.TotalSeconds, Percentile(samples, 0.50), Percentile(samples, 0.99), cpu, ok, notOk, firstFailure);
    }

-    private static string Row(string runner, string phase, PhaseResult r)
-        => $"| {runner} | {phase} | {r.EvalsPerSecond,9:N0} | {r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | {r.Ok:N0} / {r.NotOk:N0} |" +
+    private static string Row(string runner, string phase, PhaseResult r, PhaseResult? inProcess = null)
+        => $"| {runner} | {phase} | {r.EvalsPerSecond,9:N0} | {r.P50Us,8:N0} | {r.P99Us,9:N0} | {r.HostCpuPercent,6:N0} % | " +
+           (inProcess is null ? "— |" : $"{r.HostCpuPercent - inProcess.HostCpuPercent,6:+0;-0;0} % |") +
+           $" {r.Ok:N0} / {r.NotOk:N0} |" +
           (r.FirstFailure is null ? "" : $" first failure: {r.FirstFailure}");

    /// <summary>
@@ -206,9 +208,12 @@ public sealed class ScriptRunnerPerfTests
        var report = new StringBuilder()
            .AppendLine($"### Script runner perf ({Env()})")
            .AppendLine()
-            .AppendLine("| Runner | Phase | evals/s | p50 µs | p99 µs | host CPU | ok / not ok |")
-            .AppendLine("|---|---|---:|---:|---:|---:|---|");
+            .AppendLine("| Runner | Phase | evals/s | p50 µs | p99 µs | host CPU | vs InProcess | ok / not ok |")
+            .AppendLine("|---|---|---:|---:|---:|---:|---:|---|");
        PhaseResult? k2Closed = null;
+        // The in-process run of each phase is the harness's own floor (its pacing loop, its continuations, the pool):
+        // the "vs InProcess" column is what the out-of-process runner adds on top — the §8.2 "host CPU for IPC".
+        var baseline = new Dictionary<string, PhaseResult>(StringComparer.Ordinal);

        var configs = new (string Name, Func<IScriptRunner> Create)[]
        {
@@ -223,11 +228,16 @@ public sealed class ScriptRunnerPerfTests
            try
            {
                await WarmAsync(runner);
-                report.AppendLine(Row(name, "idle, sequential", await IdleLatencyAsync(runner)));
-                report.AppendLine(Row(name, "paced 5,000/s", await PacedAsync(runner, 5_000)));
-                report.AppendLine(Row(name, "paced 10,000/s", await PacedAsync(runner, 10_000)));
+                void Add(string phase, PhaseResult r)
+                {
+                    if (name == "InProcess") baseline[phase] = r;
+                    report.AppendLine(Row(name, phase, r, name == "InProcess" ? null : baseline.GetValueOrDefault(phase)));
+                }
+                Add("idle, sequential", await IdleLatencyAsync(runner));
+                Add("paced 5,000/s", await PacedAsync(runner, 5_000));
+                Add("paced 10,000/s", await PacedAsync(runner, 10_000));
                var closed = await ClosedLoopAsync(runner, concurrency: 64);
-                report.AppendLine(Row(name, "closed loop ×64", closed));
+                Add("closed loop ×64", closed);
                if (name == "OutOfProcess K=2") k2Closed = closed;
            }
            finally
@@ -242,7 +252,8 @@ public sealed class ScriptRunnerPerfTests

        report.AppendLine()
            .AppendLine("§8.2 budgets: added latency p50 ≤ 150 µs (idle), p99 ≤ 2 ms at 10k/s, ≥ 10,000 evals/s at K=2, " +
-                        "host CPU ≤ 25 % of one core at 10k/s. Accepted gate (owner, §14): ≥ 5,000 evals/s/node.");
+                        "host CPU for IPC (the \"vs InProcess\" column) ≤ 25 % of one core at 10k/s. " +
+                        "Accepted gate (owner, §14): ≥ 5,000 evals/s/node.");
        Report(report);

        k2Closed.ShouldNotBeNull();
@@ -286,7 +297,7 @@ public sealed class ScriptRunnerPerfTests

        var report = new StringBuilder().AppendLine($"### Script worker restart / RSS / kill latency ({Env()})").AppendLine();
        TimeSpan cold, warmFresh, killRestart;
-        long rssIdle, rssLoaded;
+        long rssIdle, rssLoaded, rssWarmIdle, rssWarmLoaded;
        await using (var runner = Scripts.Runner(PerfOptions(2, cacheDir), _runnerLog.CreateLogger()))
        {
            (await runner.RunAsync(Inv(0), Ct)).Status.ShouldBe(ScriptRunStatus.Ok);
@@ -296,8 +307,17 @@ public sealed class ScriptRunnerPerfTests
            rssLoaded = runner.Shards.Max(s => s.Current is { } c ? Process.GetProcessById(c.Pid).WorkingSet64 : 0);
        }
        await using (var runner = Scripts.Runner(PerfOptions(2, cacheDir), _runnerLog.CreateLogger()))
+        {
+            // A worker that has only loaded verified PEs from the cache: Roslyn is never entered in it.
+            (await runner.RunAsync(Inv(0), Ct)).Status.ShouldBe(ScriptRunStatus.Ok);
+            await Task.Delay(500, Ct);
+            rssWarmIdle = runner.Shards.Max(s => s.Current is { } c ? Process.GetProcessById(c.Pid).WorkingSet64 : 0);
+            runner.PeCacheCounts.Misses.ShouldBe(0, "every load came from the cache");
+        }
+        await using (var runner = Scripts.Runner(PerfOptions(2, cacheDir), _runnerLog.CreateLogger()))
        {
            warmFresh = await AllFirstResults(runner);
+            rssWarmLoaded = runner.Shards.Max(s => s.Current is { } c ? Process.GetProcessById(c.Pid).WorkingSet64 : 0);
            // A kill of both shard workers: time from the kill to every script's next correct result (includes the
            // 250 ms first restart backoff and the cache load).
            foreach (var shard in runner.Shards) Process.GetProcessById(shard.Current!.Pid).Kill();
@@ -316,11 +336,14 @@ public sealed class ScriptRunnerPerfTests
            {
                (await runner.RunAsync(Scripts.Vt("warm", "return 1;"), Ct)).Status.ShouldBe(ScriptRunStatus.Ok);
                var worker = runner.Shards[0].Current!;
-                var sw = Stopwatch.StartNew();
+                // Timed from the soft-timeout answer (T after the submit), not from the call: the call also compiles
+                // the cold runaway key, which on the Windows VM alone is 100–200 ms (lmxopcua#789 — the first
+                // harness counted it as kill latency). Kill latency = exit − (answer + G).
                var outcome = await runner.RunAsync(Scripts.Vt("runaway-" + i, "while (true) { }", TimeSpan.FromMilliseconds(200)), Ct);
+                var sw = Stopwatch.StartNew();
                outcome.Status.ShouldBe(ScriptRunStatus.Failed);
                await worker.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
-                killLatencies.Add((sw.Elapsed - TimeSpan.FromMilliseconds(1200)).TotalMilliseconds);
+                killLatencies.Add((sw.Elapsed - TimeSpan.FromMilliseconds(1000)).TotalMilliseconds);
            }
        }

@@ -329,8 +352,10 @@ public sealed class ScriptRunnerPerfTests
            .AppendLine($"| {scripts} scripts, empty cache, all first results (compile + write) | {cold.TotalMilliseconds:N0} ms | — |")
            .AppendLine($"| {scripts} scripts, fresh runner on a warm cache, all first results | {warmFresh.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
            .AppendLine($"| kill both shards → every script's next correct result | {killRestart.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
-            .AppendLine($"| worker RSS after first eval (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
-            .AppendLine($"| worker RSS with {scripts / 2} scripts loaded per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |")
+            .AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
+            .AppendLine($"| worker RSS with {scripts / 2} scripts compiled per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |")
+            .AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {rssWarmIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
+            .AppendLine($"| worker RSS with {scripts / 2} scripts loaded from the cache per shard (max) | {rssWarmLoaded / 1048576.0:N0} MB | — |")
            .AppendLine($"| kill latency after T+G (5 runs, ms) | {string.Join(" / ", killLatencies.Select(l => l.ToString("N0")))} | ≤ 100 ms |");
        Report(report);
    }