perf(scripting): cut out-of-process runner host CPU; compile timeout from compile start; quiet start-failure logs (lmxopcua#789, #790, #788)

#789 (partial, further tuning deferred): combining inline frame writer, buffered
frame reader, one timer per evaluation for T and T+G, slot release on the reader
before the caller resumes, a non-spinning per-connection result dispatcher, and
blocking pipe I/O on Unix (both ends) so the socket never enters the socket
engine; TerminateJobObject kill on Windows. macOS host CPU at 10k evals/s K=2:
228-240 % -> 79 % (InProcess 69 %, IPC share +10 vs budget 25); windev 153 % ->
123 % (+115, still a miss). Harness reports the "vs InProcess" share, warm-cache
worker RSS, and times kill latency from the soft-timeout answer.

#790: the worker sends LoadStarted; the compile timeout is armed then, so a load
queued behind other compiles is never killed or struck.

#788: one Error with stack per down period, rate-limited Warnings with attempt
count, one Information on recovery.

Design doc §8.2 notes + §12.2 record.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerShard.cs

@@ -45,6 +45,12 @@ internal sealed class WorkerShard : IAsyncDisposable
     internal static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(30);
     internal static readonly TimeSpan HealthyResetAfter = TimeSpan.FromSeconds(60);
 
+    /// <summary>
+    ///     While a shard keeps failing to start, at most one Warning per this interval (lmxopcua#788); the other retries
+    ///     log at Debug. The first failure of a down period is the one Error (with the exception).
+    /// </summary>
+    internal static readonly TimeSpan DefaultStartFailureWarningInterval = TimeSpan.FromSeconds(60);
+
     private static readonly Counter<long> StartsCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
         "otopcua.script.worker.starts", unit: "{process}", description: "ScriptWorker processes started (tag: shard).");
 
@@ -58,6 +64,7 @@ internal sealed class WorkerShard : IAsyncDisposable
     private readonly object _gate = new();
     private readonly Random _jitter = new();
     private readonly CancellationTokenSource _disposing = new();
+    private readonly TimeSpan _startFailureWarningInterval;
     private WorkerConnection? _current;
     private Task<WorkerConnection?>? _starting;
     private TaskCompletionSource? _recovered;
@@ -67,9 +74,15 @@ internal sealed class WorkerShard : IAsyncDisposable
     private int _consecutiveFailures;
     private bool _disposed;
 
+    // The current run of failed starts (lmxopcua#788): how many, since when, and when the last Warning went out.
+    private int _startFailures;
+    private long _startFailingSince;
+    private long _lastStartFailureWarning;
+
     public WorkerShard(string label, bool isolation, Func<Action<WorkerExit>, CancellationToken, Task<WorkerConnection>> launch,
-        Action<WorkerShard, WorkerExit> onExit, ILogger log)
+        Action<WorkerShard, WorkerExit> onExit, ILogger log, TimeSpan? startFailureWarningInterval = null)
     {
+        _startFailureWarningInterval = startFailureWarningInterval ?? DefaultStartFailureWarningInterval;
         Label = label;
         IsIsolation = isolation;
         _launch = launch;
@@ -298,7 +311,21 @@ internal sealed class WorkerShard : IAsyncDisposable
         {
             connection = await _launch(Attribute, CancellationToken.None).ConfigureAwait(false);
             StartsCounter.Add(1, new KeyValuePair<string, object?>("shard", Label));
-            _log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
+            int failedBefore;
+            long failingSince;
+            lock (_gate)
+            {
+                failedBefore = _startFailures;
+                failingSince = _startFailingSince;
+                _startFailures = 0;
+                _startFailingSince = 0;
+                _lastStartFailureWarning = 0;
+            }
+            if (failedBefore == 0)
+                _log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
+            else
+                _log.Information("ScriptWorker[{Shard}] started, pid {Pid}, after {Attempts} failed start attempt(s) over {Outage}",
+                    Label, connection.Pid, failedBefore, Stopwatch.GetElapsedTime(failingSince));
             var started = connection;
             _ = started.Exited.ContinueWith(t => OnExited(started, t.Result), CancellationToken.None,
                 TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Default);
@@ -320,13 +347,31 @@ internal sealed class WorkerShard : IAsyncDisposable
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does", Label);
+            int attempt;
+            TimeSpan failingFor;
+            bool warn;
             lock (_gate)
             {
                 _starting = null;
                 MarkDownLocked();
                 ScheduleBackoffLocked();
+                attempt = ++_startFailures;
+                var now = Stopwatch.GetTimestamp();
+                if (attempt == 1) _startFailingSince = now;
+                failingFor = Stopwatch.GetElapsedTime(_startFailingSince, now);
+                warn = attempt > 1 && (_lastStartFailureWarning == 0
+                                       || Stopwatch.GetElapsedTime(_lastStartFailureWarning, now) >= _startFailureWarningInterval);
+                if (attempt == 1 || warn) _lastStartFailureWarning = now;
             }
+            // lmxopcua#788: one Error with the stack per down period; the retries (every backoff, capped at 30 s, per
+            // shard) are a rate-limited Warning without it, and the rest Debug. Recovery is one Information line above.
+            if (attempt == 1)
+                _log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does (retrying with backoff)", Label);
+            else if (warn)
+                _log.Warning("ScriptWorker[{Shard}] still failing to start: {Attempts} failed attempt(s) over {Outage}; last error: {Error}",
+                    Label, attempt, failingFor, ex.Message);
+            else
+                _log.Debug("ScriptWorker[{Shard}] start attempt {Attempts} failed: {Error}", Label, attempt, ex.Message);
             if (connection is not null) await connection.DisposeAsync().ConfigureAwait(false);
             return null;
         }