feat(scripting): out-of-process pre-emptible script runner behind Scripting:Runner (lmxopcua#780 P1)

Opt-in (Scripting:Runner=OutOfProcess; InProcess stays the default and its code path is unchanged). Implements
the P1 slice of docs/plans/2026-11-09-out-of-process-script-runner-design.md with the owner's §14 decisions:

- Options: Scripting:Runner (InProcess | Shadow | OutOfProcess), Scripting:Runner:Seams, Scripting:Worker:*
  (Shards 2, HardGrace max(1 s,T), HeapLimitBytes 512 MiB, QuarantineAfterKills 2, UnavailableGrace 5 s,
  IsolationProbationRuns 20, PeCacheDir validated absolute + outside the content root (fail closed),
  ExecutablePath). Shadow is not implemented yet: runs InProcess with a startup Warning.
- Core.Scripting: IScriptRunner seam + InProcessScriptRunner (today's guard, the parity baseline).
- Core.Scripting.Protocol: protobuf envelopes, 4-byte LE framing (zero/oversize/truncated/garbage refused),
  Variant carrying the exact CLR type; unsupported types are explicit failures.
- ScriptWorker app: compiles in the worker, P-thread scheduler, host-driven soft cancel, fail-fast
  watchdog, heartbeats, ctx.Logger forwarding, exits on pipe EOF / parent gone, forbidden-assembly
  self-check; Akka and Grpc.Core.Api excluded from its folder. Shipped in script-worker/ (Build + Publish).
- Core.Scripting.OutOfProcess: K shards + isolation worker, soft T / hard T+G kill, attribution
  (deadline kill = one key; crash = suspects -> isolation; isolation crash = strike), kill-strike
  quarantine, backoff 250 ms -> 30 s, fail closed, cleared allow-listed env, nonce + pid handshake on a
  current-user pipe, Windows Job Object.
- VirtualTag + Calculation seams use the runner when selected; a runner-side failure (worker restarted /
  runner unavailable) publishes BadResourceUnavailable (0x80040000) — ScadaBridge#112. Script faults keep
  BadInternalError. The scripted-alarm seam stays in process (alarms last).

Tests: new ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests (Category=ScriptWorker, real processes; green on macOS and
on windev/Windows), seam tests in Host/Runtime/Calculation suites; mutation-checked.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess/WorkerShard.cs

@@ -0,0 +1,249 @@
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using Serilog;
+using ZB.MOM.WW.OtOpcUa.Commons.Observability;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
+
+namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
+
+/// <summary>
+///     Supervises one shard's worker process (design §3.3, §5.3.5): starts it lazily on first use, reports its end to
+///     the runner (for attribution), and restarts it after an exponential backoff — 250 ms doubling to a 30 s cap with
+///     ±20 % jitter, reset once a worker has stayed up for 60 s. It never runs a script in the host when the worker
+///     cannot start (fail closed, §14 #4): <see cref="GetReadyAsync"/> returns null and the caller reports
+///     <c>BadResourceUnavailable</c>.
+/// </summary>
+internal sealed class WorkerShard : IAsyncDisposable
+{
+    internal static readonly TimeSpan InitialBackoff = TimeSpan.FromMilliseconds(250);
+    internal static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(30);
+    internal static readonly TimeSpan HealthyResetAfter = TimeSpan.FromSeconds(60);
+
+    private static readonly Counter<long> StartsCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
+        "otopcua.script.worker.starts", unit: "{process}", description: "ScriptWorker processes started (tag: shard).");
+
+    private static readonly Counter<long> ExitsCounter = OtOpcUaTelemetry.Meter.CreateCounter<long>(
+        "otopcua.script.worker.exits", unit: "{process}",
+        description: "ScriptWorker processes ended (tags: shard, reason = clean | killed_deadline | killed_heartbeat | killed_load | crash | protocol | disposed).");
+
+    private readonly Func<CancellationToken, Task<WorkerConnection>> _launch;
+    private readonly Action<WorkerShard, WorkerExit> _onExit;
+    private readonly ILogger _log;
+    private readonly object _gate = new();
+    private readonly Random _jitter = new();
+    private WorkerConnection? _current;
+    private Task<WorkerConnection?>? _starting;
+    private long _backoffUntil;
+    private int _consecutiveFailures;
+    private bool _disposed;
+
+    public WorkerShard(string label, bool isolation, Func<CancellationToken, Task<WorkerConnection>> launch,
+        Action<WorkerShard, WorkerExit> onExit, ILogger log)
+    {
+        Label = label;
+        IsIsolation = isolation;
+        _launch = launch;
+        _onExit = onExit;
+        _log = log;
+        // The isolation worker runs one evaluation at a time, so a crash there names exactly one key (design §5.3.3).
+        IsolationGate = isolation ? new SemaphoreSlim(1, 1) : null;
+    }
+
+    /// <summary>The shard label (<c>0</c>…<c>K-1</c>, or <c>iso</c>).</summary>
+    public string Label { get; }
+
+    /// <summary>Whether this is the isolation ("penalty box") worker.</summary>
+    public bool IsIsolation { get; }
+
+    /// <summary>Serialises evaluations on the isolation worker; null on a normal shard.</summary>
+    public SemaphoreSlim? IsolationGate { get; }
+
+    /// <summary>The live worker, if any (diagnostics / tests).</summary>
+    public WorkerConnection? Current
+    {
+        get { lock (_gate) return _current is { IsAlive: true } c ? c : null; }
+    }
+
+    /// <summary>Consecutive start failures and crashes since the last healthy period (diagnostics / tests).</summary>
+    public int ConsecutiveFailures
+    {
+        get { lock (_gate) return _consecutiveFailures; }
+    }
+
+    /// <summary>
+    ///     The ready worker, starting one if needed. Waits at most <paramref name="grace"/> — through a start in progress
+    ///     or a backoff — and returns null if no worker is ready by then.
+    /// </summary>
+    /// <param name="grace">How long to wait.</param>
+    /// <param name="ct">Cancellation.</param>
+    /// <returns>The worker, or null.</returns>
+    public async Task<WorkerConnection?> GetReadyAsync(TimeSpan grace, CancellationToken ct)
+    {
+        var deadline = Stopwatch.GetTimestamp() + (long)(grace.TotalSeconds * Stopwatch.Frequency);
+        while (true)
+        {
+            Task<WorkerConnection?> starting;
+            TimeSpan wait;
+            lock (_gate)
+            {
+                if (_disposed) return null;
+                if (_current is { IsAlive: true } live) return live;
+                var now = Stopwatch.GetTimestamp();
+                if (_starting is null && now >= _backoffUntil)
+                {
+                    _starting = StartCoreAsync();
+                }
+                if (_starting is not null)
+                {
+                    starting = _starting;
+                    wait = TimeSpan.Zero;
+                }
+                else
+                {
+                    starting = null!;
+                    wait = Stopwatch.GetElapsedTime(now, _backoffUntil);
+                }
+            }
+
+            var remaining = Stopwatch.GetElapsedTime(Stopwatch.GetTimestamp(), deadline);
+            if (remaining <= TimeSpan.Zero) return null;
+            if (starting is not null)
+            {
+                try
+                {
+                    var started = await starting.WaitAsync(remaining, ct).ConfigureAwait(false);
+                    if (started is { IsAlive: true }) return started;
+                }
+                catch (TimeoutException)
+                {
+                    return null;
+                }
+                continue;
+            }
+            if (wait > remaining) return null;
+            await Task.Delay(wait, ct).ConfigureAwait(false);
+        }
+    }
+
+    private async Task<WorkerConnection?> StartCoreAsync()
+    {
+        await Task.Yield();
+        WorkerConnection? connection = null;
+        try
+        {
+            connection = await _launch(CancellationToken.None).ConfigureAwait(false);
+            StartsCounter.Add(1, new KeyValuePair<string, object?>("shard", Label));
+            _log.Information("ScriptWorker[{Shard}] started, pid {Pid}", Label, connection.Pid);
+            var started = connection;
+            _ = started.Exited.ContinueWith(t => OnExited(started, t.Result), CancellationToken.None,
+                TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Default);
+            lock (_gate)
+            {
+                _starting = null;
+                if (_disposed)
+                {
+                    _ = started.DisposeAsync().AsTask();
+                    return null;
+                }
+                _current = started;
+            }
+            return started;
+        }
+        catch (Exception ex)
+        {
+            _log.Error(ex, "ScriptWorker[{Shard}] failed to start; scripts on it report runner unavailable until it does", Label);
+            lock (_gate)
+            {
+                _starting = null;
+                ScheduleBackoffLocked();
+            }
+            if (connection is not null) await connection.DisposeAsync().ConfigureAwait(false);
+            return null;
+        }
+    }
+
+    private void OnExited(WorkerConnection connection, WorkerExit exit)
+    {
+        ExitsCounter.Add(1, new KeyValuePair<string, object?>("shard", Label),
+            new KeyValuePair<string, object?>("reason", MetricReason(exit.Reason)));
+        lock (_gate)
+        {
+            if (ReferenceEquals(_current, connection)) _current = null;
+            if (!_disposed)
+            {
+                if (Stopwatch.GetElapsedTime(connection.ReadyTimestamp) >= HealthyResetAfter) _consecutiveFailures = 0;
+                ScheduleBackoffLocked();
+            }
+        }
+        if (exit.Reason != WorkerExitReason.Disposed)
+        {
+            _log.Warning("ScriptWorker[{Shard}] pid {Pid} ended: {Reason} (exit code {ExitCode}, culprit {Culprit}, {InFlight} evaluation(s) in flight)",
+                Label, connection.Pid, exit.Reason, exit.ExitCode, exit.CulpritKey ?? "none", exit.InFlightKeys.Count);
+        }
+        try { _onExit(this, exit); }
+        catch (Exception ex) { _log.Error(ex, "ScriptWorker[{Shard}] exit handler threw", Label); }
+        _ = connection.DisposeAsync().AsTask();
+    }
+
+    private void ScheduleBackoffLocked()
+    {
+        _consecutiveFailures++;
+        var delay = NextBackoff(_consecutiveFailures, _jitter.NextDouble());
+        _backoffUntil = Stopwatch.GetTimestamp() + (long)(delay.TotalSeconds * Stopwatch.Frequency);
+    }
+
+    /// <summary>The restart delay after <paramref name="failures"/> consecutive failures (design §5.3.5).</summary>
+    /// <param name="failures">1 for the first failure.</param>
+    /// <param name="unitJitter">A sample in [0, 1) mapped to ±20 %.</param>
+    /// <returns>The delay.</returns>
+    internal static TimeSpan NextBackoff(int failures, double unitJitter)
+    {
+        var exponent = Math.Clamp(failures - 1, 0, 30);
+        var baseMs = Math.Min(MaxBackoff.TotalMilliseconds, InitialBackoff.TotalMilliseconds * Math.Pow(2, exponent));
+        var factor = 0.8 + (0.4 * Math.Clamp(unitJitter, 0, 1));
+        return TimeSpan.FromMilliseconds(Math.Min(MaxBackoff.TotalMilliseconds * 1.2, baseMs * factor));
+    }
+
+    private static string MetricReason(WorkerExitReason reason) => reason switch
+    {
+        WorkerExitReason.Clean => "clean",
+        WorkerExitReason.KilledDeadline => "killed_deadline",
+        WorkerExitReason.KilledHeartbeat => "killed_heartbeat",
+        WorkerExitReason.KilledLoadTimeout => "killed_load",
+        WorkerExitReason.ProtocolFault => "protocol",
+        WorkerExitReason.Disposed => "disposed",
+        _ => "crash",
+    };
+
+    /// <summary>Stops the shard's worker.</summary>
+    /// <returns>A task that completes when the worker has ended.</returns>
+    public async ValueTask DisposeAsync()
+    {
+        WorkerConnection? current;
+        Task<WorkerConnection?>? starting;
+        lock (_gate)
+        {
+            _disposed = true;
+            current = _current;
+            starting = _starting;
+            _current = null;
+        }
+        if (starting is not null)
+        {
+            try { current ??= await starting.WaitAsync(TimeSpan.FromSeconds(15)).ConfigureAwait(false); } catch { /* shutting down */ }
+        }
+        if (current is not null) await current.DisposeAsync().ConfigureAwait(false);
+        IsolationGate?.Dispose();
+    }
+
+    /// <summary>Maps a context kind to the wire enum.</summary>
+    /// <param name="kind">The kind.</param>
+    /// <returns>The wire kind.</returns>
+    internal static ContextKind ToWire(ScriptContextKind kind) => kind switch
+    {
+        ScriptContextKind.VirtualTag => ContextKind.VirtualTag,
+        ScriptContextKind.Calculation => ContextKind.Calculation,
+        ScriptContextKind.AlarmPredicate => ContextKind.AlarmPredicate,
+        _ => ContextKind.Unspecified,
+    };
+}