feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1)

A timed-out script is abandoned, not stopped, and every seam re-ran it on the
next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1).
A shared ScriptRunGuard, keyed per script instance, now:

- refuses a run while that script's previous orphan is still executing
  (Bad "previous evaluation still running" + otopcua.script.run_refused);
- quarantines after 3 consecutive non-completions or an allocation-budget
  breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB):
  no further dispatch, Bad quality, one Error, otopcua.script.quarantined,
  cleared on source change / redeploy / reset;
- starts the budget clock when the pool picks the run up, so starvation is
  no longer charged to the script (R07:P5), and cancels the evaluator token
  on timeout.

Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async
EvaluateAsync; the driver's pump and timer ticks now await the run behind a
SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate
holds state and annotates the condition BadInternalError). ScriptingOptions
(Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts)
carries the knobs; the Calculation driver config accepts the same two keys.

Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with
a healthy sibling as positive control), per-seam refusal/quarantine/reset
tests. The S12 race test now uses one vtag id per loop: twelve concurrent
evaluations of ONE id is a shape production never produces and the guard
refuses by design.

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs

@@ -92,7 +92,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus
     // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync,
     // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse.
-    private readonly object _evalGate = new();
+    // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async —
+    // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's
+    // budget clock starts only once the pool has picked the script up, so pool starvation is never
+    // reported as a script timeout).
+    private readonly SemaphoreSlim _evalGate = new(1, 1);
+
+    // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight
+    // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false).
+    private readonly CancellationTokenSource _evalCts = new();
 
     // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single
     // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an
@@ -129,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         // _logger (ILogger<CalculationDriver>) implements the non-generic ILogger, so it is passed straight
         // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution
         // throws) land on the SAME logger the driver itself was constructed with, not a separate one.
-        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout);
+        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting);
 
         RebuildTagTable(_options);
     }
@@ -225,6 +233,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
                     {
                         RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [],
                         RunTimeout = _options.RunTimeout,
+                        Scripting = _options.Scripting,
                     };
                 }
             }
@@ -235,7 +244,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
                     _driverInstanceId);
             }
         }
-        return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout };
+        return new CalculationDriverOptions
+        {
+            RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting,
+        };
     }
 
     /// <summary>Re-bind the tag table from <paramref name="driverConfigJson"/> — shared by
@@ -491,13 +503,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     {
         if (_pumpScheduled) return;
         _pumpScheduled = true;
-        _ = Task.Run(RunEvaluationPump);
+        _ = Task.Run(RunEvaluationPumpAsync);
     }
 
-    /// <summary>Drain the queued change-triggered evaluations, in arrival order, on a pool thread. At most
-    /// one pump runs at a time, so per-tag evaluation ordering is preserved exactly as it was when the
-    /// change path evaluated inline under the state lock.</summary>
-    private void RunEvaluationPump()
+    /// <summary>Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a
+    /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated
+    /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread
+    /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured
+    /// false timeouts under pool starvation).</summary>
+    private async Task RunEvaluationPumpAsync()
     {
         while (true)
         {
@@ -515,39 +529,50 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
 
             // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still
             // evaluate one at a time while no host thread can block behind a script.
-            lock (_evalGate)
+            await _evalGate.WaitAsync().ConfigureAwait(false);
+            try
             {
-                try
-                {
-                    RunEvaluation(next);
-                }
-                catch (Exception ex)
-                {
-                    // The evaluator never throws for script faults, so this is a defect path only — it
-                    // must not kill the pump and strand every queued evaluation behind it.
-                    _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
-                        _driverInstanceId, next.Definition.RawPath);
-                }
+                await RunEvaluationAsync(next).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                // The evaluator never throws for script faults, so this is a defect path only — it
+                // must not kill the pump and strand every queued evaluation behind it.
+                _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
+                    _driverInstanceId, next.Definition.RawPath);
+            }
+            finally
+            {
+                _evalGate.Release();
             }
         }
     }
 
-    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag on the scheduler's
-    /// pool thread (the scheduler's per-group in-flight guard is what bounds outstanding timer work).</summary>
-    private void EvaluateTagFromTimer(string rawPath)
+    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's
+    /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task
+    /// completes).</summary>
+    private async Task EvaluateTagFromTimerAsync(string rawPath)
     {
         if (!_running) return;
         PendingEvaluation? prepared;
         lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow);
         if (prepared is null) return;
-        lock (_evalGate) RunEvaluation(prepared);
+        await _evalGate.WaitAsync().ConfigureAwait(false);
+        try
+        {
+            await RunEvaluationAsync(prepared).ConfigureAwait(false);
+        }
+        finally
+        {
+            _evalGate.Release();
+        }
     }
 
     /// <summary>Run one prepared evaluation and publish the outcome. MUST be called under
     /// <see cref="_evalGate"/> and NOT under <see cref="_lock"/>: the script run is the whole point of the
     /// split, and the script log + <see cref="OnDataChange"/> fan-out both invoke subscriber code.</summary>
     /// <param name="pending">The prepared evaluation.</param>
-    private void RunEvaluation(PendingEvaluation pending)
+    private async Task RunEvaluationAsync(PendingEvaluation pending)
     {
         var def = pending.Definition;
 
@@ -572,7 +597,8 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
 
         // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are
         // then the input's real ones rather than a fabricated Good stamped at evaluation time.
-        var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies);
+        var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
+            .ConfigureAwait(false);
 
         string? failureReason = null;
         (object? Value, uint StatusCode)? toPublish;
@@ -688,7 +714,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
     {
         var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray();
         if (timed.Length == 0) return;
-        _timers = new CalculationTimerScheduler(EvaluateTagFromTimer, _logger);
+        _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger);
         _timers.Start(timed);
     }
 
@@ -723,6 +749,9 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
         _running = false;
         StopTimers();
         lock (_lock) _pending.Clear();
+        // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop
+        // the compiled scripts. The CTS is not disposed: a pump mid-await may still read its token.
+        try { _evalCts.Cancel(); } catch (ObjectDisposedException) { }
         _evaluator.Dispose();
     }