perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution

src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Health/SiteHealthReport.cs

@@ -74,6 +74,25 @@ public record SiteHealthReport(
     /// </summary>
     public double? ScriptOldestBusyAgeSeconds { get; init; }
 
+    /// <summary>
+    /// WP3.1: script-execution worker threads the stuck-script watchdog has DETACHED and
+    /// replaced — a script wedged past its deadline plus grace in uninterruptible blocking
+    /// I/O, so the pool started a fresh thread and the wedged one will exit only when its
+    /// body finally returns. Point-in-time, refreshed by <c>ScriptSchedulerStatsReporter</c>.
+    /// Zero is the healthy state; a value that climbs and never drains means script bodies
+    /// are permanently consuming threads and the pool is being repeatedly rebuilt around them.
+    /// </summary>
+    public int DetachedScriptThreads { get; init; }
+
+    /// <summary>
+    /// WP3.1: per-interval count of script and alarm on-trigger runs SHED because
+    /// <c>MaxConcurrentRunsPerScript</c> runs were already in flight for that script. Raw
+    /// per-interval count (drained on collect) like <see cref="ScriptErrorCount"/>. A
+    /// sustained non-zero value means a trigger is firing faster than its script completes;
+    /// the shed itself is the designed back-pressure, not an error.
+    /// </summary>
+    public int ScriptRunShedCount { get; init; }
+
     // LocalDb 2-node replication of the consolidated site database (Phase 1).
     // Additive init properties for the same reason as the scheduler gauges above:
     // the positional constructor stays untouched. Refreshed on the site by

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs

@@ -25,6 +25,20 @@ public interface ISiteHealthCollector
     /// </summary>
     void IncrementDeadLetter();
 
+    /// <summary>
+    /// WP3.1: increments the per-interval count of script/alarm runs SHED because
+    /// <c>MaxConcurrentRunsPerScript</c> runs were already in flight for that script. A raw
+    /// per-interval count like the script/alarm error counters — a sustained non-zero value
+    /// means a trigger is firing faster than its script can complete. Every shed is counted
+    /// here even when its accompanying site event is rate-limited away.
+    /// Default interface implementation is a no-op so existing test fakes continue to
+    /// compile without per-fake updates.
+    /// </summary>
+    void IncrementScriptRunShed()
+    {
+        // Default no-op so test fakes do not need to be updated.
+    }
+
     /// <summary>
     /// Increment the per-interval count of
     /// <c>FallbackAuditWriter</c> primary failures. Bridged from the
@@ -173,7 +187,13 @@ public interface ISiteHealthCollector
     /// <param name="queueDepth">Script tasks waiting to run.</param>
     /// <param name="busyThreads">Worker threads currently executing a script.</param>
     /// <param name="oldestBusyAgeSeconds">Age (seconds) of the oldest in-flight script, or <c>null</c> when idle.</param>
-    void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
+    /// <param name="detachedThreads">
+    /// WP3.1: worker threads the stuck-script watchdog has detached and replaced because
+    /// their script wedged past its deadline plus grace, and which have not yet returned.
+    /// A non-zero, non-draining value means script bodies are permanently blocking threads.
+    /// </param>
+    void SetScriptSchedulerStats(
+        int queueDepth, int busyThreads, double? oldestBusyAgeSeconds, int detachedThreads = 0)
     {
         // Default no-op so test fakes do not need to be updated.
     }

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs

@@ -39,6 +39,12 @@ public class SiteHealthCollector : ISiteHealthCollector
     private int _scriptQueueDepth;
     private int _scriptBusyThreads;
     private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
+    // WP3.1: workers detached and replaced by the stuck-script watchdog and not yet exited
+    // (point-in-time, from the same reporter tick as the gauges above), and the per-interval
+    // count of runs shed by the per-script in-flight cap (reset on collect like the error
+    // counters, and restored by AddIntervalCounters when a report fails to send).
+    private int _scriptDetachedThreads;
+    private int _scriptRunShedCount;
     // WP2.6d: cumulative alarm-publish-queue drop count, refreshed by
     // SiteStreamAlarmDropReporter. Point-in-time (not reset on collect), like the
     // scheduler gauges above.
@@ -162,12 +168,20 @@ public class SiteHealthCollector : ISiteHealthCollector
     }
 
     /// <inheritdoc />
-    public void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
+    public void SetScriptSchedulerStats(
+        int queueDepth, int busyThreads, double? oldestBusyAgeSeconds, int detachedThreads = 0)
     {
         Interlocked.Exchange(ref _scriptQueueDepth, queueDepth);
         Interlocked.Exchange(ref _scriptBusyThreads, busyThreads);
         Interlocked.Exchange(ref _scriptOldestBusyAgeBits,
             BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN));
+        Interlocked.Exchange(ref _scriptDetachedThreads, detachedThreads);
+    }
+
+    /// <inheritdoc />
+    public void IncrementScriptRunShed()
+    {
+        Interlocked.Increment(ref _scriptRunShedCount);
     }
 
     /// <summary>Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null.</summary>
@@ -283,6 +297,11 @@ public class SiteHealthCollector : ISiteHealthCollector
             ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0),
             ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0),
             ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds(),
+            // WP3.1: point-in-time (like the three gauges above), so a CompareExchange read
+            // rather than an Exchange reset.
+            DetachedScriptThreads = Interlocked.CompareExchange(ref _scriptDetachedThreads, 0, 0),
+            // WP3.1: per-interval, so drained on collect like the error counters.
+            ScriptRunShedCount = Interlocked.Exchange(ref _scriptRunShedCount, 0),
             // Both fields come from the ONE snapshot read above. Null (the reporter has
             // not run) leaves both report fields null — "no data", not "disconnected
             // with an empty backlog".

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs

@@ -25,10 +25,10 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
 /// State (active/normal) is in memory only, NOT persisted.
 /// On restart: starts normal, re-evaluates from incoming values.
 ///
-/// AlarmExecutionActor CAN call Instance.CallScript() (ask to sibling Script Actor).
+/// An alarm on-trigger run CAN call Instance.CallScript() (ask to sibling Script Actor).
 ///         Instance scripts CANNOT call alarm on-trigger scripts (no Instance.CallAlarmScript API).
 ///
-/// Supervision: Resume on exception; AlarmExecutionActor stopped on exception.
+/// Supervision: Resume on exception (from the Instance Actor). This actor has no children.
 /// </summary>
 public class AlarmActor : ReceiveActor
 {
@@ -44,12 +44,39 @@ public class AlarmActor : ReceiveActor
     /// <summary>
     /// Script-execution scheduler seam (#18): the process-wide
     /// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so this
-    /// alarm's trigger-expression evaluation and spawned on-trigger scripts run on a
-    /// caller-owned pool. Resolved lazily at each use so the null (host) path is
-    /// unchanged.
+    /// alarm's launched on-trigger scripts run on a caller-owned pool. Resolved lazily at
+    /// each use so the null (host) path is unchanged.
+    ///
+    /// <para>WP3.1: trigger-expression evaluation no longer uses this scheduler — see
+    /// <see cref="_evalGate"/>. That split is the whole point of finding #4: an alarm whose
+    /// Expression trigger should raise in milliseconds must not queue behind blocking
+    /// script bodies.</para>
     /// </summary>
     private readonly ScriptExecutionScheduler? _scheduler;
 
+    /// <summary>
+    /// WP3.1 (finding #4): the concurrency gate for trigger-expression evaluation, or null
+    /// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
+    /// async work on the shared .NET thread pool behind this gate.
+    /// </summary>
+    private readonly TriggerEvalGate? _evalGate;
+
+    /// <summary>
+    /// WP3.1: on-trigger runs launched but not yet completed. Incremented at launch,
+    /// decremented on <see cref="AlarmExecutionCompleted"/>, which every terminal path
+    /// emits — including the launch-path catch. Touched only on the actor thread.
+    /// </summary>
+    private int _runsInFlight;
+
+    /// <summary>
+    /// WP3.1: when the last shed Warning site event was emitted for this alarm. Sheds are
+    /// always counted; the event is rate-limited to one per alarm per minute.
+    /// </summary>
+    private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
+
+    /// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
+    private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
+
     /// <summary>
     /// The optional site operational-event log, resolved once from
     /// <see cref="_serviceProvider"/> at construction and cached. The
@@ -84,7 +111,7 @@ public class AlarmActor : ReceiveActor
     /// <summary>
     /// The on-trigger script's per-script execution timeout in seconds,
     /// or null to use the global default. Forwarded to each spawned
-    /// <see cref="AlarmExecutionActor"/>, which applies <c>perScript ?? global</c>
+    /// <see cref="Scripts.ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
     /// (treating ≤ 0 as "use global"). The value comes from the referenced
     /// on-trigger script's <see cref="ResolvedScript.ExecutionTimeoutSeconds"/>.
     /// </summary>
@@ -125,10 +152,10 @@ public class AlarmActor : ReceiveActor
     /// <summary>
     /// Audit Log #23 (ParentExecutionId tag-cascade): the
     /// <c>parentExecutionId</c> handed to the most recently spawned
-    /// <see cref="AlarmExecutionActor"/> — i.e. the execution whose attribute
+    /// on-trigger run — i.e. the execution whose attribute
     /// write fired this alarm, or <c>null</c> when the firing change came from
     /// the Data Connection Layer (external data has no spawning execution).
-    /// The spawned actor builds its own <see cref="ScriptRuntimeContext"/>
+    /// The launched run builds its own <see cref="ScriptRuntimeContext"/>
     /// internally, so this is exposed for regression coverage of the cascade
     /// contract (mirrors <see cref="SeedAttributesReference"/>).
     /// </summary>
@@ -159,6 +186,7 @@ public class AlarmActor : ReceiveActor
     /// execution timeout in seconds (from its <see cref="ResolvedScript.ExecutionTimeoutSeconds"/>),
     /// or null/non-positive to use the global default.</param>
     /// <param name="scheduler">Optional script-execution scheduler override (#18); null uses the process-wide shared scheduler.</param>
+    /// <param name="evalGate">Optional trigger-expression concurrency gate override (WP3.1); null uses the process-wide shared gate.</param>
     public AlarmActor(
         string alarmName,
         string instanceName,
@@ -175,7 +203,9 @@ public class AlarmActor : ReceiveActor
         // Per-script timeout for the on-trigger script (null = global).
         int? onTriggerExecutionTimeoutSeconds = null,
         // Script-execution scheduler seam (#18); null uses the process-wide shared scheduler.
-        ScriptExecutionScheduler? scheduler = null)
+        ScriptExecutionScheduler? scheduler = null,
+        // WP3.1 trigger-eval gate seam; null uses the process-wide shared gate.
+        TriggerEvalGate? evalGate = null)
     {
         _alarmName = alarmName;
         _instanceName = instanceName;
@@ -186,6 +216,7 @@ public class AlarmActor : ReceiveActor
         _healthCollector = healthCollector;
         _serviceProvider = serviceProvider;
         _scheduler = scheduler;
+        _evalGate = evalGate;
         // Resolve the optional site event logger once and cache it,
         // rather than calling GetService on every alarm transition.
         _siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
@@ -217,9 +248,8 @@ public class AlarmActor : ReceiveActor
         // Handle attribute value changes
         Receive<AttributeValueChanged>(HandleAttributeValueChanged);
 
-        // Handle alarm execution completion
-        Receive<AlarmExecutionCompleted>(_ =>
-            _logger.LogDebug("Alarm {Alarm} execution completed on {Instance}", _alarmName, _instanceName));
+        // Handle alarm execution completion (also releases the WP3.1 in-flight slot)
+        Receive<AlarmExecutionCompleted>(HandleAlarmExecutionCompleted);
 
         // Handle the off-dispatcher trigger-expression evaluation result (P1).
         Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
@@ -238,20 +268,10 @@ public class AlarmActor : ReceiveActor
             _alarmName, _instanceName, _triggerType);
     }
 
-    /// <inheritdoc />
-    protected override SupervisorStrategy SupervisorStrategy()
-    {
-        return new OneForOneStrategy(
-            maxNrOfRetries: -1,
-            withinTimeRange: TimeSpan.FromMinutes(1),
-            decider: Decider.From(ex =>
-            {
-                _logger.LogWarning(ex,
-                    "AlarmExecutionActor for {Alarm} on {Instance} failed, stopping",
-                    _alarmName, _instanceName);
-                return Directive.Stop;
-            }));
-    }
+    // WP3.1: the Stop-on-failure SupervisorStrategy override is gone with the per-run
+    // AlarmExecutionActor child it supervised. This actor has no children left; a
+    // launch-path throw is caught in SpawnAlarmExecution so it can never escalate to
+    // InstanceActor, which continues to supervise this actor with Resume exactly as before.
 
     /// <summary>
     /// Evaluates alarm condition on attribute change. Alarm evaluation errors are logged,
@@ -345,7 +365,7 @@ public class AlarmActor : ReceiveActor
             // Operational `alarm` event — raise. Severity by priority.
             LogAlarmEvent(RaiseSeverity(_priority), $"Alarm {_alarmName} activated (priority {_priority})");
 
-            // Spawn AlarmExecutionActor if on-trigger script defined
+            // Launch the on-trigger run if an on-trigger script is defined
             if (_onTriggerCompiledScript != null)
             {
                 SpawnAlarmExecution(AlarmLevel.None, _priority, string.Empty, sourceExecutionId);
@@ -442,7 +462,7 @@ public class AlarmActor : ReceiveActor
     /// <see cref="ISiteEventLogger"/> (resolved once at construction and cached
     /// in <see cref="_siteEventLogger"/>). Never awaited so a logging failure
     /// cannot affect alarm evaluation (matching the established
-    /// ScriptActor/ScriptExecutionActor pattern).
+    /// ScriptActor / script-run pattern).
     /// </summary>
     private void LogAlarmEvent(string severity, string message)
     {
@@ -589,31 +609,50 @@ public class AlarmActor : ReceiveActor
         // later change arriving while the evaluation is in flight cannot
         // mis-attribute the raise this evaluation produces.
         var sourceExecutionId = _latestSourceExecutionId;
-        Task.Factory.StartNew(async () =>
+        var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
+        // WP3.1 (finding #4): the deadline clock starts AT ENQUEUE, on the actor thread —
+        // gate-wait time burns the same budget, so a saturated gate yields a timely false
+        // instead of an unbounded stall. And the work runs on the shared .NET thread pool,
+        // NOT the blocking script pool, so it can never queue behind a blocked script body.
+        var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
+
+        Task.Run(async () =>
         {
             try
             {
-                // Bound evaluation with a short timeout. The CancellationToken covers
-                // cooperative/async cases; a pathological CPU-bound expression is not
-                // fully interruptible — acceptable because trigger expressions are
-                // authored by trusted Design-role users and compile-checked pre-deploy.
-                using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
-                var state = await expression.RunAsync(new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
-                return state.ReturnValue is bool b && b;
+                await gate.WaitAsync(cts.Token).ConfigureAwait(false);
+                try
+                {
+                    // The CancellationToken covers cooperative/async cases; a pathological
+                    // CPU-bound expression is not fully interruptible — acceptable because
+                    // trigger expressions are authored by trusted Design-role users and
+                    // compile-checked pre-deploy.
+                    var state = await expression.RunAsync(
+                        new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
+                    return state.ReturnValue is bool b && b;
+                }
+                finally
+                {
+                    gate.Release();
+                }
             }
             catch (Exception ex)
             {
-                // OperationCanceledException (timeout) falls through here too and is
-                // treated as false. _healthCollector (Interlocked) and _logger are
-                // thread-safe, so this catch is safe off the actor thread.
+                // OperationCanceledException (timeout, INCLUDING one that fired while still
+                // waiting on the gate) falls through here too and is treated as false.
+                // _healthCollector (Interlocked) and _logger are thread-safe, so this catch
+                // is safe off the actor thread.
                 _healthCollector?.IncrementAlarmError();
                 _logger.LogError(ex,
                     "Alarm {Alarm} trigger expression evaluation failed on {Instance}; treated as false",
                     _alarmName, _instanceName);
                 return false;
             }
-        }, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
-            _scheduler ?? ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
+            finally
+            {
+                cts.Dispose();
+            }
+        }).PipeTo(self,
             success: r => new ExpressionEvalResult(r, sourceExecutionId),
             failure: ex => new ExpressionEvalFailed(ex, sourceExecutionId));
     }
@@ -705,9 +744,34 @@ public class AlarmActor : ReceiveActor
     }
 
     /// <summary>
-    /// Spawns an AlarmExecutionActor to run the on-trigger script.
+    /// WP3.1: releases the in-flight slot for a completed on-trigger run. Every terminal
+    /// path emits exactly one <see cref="AlarmExecutionCompleted"/> (success / timeout /
+    /// failure / launch failure), so the counter tracks reality.
+    /// </summary>
+    private void HandleAlarmExecutionCompleted(AlarmExecutionCompleted msg)
+    {
+        if (_runsInFlight > 0) _runsInFlight--;
+        _logger.LogDebug(
+            "Alarm {Alarm} execution completed on {Instance}: success={Success}",
+            _alarmName, _instanceName, msg.Success);
+    }
+
+    /// <summary>
+    /// WP3.1: on-trigger runs launched but not yet completed. Exposed for regression
+    /// coverage of the shed cap.
+    /// </summary>
+    internal int RunsInFlight => _runsInFlight;
+
+    /// <summary>
+    /// Launches the on-trigger script run.
     /// Passes the firing alarm's level/priority/message so the script can
     /// branch on severity via the <c>Alarm</c> global.
+    ///
+    /// <para>WP3.1: launched directly via <see cref="ScriptRunLauncher"/> rather than
+    /// through a short-lived <c>AlarmExecutionActor</c> child, and bounded by
+    /// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> — a raise arriving while
+    /// the cap is reached is shed (counted, rate-limited site event) rather than piling
+    /// another run onto a saturated pool.</para>
     /// </summary>
     /// <param name="level">The firing alarm severity level.</param>
     /// <param name="priority">The firing alarm priority.</param>
@@ -727,34 +791,82 @@ public class AlarmActor : ReceiveActor
     {
         if (_onTriggerCompiledScript == null) return;
 
-        var executionId = $"{_alarmName}-alarm-exec-{_executionCounter++}";
+        if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
+        {
+            ShedAlarmRun();
+            return;
+        }
+
+        var runId = _executionCounter++;
 
         // Record what the on-trigger run was parented to (null = root); read by
-        // the tag-cascade regression tests, which cannot see inside the child.
+        // the tag-cascade regression tests, which cannot observe the run directly.
         LastOnTriggerParentExecutionId = parentExecutionId;
 
-        // The on-trigger script body runs on the dedicated
-        // ScriptExecutionScheduler, not the shared .NET thread pool.
-        var props = Props.Create(() => new AlarmExecutionActor(
-            _alarmName,
-            _instanceName,
-            level,
-            priority,
-            message,
-            _onTriggerCompiledScript,
-            _instanceActor,
-            _sharedScriptLibrary,
-            _options,
-            _logger,
-            // Per-script timeout from the on-trigger script (null = global).
-            _onTriggerExecutionTimeoutSeconds,
-            // The firing execution's id — null for DCL-originated changes.
-            parentExecutionId,
-            // Scheduler seam (#18): share this alarm's scheduler override with the
-            // spawned on-trigger script body (null = process-wide shared).
-            _scheduler));
+        // Incremented BEFORE the launch so the launch-path catch below (which always emits
+        // an AlarmExecutionCompleted) balances it on every path.
+        _runsInFlight++;
 
-        Context.ActorOf(props, executionId);
+        try
+        {
+            // The on-trigger script body runs on the dedicated
+            // ScriptExecutionScheduler, not the shared .NET thread pool.
+            ScriptRunLauncher.LaunchAlarmScript(
+                _alarmName,
+                _instanceName,
+                level,
+                priority,
+                message,
+                _onTriggerCompiledScript,
+                _instanceActor,
+                _sharedScriptLibrary,
+                _options,
+                // Completion target: this actor. Identical delivery to the old
+                // Context.Parent.Tell from the execution actor.
+                Self,
+                _logger,
+                runId,
+                // Per-script timeout from the on-trigger script (null = global).
+                _onTriggerExecutionTimeoutSeconds,
+                // The firing execution's id — null for DCL-originated changes.
+                parentExecutionId,
+                // Scheduler seam (#18): share this alarm's scheduler override with the
+                // launched on-trigger script body (null = process-wide shared).
+                _scheduler);
+        }
+        catch (Exception ex)
+        {
+            // WP3.1 supervision parity: mirrors the removed OneForOneStrategy's warn-and-stop
+            // for a per-run child that failed to construct. The alarm continues.
+            _logger.LogWarning(ex,
+                "Alarm on-trigger execution launch for {Alarm} on {Instance} failed, stopping",
+                _alarmName, _instanceName);
+            Self.Tell(new AlarmExecutionCompleted(_alarmName, false));
+        }
+    }
+
+    /// <summary>
+    /// WP3.1 shed policy for alarm on-trigger runs: refuses the newest run when
+    /// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> are already in flight.
+    /// There is no Ask caller on this path (an on-trigger run is never awaited), so the
+    /// shed surfaces as a counter plus a rate-limited Warning site event.
+    /// </summary>
+    private void ShedAlarmRun()
+    {
+        _healthCollector?.IncrementScriptRunShed();
+
+        var message = $"Alarm on-trigger script for '{_alarmName}' on instance '{_instanceName}': run shed — " +
+                      $"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
+
+        _logger.LogWarning("{Message}", message);
+
+        var now = DateTimeOffset.UtcNow;
+        if (now - _lastShedEventUtc >= ShedEventInterval)
+        {
+            _lastShedEventUtc = now;
+            _ = _siteEventLogger?.LogEventAsync(
+                "script", "Warning", _instanceName, $"AlarmActor:{_alarmName}", message);
+        }
     }
 
     private AlarmEvalConfig ParseEvalConfig(string? triggerConfigJson)

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmExecutionActor.cs

@@ -1,167 +0,0 @@
-using Akka.Actor;
-using Microsoft.CodeAnalysis.Scripting;
-using Microsoft.Extensions.Logging;
-using ZB.MOM.WW.ScadaBridge.Commons.Types;
-using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
-using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
-using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
-
-namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
-
-/// <summary>
-/// Alarm Execution Actor -- short-lived child of Alarm Actor.
-/// Same pattern as ScriptExecutionActor.
-/// CAN call Instance.CallScript() (ask to sibling Script Actor).
-///         Instance scripts CANNOT call alarm on-trigger scripts (no API for it).
-/// Supervision: Stop on unhandled exception.
-/// </summary>
-public class AlarmExecutionActor : ReceiveActor
-{
-    /// <summary>Initializes a new <see cref="AlarmExecutionActor"/> and immediately schedules execution of the alarm on-trigger script.</summary>
-    /// <param name="alarmName">The canonical name of the alarm that triggered.</param>
-    /// <param name="instanceName">The name of the owning instance.</param>
-    /// <param name="level">The alarm severity level at the time of triggering.</param>
-    /// <param name="priority">The alarm priority value.</param>
-    /// <param name="message">The alarm message to pass to the script.</param>
-    /// <param name="compiledScript">The pre-compiled on-trigger script to execute.</param>
-    /// <param name="instanceActor">Reference to the parent instance actor for attribute/script calls.</param>
-    /// <param name="sharedScriptLibrary">Shared script library providing common utilities.</param>
-    /// <param name="options">Site runtime configuration options, including the execution timeout.</param>
-    /// <param name="logger">Logger for execution diagnostics.</param>
-    /// <param name="executionTimeoutSeconds">The on-trigger script's per-script execution timeout in seconds. Null or non-positive falls back to the global <see cref="SiteRuntimeOptions.ScriptExecutionTimeoutSeconds"/>.</param>
-    /// <param name="parentExecutionId">
-    /// ParentExecutionId tag-cascade: the <c>ExecutionId</c> of
-    /// the execution whose attribute write fired this alarm, threaded into the
-    /// on-trigger script's <see cref="ScriptRuntimeContext"/> as its
-    /// <c>ParentExecutionId</c> so the alarm-triggered run chains under its
-    /// firing execution. Null when the firing value came from the Data
-    /// Connection Layer (external data has no spawning execution) — that
-    /// on-trigger run is a tree root.
-    /// </param>
-    public AlarmExecutionActor(
-        string alarmName,
-        string instanceName,
-        AlarmLevel level,
-        int priority,
-        string message,
-        Script<object?> compiledScript,
-        IActorRef instanceActor,
-        SharedScriptLibrary sharedScriptLibrary,
-        SiteRuntimeOptions options,
-        ILogger logger,
-        // Per-script execution timeout override (seconds) for the
-        // alarm on-trigger script. Null or non-positive falls back to the global.
-        int? executionTimeoutSeconds = null,
-        // The firing context's execution id (null today).
-        Guid? parentExecutionId = null,
-        // Script-execution scheduler seam (#18): the process-wide scheduler by
-        // default; null selects the shared default.
-        ScriptExecutionScheduler? scheduler = null)
-    {
-        var self = Self;
-        var parent = Context.Parent;
-
-        ExecuteAlarmScript(
-            alarmName, instanceName, level, priority, message,
-            compiledScript, instanceActor,
-            sharedScriptLibrary, options, self, parent, logger,
-            executionTimeoutSeconds, parentExecutionId, scheduler);
-    }
-
-    private static void ExecuteAlarmScript(
-        string alarmName,
-        string instanceName,
-        AlarmLevel level,
-        int priority,
-        string message,
-        Script<object?> compiledScript,
-        IActorRef instanceActor,
-        SharedScriptLibrary sharedScriptLibrary,
-        SiteRuntimeOptions options,
-        IActorRef self,
-        IActorRef parent,
-        ILogger logger,
-        int? executionTimeoutSeconds,
-        Guid? parentExecutionId,
-        ScriptExecutionScheduler? scheduler)
-    {
-        // Per-script timeout overrides the global default. A null or
-        // non-positive per-script value (≤ 0) falls back to the global.
-        var timeout = TimeSpan.FromSeconds(
-            executionTimeoutSeconds is { } perScript && perScript > 0
-                ? perScript
-                : options.ScriptExecutionTimeoutSeconds);
-
-        // Run the alarm on-trigger body on the dedicated
-        // script-execution scheduler, not the shared .NET thread pool. An injected
-        // scheduler (#18) overrides the process-wide default.
-        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
-
-        _ = Task.Factory.StartNew(async () =>
-        {
-            using var cts = new CancellationTokenSource(timeout);
-            try
-            {
-                // AlarmExecutionActor can call Instance.CallScript()
-                // via the ScriptRuntimeContext injected into globals
-                var context = new ScriptRuntimeContext(
-                    instanceActor,
-                    self,
-                    sharedScriptLibrary,
-                    currentCallDepth: 0,
-                    options.MaxScriptCallDepth,
-                    timeout,
-                    instanceName,
-                    logger,
-                    // ParentExecutionId tag-cascade: the
-                    // alarm on-trigger run mints its own fresh ExecutionId (the
-                    // ctor's `?? NewGuid()` fallback) and records the firing
-                    // execution's id as its ParentExecutionId — null (a root)
-                    // only when the firing value came from the DCL.
-                    parentExecutionId: parentExecutionId,
-                    // WaitForAttribute (spec §4.4): thread the alarm on-trigger
-                    // script's per-script execution-timeout token so a
-                    // Attributes.WaitAsync inside an on-trigger script is bounded
-                    // by the same script deadline.
-                    scriptTimeoutToken: cts.Token);
-
-                var globals = new ScriptGlobals
-                {
-                    Instance = context,
-                    Parameters = new ScriptParameters(),
-                    CancellationToken = cts.Token,
-                    Alarm = new AlarmContext
-                    {
-                        Name = alarmName,
-                        Level = level,
-                        Priority = priority,
-                        Message = message
-                    }
-                };
-
-                await compiledScript.RunAsync(globals, cts.Token);
-
-                parent.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
-            }
-            catch (OperationCanceledException)
-            {
-                logger.LogWarning(
-                    "Alarm on-trigger script for {Alarm} on {Instance} timed out",
-                    alarmName, instanceName);
-                parent.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
-            }
-            catch (Exception ex)
-            {
-                // Failures logged, alarm continues
-                logger.LogError(ex,
-                    "Alarm on-trigger script for {Alarm} on {Instance} failed",
-                    alarmName, instanceName);
-                parent.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
-            }
-            finally
-            {
-                self.Tell(PoisonPill.Instance);
-            }
-        }, CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler).Unwrap();
-    }
-}

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -142,6 +142,23 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// </summary>
     private readonly HashSet<string> _initFailedPendingRowRemoval = new();
 
+    /// <summary>
+    /// WP3.1 warm-then-gate: instances whose deploy compile is being warmed off the actor
+    /// thread, keyed by instance name. Presence is the per-instance in-flight guard that keeps
+    /// same-instance command ordering intact while OTHER instances' commands flow freely.
+    /// Entries are added in <see cref="HandleDeploy"/> and removed in
+    /// <see cref="HandleDeployCompileWarmed"/> — one or the other always runs, because the
+    /// warm task pipes its result back unconditionally (it swallows its own exceptions).
+    /// </summary>
+    private readonly Dictionary<string, DeployWarmState> _deployWarms = new();
+
+    /// <summary>
+    /// WP3.1: script-execution scheduler seam (#18) — the pool this actor grows as instances
+    /// deploy. Null selects the process-wide shared scheduler; tests inject their own so
+    /// sizing assertions do not perturb (or depend on) the process-wide singleton.
+    /// </summary>
+    private readonly ScriptExecutionScheduler? _scriptScheduler;
+
     /// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
     public ITimerScheduler Timers { get; set; } = null!;
 
@@ -172,6 +189,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// external-system changes. Optional/null in tests that do not exercise external-system
     /// caching.
     /// </param>
+    /// <param name="scriptScheduler">
+    /// WP3.1: optional script-execution scheduler override (#18). This actor grows the pool
+    /// towards <see cref="ScriptExecutionScheduler.ComputeTargetThreads"/> on every instance-
+    /// count change; null uses the process-wide shared scheduler.
+    /// </param>
     public DeploymentManagerActor(
         SiteStorageService storage,
         ScriptCompilationService compilationService,
@@ -186,8 +208,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         IDeploymentConfigFetcher? configFetcher = null,
         TimeSpan? startupLoadRetryInterval = null,
         Func<Task<List<DeployedInstance>>>? configLoader = null,
-        ExternalSystemDefinitionCache? externalSystemCache = null)
+        ExternalSystemDefinitionCache? externalSystemCache = null,
+        ScriptExecutionScheduler? scriptScheduler = null)
     {
+        _scriptScheduler = scriptScheduler;
         _storage = storage;
         _compilationService = compilationService;
         _deployCompileValidator = new DeployCompileValidator(compilationService);
@@ -212,9 +236,16 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
         // Lifecycle commands
         Receive<DeployInstanceCommand>(cmd => HandleDeploy(cmd, Sender));
-        Receive<DisableInstanceCommand>(HandleDisable);
-        Receive<EnableInstanceCommand>(HandleEnable);
-        Receive<DeleteInstanceCommand>(HandleDelete);
+        Receive<DisableInstanceCommand>(cmd => HandleDisable(cmd, Sender));
+        Receive<EnableInstanceCommand>(cmd => HandleEnable(cmd, Sender));
+        Receive<DeleteInstanceCommand>(cmd => HandleDelete(cmd, Sender));
+
+        // WP3.1 warm-then-gate: the off-thread compile warm for a deploy has finished, so
+        // the (now all-cache-hit) synchronous compile gate can run on the actor thread.
+        Receive<DeployCompileWarmed>(HandleDeployCompileWarmed);
+        // WP3.1: a staggered startup batch's compiles have been pre-warmed off-thread;
+        // create the batch's Instance Actors now that their PreStart compiles are hits.
+        Receive<BatchCompileWarmed>(msg => CreateInstanceActorBatch(msg.Batch));
 
         // Notify-and-fetch: central sends a small RefreshDeploymentCommand;
         // the active singleton fetches the flattened config over HTTP, then reuses the
@@ -461,8 +492,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     }
 
     /// <summary>
-    /// Creates Instance Actors in batches with a configurable delay between batches
-    /// to prevent reconnection storms on failover.
+    /// WP3.1: pre-warms one staggered startup batch's script and trigger-expression compiles
+    /// off the actor thread, then pipes <see cref="BatchCompileWarmed"/> so
+    /// <see cref="CreateInstanceActorBatch"/> can create the batch's Instance Actors with
+    /// every <c>PreStart</c> compile already a cache hit.
+    ///
+    /// <para>This closes the long-standing "per-instance compilation during staggered startup"
+    /// gap: on failover, each Instance Actor used to Roslyn-compile its own scripts inside
+    /// <c>PreStart</c>, serialising a site's whole recovery behind compilation. The cost is one
+    /// extra message per batch.</para>
     /// </summary>
     private void HandleStartNextBatch(StartNextBatch msg)
     {
@@ -471,6 +509,45 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         var startIdx = state.NextIndex;
         var endIdx = Math.Min(startIdx + batchSize, state.Configs.Count);
 
+        var validator = _deployCompileValidator;
+        var batchConfigs = new string[endIdx - startIdx];
+        for (var i = startIdx; i < endIdx; i++)
+            batchConfigs[i - startIdx] = state.Configs[i].ConfigJson;
+
+        Task.Run(() =>
+        {
+            foreach (var configJson in batchConfigs)
+            {
+                try
+                {
+                    // Verdict discarded: startup does not gate on compile failures (a failing
+                    // script is logged by the Instance Actor and leaves the rest running).
+                    // This call exists only to populate SiteScriptCompileCache.
+                    validator.Validate(configJson);
+                }
+                catch
+                {
+                    // Best-effort warm — a throw here just means that instance's PreStart
+                    // compiles the hard way, exactly as it did before WP3.1.
+                }
+            }
+
+            return new BatchCompileWarmed(msg);
+        }).PipeTo(Self);
+    }
+
+    /// <summary>
+    /// Creates one batch's Instance Actors (after its compiles have been pre-warmed) and
+    /// schedules the next batch, with a configurable delay between batches to prevent
+    /// reconnection storms on failover.
+    /// </summary>
+    private void CreateInstanceActorBatch(StartNextBatch msg)
+    {
+        var state = msg.State;
+        var batchSize = _options.StartupBatchSize;
+        var startIdx = state.NextIndex;
+        var endIdx = Math.Min(startIdx + batchSize, state.Configs.Count);
+
         _logger.LogDebug(
             "Creating Instance Actors batch [{Start}..{End}) of {Total}",
             startIdx, endIdx, state.Configs.Count);
@@ -520,23 +597,164 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// apply still replies to the right actor.
     /// </param>
     private void HandleDeploy(DeployInstanceCommand command, IActorRef replyTo)
+    {
+        var instanceName = command.InstanceUniqueName;
+
+        // WP3.1 warm-then-gate. The S3 gate below must stay synchronous on the actor thread
+        // (mailbox FIFO is what makes redeploy-supersede and delete-during-redeploy correct),
+        // but the Roslyn compile it performs used to hold the whole singleton — every OTHER
+        // instance's commands stalled behind one instance's compile. So the compile is WARMED
+        // off-thread first and the synchronous gate then re-runs as pure cache hits.
+        //
+        // Ordering is preserved by an explicit per-instance in-flight guard: while a warm is
+        // in flight for instance X, further mutating commands for X are queued here rather
+        // than racing ahead. Commands for OTHER instances flow freely, which is the whole
+        // point — and safe, because cross-instance ordering was never guaranteed to callers.
+        if (_deployWarms.TryGetValue(instanceName, out var warming))
+        {
+            // Nothing queued behind the pending deploy yet: plain last-write-wins, and the
+            // displaced deployer is told it was superseded so it never waits out its Ask —
+            // the same contract as the mid-termination redeploy buffer below.
+            if (warming.Buffered.Count == 0)
+            {
+                warming.ReplyTo.Tell(new DeploymentStatusResponse(
+                    warming.Command.DeploymentId, instanceName, DeploymentStatus.Failed,
+                    $"superseded by newer deployment {command.DeploymentId} before the site compile gate ran",
+                    DateTimeOffset.UtcNow));
+                warming.Command = command;
+                warming.ReplyTo = replyTo;
+                // Warm the NEW config; the in-flight warm's result is recognised as stale by
+                // the reference check in HandleDeployCompileWarmed and dropped.
+                StartDeployCompileWarm(command);
+                return;
+            }
+
+            // A delete/disable/enable is already queued behind the pending deploy —
+            // superseding now would reorder it past this deploy, so queue instead.
+            warming.Buffered.Add(new BufferedInstanceCommand(command, replyTo));
+            return;
+        }
+
+        _deployWarms[instanceName] = new DeployWarmState(command, replyTo);
+        StartDeployCompileWarm(command);
+    }
+
+    /// <summary>
+    /// Compiles the deployment's scripts off the actor thread purely to populate the
+    /// process-wide <see cref="SiteScriptCompileCache"/>, then pipes
+    /// <see cref="DeployCompileWarmed"/> back so the authoritative gate can run on the actor
+    /// thread against a warm cache. The warm is best-effort: its verdict is discarded and any
+    /// exception swallowed, because <see cref="RunDeployGateAndProceed"/> re-derives the
+    /// verdict (and reproduces any failure) exactly as it did before WP3.1.
+    /// </summary>
+    private void StartDeployCompileWarm(DeployInstanceCommand command)
+    {
+        var validator = _deployCompileValidator;
+        var configJson = command.FlattenedConfigurationJson;
+        Task.Run(() =>
+        {
+            try
+            {
+                validator.Validate(configJson);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex,
+                    "Deploy compile warm for {Instance} threw; the synchronous gate will re-run it",
+                    command.InstanceUniqueName);
+            }
+
+            return new DeployCompileWarmed(command);
+        }).PipeTo(Self);
+    }
+
+    /// <summary>
+    /// Runs the authoritative compile gate for a warmed deployment on the actor thread, then
+    /// drains any commands queued for that instance during the warm — inline and in arrival
+    /// order, NOT by re-telling <c>Self</c>, which would put them behind messages that landed
+    /// in the mailbox during the warm and so reorder same-instance commands.
+    /// </summary>
+    private void HandleDeployCompileWarmed(DeployCompileWarmed msg)
+    {
+        var instanceName = msg.Command.InstanceUniqueName;
+        if (!_deployWarms.TryGetValue(instanceName, out var warming))
+            return;
+
+        // A newer deploy superseded this one while its warm was running; that supersede
+        // started its own warm, so this result is stale and must not apply a dead command.
+        if (!ReferenceEquals(warming.Command, msg.Command))
+            return;
+
+        _deployWarms.Remove(instanceName);
+
+        RunDeployGateAndProceed(warming.Command, warming.ReplyTo);
+
+        foreach (var queued in warming.Buffered)
+            DispatchBufferedCommand(queued);
+    }
+
+    /// <summary>
+    /// Queues a mutating lifecycle command for an instance whose deploy is mid-compile-warm.
+    /// Returns <see langword="true"/> when the command was queued (the caller must return
+    /// immediately), <see langword="false"/> when no warm is in flight and the caller should
+    /// proceed normally.
+    /// </summary>
+    private bool TryBufferDuringDeployWarm(string instanceName, object command, IActorRef replyTo)
+    {
+        if (!_deployWarms.TryGetValue(instanceName, out var warming)) return false;
+        warming.Buffered.Add(new BufferedInstanceCommand(command, replyTo));
+        return true;
+    }
+
+    /// <summary>
+    /// Re-dispatches one command queued during a deploy compile warm, restoring the original
+    /// sender. A queued deploy legitimately opens a NEW warm window, in which case the
+    /// commands after it re-queue behind that one — ordering still holds.
+    /// </summary>
+    private void DispatchBufferedCommand(BufferedInstanceCommand queued)
+    {
+        switch (queued.Command)
+        {
+            case DeployInstanceCommand deploy:
+                HandleDeploy(deploy, queued.Sender);
+                break;
+            case DeleteInstanceCommand delete:
+                HandleDelete(delete, queued.Sender);
+                break;
+            case DisableInstanceCommand disable:
+                HandleDisable(disable, queued.Sender);
+                break;
+            case EnableInstanceCommand enable:
+                HandleEnable(enable, queued.Sender);
+                break;
+            default:
+                _logger.LogWarning(
+                    "Unhandled buffered command type {Type} for a deploy compile warm — dropped",
+                    queued.Command.GetType().Name);
+                break;
+        }
+    }
+
+    /// <summary>
+    /// The authoritative site-side compile gate (S3) plus the deploy application. Reached
+    /// only from <see cref="HandleDeployCompileWarmed"/>, i.e. after the same compile has
+    /// been warmed off-thread.
+    /// </summary>
+    private void RunDeployGateAndProceed(DeployInstanceCommand command, IActorRef replyTo)
     {
         // Site-side compile gate (S3): a compile failure must reject the
         // deployment with NO partial state applied (no Instance Actor, no
-        // persisted config) — the design spec's contract. Validation runs
-        // synchronously on the actor thread: it is a pure prefix step of the
-        // deploy handler, so the existing redeploy-supersede / delete-during-
-        // redeploy ordering (which depends on strict mailbox FIFO) is preserved
-        // exactly. It is NOT run off-thread — piping the verdict back to self
-        // reorders concurrent deploys relative to each other and to
-        // delete/disable commands, breaking that ordering. A deploy is an
-        // infrequent admin command, so briefly holding the singleton for a pure
-        // Roslyn compile is acceptable; the central deployer already Asks and
-        // waits for the DeploymentStatusResponse. Redeploys and multi-instance
-        // deploys of unchanged scripts hit the process-wide compile cache
-        // (SiteScriptCompileCache), so the synchronous gate recompiles only
-        // genuinely new code and the Instance Actor's PreStart reuses the gate's
-        // compile (N4).
+        // persisted config) — the design spec's contract. Validation still runs
+        // synchronously on the actor thread as a pure prefix step of the deploy
+        // application, so the existing redeploy-supersede / delete-during-redeploy
+        // ordering (which depends on strict mailbox FIFO) is preserved exactly.
+        // WP3.1 removed the COST rather than the ordering: the same compile has
+        // already been warmed off-thread (StartDeployCompileWarm), so this call is
+        // pure cache hits and no longer holds the singleton — while the per-instance
+        // warm guard keeps this instance's own command ordering intact. Redeploys and
+        // multi-instance deploys of unchanged scripts hit the process-wide compile
+        // cache (SiteScriptCompileCache) too, and the Instance Actor's PreStart reuses
+        // the gate's compile (N4).
         var errors = _deployCompileValidator.Validate(command.FlattenedConfigurationJson);
         if (errors.Count > 0)
         {
@@ -942,10 +1160,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// <summary>
     /// Disables an instance: stops the actor and marks as disabled in SQLite.
     /// </summary>
-    private void HandleDisable(DisableInstanceCommand command)
+    private void HandleDisable(DisableInstanceCommand command, IActorRef replyTo)
     {
         var instanceName = command.InstanceUniqueName;
 
+        // WP3.1 warm-then-gate: a deploy for this instance is mid-compile-warm; queue this
+        // command so same-instance ordering is preserved (see TryBufferDuringDeployWarm).
+        if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
+
         // A disable arriving mid-redeploy must cancel the buffered
         // redeploy. Otherwise HandleTerminated re-creates the Instance Actor and
         // re-stores its config with isEnabled: true when the predecessor terminates,
@@ -975,7 +1197,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
         UpdateInstanceCounts();
 
-        var sender = Sender;
+        var sender = replyTo;
         _storage.SetInstanceEnabledAsync(instanceName, false).ContinueWith(t =>
         {
             if (t.IsCompletedSuccessfully)
@@ -1005,10 +1227,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// Enables an instance: marks as enabled in SQLite and re-creates the Instance Actor
     /// from the stored config.
     /// </summary>
-    private void HandleEnable(EnableInstanceCommand command)
+    private void HandleEnable(EnableInstanceCommand command, IActorRef replyTo)
     {
         var instanceName = command.InstanceUniqueName;
-        var sender = Sender;
+
+        // WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
+        if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
+
+        var sender = replyTo;
 
         Task.Run(async () =>
         {
@@ -1061,10 +1287,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// Deletes an instance: stops the actor and removes config from SQLite.
     /// Note: store-and-forward messages are NOT cleared per design decision.
     /// </summary>
-    private void HandleDelete(DeleteInstanceCommand command)
+    private void HandleDelete(DeleteInstanceCommand command, IActorRef replyTo)
     {
         var instanceName = command.InstanceUniqueName;
 
+        // WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
+        if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
+
         // A delete arriving while a redeploy is still terminating must
         // be authoritative over the mid-redeploy bookkeeping. HandleDeploy already
         // removed the instance from _instanceActors and buffered a PendingRedeploy
@@ -1105,7 +1334,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
         _deployedInstanceNames.Remove(instanceName);
         UpdateInstanceCounts();
 
-        var sender = Sender;
+        var sender = replyTo;
         _storage.RemoveDeployedConfigAsync(instanceName).ContinueWith(t =>
         {
             if (t.IsCompletedSuccessfully)
@@ -1135,7 +1364,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     /// Fire-and-forget a <c>deployment</c> operational event to the optional
     /// <see cref="ISiteEventLogger"/> on a deploy/enable/disable/delete outcome.
     /// Resolved optionally and never awaited so a logging failure cannot affect the
-    /// deployment pipeline (matching the established ScriptActor/ScriptExecutionActor
+    /// deployment pipeline (matching the established ScriptActor / script-run
     /// pattern).
     /// <para>
     /// <b>Thread-safety:</b> the disable (<see cref="HandleDisable"/>) and delete
@@ -2089,7 +2318,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     internal int InstanceActorCount => _instanceActors.Count;
 
     /// <summary>
-    /// Updates the health collector with current instance counts.
+    /// Updates the health collector with current instance counts, and (WP3.1) grows the
+    /// script-execution pool to match.
     /// Total deployed = _deployedInstanceNames.Count, enabled = running actors, disabled = difference.
     /// </summary>
     private void UpdateInstanceCounts()
@@ -2098,6 +2328,22 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
             deployed: _deployedInstanceNames.Count,
             enabled: _instanceActors.Count,
             disabled: _deployedInstanceNames.Count - _instanceActors.Count);
+
+        // WP3.1: the blocking script pool scales with the number of running instances rather
+        // than sitting at a fixed 8 forever. This is the single call site because it already
+        // runs on every deploy / undeploy / enable / disable and once per staggered startup
+        // batch. Growth is idempotent and one-way — see ScriptExecutionScheduler.EnsureCapacity.
+        try
+        {
+            var target = ScriptExecutionScheduler.ComputeTargetThreads(_instanceActors.Count, _options);
+            (_scriptScheduler ?? ScriptExecutionScheduler.Shared(_options)).EnsureCapacity(target);
+        }
+        catch (Exception ex)
+        {
+            // Pool sizing is an optimisation, never a correctness requirement: a failure here
+            // must not fail the deploy/enable/disable that triggered it.
+            _logger.LogWarning(ex, "Failed to resize the script-execution pool; continuing at its current size.");
+        }
     }
 
     // ── Internal messages ──
@@ -2131,6 +2377,43 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     internal record SharedScriptsLoaded(
         List<DeployedInstance> EnabledConfigs, int CompiledCount, int TotalCount);
 
+    /// <summary>
+    /// WP3.1 warm-then-gate: piped back to self once a deployment's scripts have been
+    /// compiled off the actor thread into the process-wide compile cache. Carries the exact
+    /// command instance it warmed so a result superseded mid-warm is recognised by reference
+    /// and dropped.
+    /// </summary>
+    internal sealed record DeployCompileWarmed(DeployInstanceCommand Command);
+
+    /// <summary>
+    /// WP3.1: piped back to self once a staggered startup batch's compiles have been
+    /// pre-warmed, carrying the original batch message so actor creation resumes unchanged.
+    /// </summary>
+    internal sealed record BatchCompileWarmed(StartNextBatch Batch);
+
+    /// <summary>
+    /// A lifecycle command queued because its instance's deploy was mid-compile-warm, with
+    /// the sender to answer once it is re-dispatched.
+    /// </summary>
+    internal sealed record BufferedInstanceCommand(object Command, IActorRef Sender);
+
+    /// <summary>
+    /// The in-flight state of one instance's deploy compile warm: the deploy that will be
+    /// applied when the warm lands (replaceable last-write-wins while nothing is queued behind
+    /// it), the deployer to answer, and the ordered commands queued during the warm.
+    /// </summary>
+    internal sealed class DeployWarmState(DeployInstanceCommand command, IActorRef replyTo)
+    {
+        /// <summary>The deploy to apply when the warm completes.</summary>
+        public DeployInstanceCommand Command { get; set; } = command;
+
+        /// <summary>The deployer awaiting this deploy's <see cref="DeploymentStatusResponse"/>.</summary>
+        public IActorRef ReplyTo { get; set; } = replyTo;
+
+        /// <summary>Commands for this instance that arrived during the warm, in arrival order.</summary>
+        public List<BufferedInstanceCommand> Buffered { get; } = [];
+    }
+
     internal record StartNextBatch(BatchState State);
     internal record BatchState(List<DeployedInstance> Configs, int NextIndex);
     internal record EnableResult(

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/InstanceActor.cs

@@ -374,7 +374,7 @@ public class InstanceActor : ReceiveActor
     /// Fire-and-forget an <c>instance_lifecycle</c> operational event to the
     /// optional <see cref="ISiteEventLogger"/>. Resolved optionally and never
     /// awaited so a logging failure cannot affect the instance lifecycle
-    /// (matching the established ScriptActor/ScriptExecutionActor pattern).
+    /// (matching the established ScriptActor / script-run pattern).
     /// </summary>
     private void LogLifecycleEvent(string message)
     {
@@ -1666,8 +1666,8 @@ public class InstanceActor : ReceiveActor
         {
             Script<object?>? onTriggerScript = null;
             // The on-trigger script's per-script execution timeout,
-            // captured from its ResolvedScript so the AlarmExecutionActor can
-            // apply perScript ?? global. Null when there is no on-trigger script.
+            // captured from its ResolvedScript so the launched on-trigger run
+            // can apply perScript ?? global. Null when there is no on-trigger script.
             int? onTriggerTimeoutSeconds = null;
 
             // Compile on-trigger script if defined

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/NativeAlarmActor.cs

@@ -444,7 +444,7 @@ public class NativeAlarmActor : ReceiveActor
     /// condition's severity); an inactive condition is a return-to-normal; an
     /// acknowledge transition is informational. Resolved optionally and never
     /// awaited so a logging failure cannot affect the mirror (matching the
-    /// established ScriptActor/ScriptExecutionActor pattern).
+    /// established ScriptActor / script-run pattern).
     /// </summary>
     private void LogAlarmEvent(NativeAlarmTransition t, AlarmConditionState condition)
     {

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptActor.cs

@@ -15,8 +15,16 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
 
 /// <summary>
 /// Script Actor — coordinator actor, child of Instance Actor.
-/// Holds compiled script delegate, manages trigger configuration, and spawns
-/// ScriptExecutionActor children per invocation. Does not block on child completion.
+/// Holds compiled script delegate, manages trigger configuration, and launches script
+/// runs per invocation. Does not block on run completion.
+///
+/// <para>WP3.1: runs are launched directly via <see cref="ScriptRunLauncher"/> rather than
+/// through a short-lived <c>ScriptExecutionActor</c> child (which had no <c>Receive</c>
+/// handler, no <c>PostStop</c>, and whose <c>IActorRef</c> was never a message target — pure
+/// per-run actor-cell overhead). Concurrent runs are bounded by
+/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; over the cap the NEWEST
+/// trigger is shed. Trigger-expression evaluation no longer runs on the blocking script pool
+/// at all — see <see cref="TriggerEvalGate"/>.</para>
 ///
 /// Trigger types:
 ///   - Interval: uses Akka timers to fire periodically
@@ -43,20 +51,52 @@ public class ScriptActor : ReceiveActor, IWithTimers
     /// <summary>
     /// Script-execution scheduler seam (#18): the process-wide
     /// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so
-    /// this actor's trigger-expression evaluation and spawned script bodies run on a
-    /// caller-owned pool instead of the shared one. Resolved lazily at each use so the
-    /// null (host) path stays byte-for-byte identical to the previous static call.
+    /// this actor's launched script bodies run on a caller-owned pool instead of the
+    /// shared one. Resolved lazily at each use so the null (host) path stays
+    /// byte-for-byte identical to the previous static call.
+    ///
+    /// <para>WP3.1: trigger-expression evaluation no longer uses this scheduler — see
+    /// <see cref="_evalGate"/>.</para>
     /// </summary>
     private readonly ScriptExecutionScheduler? _scheduler;
 
+    /// <summary>
+    /// WP3.1 (finding #4): the concurrency gate for trigger-expression evaluation, or null
+    /// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
+    /// async work on the shared .NET thread pool behind this gate, NOT on
+    /// <see cref="_scheduler"/> — that is what stops an Expression trigger from queueing
+    /// behind blocking script bodies.
+    /// </summary>
+    private readonly TriggerEvalGate? _evalGate;
+
+    /// <summary>
+    /// WP3.1: runs launched but not yet completed (queued or executing) for this script.
+    /// Incremented at launch, decremented on <see cref="ScriptExecutionCompleted"/>, which
+    /// every terminal path emits — including the launch-path catch, so the counter cannot
+    /// leak. Touched only on the actor thread.
+    /// </summary>
+    private int _runsInFlight;
+
+    /// <summary>
+    /// WP3.1: when the last shed Warning site event was emitted for this script. Sheds are
+    /// ALWAYS counted on the health collector; the site event is rate-limited to one per
+    /// script per minute so a hot trigger against a saturated cap cannot flood
+    /// <c>site_events</c>.
+    /// </summary>
+    private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
+
+    /// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
+    private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
+
     private Script<object?>? _compiledScript;
     private ScriptTriggerConfig? _triggerConfig;
     private TimeSpan? _minTimeBetweenRuns;
 
     /// <summary>
     /// The per-script execution timeout in seconds, or null to use the
-    /// global default. Threaded down to each spawned <see cref="ScriptExecutionActor"/>,
-    /// which applies <c>perScript ?? global</c> (and treats ≤ 0 as "use global").
+    /// global default. Threaded down to each launched run via
+    /// <see cref="ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
+    /// (and treats ≤ 0 as "use global").
     /// </summary>
     private readonly int? _executionTimeoutSeconds;
     private DateTimeOffset _lastExecutionTime = DateTimeOffset.MinValue;
@@ -111,6 +151,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
     /// <param name="healthCollector">Optional health metrics collector.</param>
     /// <param name="serviceProvider">Optional DI service provider for script execution context services.</param>
     /// <param name="scheduler">Optional script-execution scheduler override (#18); null uses the process-wide shared scheduler.</param>
+    /// <param name="evalGate">Optional trigger-expression concurrency gate override (WP3.1); null uses the process-wide shared gate.</param>
     public ScriptActor(
         string scriptName,
         string instanceName,
@@ -124,7 +165,8 @@ public class ScriptActor : ReceiveActor, IWithTimers
         IReadOnlyDictionary<string, object?>? initialAttributes = null,
         ISiteHealthCollector? healthCollector = null,
         IServiceProvider? serviceProvider = null,
-        ScriptExecutionScheduler? scheduler = null)
+        ScriptExecutionScheduler? scheduler = null,
+        TriggerEvalGate? evalGate = null)
     {
         _scriptName = scriptName;
         _instanceName = instanceName;
@@ -136,6 +178,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
         _healthCollector = healthCollector;
         _serviceProvider = serviceProvider;
         _scheduler = scheduler;
+        _evalGate = evalGate;
         _minTimeBetweenRuns = scriptConfig.MinTimeBetweenRuns;
         _executionTimeoutSeconds = scriptConfig.ExecutionTimeoutSeconds;
         _scope = scriptConfig.Scope;
@@ -201,24 +244,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
             _scriptName, _instanceName);
     }
 
-    /// <inheritdoc />
-    protected override SupervisorStrategy SupervisorStrategy()
-    {
-        return new OneForOneStrategy(
-            maxNrOfRetries: -1,
-            withinTimeRange: TimeSpan.FromMinutes(1),
-            decider: Decider.From(ex =>
-            {
-                _logger.LogWarning(ex,
-                    "ScriptExecutionActor for {Script} on {Instance} failed, stopping",
-                    _scriptName, _instanceName);
-                return Directive.Stop;
-            }));
-    }
+    // WP3.1: the Stop-on-failure SupervisorStrategy override is gone with the per-run
+    // ScriptExecutionActor child it supervised. This actor has no children left; a
+    // launch-path throw is caught in SpawnExecution (which replies to the Ask caller and
+    // releases the in-flight slot) so it can never escalate to InstanceActor, which
+    // continues to supervise this actor with Resume exactly as before.
 
     /// <summary>
     /// Handles CallScript ask from ScriptRuntimeContext or Instance Actor.
-    /// Spawns a ScriptExecutionActor and forwards the sender for reply.
+    /// Launches a run and captures the sender for the eventual reply.
     /// </summary>
     private void HandleScriptCallRequest(ScriptCallRequest request)
     {
@@ -233,7 +267,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
         }
 
         // (ParentExecutionId): carry any inbound-routed ParentExecutionId
-        // through to the ScriptExecutionActor so the routed script's
+        // through to the launched run so the routed script's
         // ScriptRuntimeContext can record its spawner. Null for normal
         // (tag-change / timer) runs and nested Script.Call invocations.
         SpawnExecution(
@@ -287,12 +321,22 @@ public class ScriptActor : ReceiveActor, IWithTimers
     /// Starts an off-dispatcher evaluation of the compiled trigger expression (P1).
     /// The expression previously ran synchronously on the actor's dispatcher thread
     /// via <c>RunAsync(...).GetAwaiter().GetResult()</c>, blocking the dispatcher on
-    /// every attribute change. It now runs on the dedicated script-execution
-    /// scheduler against a point-in-time snapshot; the boolean result is piped back
-    /// to this actor as an <see cref="ExpressionEvalResult"/> so all edge state is
-    /// applied on the actor thread. Bursts coalesce: at most one evaluation is in
-    /// flight, at most one pending, so a storm of changes collapses to the latest
-    /// snapshot without unbounded task fan-out.
+    /// every attribute change. It runs off-thread against a point-in-time snapshot;
+    /// the boolean result is piped back to this actor as an
+    /// <see cref="ExpressionEvalResult"/> so all edge state is applied on the actor
+    /// thread. Bursts coalesce: at most one evaluation is in flight, at most one
+    /// pending, so a storm of changes collapses to the latest snapshot without
+    /// unbounded task fan-out.
+    ///
+    /// <para>WP3.1 (finding #4) changed WHERE it runs and WHEN its clock starts. It used
+    /// to run on the dedicated <see cref="ScriptExecutionScheduler"/> — the same bounded
+    /// pool as blocking script bodies — so N blocked scripts stalled every Expression
+    /// trigger on the node indefinitely. It now runs as plain async work on the shared
+    /// .NET thread pool behind <see cref="TriggerEvalGate"/>: trigger expressions are
+    /// non-blocking by construction, so they belong there. And the deadline
+    /// <see cref="CancellationTokenSource"/> is armed HERE, on the actor thread, before
+    /// the work is queued — so gate-wait time burns the same budget and a saturated gate
+    /// yields a timely false instead of an unbounded stall.</para>
     /// </summary>
     private void StartExpressionEvaluation()
     {
@@ -303,29 +347,45 @@ public class ScriptActor : ReceiveActor, IWithTimers
         var snapshot = new Dictionary<string, object?>(_attributeSnapshot); // point-in-time copy, actor thread
         var expression = _compiledTriggerExpression;
         var self = Self;
-        Task.Factory.StartNew(async () =>
+        var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
+        // Clock starts AT ENQUEUE, not at dequeue.
+        var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
+
+        Task.Run(async () =>
         {
             try
             {
-                // Bound evaluation with a short timeout. The CancellationToken covers
-                // cooperative/async cases; a pathological CPU-bound expression is not
-                // fully interruptible — acceptable because trigger expressions are
-                // authored by trusted Design-role users and compile-checked pre-deploy.
-                using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
-                var state = await expression.RunAsync(new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
-                return state.ReturnValue is bool b && b;
+                await gate.WaitAsync(cts.Token).ConfigureAwait(false);
+                try
+                {
+                    // The CancellationToken covers cooperative/async cases; a pathological
+                    // CPU-bound expression is not fully interruptible — acceptable because
+                    // trigger expressions are authored by trusted Design-role users and
+                    // compile-checked pre-deploy.
+                    var state = await expression.RunAsync(
+                        new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
+                    return state.ReturnValue is bool b && b;
+                }
+                finally
+                {
+                    gate.Release();
+                }
             }
             catch (Exception ex)
             {
-                // OperationCanceledException (timeout) falls through here too and is
-                // treated as false. LogExpressionError touches only thread-safe
-                // members (_healthCollector Interlocked, _logger, DI-resolved
-                // singleton logger) so it is safe off the actor thread.
+                // OperationCanceledException (timeout, INCLUDING a timeout that fired while
+                // still waiting on the gate) falls through here too and is treated as false.
+                // LogExpressionError touches only thread-safe members (_healthCollector
+                // Interlocked, _logger, DI-resolved singleton logger) so it is safe off the
+                // actor thread.
                 LogExpressionError(ex);
                 return false;
             }
-        }, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
-            _scheduler ?? ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
+            finally
+            {
+                cts.Dispose();
+            }
+        }).PipeTo(self,
             success: r => new ExpressionEvalResult(r),
             failure: ex => new ExpressionEvalFailed(ex));
     }
@@ -421,7 +481,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
 
     /// <summary>
     /// Records a trigger-expression evaluation failure to the site event log,
-    /// mirroring how ScriptExecutionActor reports script errors.
+    /// mirroring how a script run reports its own errors.
     /// </summary>
     private void LogExpressionError(Exception ex)
     {
@@ -457,8 +517,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
     }
 
     /// <summary>
-    /// Spawns a new ScriptExecutionActor child for this invocation.
-    /// Multiple concurrent executions are allowed.
+    /// Launches a run of this script. Multiple concurrent runs are allowed, up to
+    /// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; beyond that the newest
+    /// trigger is shed (see <see cref="ShedRun"/>).
+    ///
+    /// <para>WP3.1: the run is launched directly on the script-execution scheduler via
+    /// <see cref="ScriptRunLauncher"/> — the actor's mailbox stays on the default dispatcher,
+    /// but the script body runs on the bounded set of dedicated threads, so blocking script
+    /// I/O is contained there and cannot starve the shared .NET thread pool. No per-run child
+    /// actor is created.</para>
     /// </summary>
     private void SpawnExecution(
         IReadOnlyDictionary<string, object?>? parameters,
@@ -467,46 +534,123 @@ public class ScriptActor : ReceiveActor, IWithTimers
         string correlationId,
         Guid? parentExecutionId = null)
     {
-        var executionId = $"{_scriptName}-exec-{_executionCounter++}";
+        if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
+        {
+            ShedRun(replyTo, correlationId);
+            return;
+        }
 
-        // The actor's mailbox stays on the default dispatcher, but the
-        // script body itself runs on the dedicated ScriptExecutionScheduler (a bounded
-        // set of dedicated threads), so blocking script I/O is contained there and
-        // cannot starve the shared .NET thread pool.
-        var props = Props.Create(() => new ScriptExecutionActor(
-            _scriptName,
-            _instanceName,
-            _compiledScript!,
-            parameters,
-            callDepth,
-            _instanceActor,
-            _sharedScriptLibrary,
-            _options,
-            replyTo,
-            correlationId,
-            _logger,
-            _scope,
-            _healthCollector,
-            _serviceProvider,
-            // (ParentExecutionId): null for trigger-driven runs;
-            // an inbound-API-routed call supplies the inbound request's id.
-            parentExecutionId,
-            // Per-script timeout override (null = use global).
-            _executionTimeoutSeconds,
-            // Scheduler seam (#18): thread this actor's scheduler override down so
-            // spawned script bodies share the same pool (null = process-wide shared).
-            _scheduler));
+        var runId = _executionCounter++;
+        // Incremented BEFORE the launch so the launch-path catch below (which always emits a
+        // ScriptExecutionCompleted) balances it on every path — the counter cannot leak.
+        _runsInFlight++;
 
-        Context.ActorOf(props, executionId);
+        try
+        {
+            ScriptRunLauncher.LaunchScript(
+                _scriptName,
+                _instanceName,
+                _compiledScript!,
+                parameters,
+                callDepth,
+                _instanceActor,
+                _sharedScriptLibrary,
+                _options,
+                replyTo,
+                correlationId,
+                // Completion target: this actor. Identical delivery to the old
+                // Context.Parent.Tell from the execution actor, and it doubles as the
+                // in-flight release.
+                Self,
+                // Notification Outbox: the site communication actor Notify.Status queries
+                // central through. Resolved here, on the actor thread, and handed to the
+                // launcher (which has no ActorContext of its own).
+                Context.System.ActorSelection("/user/site-communication"),
+                _logger,
+                _scope,
+                runId,
+                _healthCollector,
+                _serviceProvider,
+                // (ParentExecutionId): null for trigger-driven runs;
+                // an inbound-API-routed call supplies the inbound request's id.
+                parentExecutionId,
+                // Per-script timeout override (null = use global).
+                _executionTimeoutSeconds,
+                // Scheduler seam (#18): thread this actor's scheduler override down so
+                // launched script bodies share the same pool (null = process-wide shared).
+                _scheduler);
+        }
+        catch (Exception ex)
+        {
+            // WP3.1 supervision parity: the only failure the removed per-run child could
+            // surface was a constructor throw (e.g. queueing onto a disposed scheduler),
+            // which the old OneForOneStrategy logged and stopped — leaving an Ask caller to
+            // hang with no reply. Same log shape, same "coordinator unaffected" outcome, plus
+            // a reply so the caller fails fast instead of waiting out its Ask timeout.
+            var errorMsg = $"Script '{_scriptName}' on instance '{_instanceName}' could not be launched: {ex.Message}";
+            _logger.LogWarning(ex,
+                "Script execution launch for {Script} on {Instance} failed, stopping",
+                _scriptName, _instanceName);
+
+            if (!replyTo.IsNobody())
+                replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
+
+            Self.Tell(new ScriptExecutionCompleted(_scriptName, false, errorMsg));
+        }
+    }
+
+    /// <summary>
+    /// WP3.1 shed policy: refuses the incoming run because
+    /// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> runs are already in flight.
+    /// The four already queued/running are kept — they are closest to their own deadlines and
+    /// already charged against them — so nothing is ever reordered. A trigger-driven run is
+    /// simply not launched; an Ask-based <c>CallScript</c> gets an explicit error so a nested
+    /// call or inbound-API route fails fast rather than hanging to its Ask timeout.
+    /// </summary>
+    private void ShedRun(IActorRef replyTo, string correlationId)
+    {
+        _healthCollector?.IncrementScriptRunShed();
+
+        var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
+                      $"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
+
+        _logger.LogWarning("{Message}", message);
+
+        // Rate-limited to one event per script per minute: the counter above still counts
+        // every shed, but a hot trigger against a saturated cap must not flood site_events.
+        var now = DateTimeOffset.UtcNow;
+        if (now - _lastShedEventUtc >= ShedEventInterval)
+        {
+            _lastShedEventUtc = now;
+            _ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
+                "script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
+        }
+
+        if (!replyTo.IsNobody())
+        {
+            replyTo.Tell(new ScriptCallResult(
+                correlationId, false, null,
+                $"shed: {_runsInFlight} runs already in flight"));
+        }
     }
 
     private void HandleExecutionCompleted(ScriptExecutionCompleted msg)
     {
+        // WP3.1: release the in-flight slot. Every terminal path emits exactly one of these
+        // (success / timeout / failure / launch failure), so the counter tracks reality.
+        if (_runsInFlight > 0) _runsInFlight--;
+
         _logger.LogDebug(
             "Script {Script} execution completed on {Instance}: success={Success}",
             _scriptName, _instanceName, msg.Success);
     }
 
+    /// <summary>
+    /// WP3.1: runs launched but not yet completed. Exposed for regression coverage of the
+    /// shed cap — the counter is otherwise invisible from outside the actor.
+    /// </summary>
+    internal int RunsInFlight => _runsInFlight;
+
     // internal (not private) so the culture-invariance of the non-numeric fallback
     // can be unit-tested directly on the test thread — the live path evaluates on a
     // dispatcher thread whose CurrentCulture the test cannot deterministically set.

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs

@@ -1,341 +0,0 @@
-using Akka.Actor;
-using Microsoft.CodeAnalysis.Scripting;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
-using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
-using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
-using ZB.MOM.WW.ScadaBridge.Commons.Types;
-using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
-using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
-using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
-using ZB.MOM.WW.ScadaBridge.StoreAndForward;
-
-namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
-
-/// <summary>
-/// Script Execution Actor -- short-lived child of Script Actor.
-/// Receives compiled code, params, Instance Actor ref, and call depth.
-/// Executes the script via Script Runtime API, returns result, then stops.
-///
-/// The actor itself and its mailbox run on the default Akka dispatcher; only the
-/// script body is dispatched off the actor thread, onto the dedicated
-/// <see cref="ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts.ScriptExecutionScheduler"/>,
-/// so blocking script I/O cannot starve the shared thread pool
-/// or stall other Akka dispatchers.
-///
-/// Script failures are logged but do not disable the script.
-/// Supervision: Stop on unhandled exception (parent ScriptActor decides).
-/// </summary>
-public class ScriptExecutionActor : ReceiveActor
-{
-    /// <summary>
-    /// Initializes the actor and immediately begins script execution on construction.
-    /// </summary>
-    /// <param name="scriptName">Name of the script being executed.</param>
-    /// <param name="instanceName">Name of the instance that owns the script.</param>
-    /// <param name="compiledScript">Compiled Roslyn script to execute.</param>
-    /// <param name="parameters">Optional named parameter values for the script.</param>
-    /// <param name="callDepth">Current call-nesting depth (used to enforce the max-depth limit).</param>
-    /// <param name="instanceActor">Parent instance actor reference for attribute access.</param>
-    /// <param name="sharedScriptLibrary">Library of shared scripts available during execution.</param>
-    /// <param name="options">Site runtime options applied during execution.</param>
-    /// <param name="replyTo">Actor reference that receives the script result.</param>
-    /// <param name="correlationId">Application-level correlation id threaded through the execution.</param>
-    /// <param name="logger">Logger for script execution events.</param>
-    /// <param name="scope">Script scope controlling which APIs are available.</param>
-    /// <param name="healthCollector">Optional health collector for recording execution metrics.</param>
-    /// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
-    /// <param name="parentExecutionId">ExecutionId of the spawning inbound-API execution for audit correlation; null for normal runs.</param>
-    /// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global <see cref="SiteRuntimeOptions.ScriptExecutionTimeoutSeconds"/>.</param>
-    public ScriptExecutionActor(
-        string scriptName,
-        string instanceName,
-        Script<object?> compiledScript,
-        IReadOnlyDictionary<string, object?>? parameters,
-        int callDepth,
-        IActorRef instanceActor,
-        SharedScriptLibrary sharedScriptLibrary,
-        SiteRuntimeOptions options,
-        IActorRef replyTo,
-        string correlationId,
-        ILogger logger,
-        Commons.Types.Scripts.ScriptScope scope,
-        ISiteHealthCollector? healthCollector = null,
-        IServiceProvider? serviceProvider = null,
-        // The spawning execution's
-        // ExecutionId for an inbound-API-routed call. Null for normal
-        // (tag-change / timer) runs and nested Script.Call invocations.
-        Guid? parentExecutionId = null,
-        // Per-script execution timeout override (seconds). Null or
-        // non-positive falls back to the global ScriptExecutionTimeoutSeconds.
-        int? executionTimeoutSeconds = null,
-        // Script-execution scheduler seam (#18): the process-wide
-        // ScriptExecutionScheduler by default; a test (or a future multi-site
-        // host) can inject its own instance so script bodies never run on the
-        // shared process-wide pool. Null selects the shared default.
-        ScriptExecutionScheduler? scheduler = null)
-    {
-        // Immediately begin execution
-        var self = Self;
-        var parent = Context.Parent;
-
-        ExecuteScript(
-            scriptName, instanceName, compiledScript, parameters, callDepth,
-            instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
-            self, parent, logger, scope, healthCollector, serviceProvider,
-            parentExecutionId, executionTimeoutSeconds, scheduler);
-    }
-
-    private static void ExecuteScript(
-        string scriptName,
-        string instanceName,
-        Script<object?> compiledScript,
-        IReadOnlyDictionary<string, object?>? parameters,
-        int callDepth,
-        IActorRef instanceActor,
-        SharedScriptLibrary sharedScriptLibrary,
-        SiteRuntimeOptions options,
-        IActorRef replyTo,
-        string correlationId,
-        IActorRef self,
-        IActorRef parent,
-        ILogger logger,
-        Commons.Types.Scripts.ScriptScope scope,
-        ISiteHealthCollector? healthCollector,
-        IServiceProvider? serviceProvider,
-        Guid? parentExecutionId,
-        int? executionTimeoutSeconds,
-        ScriptExecutionScheduler? scheduler)
-    {
-        // Per-script timeout overrides the global default. A null or
-        // non-positive per-script value (≤ 0) falls back to the global.
-        var timeout = TimeSpan.FromSeconds(
-            executionTimeoutSeconds is { } perScript && perScript > 0
-                ? perScript
-                : options.ScriptExecutionTimeoutSeconds);
-
-        // Run the script body on the dedicated script-execution
-        // scheduler, not the shared .NET thread pool, so blocking script I/O cannot
-        // starve the global pool and stall Akka dispatchers / HTTP handling. An
-        // injected scheduler (#18) overrides the process-wide default.
-        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
-
-        // Notification Outbox: the site communication actor that Notify.Status queries
-        // central through. Resolved by actor path so the Notify helper does not need an
-        // IActorRef threaded all the way down from the host wiring.
-        var siteCommunicationActor = Context.System.ActorSelection("/user/site-communication");
-
-        // CTS must be created inside the async lambda so it outlives this method
-        _ = Task.Factory.StartNew(async () =>
-        {
-            IServiceScope? serviceScope = null;
-            // ISiteEventLogger is a singleton; resolve from the root provider so
-            // it is available to the catch blocks regardless of scope state.
-            var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
-            using var cts = new CancellationTokenSource(timeout);
-
-            // Stuck-script watchdog (S2). The CTS firing only REQUESTS cooperative
-            // cancellation; it does NOT free a thread blocked in synchronous I/O.
-            // When the timeout elapses, wait a grace period on the thread pool and,
-            // if the body still hasn't returned, name the script loudly — this is
-            // the only signal an operator gets that one of the bounded
-            // script-execution threads is gone. `completed` is flipped in the
-            // finally below; Register fires on cancellation only (normal completion
-            // disposes the CTS with no callback). The Task.Run/Task.Delay run on the
-            // thread pool, not the (possibly saturated) script scheduler — deliberate.
-            var completed = 0;
-            var graceMs = options.StuckScriptGraceMs;
-            cts.Token.Register(() => _ = Task.Run(async () =>
-            {
-                await Task.Delay(graceMs);
-                if (Volatile.Read(ref completed) == 0)
-                {
-                    var stuckMsg = $"Script '{scriptName}' on instance '{instanceName}' exceeded its " +
-                                   $"{timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — its dedicated " +
-                                   "script-execution thread is blocked (cooperative cancellation not observed).";
-                    logger.LogError(stuckMsg);
-                    _ = siteEventLogger?.LogEventAsync("script", "Error", instanceName,
-                        $"ScriptActor:{scriptName}", stuckMsg);
-                }
-            }));
-
-            try
-            {
-                // Resolve integration services from DI (scoped lifetime)
-                IExternalSystemClient? externalSystemClient = null;
-                IDatabaseGateway? databaseGateway = null;
-                // Notification Outbox: the S&F engine is a singleton; the site identity
-                // provider supplies the site id stamped on enqueued notifications.
-                StoreAndForwardService? storeAndForward = null;
-                var siteId = string.Empty;
-                // The writer is a singleton (FallbackAuditWriter
-                // composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
-                // that haven't called AddAuditLog, which the helper handles as a no-op.
-                IAuditWriter? auditWriter = null;
-                // Site-local tracking store
-                // backing Tracking.Status(id). Singleton; null in tests / hosts
-                // that haven't wired the store, which the helper handles by
-                // throwing on access.
-                IOperationTrackingStore? operationTrackingStore = null;
-                // Site-side cached-call
-                // telemetry forwarder. Singleton bound to the AuditLog
-                // composition root; null in tests / hosts that haven't called
-                // AddAuditLog, in which case the cached-call helpers degrade
-                // to the no-emission path (the underlying S&F handoff still
-                // happens and a TrackedOperationId is still returned).
-                ICachedCallTelemetryForwarder? cachedForwarder = null;
-                // SourceNode-stamping: the local node name
-                // resolved from INodeIdentityProvider — node-a/node-b on site
-                // hosts. Null in tests / hosts that haven't registered the
-                // provider, in which case NotificationSubmit.SourceNode and
-                // SiteCallOperational.SourceNode stay null and central
-                // persists the rows with SourceNode NULL.
-                string? sourceNode = null;
-
-                if (serviceProvider != null)
-                {
-                    serviceScope = serviceProvider.CreateScope();
-                    externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
-                    databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
-                    storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
-                    siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
-                        ?? string.Empty;
-                    auditWriter = serviceScope.ServiceProvider.GetService<IAuditWriter>();
-                    operationTrackingStore = serviceScope.ServiceProvider.GetService<IOperationTrackingStore>();
-                    cachedForwarder = serviceScope.ServiceProvider.GetService<ICachedCallTelemetryForwarder>();
-                    sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
-                }
-
-                var context = new ScriptRuntimeContext(
-                    instanceActor,
-                    self,
-                    sharedScriptLibrary,
-                    callDepth,
-                    options.MaxScriptCallDepth,
-                    timeout,
-                    instanceName,
-                    logger,
-                    externalSystemClient,
-                    databaseGateway,
-                    storeAndForward,
-                    siteCommunicationActor,
-                    siteId,
-                    // Notification Outbox (FU3): stamp the executing script onto outbound
-                    // notifications using the Site Event Logging "Source" convention.
-                    sourceScript: $"ScriptActor:{scriptName}",
-                    // Emit one ApiOutbound/ApiCall row per
-                    // ExternalSystem.Call. Writer is best-effort; failures are logged
-                    // and swallowed inside the helper so the script's call path is
-                    // never aborted by an audit failure.
-                    auditWriter: auditWriter,
-                    // Site-local tracking store
-                    // backing Tracking.Status(id). Authoritative source of truth for
-                    // cached-call status — read directly by the script API.
-                    operationTrackingStore: operationTrackingStore,
-                    // Cached-call telemetry
-                    // forwarder for ExternalSystem.CachedCall / Database.CachedWrite
-                    // CachedSubmit emission + the immediate-success terminal-row
-                    // emission. Best-effort: null degrades the helpers to a
-                    // no-emission path; the S&F handoff and TrackedOperationId
-                    // return are unaffected.
-                    cachedForwarder: cachedForwarder,
-                    // The spawning execution's
-                    // id for an inbound-API-routed call. The routed script still
-                    // mints its own fresh ExecutionId — this records the spawner.
-                    // Null for normal (tag-change / timer) runs.
-                    parentExecutionId: parentExecutionId,
-                    // SourceNode-stamping: the local node name
-                    // (node-a/node-b on a site) — threaded down so Notify.Send
-                    // and the four cached-call telemetry constructors can stamp
-                    // it onto NotificationSubmit.SourceNode and
-                    // SiteCallOperational.SourceNode respectively.
-                    sourceNode: sourceNode,
-                    // Thread the singleton site event logger so
-                    // recursion-limit violations at CallScript/CallShared emit a
-                    // script Error site event in addition to ILogger.LogError.
-                    siteEventLogger: siteEventLogger,
-                    // WaitForAttribute (spec §4.3/§4.4): thread the per-script
-                    // execution-timeout token so Attributes.WaitAsync's Ask is
-                    // bounded by the script's own ExecutionTimeoutSeconds — a
-                    // shorter script deadline wins over the wait's own timeout.
-                    scriptTimeoutToken: cts.Token);
-
-                var globals = new ScriptGlobals
-                {
-                    Instance = context,
-                    Parameters = new ScriptParameters(parameters ?? new Dictionary<string, object?>()),
-                    CancellationToken = cts.Token,
-                    Scope = scope
-                };
-
-                // Operational `script` event — execution started. Fire-and-forget
-                // (the `_ =` discards the task) so the event log can never block or
-                // fault the script's own run; mirrors the existing Error-path emit.
-                _ = siteEventLogger?.LogEventAsync(
-                    "script", "Info", instanceName, $"ScriptActor:{scriptName}",
-                    $"Script '{scriptName}' on instance '{instanceName}' started");
-
-                var state = await compiledScript.RunAsync(globals, cts.Token);
-
-                // Send result to requester if this was an Ask-based call
-                if (!replyTo.IsNobody())
-                {
-                    replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
-                }
-
-                // Operational `script` event — execution completed successfully.
-                _ = siteEventLogger?.LogEventAsync(
-                    "script", "Info", instanceName, $"ScriptActor:{scriptName}",
-                    $"Script '{scriptName}' on instance '{instanceName}' completed");
-
-                // Notify parent of completion
-                parent.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
-            }
-            catch (OperationCanceledException)
-            {
-                healthCollector?.IncrementScriptError();
-                var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
-                logger.LogWarning(errorMsg);
-
-                // Failures recorded to site event log; script NOT disabled after failure.
-                _ = siteEventLogger?.LogEventAsync(
-                    "script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
-
-                if (!replyTo.IsNobody())
-                {
-                    replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
-                }
-
-                parent.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
-            }
-            catch (Exception ex)
-            {
-                healthCollector?.IncrementScriptError();
-                // Failures recorded to site event log; script NOT disabled after failure.
-                var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
-                logger.LogError(ex, "Script execution failed: {Script} on {Instance}", scriptName, instanceName);
-
-                _ = siteEventLogger?.LogEventAsync(
-                    "script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());
-
-                if (!replyTo.IsNobody())
-                {
-                    replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
-                }
-
-                parent.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
-            }
-            finally
-            {
-                // Mark the body finished so the stuck-script watchdog (registered on
-                // cts.Token above) treats a timely cancellation as NOT stuck.
-                Interlocked.Exchange(ref completed, 1);
-                // Dispose the DI scope (and scoped services) after script execution completes
-                serviceScope?.Dispose();
-                // Stop self after execution completes
-                self.Tell(PoisonPill.Instance);
-            }
-        }, CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler).Unwrap();
-    }
-}

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs

@@ -3,30 +3,106 @@ using System.Collections.Concurrent;
 namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
 
 /// <summary>
-/// A dedicated, bounded <see cref="TaskScheduler"/> for running script
+/// The outcome of a <see cref="ScriptExecutionScheduler.TryDetachWorker"/> attempt.
+/// </summary>
+public enum WorkerDetachOutcome
+{
+    /// <summary>
+    /// The recorded slot/run pair no longer identifies a running task (the script finished,
+    /// or it never held a worker because it hopped threads on an <c>await</c>). Nothing was
+    /// detached and nothing needed to be — the worker is not lost.
+    /// </summary>
+    NotRunning,
+
+    /// <summary>The worker was marked detached and a replacement thread was started.</summary>
+    Detached,
+
+    /// <summary>
+    /// The number of live detached threads already equals the pool size, so no replacement was
+    /// started. Bounded starvation is preferable to unbounded thread growth when scripts wedge
+    /// en masse; the caller is expected to log this loudly.
+    /// </summary>
+    AtCap
+}
+
+/// <summary>
+/// A dedicated, grow-only <see cref="TaskScheduler"/> for running script
 /// and alarm on-trigger bodies.
 ///
 /// Script bodies may perform synchronous blocking I/O (a database connection, a
 /// synchronous external-system call). Running them on the shared .NET
 /// <see cref="ThreadPool"/> lets a burst of blocking scripts starve the pool and stall
-/// unrelated Akka dispatchers and HTTP request handling. This scheduler owns a fixed set
+/// unrelated Akka dispatchers and HTTP request handling. This scheduler owns a set
 /// of dedicated threads, so script blocking is contained to those threads and cannot
 /// exhaust the global pool.
 ///
+/// <para>WP3.1 changed three things about that pool:</para>
+/// <list type="number">
+///   <item>It is no longer fixed-size. <see cref="EnsureCapacity"/> grows it towards
+///   <see cref="ComputeTargetThreads"/> (instance-scaled, clamped between the configured
+///   floor and ceiling). It is deliberately <em>grow-only</em>: undeploying instances
+///   leaves idle threads, which cost nothing measurable and avoid drain/steal complexity.</item>
+///   <item>A worker whose task has wedged in uninterruptible blocking I/O can be
+///   <see cref="TryDetachWorker">detached</see> and replaced, so a stuck script no longer
+///   permanently costs the pool a thread. Live detached threads are capped at the pool size.</item>
+///   <item>Trigger-expression evaluations no longer run here at all — they are non-blocking
+///   by construction and now run on the shared thread pool behind
+///   <see cref="TriggerEvalGate"/>, so an alarm's Expression trigger can never queue behind
+///   eight blocking script bodies (finding #4).</item>
+/// </list>
+///
 /// The scheduler is process-wide (one set of threads for all instances) and is sized
 /// from <see cref="SiteRuntimeOptions"/> the first time it is configured.
 /// </summary>
 public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
 {
+    /// <summary>
+    /// Deployed instances per script-execution thread used by <see cref="ComputeTargetThreads"/>.
+    /// A named constant rather than an option: no deployment needs to tune the ratio
+    /// independently of the floor (<see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/>)
+    /// and the ceiling (<see cref="SiteRuntimeOptions.ScriptExecutionMaxThreadCount"/>).
+    /// </summary>
+    internal const int InstancesPerScriptThread = 8;
+
+    /// <summary>Thread-name prefix; also the inlining guard in <see cref="TryExecuteTaskInline"/>.</summary>
+    private const string ThreadNamePrefix = "script-execution-";
+
     private readonly BlockingCollection<Task> _queue = new();
-    private readonly List<Thread> _threads;
+
+    /// <summary>
+    /// Immutable-on-read snapshot array of worker slots. Growth (EnsureCapacity, detach
+    /// replacement) publishes a NEW longer array under <see cref="_growLock"/>; existing
+    /// slot objects are carried over by reference so a worker's index stays stable for its
+    /// whole life. Readers take one volatile read and then work on that snapshot.
+    /// </summary>
+    private volatile WorkerSlot[] _slots;
+
+    /// <summary>Guards every mutation of <see cref="_slots"/>, <see cref="_configuredCount"/>, and the detach bookkeeping.</summary>
+    private readonly object _growLock = new();
+
+    /// <summary>
+    /// Number of NON-detached workers — i.e. the pool's nominal size. Unchanged by a
+    /// detach (each detach starts a replacement), grown only by <see cref="EnsureCapacity"/>.
+    /// </summary>
+    private int _configuredCount;
+
+    /// <summary>Detached workers that have not yet finished their wedged task and exited.</summary>
+    private int _detachedLive;
+
+    /// <summary>Monotonic run identity; see <see cref="WorkerSlot.RunStamp"/>.</summary>
+    private long _runStampSeed;
+
     private int _disposed;
 
-    // Per-worker "busy since" timestamp (Environment.TickCount64 ms) while a task
-    // is executing on that worker, 0 when idle. Written by the worker thread,
-    // read (lock-free) by the observability gauges below. S2/UA5: makes a
-    // saturated or stuck script-execution pool visible on the site health report.
-    private readonly long[] _busySinceTicks;
+    /// <summary>
+    /// The slot index of the script-execution worker running the current thread, or null on
+    /// any other thread. Captured by the first synchronous segment of a script body so the
+    /// stuck-script watchdog can identify — and replace — the exact worker a wedged script
+    /// is holding. A script that hopped threads on an <c>await</c> is no longer on its worker,
+    /// which is correct: it holds no thread and must not cause a detach.
+    /// </summary>
+    [ThreadStatic]
+    internal static int? CurrentWorkerSlot;
 
     private static volatile ScriptExecutionScheduler? _shared;
     private static readonly object SharedLock = new();
@@ -34,8 +110,9 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
     /// <summary>
     /// The process-wide script-execution scheduler, used as the default when no scheduler
     /// is injected. Lazily created on first use with the thread count from
-    /// <see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/>; the first caller wins,
-    /// subsequent calls reuse the existing instance.
+    /// <see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/> (the floor — the Deployment
+    /// Manager grows it from there as instances deploy); the first caller wins, subsequent
+    /// calls reuse the existing instance.
     ///
     /// If the cached instance has been disposed it is recreated rather than handed back:
     /// a disposed scheduler can execute no work, so returning it would silently poison
@@ -59,27 +136,40 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
         }
     }
 
+    /// <summary>
+    /// Pure sizing function for the blocking script-execution pool: the configured floor,
+    /// raised to one thread per <see cref="InstancesPerScriptThread"/> enabled instances,
+    /// clamped to the configured ceiling. Static and side-effect-free so the policy is
+    /// unit-testable without starting a single thread.
+    /// </summary>
+    /// <param name="enabledInstances">Number of currently-running (enabled) Instance Actors.</param>
+    /// <param name="options">Site runtime options supplying the floor and ceiling.</param>
+    /// <returns>The target worker-thread count, always at least 1.</returns>
+    public static int ComputeTargetThreads(int enabledInstances, SiteRuntimeOptions options)
+    {
+        var floor = Math.Max(1, options.ScriptExecutionThreadCount);
+        // A ceiling below the floor is rejected by SiteRuntimeOptionsValidator; clamp here
+        // too so a directly-constructed options object can never invert the range.
+        var ceiling = Math.Max(floor, options.ScriptExecutionMaxThreadCount);
+        var scaled = enabledInstances <= 0
+            ? 0
+            : (int)Math.Ceiling(enabledInstances / (double)InstancesPerScriptThread);
+        return Math.Clamp(Math.Max(floor, scaled), 1, ceiling);
+    }
+
     /// <summary>
     /// Creates a scheduler backed by <paramref name="threadCount"/> dedicated threads.
     /// </summary>
-    /// <param name="threadCount">Number of dedicated worker threads to create.</param>
+    /// <param name="threadCount">Initial number of dedicated worker threads to create.</param>
     public ScriptExecutionScheduler(int threadCount)
     {
         if (threadCount < 1)
             threadCount = 1;
 
-        _busySinceTicks = new long[threadCount];
-        _threads = new List<Thread>(threadCount);
-        for (var i = 0; i < threadCount; i++)
+        _slots = [];
+        lock (_growLock)
         {
-            var index = i; // capture per-worker slot index
-            var thread = new Thread(() => WorkerLoop(index))
-            {
-                IsBackground = true,
-                Name = $"script-execution-{i}"
-            };
-            _threads.Add(thread);
-            thread.Start();
+            GrowTo(threadCount);
         }
     }
 
@@ -88,19 +178,28 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
     public bool IsDisposed => Volatile.Read(ref _disposed) != 0;
 
     /// <inheritdoc />
-    public override int MaximumConcurrencyLevel => _threads.Count;
+    public override int MaximumConcurrencyLevel => Volatile.Read(ref _configuredCount);
 
     /// <summary>Number of tasks waiting in the queue (not counting those currently executing).</summary>
     public int QueueDepth => _queue.Count;
 
-    /// <summary>Number of worker threads currently executing a task.</summary>
+    /// <summary>
+    /// Workers whose task wedged past its deadline plus the stuck-script grace, which have been
+    /// detached and replaced but have not yet returned. Surfaced on the site health report as
+    /// <c>DetachedScriptThreads</c>: a non-zero, non-draining value means script bodies are
+    /// permanently blocking threads.
+    /// </summary>
+    public int DetachedThreadCount => Volatile.Read(ref _detachedLive);
+
+    /// <summary>Number of worker threads currently executing a task (detached workers included — they really are busy).</summary>
     public int BusyThreadCount
     {
         get
         {
+            var slots = _slots;
             var count = 0;
-            for (var i = 0; i < _busySinceTicks.Length; i++)
-                if (Volatile.Read(ref _busySinceTicks[i]) != 0) count++;
+            foreach (var slot in slots)
+                if (Volatile.Read(ref slot.BusySinceTicks) != 0) count++;
             return count;
         }
     }
@@ -115,10 +214,11 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
         get
         {
             var now = Environment.TickCount64;
+            var slots = _slots;
             long oldestSince = 0;
-            for (var i = 0; i < _busySinceTicks.Length; i++)
+            foreach (var slot in slots)
             {
-                var since = Volatile.Read(ref _busySinceTicks[i]);
+                var since = Volatile.Read(ref slot.BusySinceTicks);
                 if (since != 0 && (oldestSince == 0 || since < oldestSince))
                     oldestSince = since;
             }
@@ -126,20 +226,138 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
         }
     }
 
+    /// <summary>
+    /// Grows the pool to <paramref name="target"/> non-detached workers. Idempotent and
+    /// never shrinking: a target at or below the current size is a no-op. Called from
+    /// <c>DeploymentManagerActor.UpdateInstanceCounts</c> on every deploy/undeploy/enable/
+    /// disable and per staggered startup batch.
+    /// </summary>
+    /// <param name="target">Desired number of non-detached worker threads.</param>
+    /// <returns>The pool's non-detached worker count after the call.</returns>
+    public int EnsureCapacity(int target)
+    {
+        if (IsDisposed) return Volatile.Read(ref _configuredCount);
+        if (target <= Volatile.Read(ref _configuredCount)) return Volatile.Read(ref _configuredCount);
+
+        lock (_growLock)
+        {
+            if (IsDisposed || target <= _configuredCount) return _configuredCount;
+            GrowTo(target - _configuredCount);
+            return _configuredCount;
+        }
+    }
+
+    /// <summary>
+    /// The identity stamp of the task currently running on <paramref name="slot"/>, or 0 when
+    /// that worker is idle. Captured alongside the slot index at script-body start and handed
+    /// back to <see cref="TryDetachWorker"/>, which detaches only if the SAME task is still
+    /// running. A monotonic counter rather than a timestamp: two runs on one worker inside the
+    /// same <see cref="Environment.TickCount64"/> tick would otherwise be indistinguishable.
+    /// </summary>
+    /// <param name="slot">The worker slot index.</param>
+    /// <returns>The current run stamp, or 0 when the slot is idle or out of range.</returns>
+    internal long CurrentRunStamp(int slot)
+    {
+        var slots = _slots;
+        return slot >= 0 && slot < slots.Length ? Volatile.Read(ref slots[slot].RunStamp) : 0L;
+    }
+
+    /// <summary>
+    /// Detaches the worker at <paramref name="slot"/> — if it is still running the task
+    /// identified by <paramref name="observedRunStamp"/> — and starts a replacement thread, so
+    /// a script wedged in uninterruptible blocking I/O no longer costs the pool a thread
+    /// permanently. The detached worker exits (instead of pulling more work) as soon as its
+    /// wedged task finally returns, so capacity never silently doubles-and-drains.
+    /// </summary>
+    /// <param name="slot">The worker slot recorded when the script body started.</param>
+    /// <param name="observedRunStamp">The run stamp recorded at the same moment.</param>
+    /// <returns>What was done; see <see cref="WorkerDetachOutcome"/>.</returns>
+    internal WorkerDetachOutcome TryDetachWorker(int slot, long observedRunStamp)
+    {
+        if (observedRunStamp == 0 || IsDisposed) return WorkerDetachOutcome.NotRunning;
+
+        lock (_growLock)
+        {
+            if (IsDisposed) return WorkerDetachOutcome.NotRunning;
+
+            var slots = _slots;
+            if (slot < 0 || slot >= slots.Length) return WorkerDetachOutcome.NotRunning;
+
+            var worker = slots[slot];
+            // Same task still running on that exact worker? If the stamp moved (or went to 0)
+            // the script finished or never held this thread — there is nothing lost to replace.
+            if (Volatile.Read(ref worker.RunStamp) != observedRunStamp) return WorkerDetachOutcome.NotRunning;
+            if (Volatile.Read(ref worker.Detached) != 0) return WorkerDetachOutcome.NotRunning;
+
+            // Bound: never hold more than 2x threads (N wedged + N live).
+            if (_detachedLive >= _configuredCount) return WorkerDetachOutcome.AtCap;
+
+            Volatile.Write(ref worker.Detached, 1);
+            _detachedLive++;
+            _configuredCount--;   // the detached worker no longer counts towards the pool …
+            GrowTo(1);            // … and GrowTo puts the count back by starting its replacement.
+            return WorkerDetachOutcome.Detached;
+        }
+    }
+
+    /// <summary>Appends <paramref name="count"/> new worker slots + threads. Caller holds <see cref="_growLock"/>.</summary>
+    private void GrowTo(int count)
+    {
+        var existing = _slots;
+        var grown = new WorkerSlot[existing.Length + count];
+        Array.Copy(existing, grown, existing.Length);
+
+        for (var i = 0; i < count; i++)
+        {
+            var index = existing.Length + i;
+            grown[index] = new WorkerSlot();
+        }
+
+        // Publish the array BEFORE starting the threads: a worker's very first action is to
+        // index its own slot, which must already be visible on the published snapshot.
+        _slots = grown;
+        _configuredCount += count;
+
+        for (var i = 0; i < count; i++)
+        {
+            var index = existing.Length + i;
+            var thread = new Thread(() => WorkerLoop(index))
+            {
+                IsBackground = true,
+                Name = ThreadNamePrefix + index
+            };
+            grown[index].Thread = thread;
+            thread.Start();
+        }
+    }
+
     private void WorkerLoop(int index)
     {
+        CurrentWorkerSlot = index;
+        var slot = _slots[index];
         try
         {
             foreach (var task in _queue.GetConsumingEnumerable())
             {
-                Volatile.Write(ref _busySinceTicks[index], Environment.TickCount64);
+                Volatile.Write(ref slot.RunStamp, Interlocked.Increment(ref _runStampSeed));
+                Volatile.Write(ref slot.BusySinceTicks, Environment.TickCount64);
                 try
                 {
                     TryExecuteTask(task);
                 }
                 finally
                 {
-                    Volatile.Write(ref _busySinceTicks[index], 0);
+                    Volatile.Write(ref slot.BusySinceTicks, 0);
+                    Volatile.Write(ref slot.RunStamp, 0);
+                }
+
+                // Detached while this task ran: a replacement worker is already live, so
+                // exit rather than pulling more work — otherwise capacity would silently
+                // double once the wedged script finally returned.
+                if (Volatile.Read(ref slot.Detached) != 0)
+                {
+                    Interlocked.Decrement(ref _detachedLive);
+                    return;
                 }
             }
         }
@@ -147,6 +365,10 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
         {
             // Scheduler disposed — worker exits.
         }
+        finally
+        {
+            CurrentWorkerSlot = null;
+        }
     }
 
     /// <inheritdoc />
@@ -157,7 +379,7 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
     {
         // Only inline if we are already on one of this scheduler's worker threads,
         // so script work never escapes onto a thread-pool thread.
-        if (Thread.CurrentThread.Name?.StartsWith("script-execution-", StringComparison.Ordinal) != true)
+        if (Thread.CurrentThread.Name?.StartsWith(ThreadNamePrefix, StringComparison.Ordinal) != true)
             return false;
 
         return TryExecuteTask(task);
@@ -173,8 +395,35 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
             return;
 
         _queue.CompleteAdding();
-        foreach (var thread in _threads)
-            thread.Join(TimeSpan.FromSeconds(5));
+        foreach (var slot in _slots)
+            slot.Thread?.Join(TimeSpan.FromSeconds(5));
         _queue.Dispose();
     }
+
+    /// <summary>
+    /// Per-worker bookkeeping. One instance per slot index, created once and carried by
+    /// reference across every <see cref="_slots"/> growth so a worker's state survives
+    /// the array being replaced.
+    /// </summary>
+    private sealed class WorkerSlot
+    {
+        /// <summary>
+        /// <see cref="Environment.TickCount64"/> at which the worker picked up its current
+        /// task, 0 when idle. Written by the worker, read lock-free by the gauges.
+        /// </summary>
+        public long BusySinceTicks;
+
+        /// <summary>
+        /// Monotonic identity of the task currently running on this worker, 0 when idle.
+        /// Distinguishes two runs that start inside the same clock tick, which
+        /// <see cref="BusySinceTicks"/> alone cannot.
+        /// </summary>
+        public long RunStamp;
+
+        /// <summary>Non-zero once the worker has been detached; it exits after its current task.</summary>
+        public int Detached;
+
+        /// <summary>The worker thread, for <see cref="Dispose"/> to join.</summary>
+        public Thread? Thread;
+    }
 }

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptRunLauncher.cs

@@ -0,0 +1,570 @@
+using Akka.Actor;
+using Microsoft.CodeAnalysis.Scripting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
+using ZB.MOM.WW.ScadaBridge.Commons.Types;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
+using ZB.MOM.WW.ScadaBridge.StoreAndForward;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+
+/// <summary>
+/// WP3.1: launches one script or alarm on-trigger run directly from its owning coordinator
+/// (<see cref="ScriptActor"/> / <see cref="AlarmActor"/>), replacing the short-lived
+/// <c>ScriptExecutionActor</c> and <c>AlarmExecutionActor</c>.
+///
+/// <para>Those actors were already inert shells: neither declared a single <c>Receive</c>
+/// handler (they executed from their constructor), neither had a <c>PostStop</c>, state, or
+/// stash, and their <c>IActorRef</c> was never a message target — the whole lifecycle lived
+/// inside a detached <see cref="Task"/> the actor never observed. What they cost was a real
+/// actor cell, mailbox, and name registration per run, plus a per-spawn expression-tree
+/// <c>Props.Create</c>. Removing them changes no semantics; the run body below is the former
+/// <c>ExecuteScript</c>/<c>ExecuteAlarmScript</c> body, unified.</para>
+///
+/// <para>Two behaviours DID change, both deliberately:</para>
+/// <list type="number">
+///   <item>The deadline <see cref="CancellationTokenSource"/> is now armed by the CALLER,
+///   before the body is queued to the <see cref="ScriptExecutionScheduler"/>, so queue wait
+///   consumes the script's own budget. A body that dequeues past its deadline skips execution
+///   entirely and takes the timeout path — a saturated pool sheds stale work instead of
+///   running it late.</item>
+///   <item>The stuck-script watchdog now DETACHES and replaces the worker thread a wedged
+///   script is holding (<see cref="ScriptExecutionScheduler.TryDetachWorker"/>) instead of
+///   only naming it, so the pool recovers its capacity.</item>
+/// </list>
+///
+/// <para>Ordering, supervision, telemetry, DI scoping, Ask replies, completion messages, and
+/// the audit <c>ExecutionId</c>/<c>ParentExecutionId</c> threading are all unchanged. The
+/// launch call itself can throw only if the target scheduler is unusable (e.g. disposed); the
+/// caller wraps it so that failure replies to the Ask caller and decrements the in-flight
+/// counter rather than escalating to the coordinator's supervisor.</para>
+/// </summary>
+internal static class ScriptRunLauncher
+{
+    /// <summary>
+    /// Launches an instance script run. Returns as soon as the body is queued; the run reports
+    /// completion to <paramref name="completionTarget"/>.
+    /// </summary>
+    /// <param name="scriptName">Name of the script being executed.</param>
+    /// <param name="instanceName">Name of the instance that owns the script.</param>
+    /// <param name="compiledScript">Compiled Roslyn script to execute.</param>
+    /// <param name="parameters">Optional named parameter values for the script.</param>
+    /// <param name="callDepth">Current call-nesting depth (used to enforce the max-depth limit).</param>
+    /// <param name="instanceActor">Instance actor reference for attribute access.</param>
+    /// <param name="sharedScriptLibrary">Library of shared scripts available during execution.</param>
+    /// <param name="options">Site runtime options applied during execution.</param>
+    /// <param name="replyTo">Actor reference that receives the script result; <c>Nobody</c> for fire-and-forget.</param>
+    /// <param name="correlationId">Application-level correlation id threaded through the execution.</param>
+    /// <param name="completionTarget">The owning <see cref="ScriptActor"/>, which receives the completion message.</param>
+    /// <param name="siteCommunicationActor">Site communication actor (resolved on the actor thread) for <c>Notify.Status</c>.</param>
+    /// <param name="logger">Logger for script execution events.</param>
+    /// <param name="scope">Script scope controlling which APIs are available.</param>
+    /// <param name="runId">Per-script run counter, used only in log messages (replaces the former per-run actor name).</param>
+    /// <param name="healthCollector">Optional health collector for recording execution metrics.</param>
+    /// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
+    /// <param name="parentExecutionId">ExecutionId of the spawning execution for audit correlation; null for root runs.</param>
+    /// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global value.</param>
+    /// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
+    public static void LaunchScript(
+        string scriptName,
+        string instanceName,
+        Script<object?> compiledScript,
+        IReadOnlyDictionary<string, object?>? parameters,
+        int callDepth,
+        IActorRef instanceActor,
+        SharedScriptLibrary sharedScriptLibrary,
+        SiteRuntimeOptions options,
+        IActorRef replyTo,
+        string correlationId,
+        IActorRef completionTarget,
+        ICanTell? siteCommunicationActor,
+        ILogger logger,
+        ScriptScope scope,
+        long runId,
+        ISiteHealthCollector? healthCollector,
+        IServiceProvider? serviceProvider,
+        Guid? parentExecutionId,
+        int? executionTimeoutSeconds,
+        ScriptExecutionScheduler? scheduler)
+    {
+        var timeout = ResolveTimeout(executionTimeoutSeconds, options);
+        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
+
+        // Armed HERE, on the caller's (actor) thread, not inside the queued body: queue wait
+        // must consume the script's own budget, otherwise a saturated pool silently grants
+        // every backlogged run a fresh full timeout.
+        var cts = new CancellationTokenSource(timeout);
+        try
+        {
+            _ = Task.Factory.StartNew(
+                () => RunScriptAsync(
+                    scriptName, instanceName, compiledScript, parameters, callDepth,
+                    instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
+                    completionTarget, siteCommunicationActor, logger, scope, runId,
+                    healthCollector, serviceProvider, parentExecutionId, timeout,
+                    executionScheduler, cts),
+                CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
+                .Unwrap();
+        }
+        catch
+        {
+            // The body never got queued, so nothing will ever dispose the CTS.
+            cts.Dispose();
+            throw;
+        }
+    }
+
+    /// <summary>
+    /// Launches an alarm on-trigger run. Same contract as <see cref="LaunchScript"/>, with the
+    /// firing alarm's level/priority/message exposed to the body through the <c>Alarm</c> global.
+    /// </summary>
+    /// <param name="alarmName">The canonical name of the alarm that triggered.</param>
+    /// <param name="instanceName">The name of the owning instance.</param>
+    /// <param name="level">The alarm severity level at the time of triggering.</param>
+    /// <param name="priority">The alarm priority value.</param>
+    /// <param name="message">The alarm message to pass to the script.</param>
+    /// <param name="compiledScript">The pre-compiled on-trigger script to execute.</param>
+    /// <param name="instanceActor">Reference to the instance actor for attribute/script calls.</param>
+    /// <param name="sharedScriptLibrary">Shared script library providing common utilities.</param>
+    /// <param name="options">Site runtime configuration options, including the execution timeout.</param>
+    /// <param name="completionTarget">The owning <see cref="AlarmActor"/>, which receives the completion message.</param>
+    /// <param name="logger">Logger for execution diagnostics.</param>
+    /// <param name="runId">Per-alarm run counter, used only in log messages.</param>
+    /// <param name="executionTimeoutSeconds">The on-trigger script's per-script timeout in seconds. Null or non-positive falls back to the global value.</param>
+    /// <param name="parentExecutionId">
+    /// ParentExecutionId tag-cascade: the <c>ExecutionId</c> of the execution whose attribute
+    /// write fired this alarm. Null when the firing value came from the Data Connection Layer
+    /// (external data has no spawning execution) — that on-trigger run is a tree ROOT.
+    /// </param>
+    /// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
+    public static void LaunchAlarmScript(
+        string alarmName,
+        string instanceName,
+        AlarmLevel level,
+        int priority,
+        string message,
+        Script<object?> compiledScript,
+        IActorRef instanceActor,
+        SharedScriptLibrary sharedScriptLibrary,
+        SiteRuntimeOptions options,
+        IActorRef completionTarget,
+        ILogger logger,
+        long runId,
+        int? executionTimeoutSeconds,
+        Guid? parentExecutionId,
+        ScriptExecutionScheduler? scheduler)
+    {
+        var timeout = ResolveTimeout(executionTimeoutSeconds, options);
+        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
+
+        // Enqueue-anchored deadline; see LaunchScript.
+        var cts = new CancellationTokenSource(timeout);
+        try
+        {
+            _ = Task.Factory.StartNew(
+                () => RunAlarmScriptAsync(
+                    alarmName, instanceName, level, priority, message, compiledScript,
+                    instanceActor, sharedScriptLibrary, options, completionTarget, logger,
+                    runId, parentExecutionId, timeout, cts),
+                CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
+                .Unwrap();
+        }
+        catch
+        {
+            cts.Dispose();
+            throw;
+        }
+    }
+
+    /// <summary>
+    /// Per-script timeout overrides the global default. A null or non-positive per-script
+    /// value (≤ 0) falls back to the global.
+    /// </summary>
+    private static TimeSpan ResolveTimeout(int? executionTimeoutSeconds, SiteRuntimeOptions options)
+        => TimeSpan.FromSeconds(
+            executionTimeoutSeconds is { } perScript && perScript > 0
+                ? perScript
+                : options.ScriptExecutionTimeoutSeconds);
+
+    private static async Task RunScriptAsync(
+        string scriptName,
+        string instanceName,
+        Script<object?> compiledScript,
+        IReadOnlyDictionary<string, object?>? parameters,
+        int callDepth,
+        IActorRef instanceActor,
+        SharedScriptLibrary sharedScriptLibrary,
+        SiteRuntimeOptions options,
+        IActorRef replyTo,
+        string correlationId,
+        IActorRef completionTarget,
+        ICanTell? siteCommunicationActor,
+        ILogger logger,
+        ScriptScope scope,
+        long runId,
+        ISiteHealthCollector? healthCollector,
+        IServiceProvider? serviceProvider,
+        Guid? parentExecutionId,
+        TimeSpan timeout,
+        ScriptExecutionScheduler executionScheduler,
+        CancellationTokenSource cts)
+    {
+        IServiceScope? serviceScope = null;
+        // ISiteEventLogger is a singleton; resolve from the root provider so
+        // it is available to the catch blocks regardless of scope state.
+        var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
+
+        // WP3.1: identify the worker this body is holding, so the watchdog can detach and
+        // replace it if the body wedges. Read in the FIRST synchronous segment — a script
+        // that later hops threads on an await no longer holds this worker, and the run-stamp
+        // guard in TryDetachWorker will (correctly) refuse to detach it.
+        var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
+        var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;
+
+        var completed = 0;
+        ArmStuckScriptWatchdog(
+            cts, options, logger, siteEventLogger, executionScheduler,
+            workerSlot, workerRunStamp, timeout,
+            () => Volatile.Read(ref completed) != 0,
+            $"Script '{scriptName}' on instance '{instanceName}'",
+            instanceName, $"ScriptActor:{scriptName}");
+
+        try
+        {
+            // WP3.1 shed-at-dequeue: the deadline was armed at ENQUEUE, so an already-cancelled
+            // token here means this run spent its entire budget queueing. Running it now would
+            // burn a scarce script thread on work that is already stale — throw straight into
+            // the existing timeout path instead, BEFORE the "started" event and before the body.
+            cts.Token.ThrowIfCancellationRequested();
+
+            // Resolve integration services from DI (scoped lifetime)
+            IExternalSystemClient? externalSystemClient = null;
+            IDatabaseGateway? databaseGateway = null;
+            // Notification Outbox: the S&F engine is a singleton; the site identity
+            // provider supplies the site id stamped on enqueued notifications.
+            StoreAndForwardService? storeAndForward = null;
+            var siteId = string.Empty;
+            // The writer is a singleton (FallbackAuditWriter
+            // composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
+            // that haven't called AddAuditLog, which the helper handles as a no-op.
+            IAuditWriter? auditWriter = null;
+            // Site-local tracking store
+            // backing Tracking.Status(id). Singleton; null in tests / hosts
+            // that haven't wired the store, which the helper handles by
+            // throwing on access.
+            IOperationTrackingStore? operationTrackingStore = null;
+            // Site-side cached-call
+            // telemetry forwarder. Singleton bound to the AuditLog
+            // composition root; null in tests / hosts that haven't called
+            // AddAuditLog, in which case the cached-call helpers degrade
+            // to the no-emission path (the underlying S&F handoff still
+            // happens and a TrackedOperationId is still returned).
+            ICachedCallTelemetryForwarder? cachedForwarder = null;
+            // SourceNode-stamping: the local node name
+            // resolved from INodeIdentityProvider — node-a/node-b on site
+            // hosts. Null in tests / hosts that haven't registered the
+            // provider, in which case NotificationSubmit.SourceNode and
+            // SiteCallOperational.SourceNode stay null and central
+            // persists the rows with SourceNode NULL.
+            string? sourceNode = null;
+
+            if (serviceProvider != null)
+            {
+                serviceScope = serviceProvider.CreateScope();
+                externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
+                databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
+                storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
+                siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
+                    ?? string.Empty;
+                auditWriter = serviceScope.ServiceProvider.GetService<IAuditWriter>();
+                operationTrackingStore = serviceScope.ServiceProvider.GetService<IOperationTrackingStore>();
+                cachedForwarder = serviceScope.ServiceProvider.GetService<ICachedCallTelemetryForwarder>();
+                sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
+            }
+
+            var context = new ScriptRuntimeContext(
+                instanceActor,
+                sharedScriptLibrary,
+                callDepth,
+                options.MaxScriptCallDepth,
+                timeout,
+                instanceName,
+                logger,
+                externalSystemClient,
+                databaseGateway,
+                storeAndForward,
+                siteCommunicationActor,
+                siteId,
+                // Notification Outbox (FU3): stamp the executing script onto outbound
+                // notifications using the Site Event Logging "Source" convention.
+                sourceScript: $"ScriptActor:{scriptName}",
+                // Emit one ApiOutbound/ApiCall row per
+                // ExternalSystem.Call. Writer is best-effort; failures are logged
+                // and swallowed inside the helper so the script's call path is
+                // never aborted by an audit failure.
+                auditWriter: auditWriter,
+                // Site-local tracking store
+                // backing Tracking.Status(id). Authoritative source of truth for
+                // cached-call status — read directly by the script API.
+                operationTrackingStore: operationTrackingStore,
+                // Cached-call telemetry
+                // forwarder for ExternalSystem.CachedCall / Database.CachedWrite
+                // CachedSubmit emission + the immediate-success terminal-row
+                // emission. Best-effort: null degrades the helpers to a
+                // no-emission path; the S&F handoff and TrackedOperationId
+                // return are unaffected.
+                cachedForwarder: cachedForwarder,
+                // The spawning execution's
+                // id for an inbound-API-routed call. The routed script still
+                // mints its own fresh ExecutionId — this records the spawner.
+                // Null for normal (tag-change / timer) runs.
+                parentExecutionId: parentExecutionId,
+                // SourceNode-stamping: the local node name
+                // (node-a/node-b on a site) — threaded down so Notify.Send
+                // and the four cached-call telemetry constructors can stamp
+                // it onto NotificationSubmit.SourceNode and
+                // SiteCallOperational.SourceNode respectively.
+                sourceNode: sourceNode,
+                // Thread the singleton site event logger so
+                // recursion-limit violations at CallScript/CallShared emit a
+                // script Error site event in addition to ILogger.LogError.
+                siteEventLogger: siteEventLogger,
+                // WaitForAttribute (spec §4.3/§4.4): thread the per-script
+                // execution-timeout token so Attributes.WaitAsync's Ask is
+                // bounded by the script's own ExecutionTimeoutSeconds — a
+                // shorter script deadline wins over the wait's own timeout.
+                scriptTimeoutToken: cts.Token);
+
+            var globals = new ScriptGlobals
+            {
+                Instance = context,
+                Parameters = new ScriptParameters(parameters ?? new Dictionary<string, object?>()),
+                CancellationToken = cts.Token,
+                Scope = scope
+            };
+
+            // Operational `script` event — execution started. Fire-and-forget
+            // (the `_ =` discards the task) so the event log can never block or
+            // fault the script's own run; mirrors the existing Error-path emit.
+            _ = siteEventLogger?.LogEventAsync(
+                "script", "Info", instanceName, $"ScriptActor:{scriptName}",
+                $"Script '{scriptName}' on instance '{instanceName}' started");
+
+            var state = await compiledScript.RunAsync(globals, cts.Token);
+
+            // Send result to requester if this was an Ask-based call
+            if (!replyTo.IsNobody())
+            {
+                replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
+            }
+
+            // Operational `script` event — execution completed successfully.
+            _ = siteEventLogger?.LogEventAsync(
+                "script", "Info", instanceName, $"ScriptActor:{scriptName}",
+                $"Script '{scriptName}' on instance '{instanceName}' completed");
+
+            // Notify the owning ScriptActor of completion (also releases its in-flight slot).
+            completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
+        }
+        catch (OperationCanceledException)
+        {
+            healthCollector?.IncrementScriptError();
+            var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
+            logger.LogWarning("{Message} (run {RunId})", errorMsg, runId);
+
+            // Failures recorded to site event log; script NOT disabled after failure.
+            _ = siteEventLogger?.LogEventAsync(
+                "script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
+
+            if (!replyTo.IsNobody())
+            {
+                replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
+            }
+
+            completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
+        }
+        catch (Exception ex)
+        {
+            healthCollector?.IncrementScriptError();
+            // Failures recorded to site event log; script NOT disabled after failure.
+            var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
+            logger.LogError(ex, "Script execution failed: {Script} on {Instance} (run {RunId})",
+                scriptName, instanceName, runId);
+
+            _ = siteEventLogger?.LogEventAsync(
+                "script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());
+
+            if (!replyTo.IsNobody())
+            {
+                replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
+            }
+
+            completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
+        }
+        finally
+        {
+            // Mark the body finished so the stuck-script watchdog (registered on
+            // cts.Token above) treats a timely cancellation as NOT stuck.
+            Interlocked.Exchange(ref completed, 1);
+            // Dispose the DI scope (and scoped services) after script execution completes
+            serviceScope?.Dispose();
+            cts.Dispose();
+        }
+    }
+
+    private static async Task RunAlarmScriptAsync(
+        string alarmName,
+        string instanceName,
+        AlarmLevel level,
+        int priority,
+        string message,
+        Script<object?> compiledScript,
+        IActorRef instanceActor,
+        SharedScriptLibrary sharedScriptLibrary,
+        SiteRuntimeOptions options,
+        IActorRef completionTarget,
+        ILogger logger,
+        long runId,
+        Guid? parentExecutionId,
+        TimeSpan timeout,
+        CancellationTokenSource cts)
+    {
+        try
+        {
+            // Enqueue-anchored deadline: an already-cancelled token means this on-trigger run
+            // spent its whole budget queueing behind other script bodies. Shed it rather than
+            // run it late — the alarm it would react to is already stale.
+            cts.Token.ThrowIfCancellationRequested();
+
+            // An alarm on-trigger run can call Instance.CallScript()
+            // via the ScriptRuntimeContext injected into globals
+            var context = new ScriptRuntimeContext(
+                instanceActor,
+                sharedScriptLibrary,
+                currentCallDepth: 0,
+                options.MaxScriptCallDepth,
+                timeout,
+                instanceName,
+                logger,
+                // ParentExecutionId tag-cascade: the
+                // alarm on-trigger run mints its own fresh ExecutionId (the
+                // ctor's `?? NewGuid()` fallback) and records the firing
+                // execution's id as its ParentExecutionId — null (a root)
+                // only when the firing value came from the DCL.
+                parentExecutionId: parentExecutionId,
+                // WaitForAttribute (spec §4.4): thread the alarm on-trigger
+                // script's per-script execution-timeout token so a
+                // Attributes.WaitAsync inside an on-trigger script is bounded
+                // by the same script deadline.
+                scriptTimeoutToken: cts.Token);
+
+            var globals = new ScriptGlobals
+            {
+                Instance = context,
+                Parameters = new ScriptParameters(),
+                CancellationToken = cts.Token,
+                Alarm = new AlarmContext
+                {
+                    Name = alarmName,
+                    Level = level,
+                    Priority = priority,
+                    Message = message
+                }
+            };
+
+            await compiledScript.RunAsync(globals, cts.Token);
+
+            completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
+        }
+        catch (OperationCanceledException)
+        {
+            logger.LogWarning(
+                "Alarm on-trigger script for {Alarm} on {Instance} timed out (run {RunId})",
+                alarmName, instanceName, runId);
+            completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
+        }
+        catch (Exception ex)
+        {
+            // Failures logged, alarm continues
+            logger.LogError(ex,
+                "Alarm on-trigger script for {Alarm} on {Instance} failed (run {RunId})",
+                alarmName, instanceName, runId);
+            completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
+        }
+        finally
+        {
+            cts.Dispose();
+        }
+    }
+
+    /// <summary>
+    /// Arms the stuck-script watchdog (S2, extended by WP3.1).
+    ///
+    /// <para>The CTS firing only REQUESTS cooperative cancellation; it does NOT free a thread
+    /// blocked in synchronous I/O. When the deadline elapses, this waits
+    /// <see cref="SiteRuntimeOptions.StuckScriptGraceMs"/> and, if the body still has not
+    /// returned, (a) names the script loudly on the site event log and (b) DETACHES the worker
+    /// thread it is holding and starts a replacement — so a wedged script costs the pool one
+    /// thread only until the watchdog fires, not forever. At the detach cap it logs an Error
+    /// instead of growing threads without bound.</para>
+    ///
+    /// <para>The <c>Task.Run</c>/<c>Task.Delay</c> run on the shared thread pool, never on the
+    /// (possibly saturated) script scheduler — deliberate, and the whole point of the watchdog.
+    /// <c>Register</c> fires on cancellation only; a normally-completing run disposes its CTS
+    /// with no callback. A run cancelled while still QUEUED flips its completed flag in the
+    /// body's <c>finally</c> long before the grace elapses, so it is correctly not reported.</para>
+    /// </summary>
+    private static void ArmStuckScriptWatchdog(
+        CancellationTokenSource cts,
+        SiteRuntimeOptions options,
+        ILogger logger,
+        ISiteEventLogger? siteEventLogger,
+        ScriptExecutionScheduler scheduler,
+        int? workerSlot,
+        long workerRunStamp,
+        TimeSpan timeout,
+        Func<bool> isCompleted,
+        string subject,
+        string instanceName,
+        string source)
+    {
+        var graceMs = options.StuckScriptGraceMs;
+        cts.Token.Register(() => _ = Task.Run(async () =>
+        {
+            await Task.Delay(graceMs);
+            if (isCompleted()) return;
+
+            var stuckMsg = $"{subject} exceeded its {timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — " +
+                           "its dedicated script-execution thread is blocked (cooperative cancellation not observed).";
+
+            var outcome = workerSlot is { } slot
+                ? scheduler.TryDetachWorker(slot, workerRunStamp)
+                : WorkerDetachOutcome.NotRunning;
+
+            switch (outcome)
+            {
+                case WorkerDetachOutcome.Detached:
+                    stuckMsg += " The worker thread has been DETACHED and replaced; it will exit when the body returns.";
+                    break;
+                case WorkerDetachOutcome.AtCap:
+                    stuckMsg += $" The script-execution pool already holds {scheduler.DetachedThreadCount} detached " +
+                                "stuck threads (at cap); NOT replacing this one.";
+                    break;
+                case WorkerDetachOutcome.NotRunning:
+                    // The body is not on a worker thread (it hopped threads on an await), so no
+                    // dedicated thread is lost — report it, but there is nothing to replace.
+                    break;
+            }
+
+            logger.LogError(stuckMsg);
+            _ = siteEventLogger?.LogEventAsync("script", "Error", instanceName, source, stuckMsg);
+        }));
+    }
+}

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptRuntimeContext.cs

@@ -40,7 +40,6 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
 public class ScriptRuntimeContext
 {
     private readonly IActorRef _instanceActor;
-    private readonly IActorRef _self;
     private readonly SharedScriptLibrary _sharedScriptLibrary;
     private readonly int _currentCallDepth;
     private readonly int _maxCallDepth;
@@ -50,7 +49,7 @@ public class ScriptRuntimeContext
 
     /// <summary>
     /// WaitForAttribute (spec §4.3): the per-script execution-timeout token from
-    /// the owning <c>ScriptExecutionActor</c>/<c>AlarmExecutionActor</c>
+    /// the owning script or alarm on-trigger run
     /// (<c>cts.Token</c>). Bounds the <c>Attributes.WaitAsync</c> Ask so a script
     /// that hits its own <c>ExecutionTimeoutSeconds</c> abandons the wait. Defaults
     /// to <see cref="CancellationToken.None"/> for contexts that do not thread one
@@ -170,7 +169,6 @@ public class ScriptRuntimeContext
     /// execution, external system calls, database access, and notification delivery.
     /// </summary>
     /// <param name="instanceActor">Reference to the Instance Actor managing this instance's state.</param>
-    /// <param name="self">Reference to the executing script actor.</param>
     /// <param name="sharedScriptLibrary">Library containing shared scripts available to all instances.</param>
     /// <param name="currentCallDepth">Current recursion depth of script calls.</param>
     /// <param name="maxCallDepth">Maximum allowed recursion depth before an error is thrown.</param>
@@ -217,7 +215,6 @@ public class ScriptRuntimeContext
     /// </param>
     public ScriptRuntimeContext(
         IActorRef instanceActor,
-        IActorRef self,
         SharedScriptLibrary sharedScriptLibrary,
         int currentCallDepth,
         int maxCallDepth,
@@ -240,7 +237,6 @@ public class ScriptRuntimeContext
         CancellationToken scriptTimeoutToken = default)
     {
         _instanceActor = instanceActor;
-        _self = self;
         _sharedScriptLibrary = sharedScriptLibrary;
         _currentCallDepth = currentCallDepth;
         _maxCallDepth = maxCallDepth;
@@ -257,7 +253,7 @@ public class ScriptRuntimeContext
         _operationTrackingStore = operationTrackingStore;
         _cachedForwarder = cachedForwarder;
         // SourceNode-stamping: the local node name read from
-        // INodeIdentityProvider at the ScriptExecutionActor; null when no
+        // INodeIdentityProvider at the launching script run; null when no
         // provider was wired so the downstream callsites pass null through
         // verbatim — leaving central SourceNode as NULL.
         _sourceNode = sourceNode;
@@ -265,7 +261,7 @@ public class ScriptRuntimeContext
         // (ParentExecutionId): stored verbatim — no `?? NewGuid()`
         // fallback. A non-routed run legitimately has no parent and stays null.
         _parentExecutionId = parentExecutionId;
-        // Optional — null when not wired (tests / AlarmExecutionActor).
+        // Optional — null when not wired (tests / alarm on-trigger runs).
         _siteEventLogger = siteEventLogger;
         // WaitForAttribute (spec §4.3): default(CancellationToken) == None when
         // not threaded in — the WaitAsync Ask is then bounded only by its own timeout.
@@ -302,7 +298,6 @@ public class ScriptRuntimeContext
     {
         return new ScriptRuntimeContext(
             _instanceActor,
-            _self,
             _sharedScriptLibrary,
             childCallDepth,
             _maxCallDepth,
@@ -333,7 +328,7 @@ public class ScriptRuntimeContext
     /// <summary>
     /// Fire-and-forget emission of a <c>script</c> Error site event
     /// for a recursion-limit violation. Mirrors the call shape used by
-    /// <c>ScriptExecutionActor</c>'s catch blocks. A fault from
+    /// the run's own catch blocks. A fault from
     /// the site-event logger is observed-and-dropped (best-effort) via
     /// <c>ContinueWith(OnlyOnFaulted)</c> — it never blocks or faults the
     /// <c>_logger.LogError</c> + throw path that follows. A null logger is a no-op.
@@ -1135,7 +1130,7 @@ public class ScriptRuntimeContext
                         SourceSite: _siteId,
                         // SourceNode-stamping: the local node name
                         // (node-a/node-b) — threaded through INodeIdentityProvider
-                        // at the ScriptExecutionActor; null when no provider was
+                        // at the launching script run; null when no provider was
                         // wired so central persists SiteCalls.SourceNode as NULL.
                         SourceNode: _sourceNode,
                         Status: "Submitted",
@@ -1254,7 +1249,7 @@ public class ScriptRuntimeContext
                         SourceSite: _siteId,
                         // SourceNode-stamping: the local node name
                         // (node-a/node-b) — threaded through INodeIdentityProvider
-                        // at the ScriptExecutionActor; null when no provider was
+                        // at the launching script run; null when no provider was
                         // wired so central persists SiteCalls.SourceNode as NULL.
                         SourceNode: _sourceNode,
                         Status: "Attempted",
@@ -1322,7 +1317,7 @@ public class ScriptRuntimeContext
                         SourceSite: _siteId,
                         // SourceNode-stamping: the local node name
                         // (node-a/node-b) — threaded through INodeIdentityProvider
-                        // at the ScriptExecutionActor; null when no provider was
+                        // at the launching script run; null when no provider was
                         // wired so central persists SiteCalls.SourceNode as NULL.
                         SourceNode: _sourceNode,
                         Status: operationalTerminalStatus,
@@ -1913,7 +1908,7 @@ public class ScriptRuntimeContext
                         SourceSite: _siteId,
                         // SourceNode-stamping: the local node name
                         // (node-a/node-b) — threaded through INodeIdentityProvider
-                        // at the ScriptExecutionActor; null when no provider was
+                        // at the launching script run; null when no provider was
                         // wired so central persists SiteCalls.SourceNode as NULL.
                         SourceNode: _sourceNode,
                         Status: "Submitted",
@@ -2260,7 +2255,7 @@ public class ScriptRuntimeContext
                 OriginParentExecutionId: _parentExecutionId,
                 // SourceNode-stamping: the cluster node name on which this
                 // notification was emitted (node-a/node-b). Stamped from the local
-                // INodeIdentityProvider via ScriptExecutionActor. Rides inside the
+                // INodeIdentityProvider via the launching script run. Rides inside the
                 // serialized payload through the S&F buffer to central, where
                 // NotificationOutboxActor persists it on the Notifications row.
                 SourceNode: _sourceNode);

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptSchedulerStatsReporter.cs

@@ -80,7 +80,9 @@ public sealed class ScriptSchedulerStatsReporter : BackgroundService
             _collector.SetScriptSchedulerStats(
                 scheduler.QueueDepth,
                 scheduler.BusyThreadCount,
-                scheduler.OldestBusyAge?.TotalSeconds);
+                scheduler.OldestBusyAge?.TotalSeconds,
+                // WP3.1: workers detached and replaced by the stuck-script watchdog.
+                scheduler.DetachedThreadCount);
         }
         catch (Exception ex)
         {

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/SiteScriptCompileCache.cs

@@ -26,19 +26,41 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
 ///
 /// <para>
 /// Bounded at <see cref="MaxEntries"/> entries — deliberately smaller than the verdict cache's
-/// 4096 because entries pin compiled assemblies, not just verdict strings. On overflow the cache
-/// is cleared wholesale (results are recomputable, so a coarse reset avoids eviction bookkeeping).
-/// <see cref="Hits"/>/<see cref="Count"/>/<see cref="Clear"/> are exposed for tests and diagnostics.
+/// 4096 because entries pin compiled assemblies, not just verdict strings.
 /// </para>
+///
+/// <para><b>WP3.1 — approximate LRU replaced the overflow cliff.</b> The cache used to
+/// <c>Clear()</c> wholesale on overflow. Instance scripts AND trigger expressions share this
+/// cache, so on a site large enough to cross 1024 entries every overflow discarded up to 1023
+/// live compiled scripts and the next deploy or Instance-Actor start paid a full recompile
+/// storm — on actor threads. Now an insert at the bound evicts only the oldest
+/// <see cref="EvictionBatchDivisor">⅛</see> of entries by last-access stamp: overflow costs one
+/// 1024-element scan instead of 1023 future recompiles, and hot entries survive.</para>
+///
+/// <para>Recency is an <see cref="Interlocked"/> access sequence, not a clock: it is
+/// deterministic for tests and immune to clock steps. Hits update the stamp lock-free; the
+/// only lock is the (rare) eviction sweep, double-checked so concurrent inserts do not
+/// stampede it.</para>
 /// </summary>
 internal static class SiteScriptCompileCache
 {
-    /// <summary>Upper bound on cached entries; the cache is cleared wholesale on overflow.</summary>
+    /// <summary>Upper bound on cached entries; crossing it evicts the oldest batch.</summary>
     internal const int MaxEntries = 1024;
 
-    private static readonly ConcurrentDictionary<string, ScriptCompilationResult> Cache = new();
+    /// <summary>
+    /// Fraction of the cache evicted in one sweep (⅛ = 128 entries at
+    /// <see cref="MaxEntries"/>). Batching amortises the O(n) scan across many inserts, so
+    /// steady-state churn does not re-scan on every single miss.
+    /// </summary>
+    private const int EvictionBatchDivisor = 8;
+
+    private static readonly ConcurrentDictionary<string, CacheEntry> Cache = new();
+    private static readonly object EvictionLock = new();
     private static long _hits;
 
+    /// <summary>Monotonic access sequence; the recency stamp written onto entries.</summary>
+    private static long _accessSequence;
+
     /// <summary>Number of cache hits observed since the last <see cref="Clear"/>.</summary>
     public static long Hits => Interlocked.Read(ref _hits);
 
@@ -48,8 +70,8 @@ internal static class SiteScriptCompileCache
     /// <summary>
     /// Returns the cached compile result for <paramref name="code"/> against
     /// <paramref name="globalsType"/>, or computes it via <paramref name="factory"/> and caches
-    /// it on a miss. A hit increments <see cref="Hits"/>. Both success and failure results are
-    /// cached — the error text is name-free by construction.
+    /// it on a miss. A hit increments <see cref="Hits"/> and refreshes the entry's recency.
+    /// Both success and failure results are cached — the error text is name-free by construction.
     /// </summary>
     /// <param name="code">The script source code to look up (hashed to form the cache key).</param>
     /// <param name="globalsType">The Roslyn globals surface the script compiles against; part of the key so identical source under different globals stays distinct.</param>
@@ -59,19 +81,19 @@ internal static class SiteScriptCompileCache
     {
         var key = globalsType.FullName + ":" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(code)));
 
-        if (Cache.TryGetValue(key, out var result))
+        if (Cache.TryGetValue(key, out var entry))
         {
             Interlocked.Increment(ref _hits);
-            return result;
+            Touch(entry);
+            return entry.Result;
         }
 
-        result = factory();
+        var result = factory();
 
-        // Coarse bound: on overflow drop everything rather than track evictions.
         if (Cache.Count >= MaxEntries)
-            Cache.Clear();
+            EvictOldestBatch();
 
-        Cache[key] = result;
+        Cache[key] = new CacheEntry(result) { LastAccess = NextStamp() };
         return result;
     }
 
@@ -81,4 +103,53 @@ internal static class SiteScriptCompileCache
         Cache.Clear();
         Interlocked.Exchange(ref _hits, 0);
     }
+
+    /// <summary>Refreshes an entry's recency stamp. Lock-free — a lost race only costs accuracy, never correctness.</summary>
+    private static void Touch(CacheEntry entry) => Volatile.Write(ref entry.LastAccess, NextStamp());
+
+    private static long NextStamp() => Interlocked.Increment(ref _accessSequence);
+
+    /// <summary>
+    /// Evicts the oldest <see cref="MaxEntries"/> / <see cref="EvictionBatchDivisor"/> entries
+    /// by recency stamp. Double-checked under <see cref="EvictionLock"/> so several concurrent
+    /// inserts crossing the bound together perform ONE sweep rather than one each.
+    /// </summary>
+    private static void EvictOldestBatch()
+    {
+        lock (EvictionLock)
+        {
+            if (Cache.Count < MaxEntries) return; // another thread already swept
+
+            var batch = Math.Max(1, MaxEntries / EvictionBatchDivisor);
+
+            // ConcurrentDictionary.ToArray() takes an ATOMIC snapshot. Enumerating (or
+            // LINQ-ing) the dictionary directly does not: LINQ's ToArray picks the
+            // ICollection<KeyValuePair<,>>.CopyTo fast path, which reads Count and then
+            // copies, and throws ArgumentException when a concurrent insert lands between
+            // the two. The eviction lock only excludes other EVICTORS — inserts run
+            // lock-free by design — so the snapshot must be the thread-safe one.
+            var victims = Cache.ToArray()
+                .OrderBy(kvp => Volatile.Read(ref kvp.Value.LastAccess))
+                .Take(batch)
+                .Select(kvp => kvp.Key)
+                .ToArray();
+
+            foreach (var victim in victims)
+                Cache.TryRemove(victim, out _);
+        }
+    }
+
+    /// <summary>
+    /// A cached compile result plus its recency stamp. A class (not a struct) so
+    /// <see cref="Touch"/> can update recency in place without replacing the dictionary
+    /// value — the hot path stays a single volatile write.
+    /// </summary>
+    private sealed class CacheEntry(ScriptCompilationResult result)
+    {
+        /// <summary>The memoised compile result (success or failure).</summary>
+        public ScriptCompilationResult Result { get; } = result;
+
+        /// <summary>Access sequence number of the most recent hit; older = evicted first.</summary>
+        public long LastAccess;
+    }
 }

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/TriggerEvalGate.cs

@@ -0,0 +1,81 @@
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+
+/// <summary>
+/// WP3.1 (finding #4): the process-wide concurrency gate for trigger-expression evaluation.
+///
+/// <para>Before WP3.1, <c>ScriptActor</c> and <c>AlarmActor</c> evaluated their compiled
+/// Expression triggers on the <see cref="ScriptExecutionScheduler"/> — the same bounded pool
+/// of dedicated threads that runs script bodies. Eight scripts blocked in synchronous I/O
+/// therefore stalled EVERY Expression trigger on the node behind them, for an unbounded time,
+/// and the evaluation's own 2 s timeout did not even start until it was dequeued. An alarm
+/// that should have raised in milliseconds simply never raised.</para>
+///
+/// <para>Trigger expressions are non-blocking <em>by construction</em>:
+/// <see cref="TriggerExpressionGlobals"/> exposes only reads over an in-memory snapshot
+/// dictionary, and the script trust gate has already denied I/O, network, threading, and
+/// reflection long before the expression can deploy. They are short CPU-bound work — exactly
+/// what the shared .NET thread pool is for. So they run there, and this semaphore is the only
+/// thing bounding their fan-out. A second dedicated pool was considered and rejected: it would
+/// add threads, gauges, and a second starvation surface for no isolation gain.</para>
+///
+/// <para>Per-trigger coalescing in the actors (one evaluation in flight, one pending) already
+/// caps waiters at one per Expression trigger, so this gate's queue is bounded by trigger
+/// count.</para>
+///
+/// Mirrors <see cref="ScriptExecutionScheduler.Shared"/>'s lazy-singleton plus injectable-seam
+/// shape so tests can drive a gate of size 1 deterministically.
+/// </summary>
+public sealed class TriggerEvalGate : IDisposable
+{
+    private readonly SemaphoreSlim _gate;
+
+    private static volatile TriggerEvalGate? _shared;
+    private static readonly object SharedLock = new();
+
+    /// <summary>Creates a gate admitting <paramref name="maxConcurrency"/> concurrent evaluations.</summary>
+    /// <param name="maxConcurrency">Maximum concurrent trigger-expression evaluations; values below 1 are clamped to 1.</param>
+    public TriggerEvalGate(int maxConcurrency)
+    {
+        MaxConcurrency = Math.Max(1, maxConcurrency);
+        _gate = new SemaphoreSlim(MaxConcurrency, MaxConcurrency);
+    }
+
+    /// <summary>The configured concurrency limit.</summary>
+    public int MaxConcurrency { get; }
+
+    /// <summary>Free permits right now; 0 means the gate is saturated and new evaluations will queue.</summary>
+    public int AvailablePermits => _gate.CurrentCount;
+
+    /// <summary>
+    /// The process-wide gate, used when no gate is injected. Lazily created from
+    /// <see cref="SiteRuntimeOptions.TriggerEvalMaxConcurrency"/>; the first caller wins.
+    /// </summary>
+    /// <param name="options">Site runtime options supplying the concurrency limit.</param>
+    /// <returns>The shared gate instance.</returns>
+    public static TriggerEvalGate Shared(SiteRuntimeOptions options)
+    {
+        var existing = _shared;
+        if (existing is not null) return existing;
+
+        lock (SharedLock)
+        {
+            return _shared ??= new TriggerEvalGate(options.TriggerEvalMaxConcurrency);
+        }
+    }
+
+    /// <summary>
+    /// Waits for a permit. The caller passes the evaluation's own deadline token, which was
+    /// armed at ENQUEUE — so time spent queueing here burns the same budget the evaluation
+    /// itself would, and a saturated gate produces a timely cancellation (treated as
+    /// <see langword="false"/> by both actors) instead of an unbounded stall.
+    /// </summary>
+    /// <param name="cancellationToken">The evaluation's deadline token.</param>
+    /// <returns>A task that completes when a permit is acquired.</returns>
+    public Task WaitAsync(CancellationToken cancellationToken) => _gate.WaitAsync(cancellationToken);
+
+    /// <summary>Returns a permit. Must be called exactly once per successful <see cref="WaitAsync"/>.</summary>
+    public void Release() => _gate.Release();
+
+    /// <inheritdoc />
+    public void Dispose() => _gate.Dispose();
+}

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs

@@ -38,13 +38,64 @@ public class SiteRuntimeOptions
     public int StreamBufferSize { get; set; } = 1000;
 
     /// <summary>
-    /// Number of dedicated threads in the script-execution scheduler.
+    /// FLOOR for the number of dedicated threads in the script-execution scheduler.
     /// Script and alarm on-trigger bodies run on these threads instead of the shared
     /// .NET thread pool, so blocking script I/O cannot starve the global pool.
+    ///
+    /// <para>WP3.1: this was a fixed size and is now the lower bound of a grow-only,
+    /// instance-scaled pool — see
+    /// <see cref="Scripts.ScriptExecutionScheduler.ComputeTargetThreads"/>. Existing
+    /// configurations keep exactly the previous behaviour at or below
+    /// <c>ScriptExecutionThreadCount * InstancesPerScriptThread</c> deployed instances.</para>
+    ///
     /// Default: 8.
     /// </summary>
     public int ScriptExecutionThreadCount { get; set; } = 8;
 
+    /// <summary>
+    /// WP3.1: CEILING for the instance-scaled script-execution pool. The pool grows
+    /// towards <c>ceil(enabledInstances / 8)</c> threads but never past this value; beyond
+    /// it, <see cref="MaxConcurrentRunsPerScript"/> is the real regulator. A 1 MB-stack
+    /// dedicated thread is cheap, so 32 (≈ 256 instances at the /8 ratio) is a generous
+    /// default. Must be greater than or equal to <see cref="ScriptExecutionThreadCount"/>.
+    /// Default: 32.
+    /// </summary>
+    public int ScriptExecutionMaxThreadCount { get; set; } = 32;
+
+    /// <summary>
+    /// WP3.1 (finding #4): maximum number of trigger-expression evaluations allowed to run
+    /// concurrently across the whole process. Trigger expressions are non-blocking by
+    /// construction (<see cref="Scripts.TriggerExpressionGlobals"/> exposes only reads over an
+    /// in-memory snapshot, and the script trust gate has already denied I/O, network, and
+    /// threading), so they run as plain async work on the shared .NET thread pool behind this
+    /// gate rather than on the blocking script-execution pool. That is what keeps an alarm's
+    /// Expression trigger from queueing behind blocking script bodies.
+    /// Default: <c>max(2, Environment.ProcessorCount)</c>.
+    /// </summary>
+    public int TriggerEvalMaxConcurrency { get; set; } = Math.Max(2, Environment.ProcessorCount);
+
+    /// <summary>
+    /// WP3.1: timeout (seconds) for a single trigger-expression evaluation. Previously
+    /// hardcoded at 2 s in both <c>ScriptActor</c> and <c>AlarmActor</c>. The deadline is now
+    /// armed when the evaluation is ENQUEUED, not when it is dequeued, so time spent waiting
+    /// on <see cref="TriggerEvalMaxConcurrency"/> burns the same budget — a saturated gate
+    /// produces a timely "false" instead of an unbounded stall.
+    /// Default: 2.
+    /// </summary>
+    public int TriggerEvalTimeoutSeconds { get; set; } = 2;
+
+    /// <summary>
+    /// WP3.1: maximum number of concurrent runs (queued or executing) for any single script
+    /// or alarm on-trigger script. A trigger that arrives while the cap is reached is SHED —
+    /// the newest run is refused, the four already in flight (which are closest to their own
+    /// deadlines) are kept, and no reordering occurs. A shed increments
+    /// <c>ISiteHealthCollector.IncrementScriptRunShed</c>, emits a rate-limited Warning site
+    /// event, and — for an Ask-based <c>CallScript</c> — replies with an explicit error rather
+    /// than letting the caller hang to its Ask timeout.
+    /// Default: 4.
+    /// </summary>
+    public int MaxConcurrentRunsPerScript { get; set; } = 4;
+
     /// <summary>
     /// Max mirrored native alarms retained per source binding before older entries are dropped (logged).
     /// Default: 1000.

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptionsValidator.cs

@@ -39,7 +39,28 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
 
         builder.RequireThat(options.ScriptExecutionThreadCount > 0,
             $"ScadaBridge:SiteRuntime:ScriptExecutionThreadCount must be greater than 0 " +
-            $"(was {options.ScriptExecutionThreadCount}); it sizes the dedicated script-execution scheduler.");
+            $"(was {options.ScriptExecutionThreadCount}); it is the FLOOR for the dedicated script-execution scheduler.");
+
+        builder.RequireThat(options.ScriptExecutionMaxThreadCount >= options.ScriptExecutionThreadCount,
+            $"ScadaBridge:SiteRuntime:ScriptExecutionMaxThreadCount must be >= ScriptExecutionThreadCount " +
+            $"(was {options.ScriptExecutionMaxThreadCount} vs {options.ScriptExecutionThreadCount}); it is the " +
+            "CEILING for the instance-scaled script-execution pool and a ceiling below the floor would " +
+            "silently shrink the configured pool.");
+
+        builder.RequireThat(options.TriggerEvalMaxConcurrency > 0,
+            $"ScadaBridge:SiteRuntime:TriggerEvalMaxConcurrency must be greater than 0 " +
+            $"(was {options.TriggerEvalMaxConcurrency}); it gates concurrent trigger-expression evaluations " +
+            "and a zero gate would park every Expression trigger on the node forever.");
+
+        builder.RequireThat(options.TriggerEvalTimeoutSeconds > 0,
+            $"ScadaBridge:SiteRuntime:TriggerEvalTimeoutSeconds must be greater than 0 " +
+            $"(was {options.TriggerEvalTimeoutSeconds}); it bounds a single trigger-expression evaluation " +
+            "(measured from enqueue, so it also bounds gate-wait time).");
+
+        builder.RequireThat(options.MaxConcurrentRunsPerScript > 0,
+            $"ScadaBridge:SiteRuntime:MaxConcurrentRunsPerScript must be greater than 0 " +
+            $"(was {options.MaxConcurrentRunsPerScript}); it caps concurrent runs per script and a zero cap " +
+            "would shed every trigger.");
 
         builder.RequireThat(options.MirroredAlarmCapPerSource > 0,
             $"ScadaBridge:SiteRuntime:MirroredAlarmCapPerSource must be greater than 0 " +