perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs

@@ -25,6 +25,20 @@ public interface ISiteHealthCollector
     /// </summary>
     void IncrementDeadLetter();
 
+    /// <summary>
+    /// WP3.1: increments the per-interval count of script/alarm runs SHED because
+    /// <c>MaxConcurrentRunsPerScript</c> runs were already in flight for that script. A raw
+    /// per-interval count like the script/alarm error counters — a sustained non-zero value
+    /// means a trigger is firing faster than its script can complete. Every shed is counted
+    /// here even when its accompanying site event is rate-limited away.
+    /// Default interface implementation is a no-op so existing test fakes continue to
+    /// compile without per-fake updates.
+    /// </summary>
+    void IncrementScriptRunShed()
+    {
+        // Default no-op so test fakes do not need to be updated.
+    }
+
     /// <summary>
     /// Increment the per-interval count of
     /// <c>FallbackAuditWriter</c> primary failures. Bridged from the
@@ -173,7 +187,13 @@ public interface ISiteHealthCollector
     /// <param name="queueDepth">Script tasks waiting to run.</param>
     /// <param name="busyThreads">Worker threads currently executing a script.</param>
     /// <param name="oldestBusyAgeSeconds">Age (seconds) of the oldest in-flight script, or <c>null</c> when idle.</param>
-    void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
+    /// <param name="detachedThreads">
+    /// WP3.1: worker threads the stuck-script watchdog has detached and replaced because
+    /// their script wedged past its deadline plus grace, and which have not yet returned.
+    /// A non-zero, non-draining value means script bodies are permanently blocking threads.
+    /// </param>
+    void SetScriptSchedulerStats(
+        int queueDepth, int busyThreads, double? oldestBusyAgeSeconds, int detachedThreads = 0)
     {
         // Default no-op so test fakes do not need to be updated.
     }

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs

@@ -39,6 +39,12 @@ public class SiteHealthCollector : ISiteHealthCollector
     private int _scriptQueueDepth;
     private int _scriptBusyThreads;
     private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
+    // WP3.1: workers detached and replaced by the stuck-script watchdog and not yet exited
+    // (point-in-time, from the same reporter tick as the gauges above), and the per-interval
+    // count of runs shed by the per-script in-flight cap (reset on collect like the error
+    // counters, and restored by AddIntervalCounters when a report fails to send).
+    private int _scriptDetachedThreads;
+    private int _scriptRunShedCount;
     // WP2.6d: cumulative alarm-publish-queue drop count, refreshed by
     // SiteStreamAlarmDropReporter. Point-in-time (not reset on collect), like the
     // scheduler gauges above.
@@ -162,12 +168,20 @@ public class SiteHealthCollector : ISiteHealthCollector
     }
 
     /// <inheritdoc />
-    public void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
+    public void SetScriptSchedulerStats(
+        int queueDepth, int busyThreads, double? oldestBusyAgeSeconds, int detachedThreads = 0)
     {
         Interlocked.Exchange(ref _scriptQueueDepth, queueDepth);
         Interlocked.Exchange(ref _scriptBusyThreads, busyThreads);
         Interlocked.Exchange(ref _scriptOldestBusyAgeBits,
             BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN));
+        Interlocked.Exchange(ref _scriptDetachedThreads, detachedThreads);
+    }
+
+    /// <inheritdoc />
+    public void IncrementScriptRunShed()
+    {
+        Interlocked.Increment(ref _scriptRunShedCount);
     }
 
     /// <summary>Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null.</summary>
@@ -283,6 +297,11 @@ public class SiteHealthCollector : ISiteHealthCollector
             ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0),
             ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0),
             ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds(),
+            // WP3.1: point-in-time (like the three gauges above), so a CompareExchange read
+            // rather than an Exchange reset.
+            DetachedScriptThreads = Interlocked.CompareExchange(ref _scriptDetachedThreads, 0, 0),
+            // WP3.1: per-interval, so drained on collect like the error counters.
+            ScriptRunShedCount = Interlocked.Exchange(ref _scriptRunShedCount, 0),
             // Both fields come from the ONE snapshot read above. Null (the reporter has
             // not run) leaves both report fields null — "no data", not "disconnected
             // with an empty backlog".