feat(site-runtime): stuck-script watchdog names the script holding a scheduler thread past timeout (S2)

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs

@@ -127,6 +127,32 @@ public class ScriptExecutionActor : ReceiveActor
             // it is available to the catch blocks regardless of scope state.
             var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
             using var cts = new CancellationTokenSource(timeout);
+
+            // Stuck-script watchdog (S2). The CTS firing only REQUESTS cooperative
+            // cancellation; it does NOT free a thread blocked in synchronous I/O.
+            // When the timeout elapses, wait a grace period on the thread pool and,
+            // if the body still hasn't returned, name the script loudly — this is
+            // the only signal an operator gets that one of the bounded
+            // script-execution threads is gone. `completed` is flipped in the
+            // finally below; Register fires on cancellation only (normal completion
+            // disposes the CTS with no callback). The Task.Run/Task.Delay run on the
+            // thread pool, not the (possibly saturated) script scheduler — deliberate.
+            var completed = 0;
+            var graceMs = options.StuckScriptGraceMs;
+            cts.Token.Register(() => _ = Task.Run(async () =>
+            {
+                await Task.Delay(graceMs);
+                if (Volatile.Read(ref completed) == 0)
+                {
+                    var stuckMsg = $"Script '{scriptName}' on instance '{instanceName}' exceeded its " +
+                                   $"{timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — its dedicated " +
+                                   "script-execution thread is blocked (cooperative cancellation not observed).";
+                    logger.LogError(stuckMsg);
+                    _ = siteEventLogger?.LogEventAsync("script", "Error", instanceName,
+                        $"ScriptActor:{scriptName}", stuckMsg);
+                }
+            }));
+
             try
             {
                 // Resolve integration services from DI (scoped lifetime)
@@ -295,6 +321,9 @@ public class ScriptExecutionActor : ReceiveActor
             }
             finally
             {
+                // Mark the body finished so the stuck-script watchdog (registered on
+                // cts.Token above) treats a timely cancellation as NOT stuck.
+                Interlocked.Exchange(ref completed, 1);
                 // Dispose the DI scope (and scoped services) after script execution completes
                 serviceScope?.Dispose();
                 // Stop self after execution completes

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs

@@ -65,4 +65,14 @@ public class SiteRuntimeOptions
     /// Reserved — consumed by the standby replication fetch in a later task; not yet wired.
     /// </summary>
     public int ConfigFetchRetryCount { get; set; } = 3;
+
+    /// <summary>
+    /// Grace period (ms) after a script's execution timeout elapses before the
+    /// stuck-script watchdog declares the script's dedicated thread blocked and
+    /// emits a loud site event naming it (S2). The CTS timeout only requests
+    /// cooperative cancellation; a script blocked in synchronous I/O never
+    /// observes it, so this watchdog is the only operator signal that one of the
+    /// bounded script-execution threads is gone. Default: 30000ms.
+    /// </summary>
+    public int StuckScriptGraceMs { get; set; } = 30000;
 }