feat(site-runtime): stuck-script watchdog names the script holding a scheduler thread past timeout (S2)

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs

@@ -127,6 +127,32 @@ public class ScriptExecutionActor : ReceiveActor
             // it is available to the catch blocks regardless of scope state.
             var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
             using var cts = new CancellationTokenSource(timeout);
+
+            // Stuck-script watchdog (S2). The CTS firing only REQUESTS cooperative
+            // cancellation; it does NOT free a thread blocked in synchronous I/O.
+            // When the timeout elapses, wait a grace period on the thread pool and,
+            // if the body still hasn't returned, name the script loudly — this is
+            // the only signal an operator gets that one of the bounded
+            // script-execution threads is gone. `completed` is flipped in the
+            // finally below; Register fires on cancellation only (normal completion
+            // disposes the CTS with no callback). The Task.Run/Task.Delay run on the
+            // thread pool, not the (possibly saturated) script scheduler — deliberate.
+            var completed = 0;
+            var graceMs = options.StuckScriptGraceMs;
+            cts.Token.Register(() => _ = Task.Run(async () =>
+            {
+                await Task.Delay(graceMs);
+                if (Volatile.Read(ref completed) == 0)
+                {
+                    var stuckMsg = $"Script '{scriptName}' on instance '{instanceName}' exceeded its " +
+                                   $"{timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — its dedicated " +
+                                   "script-execution thread is blocked (cooperative cancellation not observed).";
+                    logger.LogError(stuckMsg);
+                    _ = siteEventLogger?.LogEventAsync("script", "Error", instanceName,
+                        $"ScriptActor:{scriptName}", stuckMsg);
+                }
+            }));
+
             try
             {
                 // Resolve integration services from DI (scoped lifetime)
@@ -295,6 +321,9 @@ public class ScriptExecutionActor : ReceiveActor
             }
             finally
             {
+                // Mark the body finished so the stuck-script watchdog (registered on
+                // cts.Token above) treats a timely cancellation as NOT stuck.
+                Interlocked.Exchange(ref completed, 1);
                 // Dispose the DI scope (and scoped services) after script execution completes
                 serviceScope?.Dispose();
                 // Stop self after execution completes

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs

@@ -65,4 +65,14 @@ public class SiteRuntimeOptions
     /// Reserved — consumed by the standby replication fetch in a later task; not yet wired.
     /// </summary>
     public int ConfigFetchRetryCount { get; set; } = 3;
+
+    /// <summary>
+    /// Grace period (ms) after a script's execution timeout elapses before the
+    /// stuck-script watchdog declares the script's dedicated thread blocked and
+    /// emits a loud site event naming it (S2). The CTS timeout only requests
+    /// cooperative cancellation; a script blocked in synchronous I/O never
+    /// observes it, so this watchdog is the only operator signal that one of the
+    /// bounded script-execution threads is gone. Default: 30000ms.
+    /// </summary>
+    public int StuckScriptGraceMs { get; set; } = 30000;
 }

tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ExecutionActorTests.cs

@@ -371,4 +371,70 @@ public class ExecutionActorTests : TestKit, IDisposable
         // Non-positive per-script timeout must be ignored; global (1s) must fire.
         ExpectTerminated(exec, TimeSpan.FromSeconds(10));
     }
+
+    // ── S2: stuck-script watchdog names the script holding a scheduler thread ──
+
+    /// <summary>
+    /// Compiles a raw script that can reach the test-assembly <see cref="StuckTestHooks"/>
+    /// (so a script body can block on a gate). Bypasses the trust validator, like
+    /// <see cref="CompileScript"/>.
+    /// </summary>
+    private static Script<object?> CompileRaw(string code)
+    {
+        var scriptOptions = ScriptOptions.Default
+            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(StuckTestHooks).Assembly)
+            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
+        var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
+        script.Compile();
+        return script;
+    }
+
+    [Fact]
+    public void TimedOutScript_WithBlockedThread_EmitsStuckThreadSiteEvent()
+    {
+        // A script that blocks synchronously on a gate NEVER observes cooperative
+        // cancellation, so the CTS firing at the 1s timeout does not free its
+        // scheduler thread. After the 200ms grace the watchdog must name it loudly.
+        StuckTestHooks.Gate = new SemaphoreSlim(0);
+        var compiled = CompileRaw(
+            "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StuckTestHooks.Gate.Wait(); return null;");
+        var replyTo = CreateTestProbe();
+        var instanceActor = CreateTestProbe();
+        var siteLog = new FakeSiteEventLogger();
+        var options = new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 200 };
+
+        ActorOf(Props.Create(() => new ScriptExecutionActor(
+            "StuckScript", "Inst1", compiled, null, 0,
+            instanceActor.Ref, _sharedLibrary, options,
+            replyTo.Ref, "corr-stuck", NullLogger.Instance,
+            ScriptScope.Root, null, new SingleServiceProvider(siteLog))));
+
+        try
+        {
+            AwaitAssert(() =>
+            {
+                var rows = siteLog.OfType("script");
+                Assert.Contains(rows, r =>
+                    r.Severity == "Error" &&
+                    r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) &&
+                    r.Message.Contains("StuckScript"));
+            }, TimeSpan.FromSeconds(10));
+        }
+        finally
+        {
+            // Free the blocked scheduler thread so the test run stays clean.
+            StuckTestHooks.Gate.Release();
+        }
+    }
+}
+
+/// <summary>
+/// Test hook the stuck-script watchdog test uses to block a script-execution
+/// thread deterministically: the compiled body waits on <see cref="Gate"/>, which
+/// the test releases once it has observed the stuck-thread site event.
+/// </summary>
+public static class StuckTestHooks
+{
+    /// <summary>Gate a blocking test script waits on; reset per test.</summary>
+    public static SemaphoreSlim Gate = new(0);
 }