From 467edd74aca8545f4742770cb53ce8e32411eb15 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 23:46:48 -0400 Subject: [PATCH] feat(site-runtime): stuck-script watchdog names the script holding a scheduler thread past timeout (S2) --- .../Actors/ScriptExecutionActor.cs | 29 ++++++++ .../SiteRuntimeOptions.cs | 10 +++ .../Actors/ExecutionActorTests.cs | 66 +++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs index 6a4e456a..0f705995 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs @@ -127,6 +127,32 @@ public class ScriptExecutionActor : ReceiveActor // it is available to the catch blocks regardless of scope state. var siteEventLogger = serviceProvider?.GetService(); using var cts = new CancellationTokenSource(timeout); + + // Stuck-script watchdog (S2). The CTS firing only REQUESTS cooperative + // cancellation; it does NOT free a thread blocked in synchronous I/O. + // When the timeout elapses, wait a grace period on the thread pool and, + // if the body still hasn't returned, name the script loudly — this is + // the only signal an operator gets that one of the bounded + // script-execution threads is gone. `completed` is flipped in the + // finally below; Register fires on cancellation only (normal completion + // disposes the CTS with no callback). The Task.Run/Task.Delay run on the + // thread pool, not the (possibly saturated) script scheduler — deliberate. + var completed = 0; + var graceMs = options.StuckScriptGraceMs; + cts.Token.Register(() => _ = Task.Run(async () => + { + await Task.Delay(graceMs); + if (Volatile.Read(ref completed) == 0) + { + var stuckMsg = $"Script '{scriptName}' on instance '{instanceName}' exceeded its " + + $"{timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — its dedicated " + + "script-execution thread is blocked (cooperative cancellation not observed)."; + logger.LogError(stuckMsg); + _ = siteEventLogger?.LogEventAsync("script", "Error", instanceName, + $"ScriptActor:{scriptName}", stuckMsg); + } + })); + try { // Resolve integration services from DI (scoped lifetime) @@ -295,6 +321,9 @@ public class ScriptExecutionActor : ReceiveActor } finally { + // Mark the body finished so the stuck-script watchdog (registered on + // cts.Token above) treats a timely cancellation as NOT stuck. + Interlocked.Exchange(ref completed, 1); // Dispose the DI scope (and scoped services) after script execution completes serviceScope?.Dispose(); // Stop self after execution completes diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs index 32bb85e9..66c3bf98 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/SiteRuntimeOptions.cs @@ -65,4 +65,14 @@ public class SiteRuntimeOptions /// Reserved — consumed by the standby replication fetch in a later task; not yet wired. /// public int ConfigFetchRetryCount { get; set; } = 3; + + /// + /// Grace period (ms) after a script's execution timeout elapses before the + /// stuck-script watchdog declares the script's dedicated thread blocked and + /// emits a loud site event naming it (S2). The CTS timeout only requests + /// cooperative cancellation; a script blocked in synchronous I/O never + /// observes it, so this watchdog is the only operator signal that one of the + /// bounded script-execution threads is gone. Default: 30000ms. + /// + public int StuckScriptGraceMs { get; set; } = 30000; } diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ExecutionActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ExecutionActorTests.cs index f203f589..0b1c4b5b 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ExecutionActorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ExecutionActorTests.cs @@ -371,4 +371,70 @@ public class ExecutionActorTests : TestKit, IDisposable // Non-positive per-script timeout must be ignored; global (1s) must fire. ExpectTerminated(exec, TimeSpan.FromSeconds(10)); } + + // ── S2: stuck-script watchdog names the script holding a scheduler thread ── + + /// + /// Compiles a raw script that can reach the test-assembly + /// (so a script body can block on a gate). Bypasses the trust validator, like + /// . + /// + private static Script CompileRaw(string code) + { + var scriptOptions = ScriptOptions.Default + .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(StuckTestHooks).Assembly) + .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks"); + var script = CSharpScript.Create(code, scriptOptions, typeof(ScriptGlobals)); + script.Compile(); + return script; + } + + [Fact] + public void TimedOutScript_WithBlockedThread_EmitsStuckThreadSiteEvent() + { + // A script that blocks synchronously on a gate NEVER observes cooperative + // cancellation, so the CTS firing at the 1s timeout does not free its + // scheduler thread. After the 200ms grace the watchdog must name it loudly. + StuckTestHooks.Gate = new SemaphoreSlim(0); + var compiled = CompileRaw( + "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StuckTestHooks.Gate.Wait(); return null;"); + var replyTo = CreateTestProbe(); + var instanceActor = CreateTestProbe(); + var siteLog = new FakeSiteEventLogger(); + var options = new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 200 }; + + ActorOf(Props.Create(() => new ScriptExecutionActor( + "StuckScript", "Inst1", compiled, null, 0, + instanceActor.Ref, _sharedLibrary, options, + replyTo.Ref, "corr-stuck", NullLogger.Instance, + ScriptScope.Root, null, new SingleServiceProvider(siteLog)))); + + try + { + AwaitAssert(() => + { + var rows = siteLog.OfType("script"); + Assert.Contains(rows, r => + r.Severity == "Error" && + r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) && + r.Message.Contains("StuckScript")); + }, TimeSpan.FromSeconds(10)); + } + finally + { + // Free the blocked scheduler thread so the test run stays clean. + StuckTestHooks.Gate.Release(); + } + } +} + +/// +/// Test hook the stuck-script watchdog test uses to block a script-execution +/// thread deterministically: the compiled body waits on , which +/// the test releases once it has observed the stuck-thread site event. +/// +public static class StuckTestHooks +{ + /// Gate a blocking test script waits on; reset per test. + public static SemaphoreSlim Gate = new(0); }