using Akka.Actor; using Akka.TestKit; using Akka.TestKit.Xunit2; using Microsoft.CodeAnalysis.CSharp.Scripting; using Microsoft.CodeAnalysis.Scripting; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// Alarm on-trigger runs used to bypass the stuck-script /// watchdog, the script-error health counter, and /// entirely: ScriptRunLauncher.RunAlarmScriptAsync carried none of the machinery its /// sibling RunScriptAsync (the instance-script path) carries, on the SAME bounded /// pool. A wedged alarm body cost a pool thread /// forever with no site event, no detach/replace, and DetachedScriptThreads stayed 0; /// a timed-out or failed on-trigger run never touched the health collector or the run-summary /// counters either. /// /// These tests mirror and /// exactly, but drive an /// on-trigger run (via a ValueMatch raise) instead of a /// Call. /// public class AlarmScriptWatchdogTests : TestKit, IDisposable { private readonly SharedScriptLibrary _sharedLibrary; /// Initializes the test class with a shared script library and a closed watchdog gate. public AlarmScriptWatchdogTests() { var compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedLibrary = new SharedScriptLibrary( compilationService, NullLogger.Instance); AlarmWatchdogHooks.Gate = new SemaphoreSlim(0); } /// void IDisposable.Dispose() { // Free any still-wedged worker before tearing down, so no test leaves a blocked thread. AlarmWatchdogHooks.Gate.Release(8); Shutdown(); } private static Script CompileRaw(string code) { var scriptOptions = ScriptOptions.Default .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(AlarmWatchdogHooks).Assembly) .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks"); var script = CSharpScript.Create(code, scriptOptions, typeof(ScriptGlobals)); script.Compile(); return script; } /// A body that blocks its worker thread outright — cooperative cancellation is never observed. private static Script WedgeScript() => CompileRaw( "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.AlarmWatchdogHooks.Gate.Wait(); return null;"); private TestActorRef BuildAlarmActor( string alarmName, Script compiled, SiteRuntimeOptions options, ScriptExecutionScheduler scheduler, IServiceProvider? serviceProvider, ISiteHealthCollector? healthCollector = null) { var instance = CreateTestProbe().Ref; var config = new ResolvedAlarm { CanonicalName = alarmName, TriggerType = "ValueMatch", TriggerConfiguration = "{\"attributeName\":\"Status\",\"matchValue\":\"Critical\"}", PriorityLevel = 100 }; return ActorOfAsTestActorRef( Props.Create(() => new AlarmActor( alarmName, "Inst1", instance, config, compiled, _sharedLibrary, options, NullLogger.Instance, null, null, healthCollector, serviceProvider, null, scheduler, null)), "alarm-watchdog-" + Guid.NewGuid().ToString("N")); } /// Raises the ValueMatch condition (Normal → Active), spawning the on-trigger run. private static void Raise(IActorRef alarm, string instanceName = "Inst1") => alarm.Tell(new AttributeValueChanged( instanceName, "Status", "Status", "Critical", "Good", DateTimeOffset.UtcNow)); private static SiteRuntimeOptions WatchdogOptions() => new() { ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 200, MaxScriptCallDepth = 10 }; /// Verifies that a wedged alarm script has its worker detached, a replacement started, and the queue drains once it is freed. [Fact] public void WedgedAlarmScript_DetachesItsWorker_StartsAReplacement_AndDrainsWhenFreed() { using var scheduler = new ScriptExecutionScheduler(1); var siteLog = new FakeSiteEventLogger(); var options = WatchdogOptions(); var wedged = BuildAlarmActor("Wedged", WedgeScript(), options, scheduler, new SingleServiceProvider(siteLog)); Raise(wedged); // Timeout (1 s) + grace (200 ms) later the watchdog fires: the worker is detached and // a replacement thread starts — exactly like the instance-script path. AwaitAssert( () => Assert.Equal(1, scheduler.DetachedThreadCount), TimeSpan.FromSeconds(15)); AwaitAssert( () => Assert.Contains(siteLog.OfType("script"), r => r.Severity == "Error" && r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) && r.Message.Contains("Wedged") && r.Message.Contains("DETACHED")), TimeSpan.FromSeconds(5)); // The replacement worker is live: a fresh alarm on-trigger run completes even though // the pool's only original thread is still blocked. Before this fix there was no // detach at all, so a second wedge would have starved the whole pool. var healthy = BuildAlarmActor("Healthy", CompileRaw("return 7;"), options, scheduler, null); Raise(healthy); AwaitAssert( () => Assert.Equal(0, healthy.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15)); // Freeing the wedged body lets the detached worker exit rather than pull more work, // so the pool does not silently end up double-sized. AlarmWatchdogHooks.Gate.Release(); AwaitAssert( () => Assert.Equal(0, scheduler.DetachedThreadCount), TimeSpan.FromSeconds(15)); Assert.Equal(1, scheduler.MaximumConcurrencyLevel); } /// Verifies that an alarm run that timed out at dequeue increments the health collector's script error count. [Fact] public void AlarmRunTimedOutAtDequeue_IncrementsHealthCollectorScriptErrorCount() { using var scheduler = new ScriptExecutionScheduler(1); var siteLog = new FakeSiteEventLogger(); var health = new SiteHealthCollector(); // Occupy the sole worker with a first alarm's on-trigger run, so the second alarm's // run is queued behind it (mirrors ScriptDeadlineAtEnqueueTests' proven pattern). var blockerOptions = new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 120, StuckScriptGraceMs = 120_000 }; var blocker = BuildAlarmActor("Blocker", WedgeScript(), blockerOptions, scheduler, null); Raise(blocker); AwaitAssert(() => Assert.Equal(1, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15)); // The target alarm has a 1 s budget — by the time the blocker frees the worker, this // run's deadline has already passed, so it takes the timeout (OperationCanceledException) // path at dequeue without ever executing its body. var lateOptions = new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 1000 }; var late = BuildAlarmActor("Late", CompileRaw("return 1;"), lateOptions, scheduler, new SingleServiceProvider(siteLog), health); Raise(late); AwaitCondition(() => scheduler.QueueDepth >= 1, TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(25)); var queuedFor = System.Diagnostics.Stopwatch.StartNew(); AwaitCondition( () => queuedFor.Elapsed >= TimeSpan.FromSeconds(1.5) || scheduler.QueueDepth == 0, TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(25)); Assert.True(scheduler.QueueDepth >= 1, "the late alarm run left the queue before its budget elapsed, so this run never " + "exercised the timeout-at-dequeue path"); AlarmWatchdogHooks.Gate.Release(); // Before this fix, RunAlarmScriptAsync's OperationCanceledException catch never // touched the health collector at all — this stayed 0 forever regardless of how many // alarm on-trigger runs timed out. AwaitAssert( () => Assert.Equal(1, health.CollectReport("site-1").ScriptErrorCount), TimeSpan.FromSeconds(15)); // The shed-at-dequeue run never held a worker (it was cancelled before its body ran), // so — unlike the wedged-worker test above — it must NOT be reported as stuck. Assert.DoesNotContain(siteLog.OfType("script"), r => r.Message.Contains("'Late'", StringComparison.Ordinal) && r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase)); AwaitCondition(() => scheduler.BusyThreadCount == 0 && scheduler.DetachedThreadCount == 0, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25)); } /// Verifies that an alarm run completion is recorded by the script run summary recorder. /// A task that represents the asynchronous operation. [Fact] public async Task AlarmRunCompletion_IsRecordedByScriptRunSummaryRecorder() { using var scheduler = new ScriptExecutionScheduler(2); var siteLog = new FakeSiteEventLogger(); var recorder = new ScriptRunSummaryRecorder(); var options = new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 30, StuckScriptGraceMs = 30_000 }; // Before this fix, RunAlarmScriptAsync never resolved or touched // ScriptRunSummaryRecorder at all — a flush after any number of alarm runs would // always report zero activity for them. var alarm = BuildAlarmActor("Recorded", CompileRaw("return 42;"), options, scheduler, new SingleServiceProvider(siteLog, summaryRecorder: recorder)); Raise(alarm); AwaitAssert(() => Assert.Equal(0, alarm.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15)); var emitted = await recorder.FlushAsync(siteLog); Assert.True(emitted); var rows = siteLog.OfType("script"); var summaryRow = Assert.Single(rows, r => r.Source == "ScriptRunSummary"); Assert.Equal("1 runs: 1 completed, 0 failed, 0 timed out across 1 scripts", summaryRow.Message); Assert.NotNull(summaryRow.Details); Assert.Contains("AlarmActor:Recorded", summaryRow.Details); } } /// /// Test hook the alarm stuck-script watchdog tests use to block a script-execution thread /// deterministically: the compiled body waits on , which the test releases /// once it has observed the detach (or at teardown). /// public static class AlarmWatchdogHooks { /// Gate a blocking test script waits on; reset per test. public static SemaphoreSlim Gate = new(0); }