using Akka.Actor;
using Akka.TestKit;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// Alarm on-trigger runs used to bypass the stuck-script
/// watchdog, the script-error health counter, and
/// entirely: ScriptRunLauncher.RunAlarmScriptAsync carried none of the machinery its
/// sibling RunScriptAsync (the instance-script path) carries, on the SAME bounded
/// pool. A wedged alarm body cost a pool thread
/// forever with no site event, no detach/replace, and DetachedScriptThreads stayed 0;
/// a timed-out or failed on-trigger run never touched the health collector or the run-summary
/// counters either.
///
/// These tests mirror and
/// exactly, but drive an
/// on-trigger run (via a ValueMatch raise) instead of a
/// Call .
///
public class AlarmScriptWatchdogTests : TestKit, IDisposable
{
private readonly SharedScriptLibrary _sharedLibrary;
/// Initializes the test class with a shared script library and a closed watchdog gate.
public AlarmScriptWatchdogTests()
{
var compilationService = new ScriptCompilationService(
NullLogger.Instance);
_sharedLibrary = new SharedScriptLibrary(
compilationService, NullLogger.Instance);
AlarmWatchdogHooks.Gate = new SemaphoreSlim(0);
}
///
void IDisposable.Dispose()
{
// Free any still-wedged worker before tearing down, so no test leaves a blocked thread.
AlarmWatchdogHooks.Gate.Release(8);
Shutdown();
}
private static Script CompileRaw(string code)
{
var scriptOptions = ScriptOptions.Default
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
typeof(AlarmWatchdogHooks).Assembly)
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
var script = CSharpScript.Create(code, scriptOptions, typeof(ScriptGlobals));
script.Compile();
return script;
}
/// A body that blocks its worker thread outright — cooperative cancellation is never observed.
private static Script WedgeScript() => CompileRaw(
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.AlarmWatchdogHooks.Gate.Wait(); return null;");
private TestActorRef BuildAlarmActor(
string alarmName,
Script compiled,
SiteRuntimeOptions options,
ScriptExecutionScheduler scheduler,
IServiceProvider? serviceProvider,
ISiteHealthCollector? healthCollector = null)
{
var instance = CreateTestProbe().Ref;
var config = new ResolvedAlarm
{
CanonicalName = alarmName,
TriggerType = "ValueMatch",
TriggerConfiguration = "{\"attributeName\":\"Status\",\"matchValue\":\"Critical\"}",
PriorityLevel = 100
};
return ActorOfAsTestActorRef(
Props.Create(() => new AlarmActor(
alarmName, "Inst1", instance, config, compiled, _sharedLibrary, options,
NullLogger.Instance, null, null, healthCollector, serviceProvider,
null, scheduler, null)),
"alarm-watchdog-" + Guid.NewGuid().ToString("N"));
}
/// Raises the ValueMatch condition (Normal → Active), spawning the on-trigger run.
private static void Raise(IActorRef alarm, string instanceName = "Inst1") =>
alarm.Tell(new AttributeValueChanged(
instanceName, "Status", "Status", "Critical", "Good", DateTimeOffset.UtcNow));
private static SiteRuntimeOptions WatchdogOptions() => new()
{
ScriptExecutionTimeoutSeconds = 1,
StuckScriptGraceMs = 200,
MaxScriptCallDepth = 10
};
/// Verifies that a wedged alarm script has its worker detached, a replacement started, and the queue drains once it is freed.
[Fact]
public void WedgedAlarmScript_DetachesItsWorker_StartsAReplacement_AndDrainsWhenFreed()
{
using var scheduler = new ScriptExecutionScheduler(1);
var siteLog = new FakeSiteEventLogger();
var options = WatchdogOptions();
var wedged = BuildAlarmActor("Wedged", WedgeScript(), options, scheduler,
new SingleServiceProvider(siteLog));
Raise(wedged);
// Timeout (1 s) + grace (200 ms) later the watchdog fires: the worker is detached and
// a replacement thread starts — exactly like the instance-script path.
AwaitAssert(
() => Assert.Equal(1, scheduler.DetachedThreadCount),
TimeSpan.FromSeconds(15));
AwaitAssert(
() => Assert.Contains(siteLog.OfType("script"), r =>
r.Severity == "Error" &&
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) &&
r.Message.Contains("Wedged") &&
r.Message.Contains("DETACHED")),
TimeSpan.FromSeconds(5));
// The replacement worker is live: a fresh alarm on-trigger run completes even though
// the pool's only original thread is still blocked. Before this fix there was no
// detach at all, so a second wedge would have starved the whole pool.
var healthy = BuildAlarmActor("Healthy", CompileRaw("return 7;"), options, scheduler, null);
Raise(healthy);
AwaitAssert(
() => Assert.Equal(0, healthy.UnderlyingActor.RunsInFlight),
TimeSpan.FromSeconds(15));
// Freeing the wedged body lets the detached worker exit rather than pull more work,
// so the pool does not silently end up double-sized.
AlarmWatchdogHooks.Gate.Release();
AwaitAssert(
() => Assert.Equal(0, scheduler.DetachedThreadCount),
TimeSpan.FromSeconds(15));
Assert.Equal(1, scheduler.MaximumConcurrencyLevel);
}
/// Verifies that an alarm run that timed out at dequeue increments the health collector's script error count.
[Fact]
public void AlarmRunTimedOutAtDequeue_IncrementsHealthCollectorScriptErrorCount()
{
using var scheduler = new ScriptExecutionScheduler(1);
var siteLog = new FakeSiteEventLogger();
var health = new SiteHealthCollector();
// Occupy the sole worker with a first alarm's on-trigger run, so the second alarm's
// run is queued behind it (mirrors ScriptDeadlineAtEnqueueTests' proven pattern).
var blockerOptions = new SiteRuntimeOptions
{ ScriptExecutionTimeoutSeconds = 120, StuckScriptGraceMs = 120_000 };
var blocker = BuildAlarmActor("Blocker", WedgeScript(), blockerOptions, scheduler, null);
Raise(blocker);
AwaitAssert(() => Assert.Equal(1, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15));
// The target alarm has a 1 s budget — by the time the blocker frees the worker, this
// run's deadline has already passed, so it takes the timeout (OperationCanceledException)
// path at dequeue without ever executing its body.
var lateOptions = new SiteRuntimeOptions
{ ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 1000 };
var late = BuildAlarmActor("Late", CompileRaw("return 1;"), lateOptions, scheduler,
new SingleServiceProvider(siteLog), health);
Raise(late);
AwaitCondition(() => scheduler.QueueDepth >= 1, TimeSpan.FromSeconds(15),
TimeSpan.FromMilliseconds(25));
var queuedFor = System.Diagnostics.Stopwatch.StartNew();
AwaitCondition(
() => queuedFor.Elapsed >= TimeSpan.FromSeconds(1.5) || scheduler.QueueDepth == 0,
TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(25));
Assert.True(scheduler.QueueDepth >= 1,
"the late alarm run left the queue before its budget elapsed, so this run never " +
"exercised the timeout-at-dequeue path");
AlarmWatchdogHooks.Gate.Release();
// Before this fix, RunAlarmScriptAsync's OperationCanceledException catch never
// touched the health collector at all — this stayed 0 forever regardless of how many
// alarm on-trigger runs timed out.
AwaitAssert(
() => Assert.Equal(1, health.CollectReport("site-1").ScriptErrorCount),
TimeSpan.FromSeconds(15));
// The shed-at-dequeue run never held a worker (it was cancelled before its body ran),
// so — unlike the wedged-worker test above — it must NOT be reported as stuck.
Assert.DoesNotContain(siteLog.OfType("script"),
r => r.Message.Contains("'Late'", StringComparison.Ordinal) &&
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase));
AwaitCondition(() => scheduler.BusyThreadCount == 0 && scheduler.DetachedThreadCount == 0,
TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25));
}
/// Verifies that an alarm run completion is recorded by the script run summary recorder.
/// A task that represents the asynchronous operation.
[Fact]
public async Task AlarmRunCompletion_IsRecordedByScriptRunSummaryRecorder()
{
using var scheduler = new ScriptExecutionScheduler(2);
var siteLog = new FakeSiteEventLogger();
var recorder = new ScriptRunSummaryRecorder();
var options = new SiteRuntimeOptions
{ ScriptExecutionTimeoutSeconds = 30, StuckScriptGraceMs = 30_000 };
// Before this fix, RunAlarmScriptAsync never resolved or touched
// ScriptRunSummaryRecorder at all — a flush after any number of alarm runs would
// always report zero activity for them.
var alarm = BuildAlarmActor("Recorded", CompileRaw("return 42;"), options, scheduler,
new SingleServiceProvider(siteLog, summaryRecorder: recorder));
Raise(alarm);
AwaitAssert(() => Assert.Equal(0, alarm.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
var emitted = await recorder.FlushAsync(siteLog);
Assert.True(emitted);
var rows = siteLog.OfType("script");
var summaryRow = Assert.Single(rows, r => r.Source == "ScriptRunSummary");
Assert.Equal("1 runs: 1 completed, 0 failed, 0 timed out across 1 scripts", summaryRow.Message);
Assert.NotNull(summaryRow.Details);
Assert.Contains("AlarmActor:Recorded", summaryRow.Details);
}
}
///
/// Test hook the alarm stuck-script watchdog tests use to block a script-execution thread
/// deterministically: the compiled body waits on , which the test releases
/// once it has observed the detach (or at teardown).
///
public static class AlarmWatchdogHooks
{
/// Gate a blocking test script waits on; reset per test.
public static SemaphoreSlim Gate = new(0);
}