fix(siteruntime): arm stuck-script watchdog + health/summary on alarm runs (arch-review 2026-08-19 P1-3)
Alarm on-trigger runs (ScriptRunLauncher.RunAlarmScriptAsync) bypassed the stuck-script watchdog, the script-error health counter, and ScriptRunSummaryRecorder entirely, even though they run on the same bounded ScriptExecutionScheduler pool as instance scripts. A wedged alarm body cost a pool thread forever with no site event and no detach/replace, and a timed-out or failed on-trigger run was invisible to health monitoring and the run summary. Bring the alarm path to parity with the instance-script path (RunScriptAsync): capture the worker slot/run stamp in the first synchronous segment and arm ArmStuckScriptWatchdog with the same detach-and-replace semantics and gauge; resolve ScriptRunSummaryRecorder/ISiteEventLogger from the same DI provider and thread RecordStarted/RecordCompleted/RecordFailed/RecordTimedOut; increment ISiteHealthCollector.IncrementScriptError on both the timeout and general failure catch blocks. LaunchAlarmScript/RunAlarmScriptAsync now take healthCollector and serviceProvider parameters, passed from AlarmActor's existing _healthCollector/_serviceProvider fields. No change to the enqueue-anchored deadline, the shed cap, or "a stop does NOT cancel an in-flight run" semantics. Adds AlarmScriptWatchdogTests (previously zero coverage of LaunchAlarmScript): wedged-worker detach + replacement, timeout-at-dequeue health counter increment, and successful-completion summary recording. All three fail against the pre-fix code (verified via stash) and pass after.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,237 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// arch-review 2026-08-19 P1-3 — alarm on-trigger runs used to bypass the stuck-script
|
||||
/// watchdog, the script-error health counter, and <see cref="ScriptRunSummaryRecorder"/>
|
||||
/// entirely: <c>ScriptRunLauncher.RunAlarmScriptAsync</c> carried none of the machinery its
|
||||
/// sibling <c>RunScriptAsync</c> (the instance-script path) carries, on the SAME bounded
|
||||
/// <see cref="ScriptExecutionScheduler"/> pool. A wedged alarm body cost a pool thread
|
||||
/// forever with no site event, no detach/replace, and <c>DetachedScriptThreads</c> stayed 0;
|
||||
/// a timed-out or failed on-trigger run never touched the health collector or the run-summary
|
||||
/// counters either.
|
||||
///
|
||||
/// <para>These tests mirror <see cref="StuckScriptWatchdogTests"/> and
|
||||
/// <see cref="ScriptDeadlineAtEnqueueTests"/> exactly, but drive an <see cref="AlarmActor"/>
|
||||
/// on-trigger run (via a <c>ValueMatch</c> raise) instead of a <see cref="ScriptActor"/>
|
||||
/// <c>Call</c>.</para>
|
||||
/// </summary>
|
||||
public class AlarmScriptWatchdogTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
|
||||
public AlarmScriptWatchdogTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
AlarmWatchdogHooks.Gate = new SemaphoreSlim(0);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
// Free any still-wedged worker before tearing down, so no test leaves a blocked thread.
|
||||
AlarmWatchdogHooks.Gate.Release(8);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var scriptOptions = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(AlarmWatchdogHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>A body that blocks its worker thread outright — cooperative cancellation is never observed.</summary>
|
||||
private static Script<object?> WedgeScript() => CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.AlarmWatchdogHooks.Gate.Wait(); return null;");
|
||||
|
||||
private TestActorRef<AlarmActor> BuildAlarmActor(
|
||||
string alarmName,
|
||||
Script<object?> compiled,
|
||||
SiteRuntimeOptions options,
|
||||
ScriptExecutionScheduler scheduler,
|
||||
IServiceProvider? serviceProvider,
|
||||
ISiteHealthCollector? healthCollector = null)
|
||||
{
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var config = new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = alarmName,
|
||||
TriggerType = "ValueMatch",
|
||||
TriggerConfiguration = "{\"attributeName\":\"Status\",\"matchValue\":\"Critical\"}",
|
||||
PriorityLevel = 100
|
||||
};
|
||||
return ActorOfAsTestActorRef<AlarmActor>(
|
||||
Props.Create(() => new AlarmActor(
|
||||
alarmName, "Inst1", instance, config, compiled, _sharedLibrary, options,
|
||||
NullLogger<AlarmActor>.Instance, null, null, healthCollector, serviceProvider,
|
||||
null, scheduler, null)),
|
||||
"alarm-watchdog-" + Guid.NewGuid().ToString("N"));
|
||||
}
|
||||
|
||||
/// <summary>Raises the ValueMatch condition (Normal → Active), spawning the on-trigger run.</summary>
|
||||
private static void Raise(IActorRef alarm, string instanceName = "Inst1") =>
|
||||
alarm.Tell(new AttributeValueChanged(
|
||||
instanceName, "Status", "Status", "Critical", "Good", DateTimeOffset.UtcNow));
|
||||
|
||||
private static SiteRuntimeOptions WatchdogOptions() => new()
|
||||
{
|
||||
ScriptExecutionTimeoutSeconds = 1,
|
||||
StuckScriptGraceMs = 200,
|
||||
MaxScriptCallDepth = 10
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void WedgedAlarmScript_DetachesItsWorker_StartsAReplacement_AndDrainsWhenFreed()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var options = WatchdogOptions();
|
||||
|
||||
var wedged = BuildAlarmActor("Wedged", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
Raise(wedged);
|
||||
|
||||
// Timeout (1 s) + grace (200 ms) later the watchdog fires: the worker is detached and
|
||||
// a replacement thread starts — exactly like the instance-script path.
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, scheduler.DetachedThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"), r =>
|
||||
r.Severity == "Error" &&
|
||||
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) &&
|
||||
r.Message.Contains("Wedged") &&
|
||||
r.Message.Contains("DETACHED")),
|
||||
TimeSpan.FromSeconds(5));
|
||||
|
||||
// The replacement worker is live: a fresh alarm on-trigger run completes even though
|
||||
// the pool's only original thread is still blocked. Before this fix there was no
|
||||
// detach at all, so a second wedge would have starved the whole pool.
|
||||
var healthy = BuildAlarmActor("Healthy", CompileRaw("return 7;"), options, scheduler, null);
|
||||
Raise(healthy);
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(0, healthy.UnderlyingActor.RunsInFlight),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
// Freeing the wedged body lets the detached worker exit rather than pull more work,
|
||||
// so the pool does not silently end up double-sized.
|
||||
AlarmWatchdogHooks.Gate.Release();
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(0, scheduler.DetachedThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
Assert.Equal(1, scheduler.MaximumConcurrencyLevel);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmRunTimedOutAtDequeue_IncrementsHealthCollectorScriptErrorCount()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var health = new SiteHealthCollector();
|
||||
|
||||
// Occupy the sole worker with a first alarm's on-trigger run, so the second alarm's
|
||||
// run is queued behind it (mirrors ScriptDeadlineAtEnqueueTests' proven pattern).
|
||||
var blockerOptions = new SiteRuntimeOptions
|
||||
{ ScriptExecutionTimeoutSeconds = 120, StuckScriptGraceMs = 120_000 };
|
||||
var blocker = BuildAlarmActor("Blocker", WedgeScript(), blockerOptions, scheduler, null);
|
||||
Raise(blocker);
|
||||
AwaitAssert(() => Assert.Equal(1, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15));
|
||||
|
||||
// The target alarm has a 1 s budget — by the time the blocker frees the worker, this
|
||||
// run's deadline has already passed, so it takes the timeout (OperationCanceledException)
|
||||
// path at dequeue without ever executing its body.
|
||||
var lateOptions = new SiteRuntimeOptions
|
||||
{ ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 1000 };
|
||||
var late = BuildAlarmActor("Late", CompileRaw("return 1;"), lateOptions, scheduler,
|
||||
new SingleServiceProvider(siteLog), health);
|
||||
Raise(late);
|
||||
|
||||
AwaitCondition(() => scheduler.QueueDepth >= 1, TimeSpan.FromSeconds(15),
|
||||
TimeSpan.FromMilliseconds(25));
|
||||
|
||||
var queuedFor = System.Diagnostics.Stopwatch.StartNew();
|
||||
AwaitCondition(
|
||||
() => queuedFor.Elapsed >= TimeSpan.FromSeconds(1.5) || scheduler.QueueDepth == 0,
|
||||
TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(25));
|
||||
Assert.True(scheduler.QueueDepth >= 1,
|
||||
"the late alarm run left the queue before its budget elapsed, so this run never " +
|
||||
"exercised the timeout-at-dequeue path");
|
||||
|
||||
AlarmWatchdogHooks.Gate.Release();
|
||||
|
||||
// Before this fix, RunAlarmScriptAsync's OperationCanceledException catch never
|
||||
// touched the health collector at all — this stayed 0 forever regardless of how many
|
||||
// alarm on-trigger runs timed out.
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, health.CollectReport("site-1").ScriptErrorCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
// The shed-at-dequeue run never held a worker (it was cancelled before its body ran),
|
||||
// so — unlike the wedged-worker test above — it must NOT be reported as stuck.
|
||||
Assert.DoesNotContain(siteLog.OfType("script"),
|
||||
r => r.Message.Contains("'Late'", StringComparison.Ordinal) &&
|
||||
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
AwaitCondition(() => scheduler.BusyThreadCount == 0 && scheduler.DetachedThreadCount == 0,
|
||||
TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AlarmRunCompletion_IsRecordedByScriptRunSummaryRecorder()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(2);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var recorder = new ScriptRunSummaryRecorder();
|
||||
var options = new SiteRuntimeOptions
|
||||
{ ScriptExecutionTimeoutSeconds = 30, StuckScriptGraceMs = 30_000 };
|
||||
|
||||
// Before this fix, RunAlarmScriptAsync never resolved or touched
|
||||
// ScriptRunSummaryRecorder at all — a flush after any number of alarm runs would
|
||||
// always report zero activity for them.
|
||||
var alarm = BuildAlarmActor("Recorded", CompileRaw("return 42;"), options, scheduler,
|
||||
new SingleServiceProvider(siteLog, summaryRecorder: recorder));
|
||||
Raise(alarm);
|
||||
|
||||
AwaitAssert(() => Assert.Equal(0, alarm.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
||||
|
||||
var emitted = await recorder.FlushAsync(siteLog);
|
||||
Assert.True(emitted);
|
||||
|
||||
var rows = siteLog.OfType("script");
|
||||
var summaryRow = Assert.Single(rows, r => r.Source == "ScriptRunSummary");
|
||||
Assert.Equal("1 runs: 1 completed, 0 failed, 0 timed out across 1 scripts", summaryRow.Message);
|
||||
Assert.NotNull(summaryRow.Details);
|
||||
Assert.Contains("AlarmActor:Recorded", summaryRow.Details);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test hook the alarm stuck-script watchdog tests use to block a script-execution thread
|
||||
/// deterministically: the compiled body waits on <see cref="Gate"/>, which the test releases
|
||||
/// once it has observed the detach (or at teardown).
|
||||
/// </summary>
|
||||
public static class AlarmWatchdogHooks
|
||||
{
|
||||
/// <summary>Gate a blocking test script waits on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
Reference in New Issue
Block a user