fix(siteruntime): arm stuck-script watchdog + health/summary on alarm runs (arch-review 2026-08-19 P1-3)

Alarm on-trigger runs (ScriptRunLauncher.RunAlarmScriptAsync) bypassed the
stuck-script watchdog, the script-error health counter, and
ScriptRunSummaryRecorder entirely, even though they run on the same bounded
ScriptExecutionScheduler pool as instance scripts. A wedged alarm body cost a
pool thread forever with no site event and no detach/replace, and a timed-out
or failed on-trigger run was invisible to health monitoring and the run
summary.

Bring the alarm path to parity with the instance-script path (RunScriptAsync):
capture the worker slot/run stamp in the first synchronous segment and arm
ArmStuckScriptWatchdog with the same detach-and-replace semantics and gauge;
resolve ScriptRunSummaryRecorder/ISiteEventLogger from the same DI provider and
thread RecordStarted/RecordCompleted/RecordFailed/RecordTimedOut; increment
ISiteHealthCollector.IncrementScriptError on both the timeout and general
failure catch blocks. LaunchAlarmScript/RunAlarmScriptAsync now take
healthCollector and serviceProvider parameters, passed from AlarmActor's
existing _healthCollector/_serviceProvider fields. No change to the
enqueue-anchored deadline, the shed cap, or "a stop does NOT cancel an
in-flight run" semantics.

Adds AlarmScriptWatchdogTests (previously zero coverage of LaunchAlarmScript):
wedged-worker detach + replacement, timeout-at-dequeue health counter
increment, and successful-completion summary recording. All three fail
against the pre-fix code (verified via stash) and pass after.
This commit is contained in:
Joseph Doherty
2026-08-19 14:37:24 -04:00
parent 91fcb5b951
commit fd4cb30273
3 changed files with 299 additions and 1 deletions
@@ -138,6 +138,18 @@ internal static class ScriptRunLauncher
/// <param name="options">Site runtime configuration options, including the execution timeout.</param>
/// <param name="completionTarget">The owning <see cref="AlarmActor"/>, which receives the completion message.</param>
/// <param name="logger">Logger for execution diagnostics.</param>
/// <param name="healthCollector">
/// Optional health collector for recording execution metrics (arch-review 2026-08-19
/// P1-3): the alarm on-trigger run shares the script-error counter with
/// <see cref="RunScriptAsync"/> so a wedged/failed alarm body is visible the same way a
/// wedged/failed instance script is.
/// </param>
/// <param name="serviceProvider">
/// Optional DI service provider (arch-review 2026-08-19 P1-3): resolves the singleton
/// <see cref="ISiteEventLogger"/> (stuck-watchdog reporting) and
/// <see cref="ScriptRunSummaryRecorder"/> (aggregate run counters), exactly like the
/// instance-script path.
/// </param>
/// <param name="runId">Per-alarm run counter, used only in log messages.</param>
/// <param name="executionTimeoutSeconds">The on-trigger script's per-script timeout in seconds. Null or non-positive falls back to the global value.</param>
/// <param name="parentExecutionId">
@@ -158,6 +170,8 @@ internal static class ScriptRunLauncher
SiteRuntimeOptions options,
IActorRef completionTarget,
ILogger logger,
ISiteHealthCollector? healthCollector,
IServiceProvider? serviceProvider,
long runId,
int? executionTimeoutSeconds,
Guid? parentExecutionId,
@@ -174,7 +188,8 @@ internal static class ScriptRunLauncher
() => RunAlarmScriptAsync(
alarmName, instanceName, level, priority, message, compiledScript,
instanceActor, sharedScriptLibrary, options, completionTarget, logger,
runId, parentExecutionId, timeout, cts),
healthCollector, serviceProvider, runId, parentExecutionId, timeout,
executionScheduler, cts),
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
.Unwrap();
}
@@ -463,11 +478,42 @@ internal static class ScriptRunLauncher
SiteRuntimeOptions options,
IActorRef completionTarget,
ILogger logger,
ISiteHealthCollector? healthCollector,
IServiceProvider? serviceProvider,
long runId,
Guid? parentExecutionId,
TimeSpan timeout,
ScriptExecutionScheduler executionScheduler,
CancellationTokenSource cts)
{
// arch-review 2026-08-19 P1-3: singletons resolved from the root provider up front,
// exactly like RunScriptAsync, so they are available to the catch blocks regardless
// of scope state. Null in tests that pass no serviceProvider — degrades the same way
// the instance-script path does (no site event log / no aggregate counters).
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
// The on-trigger run has no per-script name of its own in the site-event/summary
// sense — key it under the alarm, mirroring the "AlarmActor:{name}" source
// convention already used for this alarm's other site events.
var scriptName = $"AlarmActor:{alarmName}";
var runStopwatch = Stopwatch.StartNew();
// arch-review 2026-08-19 P1-3: identify the worker this body is holding, so the
// stuck-script watchdog can detach and replace it if the body wedges — same
// mechanism, same gauge, as the instance-script path. Read in the FIRST synchronous
// segment, before any await.
var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;
var completed = 0;
ArmStuckScriptWatchdog(
cts, options, logger, siteEventLogger, executionScheduler,
workerSlot, workerRunStamp, timeout,
() => Volatile.Read(ref completed) != 0,
$"Alarm on-trigger script for '{alarmName}' on instance '{instanceName}'",
instanceName, scriptName);
try
{
// Enqueue-anchored deadline: an already-cancelled token means this on-trigger run
@@ -511,12 +557,17 @@ internal static class ScriptRunLauncher
}
};
summaryRecorder?.RecordStarted(instanceName, scriptName);
await compiledScript.RunAsync(globals, cts.Token);
summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
}
catch (OperationCanceledException)
{
healthCollector?.IncrementScriptError();
summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
logger.LogWarning(
"Alarm on-trigger script for {Alarm} on {Instance} timed out (run {RunId})",
alarmName, instanceName, runId);
@@ -524,6 +575,8 @@ internal static class ScriptRunLauncher
}
catch (Exception ex)
{
healthCollector?.IncrementScriptError();
summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
// Failures logged, alarm continues
logger.LogError(ex,
"Alarm on-trigger script for {Alarm} on {Instance} failed (run {RunId})",
@@ -532,6 +585,9 @@ internal static class ScriptRunLauncher
}
finally
{
// Mark the body finished so the stuck-script watchdog (registered on
// cts.Token above) treats a timely cancellation as NOT stuck.
Interlocked.Exchange(ref completed, 1);
cts.Dispose();
}
}