fix(siteruntime): arm stuck-script watchdog + health/summary on alarm runs (arch-review 2026-08-19 P1-3)
Alarm on-trigger runs (ScriptRunLauncher.RunAlarmScriptAsync) bypassed the stuck-script watchdog, the script-error health counter, and ScriptRunSummaryRecorder entirely, even though they run on the same bounded ScriptExecutionScheduler pool as instance scripts. A wedged alarm body cost a pool thread forever with no site event and no detach/replace, and a timed-out or failed on-trigger run was invisible to health monitoring and the run summary. Bring the alarm path to parity with the instance-script path (RunScriptAsync): capture the worker slot/run stamp in the first synchronous segment and arm ArmStuckScriptWatchdog with the same detach-and-replace semantics and gauge; resolve ScriptRunSummaryRecorder/ISiteEventLogger from the same DI provider and thread RecordStarted/RecordCompleted/RecordFailed/RecordTimedOut; increment ISiteHealthCollector.IncrementScriptError on both the timeout and general failure catch blocks. LaunchAlarmScript/RunAlarmScriptAsync now take healthCollector and serviceProvider parameters, passed from AlarmActor's existing _healthCollector/_serviceProvider fields. No change to the enqueue-anchored deadline, the shed cap, or "a stop does NOT cancel an in-flight run" semantics. Adds AlarmScriptWatchdogTests (previously zero coverage of LaunchAlarmScript): wedged-worker detach + replacement, timeout-at-dequeue health counter increment, and successful-completion summary recording. All three fail against the pre-fix code (verified via stash) and pass after.
This commit is contained in:
@@ -825,6 +825,11 @@ public class AlarmActor : ReceiveActor
|
||||
// Context.Parent.Tell from the execution actor.
|
||||
Self,
|
||||
_logger,
|
||||
// arch-review 2026-08-19 P1-3: same health collector + service provider as
|
||||
// the instance-script path, so the alarm on-trigger run gets the same
|
||||
// stuck-script watchdog, script-error counting, and run-summary threading.
|
||||
_healthCollector,
|
||||
_serviceProvider,
|
||||
runId,
|
||||
// Per-script timeout from the on-trigger script (null = global).
|
||||
_onTriggerExecutionTimeoutSeconds,
|
||||
|
||||
@@ -138,6 +138,18 @@ internal static class ScriptRunLauncher
|
||||
/// <param name="options">Site runtime configuration options, including the execution timeout.</param>
|
||||
/// <param name="completionTarget">The owning <see cref="AlarmActor"/>, which receives the completion message.</param>
|
||||
/// <param name="logger">Logger for execution diagnostics.</param>
|
||||
/// <param name="healthCollector">
|
||||
/// Optional health collector for recording execution metrics (arch-review 2026-08-19
|
||||
/// P1-3): the alarm on-trigger run shares the script-error counter with
|
||||
/// <see cref="RunScriptAsync"/> so a wedged/failed alarm body is visible the same way a
|
||||
/// wedged/failed instance script is.
|
||||
/// </param>
|
||||
/// <param name="serviceProvider">
|
||||
/// Optional DI service provider (arch-review 2026-08-19 P1-3): resolves the singleton
|
||||
/// <see cref="ISiteEventLogger"/> (stuck-watchdog reporting) and
|
||||
/// <see cref="ScriptRunSummaryRecorder"/> (aggregate run counters), exactly like the
|
||||
/// instance-script path.
|
||||
/// </param>
|
||||
/// <param name="runId">Per-alarm run counter, used only in log messages.</param>
|
||||
/// <param name="executionTimeoutSeconds">The on-trigger script's per-script timeout in seconds. Null or non-positive falls back to the global value.</param>
|
||||
/// <param name="parentExecutionId">
|
||||
@@ -158,6 +170,8 @@ internal static class ScriptRunLauncher
|
||||
SiteRuntimeOptions options,
|
||||
IActorRef completionTarget,
|
||||
ILogger logger,
|
||||
ISiteHealthCollector? healthCollector,
|
||||
IServiceProvider? serviceProvider,
|
||||
long runId,
|
||||
int? executionTimeoutSeconds,
|
||||
Guid? parentExecutionId,
|
||||
@@ -174,7 +188,8 @@ internal static class ScriptRunLauncher
|
||||
() => RunAlarmScriptAsync(
|
||||
alarmName, instanceName, level, priority, message, compiledScript,
|
||||
instanceActor, sharedScriptLibrary, options, completionTarget, logger,
|
||||
runId, parentExecutionId, timeout, cts),
|
||||
healthCollector, serviceProvider, runId, parentExecutionId, timeout,
|
||||
executionScheduler, cts),
|
||||
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
|
||||
.Unwrap();
|
||||
}
|
||||
@@ -463,11 +478,42 @@ internal static class ScriptRunLauncher
|
||||
SiteRuntimeOptions options,
|
||||
IActorRef completionTarget,
|
||||
ILogger logger,
|
||||
ISiteHealthCollector? healthCollector,
|
||||
IServiceProvider? serviceProvider,
|
||||
long runId,
|
||||
Guid? parentExecutionId,
|
||||
TimeSpan timeout,
|
||||
ScriptExecutionScheduler executionScheduler,
|
||||
CancellationTokenSource cts)
|
||||
{
|
||||
// arch-review 2026-08-19 P1-3: singletons resolved from the root provider up front,
|
||||
// exactly like RunScriptAsync, so they are available to the catch blocks regardless
|
||||
// of scope state. Null in tests that pass no serviceProvider — degrades the same way
|
||||
// the instance-script path does (no site event log / no aggregate counters).
|
||||
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
|
||||
var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
|
||||
|
||||
// The on-trigger run has no per-script name of its own in the site-event/summary
|
||||
// sense — key it under the alarm, mirroring the "AlarmActor:{name}" source
|
||||
// convention already used for this alarm's other site events.
|
||||
var scriptName = $"AlarmActor:{alarmName}";
|
||||
var runStopwatch = Stopwatch.StartNew();
|
||||
|
||||
// arch-review 2026-08-19 P1-3: identify the worker this body is holding, so the
|
||||
// stuck-script watchdog can detach and replace it if the body wedges — same
|
||||
// mechanism, same gauge, as the instance-script path. Read in the FIRST synchronous
|
||||
// segment, before any await.
|
||||
var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
|
||||
var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;
|
||||
|
||||
var completed = 0;
|
||||
ArmStuckScriptWatchdog(
|
||||
cts, options, logger, siteEventLogger, executionScheduler,
|
||||
workerSlot, workerRunStamp, timeout,
|
||||
() => Volatile.Read(ref completed) != 0,
|
||||
$"Alarm on-trigger script for '{alarmName}' on instance '{instanceName}'",
|
||||
instanceName, scriptName);
|
||||
|
||||
try
|
||||
{
|
||||
// Enqueue-anchored deadline: an already-cancelled token means this on-trigger run
|
||||
@@ -511,12 +557,17 @@ internal static class ScriptRunLauncher
|
||||
}
|
||||
};
|
||||
|
||||
summaryRecorder?.RecordStarted(instanceName, scriptName);
|
||||
|
||||
await compiledScript.RunAsync(globals, cts.Token);
|
||||
|
||||
summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
||||
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
healthCollector?.IncrementScriptError();
|
||||
summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
||||
logger.LogWarning(
|
||||
"Alarm on-trigger script for {Alarm} on {Instance} timed out (run {RunId})",
|
||||
alarmName, instanceName, runId);
|
||||
@@ -524,6 +575,8 @@ internal static class ScriptRunLauncher
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
healthCollector?.IncrementScriptError();
|
||||
summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
||||
// Failures logged, alarm continues
|
||||
logger.LogError(ex,
|
||||
"Alarm on-trigger script for {Alarm} on {Instance} failed (run {RunId})",
|
||||
@@ -532,6 +585,9 @@ internal static class ScriptRunLauncher
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Mark the body finished so the stuck-script watchdog (registered on
|
||||
// cts.Token above) treats a timely cancellation as NOT stuck.
|
||||
Interlocked.Exchange(ref completed, 1);
|
||||
cts.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user