dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 22:36:15 -04:00
parent 312216ff2b
commit c4fc1f8ecd
42 changed files with 3673 additions and 1251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs
+177 -65
View File
@@ -25,10 +25,10 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// State (active/normal) is in memory only, NOT persisted.
/// On restart: starts normal, re-evaluates from incoming values.
///
/// AlarmExecutionActor CAN call Instance.CallScript() (ask to sibling Script Actor).
/// An alarm on-trigger run CAN call Instance.CallScript() (ask to sibling Script Actor).
/// Instance scripts CANNOT call alarm on-trigger scripts (no Instance.CallAlarmScript API).
///
/// Supervision: Resume on exception; AlarmExecutionActor stopped on exception.
/// Supervision: Resume on exception (from the Instance Actor). This actor has no children.
/// </summary>
public class AlarmActor : ReceiveActor
{
@@ -44,12 +44,39 @@ public class AlarmActor : ReceiveActor
/// <summary>
/// Script-execution scheduler seam (#18): the process-wide
/// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so this
/// alarm's trigger-expression evaluation and spawned on-trigger scripts run on a
/// caller-owned pool. Resolved lazily at each use so the null (host) path is
/// unchanged.
/// alarm's launched on-trigger scripts run on a caller-owned pool. Resolved lazily at
/// each use so the null (host) path is unchanged.
///
/// <para>WP3.1: trigger-expression evaluation no longer uses this scheduler — see
/// <see cref="_evalGate"/>. That split is the whole point of finding #4: an alarm whose
/// Expression trigger should raise in milliseconds must not queue behind blocking
/// script bodies.</para>
/// </summary>
private readonly ScriptExecutionScheduler? _scheduler;
/// <summary>
/// WP3.1 (finding #4): the concurrency gate for trigger-expression evaluation, or null
/// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
/// async work on the shared .NET thread pool behind this gate.
/// </summary>
private readonly TriggerEvalGate? _evalGate;
/// <summary>
/// WP3.1: on-trigger runs launched but not yet completed. Incremented at launch,
/// decremented on <see cref="AlarmExecutionCompleted"/>, which every terminal path
/// emits — including the launch-path catch. Touched only on the actor thread.
/// </summary>
private int _runsInFlight;
/// <summary>
/// WP3.1: when the last shed Warning site event was emitted for this alarm. Sheds are
/// always counted; the event is rate-limited to one per alarm per minute.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
/// <summary>
/// The optional site operational-event log, resolved once from
/// <see cref="_serviceProvider"/> at construction and cached. The
@@ -84,7 +111,7 @@ public class AlarmActor : ReceiveActor
/// <summary>
/// The on-trigger script's per-script execution timeout in seconds,
/// or null to use the global default. Forwarded to each spawned
/// <see cref="AlarmExecutionActor"/>, which applies <c>perScript ?? global</c>
/// <see cref="Scripts.ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
/// (treating ≤ 0 as "use global"). The value comes from the referenced
/// on-trigger script's <see cref="ResolvedScript.ExecutionTimeoutSeconds"/>.
/// </summary>
@@ -125,10 +152,10 @@ public class AlarmActor : ReceiveActor
/// <summary>
/// Audit Log #23 (ParentExecutionId tag-cascade): the
/// <c>parentExecutionId</c> handed to the most recently spawned
/// <see cref="AlarmExecutionActor"/> — i.e. the execution whose attribute
/// on-trigger run — i.e. the execution whose attribute
/// write fired this alarm, or <c>null</c> when the firing change came from
/// the Data Connection Layer (external data has no spawning execution).
/// The spawned actor builds its own <see cref="ScriptRuntimeContext"/>
/// The launched run builds its own <see cref="ScriptRuntimeContext"/>
/// internally, so this is exposed for regression coverage of the cascade
/// contract (mirrors <see cref="SeedAttributesReference"/>).
/// </summary>
@@ -159,6 +186,7 @@ public class AlarmActor : ReceiveActor
/// execution timeout in seconds (from its <see cref="ResolvedScript.ExecutionTimeoutSeconds"/>),
/// or null/non-positive to use the global default.</param>
/// <param name="scheduler">Optional script-execution scheduler override (#18); null uses the process-wide shared scheduler.</param>
/// <param name="evalGate">Optional trigger-expression concurrency gate override (WP3.1); null uses the process-wide shared gate.</param>
public AlarmActor(
string alarmName,
string instanceName,
@@ -175,7 +203,9 @@ public class AlarmActor : ReceiveActor
// Per-script timeout for the on-trigger script (null = global).
int? onTriggerExecutionTimeoutSeconds = null,
// Script-execution scheduler seam (#18); null uses the process-wide shared scheduler.
ScriptExecutionScheduler? scheduler = null)
ScriptExecutionScheduler? scheduler = null,
// WP3.1 trigger-eval gate seam; null uses the process-wide shared gate.
TriggerEvalGate? evalGate = null)
{
_alarmName = alarmName;
_instanceName = instanceName;
@@ -186,6 +216,7 @@ public class AlarmActor : ReceiveActor
_healthCollector = healthCollector;
_serviceProvider = serviceProvider;
_scheduler = scheduler;
_evalGate = evalGate;
// Resolve the optional site event logger once and cache it,
// rather than calling GetService on every alarm transition.
_siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
@@ -217,9 +248,8 @@ public class AlarmActor : ReceiveActor
// Handle attribute value changes
Receive<AttributeValueChanged>(HandleAttributeValueChanged);
// Handle alarm execution completion
Receive<AlarmExecutionCompleted>(_ =>
_logger.LogDebug("Alarm {Alarm} execution completed on {Instance}", _alarmName, _instanceName));
// Handle alarm execution completion (also releases the WP3.1 in-flight slot)
Receive<AlarmExecutionCompleted>(HandleAlarmExecutionCompleted);
// Handle the off-dispatcher trigger-expression evaluation result (P1).
Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
@@ -238,20 +268,10 @@ public class AlarmActor : ReceiveActor
_alarmName, _instanceName, _triggerType);
}
/// <inheritdoc />
protected override SupervisorStrategy SupervisorStrategy()
{
return new OneForOneStrategy(
maxNrOfRetries: -1,
withinTimeRange: TimeSpan.FromMinutes(1),
decider: Decider.From(ex =>
{
_logger.LogWarning(ex,
"AlarmExecutionActor for {Alarm} on {Instance} failed, stopping",
_alarmName, _instanceName);
return Directive.Stop;
}));
}
// WP3.1: the Stop-on-failure SupervisorStrategy override is gone with the per-run
// AlarmExecutionActor child it supervised. This actor has no children left; a
// launch-path throw is caught in SpawnAlarmExecution so it can never escalate to
// InstanceActor, which continues to supervise this actor with Resume exactly as before.
/// <summary>
/// Evaluates alarm condition on attribute change. Alarm evaluation errors are logged,
@@ -345,7 +365,7 @@ public class AlarmActor : ReceiveActor
// Operational `alarm` event — raise. Severity by priority.
LogAlarmEvent(RaiseSeverity(_priority), $"Alarm {_alarmName} activated (priority {_priority})");
// Spawn AlarmExecutionActor if on-trigger script defined
// Launch the on-trigger run if an on-trigger script is defined
if (_onTriggerCompiledScript != null)
{
SpawnAlarmExecution(AlarmLevel.None, _priority, string.Empty, sourceExecutionId);
@@ -442,7 +462,7 @@ public class AlarmActor : ReceiveActor
/// <see cref="ISiteEventLogger"/> (resolved once at construction and cached
/// in <see cref="_siteEventLogger"/>). Never awaited so a logging failure
/// cannot affect alarm evaluation (matching the established
/// ScriptActor/ScriptExecutionActor pattern).
/// ScriptActor / script-run pattern).
/// </summary>
private void LogAlarmEvent(string severity, string message)
{
@@ -589,31 +609,50 @@ public class AlarmActor : ReceiveActor
// later change arriving while the evaluation is in flight cannot
// mis-attribute the raise this evaluation produces.
var sourceExecutionId = _latestSourceExecutionId;
Task.Factory.StartNew(async () =>
var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
// WP3.1 (finding #4): the deadline clock starts AT ENQUEUE, on the actor thread —
// gate-wait time burns the same budget, so a saturated gate yields a timely false
// instead of an unbounded stall. And the work runs on the shared .NET thread pool,
// NOT the blocking script pool, so it can never queue behind a blocked script body.
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
Task.Run(async () =>
{
try
{
// Bound evaluation with a short timeout. The CancellationToken covers
// cooperative/async cases; a pathological CPU-bound expression is not
// fully interruptible — acceptable because trigger expressions are
// authored by trusted Design-role users and compile-checked pre-deploy.
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
var state = await expression.RunAsync(new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
await gate.WaitAsync(cts.Token).ConfigureAwait(false);
try
{
// The CancellationToken covers cooperative/async cases; a pathological
// CPU-bound expression is not fully interruptible — acceptable because
// trigger expressions are authored by trusted Design-role users and
// compile-checked pre-deploy.
var state = await expression.RunAsync(
new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
}
finally
{
gate.Release();
}
}
catch (Exception ex)
{
// OperationCanceledException (timeout) falls through here too and is
// treated as false. _healthCollector (Interlocked) and _logger are
// thread-safe, so this catch is safe off the actor thread.
// OperationCanceledException (timeout, INCLUDING one that fired while still
// waiting on the gate) falls through here too and is treated as false.
// _healthCollector (Interlocked) and _logger are thread-safe, so this catch
// is safe off the actor thread.
_healthCollector?.IncrementAlarmError();
_logger.LogError(ex,
"Alarm {Alarm} trigger expression evaluation failed on {Instance}; treated as false",
_alarmName, _instanceName);
return false;
}
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
_scheduler ?? ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
finally
{
cts.Dispose();
}
}).PipeTo(self,
success: r => new ExpressionEvalResult(r, sourceExecutionId),
failure: ex => new ExpressionEvalFailed(ex, sourceExecutionId));
}
@@ -705,9 +744,34 @@ public class AlarmActor : ReceiveActor
}
/// <summary>
/// Spawns an AlarmExecutionActor to run the on-trigger script.
/// WP3.1: releases the in-flight slot for a completed on-trigger run. Every terminal
/// path emits exactly one <see cref="AlarmExecutionCompleted"/> (success / timeout /
/// failure / launch failure), so the counter tracks reality.
/// </summary>
private void HandleAlarmExecutionCompleted(AlarmExecutionCompleted msg)
{
if (_runsInFlight > 0) _runsInFlight--;
_logger.LogDebug(
"Alarm {Alarm} execution completed on {Instance}: success={Success}",
_alarmName, _instanceName, msg.Success);
}
/// <summary>
/// WP3.1: on-trigger runs launched but not yet completed. Exposed for regression
/// coverage of the shed cap.
/// </summary>
internal int RunsInFlight => _runsInFlight;
/// <summary>
/// Launches the on-trigger script run.
/// Passes the firing alarm's level/priority/message so the script can
/// branch on severity via the <c>Alarm</c> global.
///
/// <para>WP3.1: launched directly via <see cref="ScriptRunLauncher"/> rather than
/// through a short-lived <c>AlarmExecutionActor</c> child, and bounded by
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> — a raise arriving while
/// the cap is reached is shed (counted, rate-limited site event) rather than piling
/// another run onto a saturated pool.</para>
/// </summary>
/// <param name="level">The firing alarm severity level.</param>
/// <param name="priority">The firing alarm priority.</param>
@@ -727,34 +791,82 @@ public class AlarmActor : ReceiveActor
{
if (_onTriggerCompiledScript == null) return;
var executionId = $"{_alarmName}-alarm-exec-{_executionCounter++}";
if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
{
ShedAlarmRun();
return;
}
var runId = _executionCounter++;
// Record what the on-trigger run was parented to (null = root); read by
// the tag-cascade regression tests, which cannot see inside the child.
// the tag-cascade regression tests, which cannot observe the run directly.
LastOnTriggerParentExecutionId = parentExecutionId;
// The on-trigger script body runs on the dedicated
// ScriptExecutionScheduler, not the shared .NET thread pool.
var props = Props.Create(() => new AlarmExecutionActor(
_alarmName,
_instanceName,
level,
priority,
message,
_onTriggerCompiledScript,
_instanceActor,
_sharedScriptLibrary,
_options,
_logger,
// Per-script timeout from the on-trigger script (null = global).
_onTriggerExecutionTimeoutSeconds,
// The firing execution's id — null for DCL-originated changes.
parentExecutionId,
// Scheduler seam (#18): share this alarm's scheduler override with the
// spawned on-trigger script body (null = process-wide shared).
_scheduler));
// Incremented BEFORE the launch so the launch-path catch below (which always emits
// an AlarmExecutionCompleted) balances it on every path.
_runsInFlight++;
Context.ActorOf(props, executionId);
try
{
// The on-trigger script body runs on the dedicated
// ScriptExecutionScheduler, not the shared .NET thread pool.
ScriptRunLauncher.LaunchAlarmScript(
_alarmName,
_instanceName,
level,
priority,
message,
_onTriggerCompiledScript,
_instanceActor,
_sharedScriptLibrary,
_options,
// Completion target: this actor. Identical delivery to the old
// Context.Parent.Tell from the execution actor.
Self,
_logger,
runId,
// Per-script timeout from the on-trigger script (null = global).
_onTriggerExecutionTimeoutSeconds,
// The firing execution's id — null for DCL-originated changes.
parentExecutionId,
// Scheduler seam (#18): share this alarm's scheduler override with the
// launched on-trigger script body (null = process-wide shared).
_scheduler);
}
catch (Exception ex)
{
// WP3.1 supervision parity: mirrors the removed OneForOneStrategy's warn-and-stop
// for a per-run child that failed to construct. The alarm continues.
_logger.LogWarning(ex,
"Alarm on-trigger execution launch for {Alarm} on {Instance} failed, stopping",
_alarmName, _instanceName);
Self.Tell(new AlarmExecutionCompleted(_alarmName, false));
}
}
/// <summary>
/// WP3.1 shed policy for alarm on-trigger runs: refuses the newest run when
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> are already in flight.
/// There is no Ask caller on this path (an on-trigger run is never awaited), so the
/// shed surfaces as a counter plus a rate-limited Warning site event.
/// </summary>
private void ShedAlarmRun()
{
_healthCollector?.IncrementScriptRunShed();
var message = $"Alarm on-trigger script for '{_alarmName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
_logger.LogWarning("{Message}", message);
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_ = _siteEventLogger?.LogEventAsync(
"script", "Warning", _instanceName, $"AlarmActor:{_alarmName}", message);
}
}
private AlarmEvalConfig ParseEvalConfig(string? triggerConfigJson)