dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 22:36:15 -04:00
parent 312216ff2b
commit c4fc1f8ecd
42 changed files with 3673 additions and 1251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptActor.cs
+220 -76
View File
@@ -15,8 +15,16 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <summary>
/// Script Actor — coordinator actor, child of Instance Actor.
/// Holds compiled script delegate, manages trigger configuration, and spawns
/// ScriptExecutionActor children per invocation. Does not block on child completion.
/// Holds compiled script delegate, manages trigger configuration, and launches script
/// runs per invocation. Does not block on run completion.
///
/// <para>WP3.1: runs are launched directly via <see cref="ScriptRunLauncher"/> rather than
/// through a short-lived <c>ScriptExecutionActor</c> child (which had no <c>Receive</c>
/// handler, no <c>PostStop</c>, and whose <c>IActorRef</c> was never a message target — pure
/// per-run actor-cell overhead). Concurrent runs are bounded by
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; over the cap the NEWEST
/// trigger is shed. Trigger-expression evaluation no longer runs on the blocking script pool
/// at all — see <see cref="TriggerEvalGate"/>.</para>
///
/// Trigger types:
/// - Interval: uses Akka timers to fire periodically
@@ -43,20 +51,52 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <summary>
/// Script-execution scheduler seam (#18): the process-wide
/// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so
/// this actor's trigger-expression evaluation and spawned script bodies run on a
/// caller-owned pool instead of the shared one. Resolved lazily at each use so the
/// null (host) path stays byte-for-byte identical to the previous static call.
/// this actor's launched script bodies run on a caller-owned pool instead of the
/// shared one. Resolved lazily at each use so the null (host) path stays
/// byte-for-byte identical to the previous static call.
///
/// <para>WP3.1: trigger-expression evaluation no longer uses this scheduler — see
/// <see cref="_evalGate"/>.</para>
/// </summary>
private readonly ScriptExecutionScheduler? _scheduler;
/// <summary>
/// WP3.1 (finding #4): the concurrency gate for trigger-expression evaluation, or null
/// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
/// async work on the shared .NET thread pool behind this gate, NOT on
/// <see cref="_scheduler"/> — that is what stops an Expression trigger from queueing
/// behind blocking script bodies.
/// </summary>
private readonly TriggerEvalGate? _evalGate;
/// <summary>
/// WP3.1: runs launched but not yet completed (queued or executing) for this script.
/// Incremented at launch, decremented on <see cref="ScriptExecutionCompleted"/>, which
/// every terminal path emits — including the launch-path catch, so the counter cannot
/// leak. Touched only on the actor thread.
/// </summary>
private int _runsInFlight;
/// <summary>
/// WP3.1: when the last shed Warning site event was emitted for this script. Sheds are
/// ALWAYS counted on the health collector; the site event is rate-limited to one per
/// script per minute so a hot trigger against a saturated cap cannot flood
/// <c>site_events</c>.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
private Script<object?>? _compiledScript;
private ScriptTriggerConfig? _triggerConfig;
private TimeSpan? _minTimeBetweenRuns;
/// <summary>
/// The per-script execution timeout in seconds, or null to use the
/// global default. Threaded down to each spawned <see cref="ScriptExecutionActor"/>,
/// which applies <c>perScript ?? global</c> (and treats ≤ 0 as "use global").
/// global default. Threaded down to each launched run via
/// <see cref="ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
/// (and treats ≤ 0 as "use global").
/// </summary>
private readonly int? _executionTimeoutSeconds;
private DateTimeOffset _lastExecutionTime = DateTimeOffset.MinValue;
@@ -111,6 +151,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <param name="healthCollector">Optional health metrics collector.</param>
/// <param name="serviceProvider">Optional DI service provider for script execution context services.</param>
/// <param name="scheduler">Optional script-execution scheduler override (#18); null uses the process-wide shared scheduler.</param>
/// <param name="evalGate">Optional trigger-expression concurrency gate override (WP3.1); null uses the process-wide shared gate.</param>
public ScriptActor(
string scriptName,
string instanceName,
@@ -124,7 +165,8 @@ public class ScriptActor : ReceiveActor, IWithTimers
IReadOnlyDictionary<string, object?>? initialAttributes = null,
ISiteHealthCollector? healthCollector = null,
IServiceProvider? serviceProvider = null,
ScriptExecutionScheduler? scheduler = null)
ScriptExecutionScheduler? scheduler = null,
TriggerEvalGate? evalGate = null)
{
_scriptName = scriptName;
_instanceName = instanceName;
@@ -136,6 +178,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
_healthCollector = healthCollector;
_serviceProvider = serviceProvider;
_scheduler = scheduler;
_evalGate = evalGate;
_minTimeBetweenRuns = scriptConfig.MinTimeBetweenRuns;
_executionTimeoutSeconds = scriptConfig.ExecutionTimeoutSeconds;
_scope = scriptConfig.Scope;
@@ -201,24 +244,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
_scriptName, _instanceName);
}
/// <inheritdoc />
protected override SupervisorStrategy SupervisorStrategy()
{
return new OneForOneStrategy(
maxNrOfRetries: -1,
withinTimeRange: TimeSpan.FromMinutes(1),
decider: Decider.From(ex =>
{
_logger.LogWarning(ex,
"ScriptExecutionActor for {Script} on {Instance} failed, stopping",
_scriptName, _instanceName);
return Directive.Stop;
}));
}
// WP3.1: the Stop-on-failure SupervisorStrategy override is gone with the per-run
// ScriptExecutionActor child it supervised. This actor has no children left; a
// launch-path throw is caught in SpawnExecution (which replies to the Ask caller and
// releases the in-flight slot) so it can never escalate to InstanceActor, which
// continues to supervise this actor with Resume exactly as before.
/// <summary>
/// Handles CallScript ask from ScriptRuntimeContext or Instance Actor.
/// Spawns a ScriptExecutionActor and forwards the sender for reply.
/// Launches a run and captures the sender for the eventual reply.
/// </summary>
private void HandleScriptCallRequest(ScriptCallRequest request)
{
@@ -233,7 +267,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
}
// (ParentExecutionId): carry any inbound-routed ParentExecutionId
// through to the ScriptExecutionActor so the routed script's
// through to the launched run so the routed script's
// ScriptRuntimeContext can record its spawner. Null for normal
// (tag-change / timer) runs and nested Script.Call invocations.
SpawnExecution(
@@ -287,12 +321,22 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// Starts an off-dispatcher evaluation of the compiled trigger expression (P1).
/// The expression previously ran synchronously on the actor's dispatcher thread
/// via <c>RunAsync(...).GetAwaiter().GetResult()</c>, blocking the dispatcher on
/// every attribute change. It now runs on the dedicated script-execution
/// scheduler against a point-in-time snapshot; the boolean result is piped back
/// to this actor as an <see cref="ExpressionEvalResult"/> so all edge state is
/// applied on the actor thread. Bursts coalesce: at most one evaluation is in
/// flight, at most one pending, so a storm of changes collapses to the latest
/// snapshot without unbounded task fan-out.
/// every attribute change. It runs off-thread against a point-in-time snapshot;
/// the boolean result is piped back to this actor as an
/// <see cref="ExpressionEvalResult"/> so all edge state is applied on the actor
/// thread. Bursts coalesce: at most one evaluation is in flight, at most one
/// pending, so a storm of changes collapses to the latest snapshot without
/// unbounded task fan-out.
///
/// <para>WP3.1 (finding #4) changed WHERE it runs and WHEN its clock starts. It used
/// to run on the dedicated <see cref="ScriptExecutionScheduler"/> — the same bounded
/// pool as blocking script bodies — so N blocked scripts stalled every Expression
/// trigger on the node indefinitely. It now runs as plain async work on the shared
/// .NET thread pool behind <see cref="TriggerEvalGate"/>: trigger expressions are
/// non-blocking by construction, so they belong there. And the deadline
/// <see cref="CancellationTokenSource"/> is armed HERE, on the actor thread, before
/// the work is queued — so gate-wait time burns the same budget and a saturated gate
/// yields a timely false instead of an unbounded stall.</para>
/// </summary>
private void StartExpressionEvaluation()
{
@@ -303,29 +347,45 @@ public class ScriptActor : ReceiveActor, IWithTimers
var snapshot = new Dictionary<string, object?>(_attributeSnapshot); // point-in-time copy, actor thread
var expression = _compiledTriggerExpression;
var self = Self;
Task.Factory.StartNew(async () =>
var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
// Clock starts AT ENQUEUE, not at dequeue.
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
Task.Run(async () =>
{
try
{
// Bound evaluation with a short timeout. The CancellationToken covers
// cooperative/async cases; a pathological CPU-bound expression is not
// fully interruptible — acceptable because trigger expressions are
// authored by trusted Design-role users and compile-checked pre-deploy.
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
var state = await expression.RunAsync(new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
await gate.WaitAsync(cts.Token).ConfigureAwait(false);
try
{
// The CancellationToken covers cooperative/async cases; a pathological
// CPU-bound expression is not fully interruptible — acceptable because
// trigger expressions are authored by trusted Design-role users and
// compile-checked pre-deploy.
var state = await expression.RunAsync(
new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
}
finally
{
gate.Release();
}
}
catch (Exception ex)
{
// OperationCanceledException (timeout) falls through here too and is
// treated as false. LogExpressionError touches only thread-safe
// members (_healthCollector Interlocked, _logger, DI-resolved
// singleton logger) so it is safe off the actor thread.
// OperationCanceledException (timeout, INCLUDING a timeout that fired while
// still waiting on the gate) falls through here too and is treated as false.
// LogExpressionError touches only thread-safe members (_healthCollector
// Interlocked, _logger, DI-resolved singleton logger) so it is safe off the
// actor thread.
LogExpressionError(ex);
return false;
}
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
_scheduler ?? ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
finally
{
cts.Dispose();
}
}).PipeTo(self,
success: r => new ExpressionEvalResult(r),
failure: ex => new ExpressionEvalFailed(ex));
}
@@ -421,7 +481,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <summary>
/// Records a trigger-expression evaluation failure to the site event log,
/// mirroring how ScriptExecutionActor reports script errors.
/// mirroring how a script run reports its own errors.
/// </summary>
private void LogExpressionError(Exception ex)
{
@@ -457,8 +517,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Spawns a new ScriptExecutionActor child for this invocation.
/// Multiple concurrent executions are allowed.
/// Launches a run of this script. Multiple concurrent runs are allowed, up to
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; beyond that the newest
/// trigger is shed (see <see cref="ShedRun"/>).
///
/// <para>WP3.1: the run is launched directly on the script-execution scheduler via
/// <see cref="ScriptRunLauncher"/> — the actor's mailbox stays on the default dispatcher,
/// but the script body runs on the bounded set of dedicated threads, so blocking script
/// I/O is contained there and cannot starve the shared .NET thread pool. No per-run child
/// actor is created.</para>
/// </summary>
private void SpawnExecution(
IReadOnlyDictionary<string, object?>? parameters,
@@ -467,46 +534,123 @@ public class ScriptActor : ReceiveActor, IWithTimers
string correlationId,
Guid? parentExecutionId = null)
{
var executionId = $"{_scriptName}-exec-{_executionCounter++}";
if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
{
ShedRun(replyTo, correlationId);
return;
}
// The actor's mailbox stays on the default dispatcher, but the
// script body itself runs on the dedicated ScriptExecutionScheduler (a bounded
// set of dedicated threads), so blocking script I/O is contained there and
// cannot starve the shared .NET thread pool.
var props = Props.Create(() => new ScriptExecutionActor(
_scriptName,
_instanceName,
_compiledScript!,
parameters,
callDepth,
_instanceActor,
_sharedScriptLibrary,
_options,
replyTo,
correlationId,
_logger,
_scope,
_healthCollector,
_serviceProvider,
// (ParentExecutionId): null for trigger-driven runs;
// an inbound-API-routed call supplies the inbound request's id.
parentExecutionId,
// Per-script timeout override (null = use global).
_executionTimeoutSeconds,
// Scheduler seam (#18): thread this actor's scheduler override down so
// spawned script bodies share the same pool (null = process-wide shared).
_scheduler));
var runId = _executionCounter++;
// Incremented BEFORE the launch so the launch-path catch below (which always emits a
// ScriptExecutionCompleted) balances it on every path — the counter cannot leak.
_runsInFlight++;
Context.ActorOf(props, executionId);
try
{
ScriptRunLauncher.LaunchScript(
_scriptName,
_instanceName,
_compiledScript!,
parameters,
callDepth,
_instanceActor,
_sharedScriptLibrary,
_options,
replyTo,
correlationId,
// Completion target: this actor. Identical delivery to the old
// Context.Parent.Tell from the execution actor, and it doubles as the
// in-flight release.
Self,
// Notification Outbox: the site communication actor Notify.Status queries
// central through. Resolved here, on the actor thread, and handed to the
// launcher (which has no ActorContext of its own).
Context.System.ActorSelection("/user/site-communication"),
_logger,
_scope,
runId,
_healthCollector,
_serviceProvider,
// (ParentExecutionId): null for trigger-driven runs;
// an inbound-API-routed call supplies the inbound request's id.
parentExecutionId,
// Per-script timeout override (null = use global).
_executionTimeoutSeconds,
// Scheduler seam (#18): thread this actor's scheduler override down so
// launched script bodies share the same pool (null = process-wide shared).
_scheduler);
}
catch (Exception ex)
{
// WP3.1 supervision parity: the only failure the removed per-run child could
// surface was a constructor throw (e.g. queueing onto a disposed scheduler),
// which the old OneForOneStrategy logged and stopped — leaving an Ask caller to
// hang with no reply. Same log shape, same "coordinator unaffected" outcome, plus
// a reply so the caller fails fast instead of waiting out its Ask timeout.
var errorMsg = $"Script '{_scriptName}' on instance '{_instanceName}' could not be launched: {ex.Message}";
_logger.LogWarning(ex,
"Script execution launch for {Script} on {Instance} failed, stopping",
_scriptName, _instanceName);
if (!replyTo.IsNobody())
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
Self.Tell(new ScriptExecutionCompleted(_scriptName, false, errorMsg));
}
}
/// <summary>
/// WP3.1 shed policy: refuses the incoming run because
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> runs are already in flight.
/// The four already queued/running are kept — they are closest to their own deadlines and
/// already charged against them — so nothing is ever reordered. A trigger-driven run is
/// simply not launched; an Ask-based <c>CallScript</c> gets an explicit error so a nested
/// call or inbound-API route fails fast rather than hanging to its Ask timeout.
/// </summary>
private void ShedRun(IActorRef replyTo, string correlationId)
{
_healthCollector?.IncrementScriptRunShed();
var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
_logger.LogWarning("{Message}", message);
// Rate-limited to one event per script per minute: the counter above still counts
// every shed, but a hot trigger against a saturated cap must not flood site_events.
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
}
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(
correlationId, false, null,
$"shed: {_runsInFlight} runs already in flight"));
}
}
private void HandleExecutionCompleted(ScriptExecutionCompleted msg)
{
// WP3.1: release the in-flight slot. Every terminal path emits exactly one of these
// (success / timeout / failure / launch failure), so the counter tracks reality.
if (_runsInFlight > 0) _runsInFlight--;
_logger.LogDebug(
"Script {Script} execution completed on {Instance}: success={Success}",
_scriptName, _instanceName, msg.Success);
}
/// <summary>
/// WP3.1: runs launched but not yet completed. Exposed for regression coverage of the
/// shed cap — the counter is otherwise invisible from outside the actor.
/// </summary>
internal int RunsInFlight => _runsInFlight;
// internal (not private) so the culture-invariance of the non-numeric fallback
// can be unit-tested directly on the test thread — the live path evaluates on a
// dispatcher thread whose CurrentCulture the test cannot deterministically set.