dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 22:36:15 -04:00
parent 312216ff2b
commit c4fc1f8ecd
42 changed files with 3673 additions and 1251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs
+177 -65
View File
@@ -25,10 +25,10 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// State (active/normal) is in memory only, NOT persisted.
/// On restart: starts normal, re-evaluates from incoming values.
///
/// AlarmExecutionActor CAN call Instance.CallScript() (ask to sibling Script Actor).
/// An alarm on-trigger run CAN call Instance.CallScript() (ask to sibling Script Actor).
/// Instance scripts CANNOT call alarm on-trigger scripts (no Instance.CallAlarmScript API).
///
/// Supervision: Resume on exception; AlarmExecutionActor stopped on exception.
/// Supervision: Resume on exception (from the Instance Actor). This actor has no children.
/// </summary>
public class AlarmActor : ReceiveActor
{
@@ -44,12 +44,39 @@ public class AlarmActor : ReceiveActor
/// <summary>
/// Script-execution scheduler seam (#18): the process-wide
/// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so this
/// alarm's trigger-expression evaluation and spawned on-trigger scripts run on a
/// caller-owned pool. Resolved lazily at each use so the null (host) path is
/// unchanged.
/// alarm's launched on-trigger scripts run on a caller-owned pool. Resolved lazily at
/// each use so the null (host) path is unchanged.
///
/// <para>WP3.1: trigger-expression evaluation no longer uses this scheduler — see
/// <see cref="_evalGate"/>. That split is the whole point of finding #4: an alarm whose
/// Expression trigger should raise in milliseconds must not queue behind blocking
/// script bodies.</para>
/// </summary>
private readonly ScriptExecutionScheduler? _scheduler;
/// <summary>
/// WP3.1 (finding #4): the concurrency gate for trigger-expression evaluation, or null
/// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
/// async work on the shared .NET thread pool behind this gate.
/// </summary>
private readonly TriggerEvalGate? _evalGate;
/// <summary>
/// WP3.1: on-trigger runs launched but not yet completed. Incremented at launch,
/// decremented on <see cref="AlarmExecutionCompleted"/>, which every terminal path
/// emits — including the launch-path catch. Touched only on the actor thread.
/// </summary>
private int _runsInFlight;
/// <summary>
/// WP3.1: when the last shed Warning site event was emitted for this alarm. Sheds are
/// always counted; the event is rate-limited to one per alarm per minute.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
/// <summary>
/// The optional site operational-event log, resolved once from
/// <see cref="_serviceProvider"/> at construction and cached. The
@@ -84,7 +111,7 @@ public class AlarmActor : ReceiveActor
/// <summary>
/// The on-trigger script's per-script execution timeout in seconds,
/// or null to use the global default. Forwarded to each spawned
/// <see cref="AlarmExecutionActor"/>, which applies <c>perScript ?? global</c>
/// <see cref="Scripts.ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
/// (treating ≤ 0 as "use global"). The value comes from the referenced
/// on-trigger script's <see cref="ResolvedScript.ExecutionTimeoutSeconds"/>.
/// </summary>
@@ -125,10 +152,10 @@ public class AlarmActor : ReceiveActor
/// <summary>
/// Audit Log #23 (ParentExecutionId tag-cascade): the
/// <c>parentExecutionId</c> handed to the most recently spawned
/// <see cref="AlarmExecutionActor"/> — i.e. the execution whose attribute
/// on-trigger run — i.e. the execution whose attribute
/// write fired this alarm, or <c>null</c> when the firing change came from
/// the Data Connection Layer (external data has no spawning execution).
/// The spawned actor builds its own <see cref="ScriptRuntimeContext"/>
/// The launched run builds its own <see cref="ScriptRuntimeContext"/>
/// internally, so this is exposed for regression coverage of the cascade
/// contract (mirrors <see cref="SeedAttributesReference"/>).
/// </summary>
@@ -159,6 +186,7 @@ public class AlarmActor : ReceiveActor
/// execution timeout in seconds (from its <see cref="ResolvedScript.ExecutionTimeoutSeconds"/>),
/// or null/non-positive to use the global default.</param>
/// <param name="scheduler">Optional script-execution scheduler override (#18); null uses the process-wide shared scheduler.</param>
/// <param name="evalGate">Optional trigger-expression concurrency gate override (WP3.1); null uses the process-wide shared gate.</param>
public AlarmActor(
string alarmName,
string instanceName,
@@ -175,7 +203,9 @@ public class AlarmActor : ReceiveActor
// Per-script timeout for the on-trigger script (null = global).
int? onTriggerExecutionTimeoutSeconds = null,
// Script-execution scheduler seam (#18); null uses the process-wide shared scheduler.
ScriptExecutionScheduler? scheduler = null)
ScriptExecutionScheduler? scheduler = null,
// WP3.1 trigger-eval gate seam; null uses the process-wide shared gate.
TriggerEvalGate? evalGate = null)
{
_alarmName = alarmName;
_instanceName = instanceName;
@@ -186,6 +216,7 @@ public class AlarmActor : ReceiveActor
_healthCollector = healthCollector;
_serviceProvider = serviceProvider;
_scheduler = scheduler;
_evalGate = evalGate;
// Resolve the optional site event logger once and cache it,
// rather than calling GetService on every alarm transition.
_siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
@@ -217,9 +248,8 @@ public class AlarmActor : ReceiveActor
// Handle attribute value changes
Receive<AttributeValueChanged>(HandleAttributeValueChanged);
// Handle alarm execution completion
Receive<AlarmExecutionCompleted>(_ =>
_logger.LogDebug("Alarm {Alarm} execution completed on {Instance}", _alarmName, _instanceName));
// Handle alarm execution completion (also releases the WP3.1 in-flight slot)
Receive<AlarmExecutionCompleted>(HandleAlarmExecutionCompleted);
// Handle the off-dispatcher trigger-expression evaluation result (P1).
Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
@@ -238,20 +268,10 @@ public class AlarmActor : ReceiveActor
_alarmName, _instanceName, _triggerType);
}
/// <inheritdoc />
protected override SupervisorStrategy SupervisorStrategy()
{
return new OneForOneStrategy(
maxNrOfRetries: -1,
withinTimeRange: TimeSpan.FromMinutes(1),
decider: Decider.From(ex =>
{
_logger.LogWarning(ex,
"AlarmExecutionActor for {Alarm} on {Instance} failed, stopping",
_alarmName, _instanceName);
return Directive.Stop;
}));
}
// WP3.1: the Stop-on-failure SupervisorStrategy override is gone with the per-run
// AlarmExecutionActor child it supervised. This actor has no children left; a
// launch-path throw is caught in SpawnAlarmExecution so it can never escalate to
// InstanceActor, which continues to supervise this actor with Resume exactly as before.
/// <summary>
/// Evaluates alarm condition on attribute change. Alarm evaluation errors are logged,
@@ -345,7 +365,7 @@ public class AlarmActor : ReceiveActor
// Operational `alarm` event — raise. Severity by priority.
LogAlarmEvent(RaiseSeverity(_priority), $"Alarm {_alarmName} activated (priority {_priority})");
// Spawn AlarmExecutionActor if on-trigger script defined
// Launch the on-trigger run if an on-trigger script is defined
if (_onTriggerCompiledScript != null)
{
SpawnAlarmExecution(AlarmLevel.None, _priority, string.Empty, sourceExecutionId);
@@ -442,7 +462,7 @@ public class AlarmActor : ReceiveActor
/// <see cref="ISiteEventLogger"/> (resolved once at construction and cached
/// in <see cref="_siteEventLogger"/>). Never awaited so a logging failure
/// cannot affect alarm evaluation (matching the established
/// ScriptActor/ScriptExecutionActor pattern).
/// ScriptActor / script-run pattern).
/// </summary>
private void LogAlarmEvent(string severity, string message)
{
@@ -589,31 +609,50 @@ public class AlarmActor : ReceiveActor
// later change arriving while the evaluation is in flight cannot
// mis-attribute the raise this evaluation produces.
var sourceExecutionId = _latestSourceExecutionId;
Task.Factory.StartNew(async () =>
var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
// WP3.1 (finding #4): the deadline clock starts AT ENQUEUE, on the actor thread —
// gate-wait time burns the same budget, so a saturated gate yields a timely false
// instead of an unbounded stall. And the work runs on the shared .NET thread pool,
// NOT the blocking script pool, so it can never queue behind a blocked script body.
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
Task.Run(async () =>
{
try
{
// Bound evaluation with a short timeout. The CancellationToken covers
// cooperative/async cases; a pathological CPU-bound expression is not
// fully interruptible — acceptable because trigger expressions are
// authored by trusted Design-role users and compile-checked pre-deploy.
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
var state = await expression.RunAsync(new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
await gate.WaitAsync(cts.Token).ConfigureAwait(false);
try
{
// The CancellationToken covers cooperative/async cases; a pathological
// CPU-bound expression is not fully interruptible — acceptable because
// trigger expressions are authored by trusted Design-role users and
// compile-checked pre-deploy.
var state = await expression.RunAsync(
new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
}
finally
{
gate.Release();
}
}
catch (Exception ex)
{
// OperationCanceledException (timeout) falls through here too and is
// treated as false. _healthCollector (Interlocked) and _logger are
// thread-safe, so this catch is safe off the actor thread.
// OperationCanceledException (timeout, INCLUDING one that fired while still
// waiting on the gate) falls through here too and is treated as false.
// _healthCollector (Interlocked) and _logger are thread-safe, so this catch
// is safe off the actor thread.
_healthCollector?.IncrementAlarmError();
_logger.LogError(ex,
"Alarm {Alarm} trigger expression evaluation failed on {Instance}; treated as false",
_alarmName, _instanceName);
return false;
}
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
_scheduler ?? ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
finally
{
cts.Dispose();
}
}).PipeTo(self,
success: r => new ExpressionEvalResult(r, sourceExecutionId),
failure: ex => new ExpressionEvalFailed(ex, sourceExecutionId));
}
@@ -705,9 +744,34 @@ public class AlarmActor : ReceiveActor
}
/// <summary>
/// Spawns an AlarmExecutionActor to run the on-trigger script.
/// WP3.1: releases the in-flight slot for a completed on-trigger run. Every terminal
/// path emits exactly one <see cref="AlarmExecutionCompleted"/> (success / timeout /
/// failure / launch failure), so the counter tracks reality.
/// </summary>
private void HandleAlarmExecutionCompleted(AlarmExecutionCompleted msg)
{
if (_runsInFlight > 0) _runsInFlight--;
_logger.LogDebug(
"Alarm {Alarm} execution completed on {Instance}: success={Success}",
_alarmName, _instanceName, msg.Success);
}
/// <summary>
/// WP3.1: on-trigger runs launched but not yet completed. Exposed for regression
/// coverage of the shed cap.
/// </summary>
internal int RunsInFlight => _runsInFlight;
/// <summary>
/// Launches the on-trigger script run.
/// Passes the firing alarm's level/priority/message so the script can
/// branch on severity via the <c>Alarm</c> global.
///
/// <para>WP3.1: launched directly via <see cref="ScriptRunLauncher"/> rather than
/// through a short-lived <c>AlarmExecutionActor</c> child, and bounded by
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> — a raise arriving while
/// the cap is reached is shed (counted, rate-limited site event) rather than piling
/// another run onto a saturated pool.</para>
/// </summary>
/// <param name="level">The firing alarm severity level.</param>
/// <param name="priority">The firing alarm priority.</param>
@@ -727,34 +791,82 @@ public class AlarmActor : ReceiveActor
{
if (_onTriggerCompiledScript == null) return;
var executionId = $"{_alarmName}-alarm-exec-{_executionCounter++}";
if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
{
ShedAlarmRun();
return;
}
var runId = _executionCounter++;
// Record what the on-trigger run was parented to (null = root); read by
// the tag-cascade regression tests, which cannot see inside the child.
// the tag-cascade regression tests, which cannot observe the run directly.
LastOnTriggerParentExecutionId = parentExecutionId;
// The on-trigger script body runs on the dedicated
// ScriptExecutionScheduler, not the shared .NET thread pool.
var props = Props.Create(() => new AlarmExecutionActor(
_alarmName,
_instanceName,
level,
priority,
message,
_onTriggerCompiledScript,
_instanceActor,
_sharedScriptLibrary,
_options,
_logger,
// Per-script timeout from the on-trigger script (null = global).
_onTriggerExecutionTimeoutSeconds,
// The firing execution's id — null for DCL-originated changes.
parentExecutionId,
// Scheduler seam (#18): share this alarm's scheduler override with the
// spawned on-trigger script body (null = process-wide shared).
_scheduler));
// Incremented BEFORE the launch so the launch-path catch below (which always emits
// an AlarmExecutionCompleted) balances it on every path.
_runsInFlight++;
Context.ActorOf(props, executionId);
try
{
// The on-trigger script body runs on the dedicated
// ScriptExecutionScheduler, not the shared .NET thread pool.
ScriptRunLauncher.LaunchAlarmScript(
_alarmName,
_instanceName,
level,
priority,
message,
_onTriggerCompiledScript,
_instanceActor,
_sharedScriptLibrary,
_options,
// Completion target: this actor. Identical delivery to the old
// Context.Parent.Tell from the execution actor.
Self,
_logger,
runId,
// Per-script timeout from the on-trigger script (null = global).
_onTriggerExecutionTimeoutSeconds,
// The firing execution's id — null for DCL-originated changes.
parentExecutionId,
// Scheduler seam (#18): share this alarm's scheduler override with the
// launched on-trigger script body (null = process-wide shared).
_scheduler);
}
catch (Exception ex)
{
// WP3.1 supervision parity: mirrors the removed OneForOneStrategy's warn-and-stop
// for a per-run child that failed to construct. The alarm continues.
_logger.LogWarning(ex,
"Alarm on-trigger execution launch for {Alarm} on {Instance} failed, stopping",
_alarmName, _instanceName);
Self.Tell(new AlarmExecutionCompleted(_alarmName, false));
}
}
/// <summary>
/// WP3.1 shed policy for alarm on-trigger runs: refuses the newest run when
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> are already in flight.
/// There is no Ask caller on this path (an on-trigger run is never awaited), so the
/// shed surfaces as a counter plus a rate-limited Warning site event.
/// </summary>
private void ShedAlarmRun()
{
_healthCollector?.IncrementScriptRunShed();
var message = $"Alarm on-trigger script for '{_alarmName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
_logger.LogWarning("{Message}", message);
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_ = _siteEventLogger?.LogEventAsync(
"script", "Warning", _instanceName, $"AlarmActor:{_alarmName}", message);
}
}
private AlarmEvalConfig ParseEvalConfig(string? triggerConfigJson)
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmExecutionActor.cs
-167
View File
@@ -1,167 +0,0 @@
using Akka.Actor;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <summary>
/// Alarm Execution Actor -- short-lived child of Alarm Actor.
/// Same pattern as ScriptExecutionActor.
/// CAN call Instance.CallScript() (ask to sibling Script Actor).
/// Instance scripts CANNOT call alarm on-trigger scripts (no API for it).
/// Supervision: Stop on unhandled exception.
/// </summary>
public class AlarmExecutionActor : ReceiveActor
{
/// <summary>Initializes a new <see cref="AlarmExecutionActor"/> and immediately schedules execution of the alarm on-trigger script.</summary>
/// <param name="alarmName">The canonical name of the alarm that triggered.</param>
/// <param name="instanceName">The name of the owning instance.</param>
/// <param name="level">The alarm severity level at the time of triggering.</param>
/// <param name="priority">The alarm priority value.</param>
/// <param name="message">The alarm message to pass to the script.</param>
/// <param name="compiledScript">The pre-compiled on-trigger script to execute.</param>
/// <param name="instanceActor">Reference to the parent instance actor for attribute/script calls.</param>
/// <param name="sharedScriptLibrary">Shared script library providing common utilities.</param>
/// <param name="options">Site runtime configuration options, including the execution timeout.</param>
/// <param name="logger">Logger for execution diagnostics.</param>
/// <param name="executionTimeoutSeconds">The on-trigger script's per-script execution timeout in seconds. Null or non-positive falls back to the global <see cref="SiteRuntimeOptions.ScriptExecutionTimeoutSeconds"/>.</param>
/// <param name="parentExecutionId">
/// ParentExecutionId tag-cascade: the <c>ExecutionId</c> of
/// the execution whose attribute write fired this alarm, threaded into the
/// on-trigger script's <see cref="ScriptRuntimeContext"/> as its
/// <c>ParentExecutionId</c> so the alarm-triggered run chains under its
/// firing execution. Null when the firing value came from the Data
/// Connection Layer (external data has no spawning execution) — that
/// on-trigger run is a tree root.
/// </param>
public AlarmExecutionActor(
string alarmName,
string instanceName,
AlarmLevel level,
int priority,
string message,
Script<object?> compiledScript,
IActorRef instanceActor,
SharedScriptLibrary sharedScriptLibrary,
SiteRuntimeOptions options,
ILogger logger,
// Per-script execution timeout override (seconds) for the
// alarm on-trigger script. Null or non-positive falls back to the global.
int? executionTimeoutSeconds = null,
// The firing context's execution id (null today).
Guid? parentExecutionId = null,
// Script-execution scheduler seam (#18): the process-wide scheduler by
// default; null selects the shared default.
ScriptExecutionScheduler? scheduler = null)
{
var self = Self;
var parent = Context.Parent;
ExecuteAlarmScript(
alarmName, instanceName, level, priority, message,
compiledScript, instanceActor,
sharedScriptLibrary, options, self, parent, logger,
executionTimeoutSeconds, parentExecutionId, scheduler);
}
private static void ExecuteAlarmScript(
string alarmName,
string instanceName,
AlarmLevel level,
int priority,
string message,
Script<object?> compiledScript,
IActorRef instanceActor,
SharedScriptLibrary sharedScriptLibrary,
SiteRuntimeOptions options,
IActorRef self,
IActorRef parent,
ILogger logger,
int? executionTimeoutSeconds,
Guid? parentExecutionId,
ScriptExecutionScheduler? scheduler)
{
// Per-script timeout overrides the global default. A null or
// non-positive per-script value (≤ 0) falls back to the global.
var timeout = TimeSpan.FromSeconds(
executionTimeoutSeconds is { } perScript && perScript > 0
? perScript
: options.ScriptExecutionTimeoutSeconds);
// Run the alarm on-trigger body on the dedicated
// script-execution scheduler, not the shared .NET thread pool. An injected
// scheduler (#18) overrides the process-wide default.
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
_ = Task.Factory.StartNew(async () =>
{
using var cts = new CancellationTokenSource(timeout);
try
{
// AlarmExecutionActor can call Instance.CallScript()
// via the ScriptRuntimeContext injected into globals
var context = new ScriptRuntimeContext(
instanceActor,
self,
sharedScriptLibrary,
currentCallDepth: 0,
options.MaxScriptCallDepth,
timeout,
instanceName,
logger,
// ParentExecutionId tag-cascade: the
// alarm on-trigger run mints its own fresh ExecutionId (the
// ctor's `?? NewGuid()` fallback) and records the firing
// execution's id as its ParentExecutionId — null (a root)
// only when the firing value came from the DCL.
parentExecutionId: parentExecutionId,
// WaitForAttribute (spec §4.4): thread the alarm on-trigger
// script's per-script execution-timeout token so a
// Attributes.WaitAsync inside an on-trigger script is bounded
// by the same script deadline.
scriptTimeoutToken: cts.Token);
var globals = new ScriptGlobals
{
Instance = context,
Parameters = new ScriptParameters(),
CancellationToken = cts.Token,
Alarm = new AlarmContext
{
Name = alarmName,
Level = level,
Priority = priority,
Message = message
}
};
await compiledScript.RunAsync(globals, cts.Token);
parent.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
}
catch (OperationCanceledException)
{
logger.LogWarning(
"Alarm on-trigger script for {Alarm} on {Instance} timed out",
alarmName, instanceName);
parent.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
}
catch (Exception ex)
{
// Failures logged, alarm continues
logger.LogError(ex,
"Alarm on-trigger script for {Alarm} on {Instance} failed",
alarmName, instanceName);
parent.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
}
finally
{
self.Tell(PoisonPill.Instance);
}
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler).Unwrap();
}
}
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
+311 -28
View File
@@ -142,6 +142,23 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// </summary>
private readonly HashSet<string> _initFailedPendingRowRemoval = new();
/// <summary>
/// WP3.1 warm-then-gate: instances whose deploy compile is being warmed off the actor
/// thread, keyed by instance name. Presence is the per-instance in-flight guard that keeps
/// same-instance command ordering intact while OTHER instances' commands flow freely.
/// Entries are added in <see cref="HandleDeploy"/> and removed in
/// <see cref="HandleDeployCompileWarmed"/> — one or the other always runs, because the
/// warm task pipes its result back unconditionally (it swallows its own exceptions).
/// </summary>
private readonly Dictionary<string, DeployWarmState> _deployWarms = new();
/// <summary>
/// WP3.1: script-execution scheduler seam (#18) — the pool this actor grows as instances
/// deploy. Null selects the process-wide shared scheduler; tests inject their own so
/// sizing assertions do not perturb (or depend on) the process-wide singleton.
/// </summary>
private readonly ScriptExecutionScheduler? _scriptScheduler;
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
public ITimerScheduler Timers { get; set; } = null!;
@@ -172,6 +189,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// external-system changes. Optional/null in tests that do not exercise external-system
/// caching.
/// </param>
/// <param name="scriptScheduler">
/// WP3.1: optional script-execution scheduler override (#18). This actor grows the pool
/// towards <see cref="ScriptExecutionScheduler.ComputeTargetThreads"/> on every instance-
/// count change; null uses the process-wide shared scheduler.
/// </param>
public DeploymentManagerActor(
SiteStorageService storage,
ScriptCompilationService compilationService,
@@ -186,8 +208,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
IDeploymentConfigFetcher? configFetcher = null,
TimeSpan? startupLoadRetryInterval = null,
Func<Task<List<DeployedInstance>>>? configLoader = null,
ExternalSystemDefinitionCache? externalSystemCache = null)
ExternalSystemDefinitionCache? externalSystemCache = null,
ScriptExecutionScheduler? scriptScheduler = null)
{
_scriptScheduler = scriptScheduler;
_storage = storage;
_compilationService = compilationService;
_deployCompileValidator = new DeployCompileValidator(compilationService);
@@ -212,9 +236,16 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// Lifecycle commands
Receive<DeployInstanceCommand>(cmd => HandleDeploy(cmd, Sender));
Receive<DisableInstanceCommand>(HandleDisable);
Receive<EnableInstanceCommand>(HandleEnable);
Receive<DeleteInstanceCommand>(HandleDelete);
Receive<DisableInstanceCommand>(cmd => HandleDisable(cmd, Sender));
Receive<EnableInstanceCommand>(cmd => HandleEnable(cmd, Sender));
Receive<DeleteInstanceCommand>(cmd => HandleDelete(cmd, Sender));
// WP3.1 warm-then-gate: the off-thread compile warm for a deploy has finished, so
// the (now all-cache-hit) synchronous compile gate can run on the actor thread.
Receive<DeployCompileWarmed>(HandleDeployCompileWarmed);
// WP3.1: a staggered startup batch's compiles have been pre-warmed off-thread;
// create the batch's Instance Actors now that their PreStart compiles are hits.
Receive<BatchCompileWarmed>(msg => CreateInstanceActorBatch(msg.Batch));
// Notify-and-fetch: central sends a small RefreshDeploymentCommand;
// the active singleton fetches the flattened config over HTTP, then reuses the
@@ -461,8 +492,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Creates Instance Actors in batches with a configurable delay between batches
/// to prevent reconnection storms on failover.
/// WP3.1: pre-warms one staggered startup batch's script and trigger-expression compiles
/// off the actor thread, then pipes <see cref="BatchCompileWarmed"/> so
/// <see cref="CreateInstanceActorBatch"/> can create the batch's Instance Actors with
/// every <c>PreStart</c> compile already a cache hit.
///
/// <para>This closes the long-standing "per-instance compilation during staggered startup"
/// gap: on failover, each Instance Actor used to Roslyn-compile its own scripts inside
/// <c>PreStart</c>, serialising a site's whole recovery behind compilation. The cost is one
/// extra message per batch.</para>
/// </summary>
private void HandleStartNextBatch(StartNextBatch msg)
{
@@ -471,6 +509,45 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
var startIdx = state.NextIndex;
var endIdx = Math.Min(startIdx + batchSize, state.Configs.Count);
var validator = _deployCompileValidator;
var batchConfigs = new string[endIdx - startIdx];
for (var i = startIdx; i < endIdx; i++)
batchConfigs[i - startIdx] = state.Configs[i].ConfigJson;
Task.Run(() =>
{
foreach (var configJson in batchConfigs)
{
try
{
// Verdict discarded: startup does not gate on compile failures (a failing
// script is logged by the Instance Actor and leaves the rest running).
// This call exists only to populate SiteScriptCompileCache.
validator.Validate(configJson);
}
catch
{
// Best-effort warm — a throw here just means that instance's PreStart
// compiles the hard way, exactly as it did before WP3.1.
}
}
return new BatchCompileWarmed(msg);
}).PipeTo(Self);
}
/// <summary>
/// Creates one batch's Instance Actors (after its compiles have been pre-warmed) and
/// schedules the next batch, with a configurable delay between batches to prevent
/// reconnection storms on failover.
/// </summary>
private void CreateInstanceActorBatch(StartNextBatch msg)
{
var state = msg.State;
var batchSize = _options.StartupBatchSize;
var startIdx = state.NextIndex;
var endIdx = Math.Min(startIdx + batchSize, state.Configs.Count);
_logger.LogDebug(
"Creating Instance Actors batch [{Start}..{End}) of {Total}",
startIdx, endIdx, state.Configs.Count);
@@ -520,23 +597,164 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// apply still replies to the right actor.
/// </param>
private void HandleDeploy(DeployInstanceCommand command, IActorRef replyTo)
{
var instanceName = command.InstanceUniqueName;
// WP3.1 warm-then-gate. The S3 gate below must stay synchronous on the actor thread
// (mailbox FIFO is what makes redeploy-supersede and delete-during-redeploy correct),
// but the Roslyn compile it performs used to hold the whole singleton — every OTHER
// instance's commands stalled behind one instance's compile. So the compile is WARMED
// off-thread first and the synchronous gate then re-runs as pure cache hits.
//
// Ordering is preserved by an explicit per-instance in-flight guard: while a warm is
// in flight for instance X, further mutating commands for X are queued here rather
// than racing ahead. Commands for OTHER instances flow freely, which is the whole
// point — and safe, because cross-instance ordering was never guaranteed to callers.
if (_deployWarms.TryGetValue(instanceName, out var warming))
{
// Nothing queued behind the pending deploy yet: plain last-write-wins, and the
// displaced deployer is told it was superseded so it never waits out its Ask —
// the same contract as the mid-termination redeploy buffer below.
if (warming.Buffered.Count == 0)
{
warming.ReplyTo.Tell(new DeploymentStatusResponse(
warming.Command.DeploymentId, instanceName, DeploymentStatus.Failed,
$"superseded by newer deployment {command.DeploymentId} before the site compile gate ran",
DateTimeOffset.UtcNow));
warming.Command = command;
warming.ReplyTo = replyTo;
// Warm the NEW config; the in-flight warm's result is recognised as stale by
// the reference check in HandleDeployCompileWarmed and dropped.
StartDeployCompileWarm(command);
return;
}
// A delete/disable/enable is already queued behind the pending deploy —
// superseding now would reorder it past this deploy, so queue instead.
warming.Buffered.Add(new BufferedInstanceCommand(command, replyTo));
return;
}
_deployWarms[instanceName] = new DeployWarmState(command, replyTo);
StartDeployCompileWarm(command);
}
/// <summary>
/// Compiles the deployment's scripts off the actor thread purely to populate the
/// process-wide <see cref="SiteScriptCompileCache"/>, then pipes
/// <see cref="DeployCompileWarmed"/> back so the authoritative gate can run on the actor
/// thread against a warm cache. The warm is best-effort: its verdict is discarded and any
/// exception swallowed, because <see cref="RunDeployGateAndProceed"/> re-derives the
/// verdict (and reproduces any failure) exactly as it did before WP3.1.
/// </summary>
private void StartDeployCompileWarm(DeployInstanceCommand command)
{
var validator = _deployCompileValidator;
var configJson = command.FlattenedConfigurationJson;
Task.Run(() =>
{
try
{
validator.Validate(configJson);
}
catch (Exception ex)
{
_logger.LogDebug(ex,
"Deploy compile warm for {Instance} threw; the synchronous gate will re-run it",
command.InstanceUniqueName);
}
return new DeployCompileWarmed(command);
}).PipeTo(Self);
}
/// <summary>
/// Runs the authoritative compile gate for a warmed deployment on the actor thread, then
/// drains any commands queued for that instance during the warm — inline and in arrival
/// order, NOT by re-telling <c>Self</c>, which would put them behind messages that landed
/// in the mailbox during the warm and so reorder same-instance commands.
/// </summary>
private void HandleDeployCompileWarmed(DeployCompileWarmed msg)
{
var instanceName = msg.Command.InstanceUniqueName;
if (!_deployWarms.TryGetValue(instanceName, out var warming))
return;
// A newer deploy superseded this one while its warm was running; that supersede
// started its own warm, so this result is stale and must not apply a dead command.
if (!ReferenceEquals(warming.Command, msg.Command))
return;
_deployWarms.Remove(instanceName);
RunDeployGateAndProceed(warming.Command, warming.ReplyTo);
foreach (var queued in warming.Buffered)
DispatchBufferedCommand(queued);
}
/// <summary>
/// Queues a mutating lifecycle command for an instance whose deploy is mid-compile-warm.
/// Returns <see langword="true"/> when the command was queued (the caller must return
/// immediately), <see langword="false"/> when no warm is in flight and the caller should
/// proceed normally.
/// </summary>
private bool TryBufferDuringDeployWarm(string instanceName, object command, IActorRef replyTo)
{
if (!_deployWarms.TryGetValue(instanceName, out var warming)) return false;
warming.Buffered.Add(new BufferedInstanceCommand(command, replyTo));
return true;
}
/// <summary>
/// Re-dispatches one command queued during a deploy compile warm, restoring the original
/// sender. A queued deploy legitimately opens a NEW warm window, in which case the
/// commands after it re-queue behind that one — ordering still holds.
/// </summary>
private void DispatchBufferedCommand(BufferedInstanceCommand queued)
{
switch (queued.Command)
{
case DeployInstanceCommand deploy:
HandleDeploy(deploy, queued.Sender);
break;
case DeleteInstanceCommand delete:
HandleDelete(delete, queued.Sender);
break;
case DisableInstanceCommand disable:
HandleDisable(disable, queued.Sender);
break;
case EnableInstanceCommand enable:
HandleEnable(enable, queued.Sender);
break;
default:
_logger.LogWarning(
"Unhandled buffered command type {Type} for a deploy compile warm — dropped",
queued.Command.GetType().Name);
break;
}
}
/// <summary>
/// The authoritative site-side compile gate (S3) plus the deploy application. Reached
/// only from <see cref="HandleDeployCompileWarmed"/>, i.e. after the same compile has
/// been warmed off-thread.
/// </summary>
private void RunDeployGateAndProceed(DeployInstanceCommand command, IActorRef replyTo)
{
// Site-side compile gate (S3): a compile failure must reject the
// deployment with NO partial state applied (no Instance Actor, no
// persisted config) — the design spec's contract. Validation runs
// synchronously on the actor thread: it is a pure prefix step of the
// deploy handler, so the existing redeploy-supersede / delete-during-
// redeploy ordering (which depends on strict mailbox FIFO) is preserved
// exactly. It is NOT run off-thread — piping the verdict back to self
// reorders concurrent deploys relative to each other and to
// delete/disable commands, breaking that ordering. A deploy is an
// infrequent admin command, so briefly holding the singleton for a pure
// Roslyn compile is acceptable; the central deployer already Asks and
// waits for the DeploymentStatusResponse. Redeploys and multi-instance
// deploys of unchanged scripts hit the process-wide compile cache
// (SiteScriptCompileCache), so the synchronous gate recompiles only
// genuinely new code and the Instance Actor's PreStart reuses the gate's
// compile (N4).
// persisted config) — the design spec's contract. Validation still runs
// synchronously on the actor thread as a pure prefix step of the deploy
// application, so the existing redeploy-supersede / delete-during-redeploy
// ordering (which depends on strict mailbox FIFO) is preserved exactly.
// WP3.1 removed the COST rather than the ordering: the same compile has
// already been warmed off-thread (StartDeployCompileWarm), so this call is
// pure cache hits and no longer holds the singleton — while the per-instance
// warm guard keeps this instance's own command ordering intact. Redeploys and
// multi-instance deploys of unchanged scripts hit the process-wide compile
// cache (SiteScriptCompileCache) too, and the Instance Actor's PreStart reuses
// the gate's compile (N4).
var errors = _deployCompileValidator.Validate(command.FlattenedConfigurationJson);
if (errors.Count > 0)
{
@@ -942,10 +1160,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// <summary>
/// Disables an instance: stops the actor and marks as disabled in SQLite.
/// </summary>
private void HandleDisable(DisableInstanceCommand command)
private void HandleDisable(DisableInstanceCommand command, IActorRef replyTo)
{
var instanceName = command.InstanceUniqueName;
// WP3.1 warm-then-gate: a deploy for this instance is mid-compile-warm; queue this
// command so same-instance ordering is preserved (see TryBufferDuringDeployWarm).
if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
// A disable arriving mid-redeploy must cancel the buffered
// redeploy. Otherwise HandleTerminated re-creates the Instance Actor and
// re-stores its config with isEnabled: true when the predecessor terminates,
@@ -975,7 +1197,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
UpdateInstanceCounts();
var sender = Sender;
var sender = replyTo;
_storage.SetInstanceEnabledAsync(instanceName, false).ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
@@ -1005,10 +1227,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// Enables an instance: marks as enabled in SQLite and re-creates the Instance Actor
/// from the stored config.
/// </summary>
private void HandleEnable(EnableInstanceCommand command)
private void HandleEnable(EnableInstanceCommand command, IActorRef replyTo)
{
var instanceName = command.InstanceUniqueName;
var sender = Sender;
// WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
var sender = replyTo;
Task.Run(async () =>
{
@@ -1061,10 +1287,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// Deletes an instance: stops the actor and removes config from SQLite.
/// Note: store-and-forward messages are NOT cleared per design decision.
/// </summary>
private void HandleDelete(DeleteInstanceCommand command)
private void HandleDelete(DeleteInstanceCommand command, IActorRef replyTo)
{
var instanceName = command.InstanceUniqueName;
// WP3.1 warm-then-gate: see TryBufferDuringDeployWarm.
if (TryBufferDuringDeployWarm(instanceName, command, replyTo)) return;
// A delete arriving while a redeploy is still terminating must
// be authoritative over the mid-redeploy bookkeeping. HandleDeploy already
// removed the instance from _instanceActors and buffered a PendingRedeploy
@@ -1105,7 +1334,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
_deployedInstanceNames.Remove(instanceName);
UpdateInstanceCounts();
var sender = Sender;
var sender = replyTo;
_storage.RemoveDeployedConfigAsync(instanceName).ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
@@ -1135,7 +1364,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// Fire-and-forget a <c>deployment</c> operational event to the optional
/// <see cref="ISiteEventLogger"/> on a deploy/enable/disable/delete outcome.
/// Resolved optionally and never awaited so a logging failure cannot affect the
/// deployment pipeline (matching the established ScriptActor/ScriptExecutionActor
/// deployment pipeline (matching the established ScriptActor / script-run
/// pattern).
/// <para>
/// <b>Thread-safety:</b> the disable (<see cref="HandleDisable"/>) and delete
@@ -2089,7 +2318,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
internal int InstanceActorCount => _instanceActors.Count;
/// <summary>
/// Updates the health collector with current instance counts.
/// Updates the health collector with current instance counts, and (WP3.1) grows the
/// script-execution pool to match.
/// Total deployed = _deployedInstanceNames.Count, enabled = running actors, disabled = difference.
/// </summary>
private void UpdateInstanceCounts()
@@ -2098,6 +2328,22 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
deployed: _deployedInstanceNames.Count,
enabled: _instanceActors.Count,
disabled: _deployedInstanceNames.Count - _instanceActors.Count);
// WP3.1: the blocking script pool scales with the number of running instances rather
// than sitting at a fixed 8 forever. This is the single call site because it already
// runs on every deploy / undeploy / enable / disable and once per staggered startup
// batch. Growth is idempotent and one-way — see ScriptExecutionScheduler.EnsureCapacity.
try
{
var target = ScriptExecutionScheduler.ComputeTargetThreads(_instanceActors.Count, _options);
(_scriptScheduler ?? ScriptExecutionScheduler.Shared(_options)).EnsureCapacity(target);
}
catch (Exception ex)
{
// Pool sizing is an optimisation, never a correctness requirement: a failure here
// must not fail the deploy/enable/disable that triggered it.
_logger.LogWarning(ex, "Failed to resize the script-execution pool; continuing at its current size.");
}
}
// ── Internal messages ──
@@ -2131,6 +2377,43 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
internal record SharedScriptsLoaded(
List<DeployedInstance> EnabledConfigs, int CompiledCount, int TotalCount);
/// <summary>
/// WP3.1 warm-then-gate: piped back to self once a deployment's scripts have been
/// compiled off the actor thread into the process-wide compile cache. Carries the exact
/// command instance it warmed so a result superseded mid-warm is recognised by reference
/// and dropped.
/// </summary>
internal sealed record DeployCompileWarmed(DeployInstanceCommand Command);
/// <summary>
/// WP3.1: piped back to self once a staggered startup batch's compiles have been
/// pre-warmed, carrying the original batch message so actor creation resumes unchanged.
/// </summary>
internal sealed record BatchCompileWarmed(StartNextBatch Batch);
/// <summary>
/// A lifecycle command queued because its instance's deploy was mid-compile-warm, with
/// the sender to answer once it is re-dispatched.
/// </summary>
internal sealed record BufferedInstanceCommand(object Command, IActorRef Sender);
/// <summary>
/// The in-flight state of one instance's deploy compile warm: the deploy that will be
/// applied when the warm lands (replaceable last-write-wins while nothing is queued behind
/// it), the deployer to answer, and the ordered commands queued during the warm.
/// </summary>
internal sealed class DeployWarmState(DeployInstanceCommand command, IActorRef replyTo)
{
/// <summary>The deploy to apply when the warm completes.</summary>
public DeployInstanceCommand Command { get; set; } = command;
/// <summary>The deployer awaiting this deploy's <see cref="DeploymentStatusResponse"/>.</summary>
public IActorRef ReplyTo { get; set; } = replyTo;
/// <summary>Commands for this instance that arrived during the warm, in arrival order.</summary>
public List<BufferedInstanceCommand> Buffered { get; } = [];
}
internal record StartNextBatch(BatchState State);
internal record BatchState(List<DeployedInstance> Configs, int NextIndex);
internal record EnableResult(
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/InstanceActor.cs
+3 -3
View File
@@ -374,7 +374,7 @@ public class InstanceActor : ReceiveActor
/// Fire-and-forget an <c>instance_lifecycle</c> operational event to the
/// optional <see cref="ISiteEventLogger"/>. Resolved optionally and never
/// awaited so a logging failure cannot affect the instance lifecycle
/// (matching the established ScriptActor/ScriptExecutionActor pattern).
/// (matching the established ScriptActor / script-run pattern).
/// </summary>
private void LogLifecycleEvent(string message)
{
@@ -1666,8 +1666,8 @@ public class InstanceActor : ReceiveActor
{
Script<object?>? onTriggerScript = null;
// The on-trigger script's per-script execution timeout,
// captured from its ResolvedScript so the AlarmExecutionActor can
// apply perScript ?? global. Null when there is no on-trigger script.
// captured from its ResolvedScript so the launched on-trigger run
// can apply perScript ?? global. Null when there is no on-trigger script.
int? onTriggerTimeoutSeconds = null;
// Compile on-trigger script if defined
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/NativeAlarmActor.cs
+1 -1
View File
@@ -444,7 +444,7 @@ public class NativeAlarmActor : ReceiveActor
/// condition's severity); an inactive condition is a return-to-normal; an
/// acknowledge transition is informational. Resolved optionally and never
/// awaited so a logging failure cannot affect the mirror (matching the
/// established ScriptActor/ScriptExecutionActor pattern).
/// established ScriptActor / script-run pattern).
/// </summary>
private void LogAlarmEvent(NativeAlarmTransition t, AlarmConditionState condition)
{
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptActor.cs
+220 -76
View File
@@ -15,8 +15,16 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <summary>
/// Script Actor — coordinator actor, child of Instance Actor.
/// Holds compiled script delegate, manages trigger configuration, and spawns
/// ScriptExecutionActor children per invocation. Does not block on child completion.
/// Holds compiled script delegate, manages trigger configuration, and launches script
/// runs per invocation. Does not block on run completion.
///
/// <para>WP3.1: runs are launched directly via <see cref="ScriptRunLauncher"/> rather than
/// through a short-lived <c>ScriptExecutionActor</c> child (which had no <c>Receive</c>
/// handler, no <c>PostStop</c>, and whose <c>IActorRef</c> was never a message target — pure
/// per-run actor-cell overhead). Concurrent runs are bounded by
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; over the cap the NEWEST
/// trigger is shed. Trigger-expression evaluation no longer runs on the blocking script pool
/// at all — see <see cref="TriggerEvalGate"/>.</para>
///
/// Trigger types:
/// - Interval: uses Akka timers to fire periodically
@@ -43,20 +51,52 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <summary>
/// Script-execution scheduler seam (#18): the process-wide
/// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so
/// this actor's trigger-expression evaluation and spawned script bodies run on a
/// caller-owned pool instead of the shared one. Resolved lazily at each use so the
/// null (host) path stays byte-for-byte identical to the previous static call.
/// this actor's launched script bodies run on a caller-owned pool instead of the
/// shared one. Resolved lazily at each use so the null (host) path stays
/// byte-for-byte identical to the previous static call.
///
/// <para>WP3.1: trigger-expression evaluation no longer uses this scheduler — see
/// <see cref="_evalGate"/>.</para>
/// </summary>
private readonly ScriptExecutionScheduler? _scheduler;
/// <summary>
/// WP3.1 (finding #4): the concurrency gate for trigger-expression evaluation, or null
/// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
/// async work on the shared .NET thread pool behind this gate, NOT on
/// <see cref="_scheduler"/> — that is what stops an Expression trigger from queueing
/// behind blocking script bodies.
/// </summary>
private readonly TriggerEvalGate? _evalGate;
/// <summary>
/// WP3.1: runs launched but not yet completed (queued or executing) for this script.
/// Incremented at launch, decremented on <see cref="ScriptExecutionCompleted"/>, which
/// every terminal path emits — including the launch-path catch, so the counter cannot
/// leak. Touched only on the actor thread.
/// </summary>
private int _runsInFlight;
/// <summary>
/// WP3.1: when the last shed Warning site event was emitted for this script. Sheds are
/// ALWAYS counted on the health collector; the site event is rate-limited to one per
/// script per minute so a hot trigger against a saturated cap cannot flood
/// <c>site_events</c>.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
private Script<object?>? _compiledScript;
private ScriptTriggerConfig? _triggerConfig;
private TimeSpan? _minTimeBetweenRuns;
/// <summary>
/// The per-script execution timeout in seconds, or null to use the
/// global default. Threaded down to each spawned <see cref="ScriptExecutionActor"/>,
/// which applies <c>perScript ?? global</c> (and treats ≤ 0 as "use global").
/// global default. Threaded down to each launched run via
/// <see cref="ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
/// (and treats ≤ 0 as "use global").
/// </summary>
private readonly int? _executionTimeoutSeconds;
private DateTimeOffset _lastExecutionTime = DateTimeOffset.MinValue;
@@ -111,6 +151,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <param name="healthCollector">Optional health metrics collector.</param>
/// <param name="serviceProvider">Optional DI service provider for script execution context services.</param>
/// <param name="scheduler">Optional script-execution scheduler override (#18); null uses the process-wide shared scheduler.</param>
/// <param name="evalGate">Optional trigger-expression concurrency gate override (WP3.1); null uses the process-wide shared gate.</param>
public ScriptActor(
string scriptName,
string instanceName,
@@ -124,7 +165,8 @@ public class ScriptActor : ReceiveActor, IWithTimers
IReadOnlyDictionary<string, object?>? initialAttributes = null,
ISiteHealthCollector? healthCollector = null,
IServiceProvider? serviceProvider = null,
ScriptExecutionScheduler? scheduler = null)
ScriptExecutionScheduler? scheduler = null,
TriggerEvalGate? evalGate = null)
{
_scriptName = scriptName;
_instanceName = instanceName;
@@ -136,6 +178,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
_healthCollector = healthCollector;
_serviceProvider = serviceProvider;
_scheduler = scheduler;
_evalGate = evalGate;
_minTimeBetweenRuns = scriptConfig.MinTimeBetweenRuns;
_executionTimeoutSeconds = scriptConfig.ExecutionTimeoutSeconds;
_scope = scriptConfig.Scope;
@@ -201,24 +244,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
_scriptName, _instanceName);
}
/// <inheritdoc />
protected override SupervisorStrategy SupervisorStrategy()
{
return new OneForOneStrategy(
maxNrOfRetries: -1,
withinTimeRange: TimeSpan.FromMinutes(1),
decider: Decider.From(ex =>
{
_logger.LogWarning(ex,
"ScriptExecutionActor for {Script} on {Instance} failed, stopping",
_scriptName, _instanceName);
return Directive.Stop;
}));
}
// WP3.1: the Stop-on-failure SupervisorStrategy override is gone with the per-run
// ScriptExecutionActor child it supervised. This actor has no children left; a
// launch-path throw is caught in SpawnExecution (which replies to the Ask caller and
// releases the in-flight slot) so it can never escalate to InstanceActor, which
// continues to supervise this actor with Resume exactly as before.
/// <summary>
/// Handles CallScript ask from ScriptRuntimeContext or Instance Actor.
/// Spawns a ScriptExecutionActor and forwards the sender for reply.
/// Launches a run and captures the sender for the eventual reply.
/// </summary>
private void HandleScriptCallRequest(ScriptCallRequest request)
{
@@ -233,7 +267,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
}
// (ParentExecutionId): carry any inbound-routed ParentExecutionId
// through to the ScriptExecutionActor so the routed script's
// through to the launched run so the routed script's
// ScriptRuntimeContext can record its spawner. Null for normal
// (tag-change / timer) runs and nested Script.Call invocations.
SpawnExecution(
@@ -287,12 +321,22 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// Starts an off-dispatcher evaluation of the compiled trigger expression (P1).
/// The expression previously ran synchronously on the actor's dispatcher thread
/// via <c>RunAsync(...).GetAwaiter().GetResult()</c>, blocking the dispatcher on
/// every attribute change. It now runs on the dedicated script-execution
/// scheduler against a point-in-time snapshot; the boolean result is piped back
/// to this actor as an <see cref="ExpressionEvalResult"/> so all edge state is
/// applied on the actor thread. Bursts coalesce: at most one evaluation is in
/// flight, at most one pending, so a storm of changes collapses to the latest
/// snapshot without unbounded task fan-out.
/// every attribute change. It runs off-thread against a point-in-time snapshot;
/// the boolean result is piped back to this actor as an
/// <see cref="ExpressionEvalResult"/> so all edge state is applied on the actor
/// thread. Bursts coalesce: at most one evaluation is in flight, at most one
/// pending, so a storm of changes collapses to the latest snapshot without
/// unbounded task fan-out.
///
/// <para>WP3.1 (finding #4) changed WHERE it runs and WHEN its clock starts. It used
/// to run on the dedicated <see cref="ScriptExecutionScheduler"/> — the same bounded
/// pool as blocking script bodies — so N blocked scripts stalled every Expression
/// trigger on the node indefinitely. It now runs as plain async work on the shared
/// .NET thread pool behind <see cref="TriggerEvalGate"/>: trigger expressions are
/// non-blocking by construction, so they belong there. And the deadline
/// <see cref="CancellationTokenSource"/> is armed HERE, on the actor thread, before
/// the work is queued — so gate-wait time burns the same budget and a saturated gate
/// yields a timely false instead of an unbounded stall.</para>
/// </summary>
private void StartExpressionEvaluation()
{
@@ -303,29 +347,45 @@ public class ScriptActor : ReceiveActor, IWithTimers
var snapshot = new Dictionary<string, object?>(_attributeSnapshot); // point-in-time copy, actor thread
var expression = _compiledTriggerExpression;
var self = Self;
Task.Factory.StartNew(async () =>
var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
// Clock starts AT ENQUEUE, not at dequeue.
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
Task.Run(async () =>
{
try
{
// Bound evaluation with a short timeout. The CancellationToken covers
// cooperative/async cases; a pathological CPU-bound expression is not
// fully interruptible — acceptable because trigger expressions are
// authored by trusted Design-role users and compile-checked pre-deploy.
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
var state = await expression.RunAsync(new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
await gate.WaitAsync(cts.Token).ConfigureAwait(false);
try
{
// The CancellationToken covers cooperative/async cases; a pathological
// CPU-bound expression is not fully interruptible — acceptable because
// trigger expressions are authored by trusted Design-role users and
// compile-checked pre-deploy.
var state = await expression.RunAsync(
new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
}
finally
{
gate.Release();
}
}
catch (Exception ex)
{
// OperationCanceledException (timeout) falls through here too and is
// treated as false. LogExpressionError touches only thread-safe
// members (_healthCollector Interlocked, _logger, DI-resolved
// singleton logger) so it is safe off the actor thread.
// OperationCanceledException (timeout, INCLUDING a timeout that fired while
// still waiting on the gate) falls through here too and is treated as false.
// LogExpressionError touches only thread-safe members (_healthCollector
// Interlocked, _logger, DI-resolved singleton logger) so it is safe off the
// actor thread.
LogExpressionError(ex);
return false;
}
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
_scheduler ?? ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
finally
{
cts.Dispose();
}
}).PipeTo(self,
success: r => new ExpressionEvalResult(r),
failure: ex => new ExpressionEvalFailed(ex));
}
@@ -421,7 +481,7 @@ public class ScriptActor : ReceiveActor, IWithTimers
/// <summary>
/// Records a trigger-expression evaluation failure to the site event log,
/// mirroring how ScriptExecutionActor reports script errors.
/// mirroring how a script run reports its own errors.
/// </summary>
private void LogExpressionError(Exception ex)
{
@@ -457,8 +517,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Spawns a new ScriptExecutionActor child for this invocation.
/// Multiple concurrent executions are allowed.
/// Launches a run of this script. Multiple concurrent runs are allowed, up to
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; beyond that the newest
/// trigger is shed (see <see cref="ShedRun"/>).
///
/// <para>WP3.1: the run is launched directly on the script-execution scheduler via
/// <see cref="ScriptRunLauncher"/> — the actor's mailbox stays on the default dispatcher,
/// but the script body runs on the bounded set of dedicated threads, so blocking script
/// I/O is contained there and cannot starve the shared .NET thread pool. No per-run child
/// actor is created.</para>
/// </summary>
private void SpawnExecution(
IReadOnlyDictionary<string, object?>? parameters,
@@ -467,46 +534,123 @@ public class ScriptActor : ReceiveActor, IWithTimers
string correlationId,
Guid? parentExecutionId = null)
{
var executionId = $"{_scriptName}-exec-{_executionCounter++}";
if (_runsInFlight >= _options.MaxConcurrentRunsPerScript)
{
ShedRun(replyTo, correlationId);
return;
}
// The actor's mailbox stays on the default dispatcher, but the
// script body itself runs on the dedicated ScriptExecutionScheduler (a bounded
// set of dedicated threads), so blocking script I/O is contained there and
// cannot starve the shared .NET thread pool.
var props = Props.Create(() => new ScriptExecutionActor(
_scriptName,
_instanceName,
_compiledScript!,
parameters,
callDepth,
_instanceActor,
_sharedScriptLibrary,
_options,
replyTo,
correlationId,
_logger,
_scope,
_healthCollector,
_serviceProvider,
// (ParentExecutionId): null for trigger-driven runs;
// an inbound-API-routed call supplies the inbound request's id.
parentExecutionId,
// Per-script timeout override (null = use global).
_executionTimeoutSeconds,
// Scheduler seam (#18): thread this actor's scheduler override down so
// spawned script bodies share the same pool (null = process-wide shared).
_scheduler));
var runId = _executionCounter++;
// Incremented BEFORE the launch so the launch-path catch below (which always emits a
// ScriptExecutionCompleted) balances it on every path — the counter cannot leak.
_runsInFlight++;
Context.ActorOf(props, executionId);
try
{
ScriptRunLauncher.LaunchScript(
_scriptName,
_instanceName,
_compiledScript!,
parameters,
callDepth,
_instanceActor,
_sharedScriptLibrary,
_options,
replyTo,
correlationId,
// Completion target: this actor. Identical delivery to the old
// Context.Parent.Tell from the execution actor, and it doubles as the
// in-flight release.
Self,
// Notification Outbox: the site communication actor Notify.Status queries
// central through. Resolved here, on the actor thread, and handed to the
// launcher (which has no ActorContext of its own).
Context.System.ActorSelection("/user/site-communication"),
_logger,
_scope,
runId,
_healthCollector,
_serviceProvider,
// (ParentExecutionId): null for trigger-driven runs;
// an inbound-API-routed call supplies the inbound request's id.
parentExecutionId,
// Per-script timeout override (null = use global).
_executionTimeoutSeconds,
// Scheduler seam (#18): thread this actor's scheduler override down so
// launched script bodies share the same pool (null = process-wide shared).
_scheduler);
}
catch (Exception ex)
{
// WP3.1 supervision parity: the only failure the removed per-run child could
// surface was a constructor throw (e.g. queueing onto a disposed scheduler),
// which the old OneForOneStrategy logged and stopped — leaving an Ask caller to
// hang with no reply. Same log shape, same "coordinator unaffected" outcome, plus
// a reply so the caller fails fast instead of waiting out its Ask timeout.
var errorMsg = $"Script '{_scriptName}' on instance '{_instanceName}' could not be launched: {ex.Message}";
_logger.LogWarning(ex,
"Script execution launch for {Script} on {Instance} failed, stopping",
_scriptName, _instanceName);
if (!replyTo.IsNobody())
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
Self.Tell(new ScriptExecutionCompleted(_scriptName, false, errorMsg));
}
}
/// <summary>
/// WP3.1 shed policy: refuses the incoming run because
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> runs are already in flight.
/// The four already queued/running are kept — they are closest to their own deadlines and
/// already charged against them — so nothing is ever reordered. A trigger-driven run is
/// simply not launched; an Ask-based <c>CallScript</c> gets an explicit error so a nested
/// call or inbound-API route fails fast rather than hanging to its Ask timeout.
/// </summary>
private void ShedRun(IActorRef replyTo, string correlationId)
{
_healthCollector?.IncrementScriptRunShed();
var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
_logger.LogWarning("{Message}", message);
// Rate-limited to one event per script per minute: the counter above still counts
// every shed, but a hot trigger against a saturated cap must not flood site_events.
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
}
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(
correlationId, false, null,
$"shed: {_runsInFlight} runs already in flight"));
}
}
private void HandleExecutionCompleted(ScriptExecutionCompleted msg)
{
// WP3.1: release the in-flight slot. Every terminal path emits exactly one of these
// (success / timeout / failure / launch failure), so the counter tracks reality.
if (_runsInFlight > 0) _runsInFlight--;
_logger.LogDebug(
"Script {Script} execution completed on {Instance}: success={Success}",
_scriptName, _instanceName, msg.Success);
}
/// <summary>
/// WP3.1: runs launched but not yet completed. Exposed for regression coverage of the
/// shed cap — the counter is otherwise invisible from outside the actor.
/// </summary>
internal int RunsInFlight => _runsInFlight;
// internal (not private) so the culture-invariance of the non-numeric fallback
// can be unit-tested directly on the test thread — the live path evaluates on a
// dispatcher thread whose CurrentCulture the test cannot deterministically set.
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptExecutionActor.cs
-341
View File
@@ -1,341 +0,0 @@
using Akka.Actor;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <summary>
/// Script Execution Actor -- short-lived child of Script Actor.
/// Receives compiled code, params, Instance Actor ref, and call depth.
/// Executes the script via Script Runtime API, returns result, then stops.
///
/// The actor itself and its mailbox run on the default Akka dispatcher; only the
/// script body is dispatched off the actor thread, onto the dedicated
/// <see cref="ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts.ScriptExecutionScheduler"/>,
/// so blocking script I/O cannot starve the shared thread pool
/// or stall other Akka dispatchers.
///
/// Script failures are logged but do not disable the script.
/// Supervision: Stop on unhandled exception (parent ScriptActor decides).
/// </summary>
public class ScriptExecutionActor : ReceiveActor
{
/// <summary>
/// Initializes the actor and immediately begins script execution on construction.
/// </summary>
/// <param name="scriptName">Name of the script being executed.</param>
/// <param name="instanceName">Name of the instance that owns the script.</param>
/// <param name="compiledScript">Compiled Roslyn script to execute.</param>
/// <param name="parameters">Optional named parameter values for the script.</param>
/// <param name="callDepth">Current call-nesting depth (used to enforce the max-depth limit).</param>
/// <param name="instanceActor">Parent instance actor reference for attribute access.</param>
/// <param name="sharedScriptLibrary">Library of shared scripts available during execution.</param>
/// <param name="options">Site runtime options applied during execution.</param>
/// <param name="replyTo">Actor reference that receives the script result.</param>
/// <param name="correlationId">Application-level correlation id threaded through the execution.</param>
/// <param name="logger">Logger for script execution events.</param>
/// <param name="scope">Script scope controlling which APIs are available.</param>
/// <param name="healthCollector">Optional health collector for recording execution metrics.</param>
/// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
/// <param name="parentExecutionId">ExecutionId of the spawning inbound-API execution for audit correlation; null for normal runs.</param>
/// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global <see cref="SiteRuntimeOptions.ScriptExecutionTimeoutSeconds"/>.</param>
public ScriptExecutionActor(
string scriptName,
string instanceName,
Script<object?> compiledScript,
IReadOnlyDictionary<string, object?>? parameters,
int callDepth,
IActorRef instanceActor,
SharedScriptLibrary sharedScriptLibrary,
SiteRuntimeOptions options,
IActorRef replyTo,
string correlationId,
ILogger logger,
Commons.Types.Scripts.ScriptScope scope,
ISiteHealthCollector? healthCollector = null,
IServiceProvider? serviceProvider = null,
// The spawning execution's
// ExecutionId for an inbound-API-routed call. Null for normal
// (tag-change / timer) runs and nested Script.Call invocations.
Guid? parentExecutionId = null,
// Per-script execution timeout override (seconds). Null or
// non-positive falls back to the global ScriptExecutionTimeoutSeconds.
int? executionTimeoutSeconds = null,
// Script-execution scheduler seam (#18): the process-wide
// ScriptExecutionScheduler by default; a test (or a future multi-site
// host) can inject its own instance so script bodies never run on the
// shared process-wide pool. Null selects the shared default.
ScriptExecutionScheduler? scheduler = null)
{
// Immediately begin execution
var self = Self;
var parent = Context.Parent;
ExecuteScript(
scriptName, instanceName, compiledScript, parameters, callDepth,
instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
self, parent, logger, scope, healthCollector, serviceProvider,
parentExecutionId, executionTimeoutSeconds, scheduler);
}
private static void ExecuteScript(
string scriptName,
string instanceName,
Script<object?> compiledScript,
IReadOnlyDictionary<string, object?>? parameters,
int callDepth,
IActorRef instanceActor,
SharedScriptLibrary sharedScriptLibrary,
SiteRuntimeOptions options,
IActorRef replyTo,
string correlationId,
IActorRef self,
IActorRef parent,
ILogger logger,
Commons.Types.Scripts.ScriptScope scope,
ISiteHealthCollector? healthCollector,
IServiceProvider? serviceProvider,
Guid? parentExecutionId,
int? executionTimeoutSeconds,
ScriptExecutionScheduler? scheduler)
{
// Per-script timeout overrides the global default. A null or
// non-positive per-script value (≤ 0) falls back to the global.
var timeout = TimeSpan.FromSeconds(
executionTimeoutSeconds is { } perScript && perScript > 0
? perScript
: options.ScriptExecutionTimeoutSeconds);
// Run the script body on the dedicated script-execution
// scheduler, not the shared .NET thread pool, so blocking script I/O cannot
// starve the global pool and stall Akka dispatchers / HTTP handling. An
// injected scheduler (#18) overrides the process-wide default.
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
// Notification Outbox: the site communication actor that Notify.Status queries
// central through. Resolved by actor path so the Notify helper does not need an
// IActorRef threaded all the way down from the host wiring.
var siteCommunicationActor = Context.System.ActorSelection("/user/site-communication");
// CTS must be created inside the async lambda so it outlives this method
_ = Task.Factory.StartNew(async () =>
{
IServiceScope? serviceScope = null;
// ISiteEventLogger is a singleton; resolve from the root provider so
// it is available to the catch blocks regardless of scope state.
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
using var cts = new CancellationTokenSource(timeout);
// Stuck-script watchdog (S2). The CTS firing only REQUESTS cooperative
// cancellation; it does NOT free a thread blocked in synchronous I/O.
// When the timeout elapses, wait a grace period on the thread pool and,
// if the body still hasn't returned, name the script loudly — this is
// the only signal an operator gets that one of the bounded
// script-execution threads is gone. `completed` is flipped in the
// finally below; Register fires on cancellation only (normal completion
// disposes the CTS with no callback). The Task.Run/Task.Delay run on the
// thread pool, not the (possibly saturated) script scheduler — deliberate.
var completed = 0;
var graceMs = options.StuckScriptGraceMs;
cts.Token.Register(() => _ = Task.Run(async () =>
{
await Task.Delay(graceMs);
if (Volatile.Read(ref completed) == 0)
{
var stuckMsg = $"Script '{scriptName}' on instance '{instanceName}' exceeded its " +
$"{timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — its dedicated " +
"script-execution thread is blocked (cooperative cancellation not observed).";
logger.LogError(stuckMsg);
_ = siteEventLogger?.LogEventAsync("script", "Error", instanceName,
$"ScriptActor:{scriptName}", stuckMsg);
}
}));
try
{
// Resolve integration services from DI (scoped lifetime)
IExternalSystemClient? externalSystemClient = null;
IDatabaseGateway? databaseGateway = null;
// Notification Outbox: the S&F engine is a singleton; the site identity
// provider supplies the site id stamped on enqueued notifications.
StoreAndForwardService? storeAndForward = null;
var siteId = string.Empty;
// The writer is a singleton (FallbackAuditWriter
// composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
// that haven't called AddAuditLog, which the helper handles as a no-op.
IAuditWriter? auditWriter = null;
// Site-local tracking store
// backing Tracking.Status(id). Singleton; null in tests / hosts
// that haven't wired the store, which the helper handles by
// throwing on access.
IOperationTrackingStore? operationTrackingStore = null;
// Site-side cached-call
// telemetry forwarder. Singleton bound to the AuditLog
// composition root; null in tests / hosts that haven't called
// AddAuditLog, in which case the cached-call helpers degrade
// to the no-emission path (the underlying S&F handoff still
// happens and a TrackedOperationId is still returned).
ICachedCallTelemetryForwarder? cachedForwarder = null;
// SourceNode-stamping: the local node name
// resolved from INodeIdentityProvider — node-a/node-b on site
// hosts. Null in tests / hosts that haven't registered the
// provider, in which case NotificationSubmit.SourceNode and
// SiteCallOperational.SourceNode stay null and central
// persists the rows with SourceNode NULL.
string? sourceNode = null;
if (serviceProvider != null)
{
serviceScope = serviceProvider.CreateScope();
externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
?? string.Empty;
auditWriter = serviceScope.ServiceProvider.GetService<IAuditWriter>();
operationTrackingStore = serviceScope.ServiceProvider.GetService<IOperationTrackingStore>();
cachedForwarder = serviceScope.ServiceProvider.GetService<ICachedCallTelemetryForwarder>();
sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
}
var context = new ScriptRuntimeContext(
instanceActor,
self,
sharedScriptLibrary,
callDepth,
options.MaxScriptCallDepth,
timeout,
instanceName,
logger,
externalSystemClient,
databaseGateway,
storeAndForward,
siteCommunicationActor,
siteId,
// Notification Outbox (FU3): stamp the executing script onto outbound
// notifications using the Site Event Logging "Source" convention.
sourceScript: $"ScriptActor:{scriptName}",
// Emit one ApiOutbound/ApiCall row per
// ExternalSystem.Call. Writer is best-effort; failures are logged
// and swallowed inside the helper so the script's call path is
// never aborted by an audit failure.
auditWriter: auditWriter,
// Site-local tracking store
// backing Tracking.Status(id). Authoritative source of truth for
// cached-call status — read directly by the script API.
operationTrackingStore: operationTrackingStore,
// Cached-call telemetry
// forwarder for ExternalSystem.CachedCall / Database.CachedWrite
// CachedSubmit emission + the immediate-success terminal-row
// emission. Best-effort: null degrades the helpers to a
// no-emission path; the S&F handoff and TrackedOperationId
// return are unaffected.
cachedForwarder: cachedForwarder,
// The spawning execution's
// id for an inbound-API-routed call. The routed script still
// mints its own fresh ExecutionId — this records the spawner.
// Null for normal (tag-change / timer) runs.
parentExecutionId: parentExecutionId,
// SourceNode-stamping: the local node name
// (node-a/node-b on a site) — threaded down so Notify.Send
// and the four cached-call telemetry constructors can stamp
// it onto NotificationSubmit.SourceNode and
// SiteCallOperational.SourceNode respectively.
sourceNode: sourceNode,
// Thread the singleton site event logger so
// recursion-limit violations at CallScript/CallShared emit a
// script Error site event in addition to ILogger.LogError.
siteEventLogger: siteEventLogger,
// WaitForAttribute (spec §4.3/§4.4): thread the per-script
// execution-timeout token so Attributes.WaitAsync's Ask is
// bounded by the script's own ExecutionTimeoutSeconds — a
// shorter script deadline wins over the wait's own timeout.
scriptTimeoutToken: cts.Token);
var globals = new ScriptGlobals
{
Instance = context,
Parameters = new ScriptParameters(parameters ?? new Dictionary<string, object?>()),
CancellationToken = cts.Token,
Scope = scope
};
// Operational `script` event — execution started. Fire-and-forget
// (the `_ =` discards the task) so the event log can never block or
// fault the script's own run; mirrors the existing Error-path emit.
_ = siteEventLogger?.LogEventAsync(
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
$"Script '{scriptName}' on instance '{instanceName}' started");
var state = await compiledScript.RunAsync(globals, cts.Token);
// Send result to requester if this was an Ask-based call
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
}
// Operational `script` event — execution completed successfully.
_ = siteEventLogger?.LogEventAsync(
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
$"Script '{scriptName}' on instance '{instanceName}' completed");
// Notify parent of completion
parent.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
}
catch (OperationCanceledException)
{
healthCollector?.IncrementScriptError();
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
logger.LogWarning(errorMsg);
// Failures recorded to site event log; script NOT disabled after failure.
_ = siteEventLogger?.LogEventAsync(
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
}
parent.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
}
catch (Exception ex)
{
healthCollector?.IncrementScriptError();
// Failures recorded to site event log; script NOT disabled after failure.
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
logger.LogError(ex, "Script execution failed: {Script} on {Instance}", scriptName, instanceName);
_ = siteEventLogger?.LogEventAsync(
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
}
parent.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
}
finally
{
// Mark the body finished so the stuck-script watchdog (registered on
// cts.Token above) treats a timely cancellation as NOT stuck.
Interlocked.Exchange(ref completed, 1);
// Dispose the DI scope (and scoped services) after script execution completes
serviceScope?.Dispose();
// Stop self after execution completes
self.Tell(PoisonPill.Instance);
}
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler).Unwrap();
}
}