Files
ScadaBridge/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/ScriptActor.cs
T
Joseph Doherty edd0ca672f feat(logging): carry ExecutionId across actor hops onto other actors' log lines (ScadaBridge#56)
LogContext is an AsyncLocal and does not cross an Akka message hop, so the run's id now rides the
message: Commons.Observability.IExecutionCorrelated (explicit interface member, no wire change) on
SetStaticAttributeCommand, AttributeValueChanged (not for an inferred writer), ScriptCallRequest and
the four RouteTo*Requests, plus a new additive SourceExecutionId on GetAttributeRequest,
WaitForAttributeRequest, WriteAttributeBatchRequest, WriteTagRequest and WriteTagBatchRequest.
ScriptRuntimeContext stamps its id on the requests it sends, the Instance Actor hands it on to the
DCL, and the Instance, Script, Alarm, Deployment Manager and both DCL actors re-push it per message in
AroundReceive (ExecutionLogContext.PushFor: one type test when the message names no execution). The
inbound executor pushes the request's ExecutionId for the method execution.

Tests (TestKit): Instance Actor line carries the message's id, no leak from the sender's ambient
context, scope ends with the message, per-write ids inside a batched SetAttributes, ids forwarded on
DCL writes, ScriptRuntimeContext stamps; DCL adapter lines on single/batch writes carry the id;
inbound executor lines carry the request id; Commons mapping + no serialized member. Each mutation-
proved. Still open on #56: a queryable per-execution sink and a Flow Debugger section. The DCL
actors' own Akka _log lines bypass Serilog (filed #120).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-10 05:28:20 -05:00

986 lines
46 KiB
C#

using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using Akka.Actor;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using System.Globalization;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <summary>
/// Script Actor — coordinator actor, child of Instance Actor.
/// Holds compiled script delegate, manages trigger configuration, and launches script
/// runs per invocation. Does not block on run completion.
///
/// <para>Runs are launched directly via <see cref="ScriptRunLauncher"/> rather than
/// through a short-lived <c>ScriptExecutionActor</c> child (which had no <c>Receive</c>
/// handler, no <c>PostStop</c>, and whose <c>IActorRef</c> was never a message target — pure
/// per-run actor-cell overhead). Concurrent runs are bounded by
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; over the cap the NEWEST
/// trigger is shed. Trigger-expression evaluation no longer runs on the blocking script pool
/// at all — see <see cref="TriggerEvalGate"/>.</para>
///
/// Trigger types:
/// - Interval: uses Akka timers to fire periodically
/// - ValueChange: receives attribute change notifications from Instance Actor
/// - Conditional: evaluates a threshold comparison on attribute change
/// - Expression: evaluates a compiled boolean expression on attribute change
/// Conditional and Expression triggers carry a <see cref="TriggerMode"/>:
/// OnTrue fires as the condition becomes true; WhileTrue additionally re-fires
/// on a timer (cadence = MinTimeBetweenRuns) while the condition stays true.
///
/// Supervision strategy: Resume on exception (coordinator preserves state).
/// </summary>
public class ScriptActor : ReceiveActor, IWithTimers
{
private readonly string _scriptName;
private readonly string _instanceName;
private readonly IActorRef _instanceActor;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly SiteRuntimeOptions _options;
private readonly ILogger _logger;
private readonly ISiteHealthCollector? _healthCollector;
private readonly IServiceProvider? _serviceProvider;
/// <summary>
/// Script-execution scheduler seam: the process-wide
/// <see cref="ScriptExecutionScheduler"/> when null, or an injected instance so
/// this actor's launched script bodies run on a caller-owned pool instead of the
/// shared one. Resolved lazily at each use so the null (host) path stays
/// byte-for-byte identical to the previous static call.
///
/// <para>Trigger-expression evaluation no longer uses this scheduler — see
/// <see cref="_evalGate"/>.</para>
/// </summary>
private readonly ScriptExecutionScheduler? _scheduler;
/// <summary>
/// The concurrency gate for trigger-expression evaluation, or null
/// for the process-wide <see cref="TriggerEvalGate.Shared"/>. Evaluations run as plain
/// async work on the shared .NET thread pool behind this gate, NOT on
/// <see cref="_scheduler"/> — that is what stops an Expression trigger from queueing
/// behind blocking script bodies.
/// </summary>
private readonly TriggerEvalGate? _evalGate;
/// <summary>
/// Runs launched but not yet completed (queued or executing) for this script.
/// Incremented at launch, decremented on <see cref="ScriptExecutionCompleted"/>, which
/// every terminal path emits — including the launch-path catch, so the counter cannot
/// leak. Touched only on the actor thread.
/// </summary>
private int _runsInFlight;
/// <summary>
/// When the last shed Warning site event was emitted for this script. Sheds are
/// ALWAYS counted on the health collector; the site event is rate-limited to one per
/// script per minute so a hot trigger against a saturated cap cannot flood
/// <c>site_events</c>.
/// </summary>
private DateTimeOffset _lastShedEventUtc = DateTimeOffset.MinValue;
private int _shedSuppressed;
/// <summary>Rate limit for the shed site event (the counter still counts every shed).</summary>
private static readonly TimeSpan ShedEventInterval = TimeSpan.FromMinutes(1);
private Script<object?>? _compiledScript;
private ScriptTriggerConfig? _triggerConfig;
private TimeSpan? _minTimeBetweenRuns;
/// <summary>
/// The per-script execution timeout in seconds, or null to use the
/// global default. Threaded down to each launched run via
/// <see cref="ScriptRunLauncher"/>, which applies <c>perScript ?? global</c>
/// (and treats ≤ 0 as "use global").
/// </summary>
private readonly int? _executionTimeoutSeconds;
private DateTimeOffset _lastExecutionTime = DateTimeOffset.MinValue;
private int _executionCounter;
private readonly Commons.Types.Scripts.ScriptScope _scope;
// Expression trigger state: compiled expression, edge-tracking, and the
// attribute snapshot the expression evaluates against.
private readonly Script<object?>? _compiledTriggerExpression;
private bool _lastExpressionResult;
private readonly Dictionary<string, object?> _attributeSnapshot = new();
// Coalescing guard for off-dispatcher expression evaluation (P1): at most one
// evaluation runs on the script scheduler at a time; a change arriving while
// one is in flight sets _evalPending so exactly one more runs afterwards
// against the latest snapshot. Both flags are touched only on the actor thread.
private bool _evalInFlight;
private bool _evalPending;
// 01/SR-U1: true when the pending evaluation was requested only by seeds (never fires).
private bool _evalPendingSeedOnly;
// WhileTrue trigger state: the most recent truth value of a Conditional
// trigger's comparison, used to detect false->true / true->false edges.
// (Expression triggers reuse _lastExpressionResult for the same purpose.)
private bool _conditionState;
/// <summary>Timer key for the WhileTrue re-fire timer (cadence = MinTimeBetweenRuns).</summary>
private const string WhileTrueTimerKey = "whiletrue-trigger";
/// <summary>
/// The exact dictionary instance this actor was seeded from
/// at construction. The Instance Actor must pass a private snapshot here, not
/// its live <c>_attributes</c> field — sharing the live dictionary lets this
/// constructor enumerate it while the Instance Actor mutates it on another
/// thread. Exposed for regression coverage of that isolation contract.
/// </summary>
internal IReadOnlyDictionary<string, object?>? SeedAttributesReference { get; }
/// <summary>Gets or sets the Akka timer scheduler used to schedule interval and WhileTrue triggers.</summary>
public ITimerScheduler Timers { get; set; } = null!;
/// <summary>
/// Initializes the ScriptActor with its compiled script, trigger configuration, and supporting services.
/// </summary>
/// <param name="scriptName">Name of the script this actor manages.</param>
/// <param name="instanceName">Unique name of the owning instance.</param>
/// <param name="instanceActor">Reference to the parent Instance Actor.</param>
/// <param name="compiledScript">Pre-compiled Roslyn script delegate, or null when compilation failed.</param>
/// <param name="scriptConfig">Resolved script metadata including trigger type and configuration.</param>
/// <param name="sharedScriptLibrary">Library of compiled shared scripts available for inline execution.</param>
/// <param name="options">Site runtime configuration options.</param>
/// <param name="logger">Logger for diagnostics.</param>
/// <param name="compiledTriggerExpression">Pre-compiled boolean trigger expression, or null when not an expression trigger.</param>
/// <param name="initialAttributes">Initial attribute snapshot used to seed expression trigger evaluation state.</param>
/// <param name="healthCollector">Optional health metrics collector.</param>
/// <param name="serviceProvider">Optional DI service provider for script execution context services.</param>
/// <param name="scheduler">Optional script-execution scheduler override; null uses the process-wide shared scheduler.</param>
/// <param name="evalGate">Optional trigger-expression concurrency gate override; null uses the process-wide shared gate.</param>
public ScriptActor(
string scriptName,
string instanceName,
IActorRef instanceActor,
Script<object?>? compiledScript,
ResolvedScript scriptConfig,
SharedScriptLibrary sharedScriptLibrary,
SiteRuntimeOptions options,
ILogger logger,
Script<object?>? compiledTriggerExpression = null,
IReadOnlyDictionary<string, object?>? initialAttributes = null,
ISiteHealthCollector? healthCollector = null,
IServiceProvider? serviceProvider = null,
ScriptExecutionScheduler? scheduler = null,
TriggerEvalGate? evalGate = null)
{
_scriptName = scriptName;
_instanceName = instanceName;
_instanceActor = instanceActor;
_compiledScript = compiledScript;
_sharedScriptLibrary = sharedScriptLibrary;
_options = options;
_logger = logger;
_healthCollector = healthCollector;
_serviceProvider = serviceProvider;
_scheduler = scheduler;
_evalGate = evalGate;
_minTimeBetweenRuns = scriptConfig.MinTimeBetweenRuns;
_executionTimeoutSeconds = scriptConfig.ExecutionTimeoutSeconds;
_scope = scriptConfig.Scope;
_compiledTriggerExpression = compiledTriggerExpression;
// Seed the trigger-expression attribute snapshot from the instance's
// initial attribute set so static attributes (which never re-emit an
// AttributeValueChanged after deploy) evaluate correctly at startup.
SeedAttributesReference = initialAttributes;
if (initialAttributes != null)
{
foreach (var kvp in initialAttributes)
_attributeSnapshot[kvp.Key] = kvp.Value;
}
// Parse trigger configuration
_triggerConfig = ParseTriggerConfig(scriptConfig.TriggerType, scriptConfig.TriggerConfiguration);
// Handle script call requests (Ask pattern from Instance Actor or ScriptRuntimeContext)
Receive<ScriptCallRequest>(HandleScriptCallRequest);
// Handle attribute value changes for value-change and conditional triggers
Receive<AttributeValueChanged>(HandleAttributeValueChanged);
// Review 2026-10-21 01/SR-U1: a value seeded at instance start updates trigger state only.
Receive<Messages.AttributeValueSeed>(HandleAttributeValueSeed);
// Handle interval tick
Receive<IntervalTick>(_ => TrySpawnExecution(null));
// Handle WhileTrue re-fire tick
Receive<WhileTrueTick>(_ => FireWhileTrueTick());
// Handle execution completion (for logging/metrics)
Receive<ScriptExecutionCompleted>(HandleExecutionCompleted);
// Handle the off-dispatcher trigger-expression evaluation result (P1).
Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
// Handle a faulted off-dispatcher evaluation task (N2) — clears in-flight
// instead of stranding the trigger with an unhandled Status.Failure.
Receive<ExpressionEvalFailed>(HandleExpressionEvalFailed);
}
/// <summary>
/// ScadaBridge#56: while handling a message sent on behalf of an execution
/// (<see cref="IExecutionCorrelated"/>), every line this actor logs - and every line logged by
/// a continuation it starts - carries that execution's <c>ExecutionId</c>. A message that names
/// no execution costs one type test.
/// </summary>
/// <param name="receive">The behaviour that handles the message.</param>
/// <param name="message">The message.</param>
/// <returns>Whether the message was handled.</returns>
protected override bool AroundReceive(Receive receive, object message)
{
using (ExecutionLogContext.PushFor(message))
return base.AroundReceive(receive, message);
}
/// <inheritdoc />
protected override void PreStart()
{
base.PreStart();
// Set up interval trigger if configured
if (_triggerConfig is IntervalTriggerConfig interval)
{
Timers.StartPeriodicTimer(
"interval-trigger",
IntervalTick.Instance,
interval.Interval,
interval.Interval);
_logger.LogDebug(
"ScriptActor {Script} on {Instance}: interval trigger set to {Interval}",
_scriptName, _instanceName, interval.Interval);
}
_logger.LogInformation(
"ScriptActor {Script} started on instance {Instance}",
_scriptName, _instanceName);
}
// The Stop-on-failure SupervisorStrategy override is gone with the per-run
// ScriptExecutionActor child it supervised. This actor has no children left; a
// launch-path throw is caught in SpawnExecution (which replies to the Ask caller and
// releases the in-flight slot) so it can never escalate to InstanceActor, which
// continues to supervise this actor with Resume exactly as before.
/// <summary>
/// Handles CallScript ask from ScriptRuntimeContext or Instance Actor.
/// Launches a run and captures the sender for the eventual reply.
/// </summary>
private void HandleScriptCallRequest(ScriptCallRequest request)
{
if (_compiledScript == null)
{
Sender.Tell(new ScriptCallResult(
request.CorrelationId,
false,
null,
$"Script '{_scriptName}' is not compiled."));
return;
}
// (ParentExecutionId): carry any inbound-routed ParentExecutionId
// through to the launched run so the routed script's
// ScriptRuntimeContext can record its spawner. Null for normal
// (tag-change / timer) runs and nested Script.Call invocations.
SpawnExecution(
request.Parameters, request.CurrentCallDepth, Sender, request.CorrelationId,
request.ParentExecutionId,
// Flow Debugger (design §5.2): the trace flag rides beside the parent id.
request.Trace);
}
/// <summary>
/// Handles attribute value changes — triggers script if configured for
/// value-change, conditional, or expression. The attribute snapshot is
/// updated for every change before any trigger logic runs.
/// </summary>
private void HandleAttributeValueChanged(AttributeValueChanged changed)
{
// Keep the snapshot current for every change, regardless of trigger type.
_attributeSnapshot[changed.AttributeName] = changed.Value;
if (_triggerConfig is ValueChangeTriggerConfig valueTrigger)
{
if (valueTrigger.AttributeName == changed.AttributeName)
{
// Flow Debugger (design §5.2/§5.3): a change stamped by a traced write parents
// the spawned run to the writing execution and carries the trace flag; every
// other value-change run stays a root by design (CLAUDE.md, alarm-leg note).
if (changed.Trace && changed.SourceExecutionId is { } sourceExecutionId)
{
TrySpawnExecution(null, sourceExecutionId, trace: true, new ScriptTraceExtra
{
TriggerKind = "valuechange",
TriggerAttribute = changed.AttributeName,
TriggerValue = AttributeValueCodec.Encode(changed.Value),
TriggerSource = changed.SourceExecutionInferred ? "inferred-device-update" : "value-change",
});
}
else
{
TrySpawnExecution(null);
}
}
}
else if (_triggerConfig is ConditionalTriggerConfig conditional)
{
if (conditional.AttributeName == changed.AttributeName)
{
var conditionMet = EvaluateCondition(conditional, changed.Value);
if (conditional.Mode == TriggerMode.WhileTrue)
{
// Edge-detect against the prior truth value; the timer does
// the repeated firing while the condition stays true.
HandleWhileTrueTransition(conditionMet, _conditionState);
_conditionState = conditionMet;
}
else if (conditionMet)
{
// OnTrue: fire on each matching change (existing behavior).
TrySpawnExecution(null);
}
}
}
else if (_triggerConfig is ExpressionTriggerConfig)
{
StartExpressionEvaluation();
}
}
/// <summary>
/// Review 2026-10-21 01/SR-U1 (user decision M1): adopts a value the Instance Actor seeded at
/// start (a persisted static override loaded after this actor was created with the deploy
/// default) WITHOUT launching a run. The snapshot and the trigger's edge state are brought up
/// to date — a Conditional's truth value, an Expression's last result (evaluated, never fired) —
/// so the NEXT real change is judged against the real value: no spurious edge from the replay,
/// and no missed edge afterwards. A WhileTrue trigger whose condition already holds is seeded
/// true without starting its re-fire timer; it fires on its next false→true edge.
/// </summary>
private void HandleAttributeValueSeed(Messages.AttributeValueSeed seed)
{
var changed = seed.Change;
_attributeSnapshot[changed.AttributeName] = changed.Value;
if (_triggerConfig is ConditionalTriggerConfig conditional &&
conditional.AttributeName == changed.AttributeName)
{
_conditionState = EvaluateCondition(conditional, changed.Value);
}
else if (_triggerConfig is ExpressionTriggerConfig)
{
StartExpressionEvaluation(seedOnly: true);
}
}
/// <summary>
/// Starts an off-dispatcher evaluation of the compiled trigger expression (P1).
/// The expression previously ran synchronously on the actor's dispatcher thread
/// via <c>RunAsync(...).GetAwaiter().GetResult()</c>, blocking the dispatcher on
/// every attribute change. It runs off-thread against a point-in-time snapshot;
/// the boolean result is piped back to this actor as an
/// <see cref="ExpressionEvalResult"/> so all edge state is applied on the actor
/// thread. Bursts coalesce: at most one evaluation is in flight, at most one
/// pending, so a storm of changes collapses to the latest snapshot without
/// unbounded task fan-out.
///
/// <para>Trigger evaluation runs off the script pool and its clock starts at enqueue. It used
/// to run on the dedicated <see cref="ScriptExecutionScheduler"/> — the same bounded
/// pool as blocking script bodies — so N blocked scripts stalled every Expression
/// trigger on the node indefinitely. It now runs as plain async work on the shared
/// .NET thread pool behind <see cref="TriggerEvalGate"/>: trigger expressions are
/// non-blocking by construction, so they belong there. And the deadline
/// <see cref="CancellationTokenSource"/> is armed HERE, on the actor thread, before
/// the work is queued — so gate-wait time burns the same budget and a saturated gate
/// yields a timely false instead of an unbounded stall.</para>
/// </summary>
/// <param name="seedOnly">
/// 01/SR-U1: the result only seeds <see cref="_lastExpressionResult"/> and never fires. A
/// coalesced pending evaluation stays seed-only unless a real change also asked for it.
/// </param>
private void StartExpressionEvaluation(bool seedOnly = false)
{
if (_compiledTriggerExpression == null || _triggerConfig is not ExpressionTriggerConfig) return;
if (_evalInFlight)
{
// coalesce bursts: one in flight, one pending (seed-only only if every request was)
_evalPendingSeedOnly = _evalPending ? _evalPendingSeedOnly && seedOnly : seedOnly;
_evalPending = true;
return;
}
_evalInFlight = true;
var snapshot = new Dictionary<string, object?>(_attributeSnapshot); // point-in-time copy, actor thread
var expression = _compiledTriggerExpression;
var self = Self;
var gate = _evalGate ?? TriggerEvalGate.Shared(_options);
// Clock starts AT ENQUEUE, not at dequeue.
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_options.TriggerEvalTimeoutSeconds));
Task.Run(async () =>
{
try
{
await gate.WaitAsync(cts.Token).ConfigureAwait(false);
try
{
// The CancellationToken covers cooperative/async cases; a pathological
// CPU-bound expression is not fully interruptible — acceptable because
// trigger expressions are authored by trusted Design-role users and
// compile-checked pre-deploy.
var state = await expression.RunAsync(
new TriggerExpressionGlobals(snapshot), cancellationToken: cts.Token);
return state.ReturnValue is bool b && b;
}
finally
{
gate.Release();
}
}
catch (Exception ex)
{
// OperationCanceledException (timeout, INCLUDING a timeout that fired while
// still waiting on the gate) falls through here too and is treated as false.
// LogExpressionError touches only thread-safe members (_healthCollector
// Interlocked, _logger, DI-resolved singleton logger) so it is safe off the
// actor thread.
LogExpressionError(ex);
return false;
}
finally
{
cts.Dispose();
}
}).PipeTo(self,
success: r => new ExpressionEvalResult(r, seedOnly),
failure: ex => new ExpressionEvalFailed(ex, seedOnly));
}
/// <summary>
/// Applies an off-dispatcher trigger-expression result on the actor thread:
/// runs the OnTrue / WhileTrue edge logic against <see cref="_lastExpressionResult"/>,
/// clears the in-flight flag, and drains a coalesced pending evaluation (against
/// the now-current snapshot) if one was requested while this one ran.
/// </summary>
private void HandleExpressionEvalResult(ExpressionEvalResult msg)
{
_evalInFlight = false;
if (_triggerConfig is ExpressionTriggerConfig exprConfig)
{
if (msg.SeedOnly)
{
// 01/SR-U1: a seed adopts the truth value and never fires.
}
else if (exprConfig.Mode == TriggerMode.WhileTrue)
HandleWhileTrueTransition(msg.Result, _lastExpressionResult);
else if (msg.Result && !_lastExpressionResult)
TrySpawnExecution(null);
_lastExpressionResult = msg.Result;
}
if (_evalPending)
{
_evalPending = false;
StartExpressionEvaluation(_evalPendingSeedOnly);
}
}
/// <summary>
/// Applies a faulted off-dispatcher evaluation task (N2): logs the fault via the
/// existing error path (which also increments the health counter), then reuses the
/// false-result path — clearing in-flight, applying the false edge, and draining any
/// pending evaluation — so a transient scheduler fault cannot permanently park the trigger.
/// A faulted SEED evaluation (review 2026-10-21 follow-up L4) carries no truth value at all:
/// it only clears in-flight and drains the pending evaluation, adopting nothing — replaying it
/// as a real false stopped a running WhileTrue timer, and even as a seeded false it would make
/// the next true evaluation look like a fresh edge and fire spuriously.
/// </summary>
private void HandleExpressionEvalFailed(ExpressionEvalFailed msg)
{
LogExpressionError(msg.Cause);
if (!msg.SeedOnly)
{
HandleExpressionEvalResult(new ExpressionEvalResult(false));
return;
}
_evalInFlight = false;
if (_evalPending)
{
_evalPending = false;
StartExpressionEvaluation(_evalPendingSeedOnly);
}
}
/// <summary>
/// Applies a WhileTrue trigger's condition-state transition: on the
/// false→true edge, fire once and start the re-fire timer; on the
/// true→false edge, stop the timer. While the state is unchanged, the
/// already-running timer continues to drive re-firing.
/// </summary>
private void HandleWhileTrueTransition(bool nowTrue, bool wasTrue)
{
if (nowTrue && !wasTrue)
{
TrySpawnExecution(null);
StartWhileTrueTimer();
}
else if (!nowTrue && wasTrue)
{
StopWhileTrueTimer();
}
}
/// <summary>
/// Starts the periodic WhileTrue re-fire timer. The cadence is the script's
/// <c>MinTimeBetweenRuns</c>; with none configured the trigger cannot
/// re-fire, so it degrades to the single edge fire and logs a warning.
/// </summary>
private void StartWhileTrueTimer()
{
if (_compiledScript == null) return;
if (_minTimeBetweenRuns is not { } interval)
{
_logger.LogWarning(
"ScriptActor {Script} on {Instance}: WhileTrue trigger has no MinTimeBetweenRuns — " +
"firing once on the edge only, no re-fire timer.",
_scriptName, _instanceName);
return;
}
Timers.StartPeriodicTimer(WhileTrueTimerKey, WhileTrueTick.Instance, interval, interval);
}
/// <summary>Cancels the WhileTrue re-fire timer (a no-op if it is not running).</summary>
private void StopWhileTrueTimer() => Timers.Cancel(WhileTrueTimerKey);
/// <summary>
/// Fires the script for a WhileTrue re-fire tick. The timer interval is
/// itself the cadence, so this spawns directly — bypassing the
/// MinTimeBetweenRuns skip-check that gates change-driven spawns (which
/// could otherwise drop a tick to sub-millisecond timing jitter).
/// </summary>
private void FireWhileTrueTick()
{
if (_compiledScript == null) return;
_lastExecutionTime = DateTimeOffset.UtcNow;
SpawnExecution(null, 0, ActorRefs.NoSender!, Guid.NewGuid().ToString());
}
/// <summary>
/// Records a trigger-expression evaluation failure to the site event log,
/// mirroring how a script run reports its own errors.
/// </summary>
private void LogExpressionError(Exception ex)
{
_healthCollector?.IncrementScriptError();
var errorMsg = $"Trigger expression for script '{_scriptName}' on instance '{_instanceName}' failed: {ex.Message}";
_logger.LogError(ex, "Trigger expression evaluation failed: {Script} on {Instance}", _scriptName, _instanceName);
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"script", "Error", _instanceName, $"ScriptActor:{_scriptName}", errorMsg, ex.ToString());
}
/// <summary>
/// Attempts to spawn a script execution, respecting MinTimeBetweenRuns.
/// </summary>
private void TrySpawnExecution(
IReadOnlyDictionary<string, object?>? parameters,
Guid? parentExecutionId = null,
bool trace = false,
ScriptTraceExtra? triggerExtra = null)
{
if (_compiledScript == null) return;
if (_minTimeBetweenRuns.HasValue)
{
var elapsed = DateTimeOffset.UtcNow - _lastExecutionTime;
if (elapsed < _minTimeBetweenRuns.Value)
{
_logger.LogDebug(
"Script {Script} on {Instance}: skipping execution, min time between runs not elapsed ({Elapsed} < {Min})",
_scriptName, _instanceName, elapsed, _minTimeBetweenRuns.Value);
// Arch-review F7: a TRACED run refused here is otherwise invisible to the Flow
// Debugger, which then blames a missing trigger. Leave a Failed ScriptRun row
// naming the guard instead.
if (trace)
{
ScriptRunLauncher.EmitSuppressedRun(
_serviceProvider, _logger, _scriptName, _instanceName, parentExecutionId,
$"suppressed: MinTimeBetweenRuns ({_minTimeBetweenRuns.Value}) not elapsed " +
$"since the previous run ({elapsed} ago).",
triggerExtra);
}
return;
}
}
_lastExecutionTime = DateTimeOffset.UtcNow;
SpawnExecution(parameters, 0, ActorRefs.NoSender!, Guid.NewGuid().ToString(), parentExecutionId, trace, triggerExtra);
}
/// <summary>
/// Launches a run of this script. Multiple concurrent runs are allowed, up to
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>; beyond that the newest
/// trigger is shed (see <see cref="ShedRun"/>).
///
/// <para>The run is launched directly on the script-execution scheduler via
/// <see cref="ScriptRunLauncher"/> — the actor's mailbox stays on the default dispatcher,
/// but the script body runs on the bounded set of dedicated threads, so blocking script
/// I/O is contained there and cannot starve the shared .NET thread pool. No per-run child
/// actor is created.</para>
///
/// <para>The cap governs NEW work only (<paramref name="callDepth"/> 0 — trigger-driven
/// runs and top-level Ask calls). A nested <c>CallScript</c> is exempt: the calling run is
/// itself still counted in <see cref="_runsInFlight"/> while it awaits its callee (the
/// slot is released only by <see cref="ScriptExecutionCompleted"/>, sent after the body
/// returns), and a script calling ITSELF routes back to this same actor — so counting the
/// nested launch against the cap would refuse legitimate self-recursion at depth
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> with a misleading "shed",
/// making <see cref="SiteRuntimeOptions.MaxScriptCallDepth"/> — the limit that actually
/// owns this path, enforced in <c>ScriptRuntimeContext.CallScript</c> — unreachable.
/// Nested depth is bounded by MaxScriptCallDepth instead, which is what the cap would
/// otherwise be doing badly.</para>
/// </summary>
private void SpawnExecution(
IReadOnlyDictionary<string, object?>? parameters,
int callDepth,
IActorRef replyTo,
string correlationId,
Guid? parentExecutionId = null,
bool trace = false,
ScriptTraceExtra? triggerExtra = null)
{
if (callDepth == 0 && _runsInFlight >= _options.MaxConcurrentRunsPerScript)
{
ShedRun(replyTo, correlationId, parentExecutionId, trace, triggerExtra);
return;
}
var runId = _executionCounter++;
// Incremented BEFORE the launch so the launch-path catch below (which always emits a
// ScriptExecutionCompleted) balances it on every path — the counter cannot leak.
_runsInFlight++;
try
{
ScriptRunLauncher.LaunchScript(
_scriptName,
_instanceName,
_compiledScript!,
parameters,
callDepth,
_instanceActor,
_sharedScriptLibrary,
_options,
replyTo,
correlationId,
// Completion target: this actor. Identical delivery to the old
// Context.Parent.Tell from the execution actor, and it doubles as the
// in-flight release.
Self,
// Notification Outbox: the site communication actor Notify.Status queries
// central through. Resolved here, on the actor thread, and handed to the
// launcher (which has no ActorContext of its own).
Context.System.ActorSelection("/user/site-communication"),
_logger,
_scope,
runId,
_healthCollector,
_serviceProvider,
// (ParentExecutionId): null for trigger-driven runs;
// an inbound-API-routed call supplies the inbound request's id.
parentExecutionId,
// Per-script timeout override (null = use global).
_executionTimeoutSeconds,
// Scheduler seam: thread this actor's scheduler override down so
// launched script bodies share the same pool (null = process-wide shared).
_scheduler,
// Flow Debugger (design §5.2): trace flag + value-change trigger detail.
trace,
triggerExtra);
}
catch (Exception ex)
{
// Supervision parity: the only failure the removed per-run child could
// surface was a constructor throw (e.g. queueing onto a disposed scheduler),
// which the old OneForOneStrategy logged and stopped — leaving an Ask caller to
// hang with no reply. Same log shape, same "coordinator unaffected" outcome, plus
// a reply so the caller fails fast instead of waiting out its Ask timeout.
var errorMsg = $"Script '{_scriptName}' on instance '{_instanceName}' could not be launched: {ex.Message}";
_logger.LogWarning(ex,
"Script execution launch for {Script} on {Instance} failed, stopping",
_scriptName, _instanceName);
if (!replyTo.IsNobody())
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
Self.Tell(new ScriptExecutionCompleted(_scriptName, false, errorMsg));
}
}
/// <summary>
/// Shed policy: refuses the incoming run because
/// <see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/> runs are already in flight.
/// The four already queued/running are kept — they are closest to their own deadlines and
/// already charged against them — so nothing is ever reordered. A trigger-driven run is
/// simply not launched; a top-level Ask (an inbound-API route, or a <c>CallScript</c> from
/// an unrelated script's run) gets an explicit error so the caller fails fast rather than
/// hanging to its Ask timeout. Nested (<c>callDepth &gt; 0</c>) launches never reach here —
/// see <see cref="SpawnExecution"/>.
///
/// <para>Arch-review F7: a traced run shed here also leaves a Failed <c>ScriptRun</c> audit
/// row, so the Flow Debugger names the cap rather than reporting a missing trigger. The
/// trigger detail (<paramref name="triggerExtra"/>) rides along so the row still says which
/// attribute change would have fired the run.</para>
/// </summary>
private void ShedRun(
IActorRef replyTo,
string correlationId,
Guid? parentExecutionId = null,
bool trace = false,
ScriptTraceExtra? triggerExtra = null)
{
_healthCollector?.IncrementScriptRunShed();
var message = $"Script '{_scriptName}' on instance '{_instanceName}': run shed — " +
$"{_runsInFlight} runs already in flight (cap {_options.MaxConcurrentRunsPerScript}).";
// Rate-limited to one log line + one event per script per minute: the counter above
// still counts every shed, but a hot trigger against a saturated cap must not flood
// site_events or the node log (review 2026-10-21 01/SR-P6). The first shed logs at once;
// later lines carry the number of sheds suppressed since the previous one.
_shedSuppressed++;
var now = DateTimeOffset.UtcNow;
if (now - _lastShedEventUtc >= ShedEventInterval)
{
_lastShedEventUtc = now;
_logger.LogWarning("{Message} ({Shed} shed(s) since the last report)", message, _shedSuppressed);
_shedSuppressed = 0;
_ = _serviceProvider?.GetService<ISiteEventLogger>()?.LogEventAsync(
"script", "Warning", _instanceName, $"ScriptActor:{_scriptName}", message);
}
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(
correlationId, false, null,
$"shed: {_runsInFlight} runs already in flight"));
}
if (trace)
{
ScriptRunLauncher.EmitSuppressedRun(
_serviceProvider, _logger, _scriptName, _instanceName, parentExecutionId,
$"shed: MaxConcurrentRunsPerScript reached — {_runsInFlight} runs already in " +
$"flight (cap {_options.MaxConcurrentRunsPerScript}).",
triggerExtra);
}
}
private void HandleExecutionCompleted(ScriptExecutionCompleted msg)
{
// Release the in-flight slot. Every terminal path emits exactly one of these
// (success / timeout / failure / launch failure), so the counter tracks reality.
if (_runsInFlight > 0) _runsInFlight--;
_logger.LogDebug(
"Script {Script} execution completed on {Instance}: success={Success}",
_scriptName, _instanceName, msg.Success);
}
/// <summary>
/// Runs launched but not yet completed. Exposed for regression coverage of the
/// shed cap — the counter is otherwise invisible from outside the actor.
/// </summary>
internal int RunsInFlight => _runsInFlight;
// internal (not private) so the culture-invariance of the non-numeric fallback
// can be unit-tested directly on the test thread — the live path evaluates on a
// dispatcher thread whose CurrentCulture the test cannot deterministically set.
/// <summary>
/// Evaluates a conditional trigger's operator/threshold against an attribute value,
/// converting the value to a culture-invariant double before comparing.
/// </summary>
/// <param name="config">The trigger configuration specifying the operator and threshold.</param>
/// <param name="value">The attribute value to evaluate, or null.</param>
/// <returns><see langword="true"/> if the value satisfies the configured condition; otherwise <see langword="false"/>.</returns>
internal static bool EvaluateCondition(ConditionalTriggerConfig config, object? value)
{
if (value == null) return false;
try
{
// Use InvariantCulture so a string attribute value like "1.5" parses
// consistently regardless of the host locale. For
// purely-numeric inputs the culture argument is a no-op, but it is
// safe and future-proof for string-typed attribute values arriving
// from scripts or the data connection layer.
var numericValue = Convert.ToDouble(value, CultureInfo.InvariantCulture);
return config.Operator switch
{
">" => numericValue > config.Threshold,
">=" => numericValue >= config.Threshold,
"<" => numericValue < config.Threshold,
"<=" => numericValue <= config.Threshold,
"==" => Math.Abs(numericValue - config.Threshold) < 0.0001,
"!=" => Math.Abs(numericValue - config.Threshold) >= 0.0001,
_ => false
};
}
catch
{
// Render the threshold with InvariantCulture to match the invariant numeric
// parse two lines up — otherwise a de-DE host renders 1.5 as "1,5" and the
// string fallback fires spuriously on locale-formatted values (C6).
return string.Equals(value.ToString(), config.Threshold.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal);
}
}
private static ScriptTriggerConfig? ParseTriggerConfig(string? triggerType, string? triggerConfigJson)
{
if (string.IsNullOrEmpty(triggerType)) return null;
return triggerType.ToLowerInvariant() switch
{
"interval" => ParseIntervalTrigger(triggerConfigJson),
"valuechange" => ParseValueChangeTrigger(triggerConfigJson),
"conditional" => ParseConditionalTrigger(triggerConfigJson),
"expression" => ParseExpressionTrigger(triggerConfigJson),
"call" => null, // No automatic trigger — invoked only via Instance.CallScript()
_ => null
};
}
private static ExpressionTriggerConfig? ParseExpressionTrigger(string? json)
{
var expr = TriggerExpressionGlobals.ExtractExpression(json);
if (expr == null) return null;
// ExtractExpression already proved the JSON parses; read the mode too.
var mode = TriggerMode.OnTrue;
try
{
using var doc = JsonDocument.Parse(json!);
mode = ParseTriggerMode(doc.RootElement);
}
catch (JsonException) { /* keep OnTrue */ }
return new ExpressionTriggerConfig(expr, mode);
}
/// <summary>
/// Reads the optional <c>mode</c> field (Conditional + Expression triggers).
/// An absent or unrecognized value (case-insensitive) yields
/// <see cref="TriggerMode.OnTrue"/>, so pre-WhileTrue configs are unchanged.
/// </summary>
private static TriggerMode ParseTriggerMode(JsonElement root)
{
var raw = root.TryGetProperty("mode", out var m) ? m.GetString() : null;
return string.Equals(raw?.Trim(), "WhileTrue", StringComparison.OrdinalIgnoreCase)
? TriggerMode.WhileTrue
: TriggerMode.OnTrue;
}
private static IntervalTriggerConfig? ParseIntervalTrigger(string? json)
{
if (string.IsNullOrEmpty(json)) return null;
try
{
var doc = JsonDocument.Parse(json);
var ms = doc.RootElement.GetProperty("intervalMs").GetInt64();
return new IntervalTriggerConfig(TimeSpan.FromMilliseconds(ms));
}
catch { return null; }
}
private static ValueChangeTriggerConfig? ParseValueChangeTrigger(string? json)
{
// Share the monitored-attribute parser with InstanceActor's routing map (P2) so both
// agree on which attribute this trigger reacts to.
return TriggerRouting.TryReadAttributeName(json, out var attr)
? new ValueChangeTriggerConfig(attr)
: null;
}
private static ConditionalTriggerConfig? ParseConditionalTrigger(string? json)
{
if (string.IsNullOrEmpty(json)) return null;
// Attribute name comes from the shared parser (P2); operator/threshold/mode stay local.
if (!TriggerRouting.TryReadAttributeName(json, out var attr)) return null;
try
{
var doc = JsonDocument.Parse(json);
var op = doc.RootElement.GetProperty("operator").GetString()!;
var threshold = doc.RootElement.GetProperty("threshold").GetDouble();
return new ConditionalTriggerConfig(
attr, op, threshold, ParseTriggerMode(doc.RootElement));
}
catch { return null; }
}
// ── Internal messages ──
internal sealed class IntervalTick
{
public static readonly IntervalTick Instance = new();
private IntervalTick() { }
}
internal sealed class WhileTrueTick
{
public static readonly WhileTrueTick Instance = new();
private WhileTrueTick() { }
}
internal record ScriptExecutionCompleted(string ScriptName, bool Success, string? Error);
/// <summary>
/// Piped back to self from an off-dispatcher trigger-expression evaluation (P1);
/// carries the boolean truth value so the OnTrue/WhileTrue edge logic runs on
/// the actor thread.
/// </summary>
private sealed record ExpressionEvalResult(bool Result, bool SeedOnly = false);
/// <summary>
/// Piped back to self when the off-dispatcher evaluation TASK itself faults
/// (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler
/// during shutdown) — the inner body's catch never sees that. Without this
/// mapping the actor receives an unhandled Status.Failure and _evalInFlight
/// stays true forever, permanently parking the expression trigger (N2).
/// </summary>
/// <param name="Cause">The task fault.</param>
/// <param name="SeedOnly">
/// Whether the faulted evaluation was a seed (01/SR-U1); a faulted seed adopts nothing
/// (review 2026-10-21 follow-up L4).
/// </param>
internal sealed record ExpressionEvalFailed(Exception Cause, bool SeedOnly = false);
}
// ── Trigger config types ──
/// <summary>
/// When a Conditional/Expression trigger fires. <see cref="OnTrue"/> fires once
/// as the condition becomes true; <see cref="WhileTrue"/> additionally re-fires
/// on a timer (cadence = the script's MinTimeBetweenRuns) until it goes false.
/// </summary>
internal enum TriggerMode { OnTrue, WhileTrue }
internal record IntervalTriggerConfig(TimeSpan Interval) : ScriptTriggerConfig;
internal record ValueChangeTriggerConfig(string AttributeName) : ScriptTriggerConfig;
internal record ConditionalTriggerConfig(string AttributeName, string Operator, double Threshold, TriggerMode Mode) : ScriptTriggerConfig;
internal record ExpressionTriggerConfig(string Expression, TriggerMode Mode) : ScriptTriggerConfig;
internal abstract record ScriptTriggerConfig;