Joseph Doherty
807 lines
43 KiB
C#
807 lines
43 KiB
C#
using System.Diagnostics;
|
|
using Akka.Actor;
|
|
using Microsoft.CodeAnalysis.Scripting;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
|
using System.Text.Json;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
|
|
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
|
|
|
/// <summary>
|
|
/// WP3.1: launches one script or alarm on-trigger run directly from its owning coordinator
|
|
/// (<see cref="ScriptActor"/> / <see cref="AlarmActor"/>), replacing the short-lived
|
|
/// <c>ScriptExecutionActor</c> and <c>AlarmExecutionActor</c>.
|
|
///
|
|
/// <para>Those actors were already inert shells: neither declared a single <c>Receive</c>
|
|
/// handler (they executed from their constructor), neither had a <c>PostStop</c>, state, or
|
|
/// stash, and their <c>IActorRef</c> was never a message target — the whole lifecycle lived
|
|
/// inside a detached <see cref="Task"/> the actor never observed. What they cost was a real
|
|
/// actor cell, mailbox, and name registration per run, plus a per-spawn expression-tree
|
|
/// <c>Props.Create</c>. Removing them changes no semantics; the run body below is the former
|
|
/// <c>ExecuteScript</c>/<c>ExecuteAlarmScript</c> body, unified.</para>
|
|
///
|
|
/// <para>Two behaviours DID change, both deliberately:</para>
|
|
/// <list type="number">
|
|
/// <item>The deadline <see cref="CancellationTokenSource"/> is now armed by the CALLER,
|
|
/// before the body is queued to the <see cref="ScriptExecutionScheduler"/>, so queue wait
|
|
/// consumes the script's own budget. A body that dequeues past its deadline skips execution
|
|
/// entirely and takes the timeout path — a saturated pool sheds stale work instead of
|
|
/// running it late.</item>
|
|
/// <item>The stuck-script watchdog now DETACHES and replaces the worker thread a wedged
|
|
/// script is holding (<see cref="ScriptExecutionScheduler.TryDetachWorker"/>) instead of
|
|
/// only naming it, so the pool recovers its capacity.</item>
|
|
/// </list>
|
|
///
|
|
/// <para>Ordering, supervision, telemetry, DI scoping, Ask replies, completion messages, and
|
|
/// the audit <c>ExecutionId</c>/<c>ParentExecutionId</c> threading are all unchanged. The
|
|
/// launch call itself can throw only if the target scheduler is unusable (e.g. disposed); the
|
|
/// caller wraps it so that failure replies to the Ask caller and decrements the in-flight
|
|
/// counter rather than escalating to the coordinator's supervisor.</para>
|
|
/// </summary>
|
|
internal static class ScriptRunLauncher
|
|
{
|
|
/// <summary>
|
|
/// Launches an instance script run. Returns as soon as the body is queued; the run reports
|
|
/// completion to <paramref name="completionTarget"/>.
|
|
/// </summary>
|
|
/// <param name="scriptName">Name of the script being executed.</param>
|
|
/// <param name="instanceName">Name of the instance that owns the script.</param>
|
|
/// <param name="compiledScript">Compiled Roslyn script to execute.</param>
|
|
/// <param name="parameters">Optional named parameter values for the script.</param>
|
|
/// <param name="callDepth">Current call-nesting depth (used to enforce the max-depth limit).</param>
|
|
/// <param name="instanceActor">Instance actor reference for attribute access.</param>
|
|
/// <param name="sharedScriptLibrary">Library of shared scripts available during execution.</param>
|
|
/// <param name="options">Site runtime options applied during execution.</param>
|
|
/// <param name="replyTo">Actor reference that receives the script result; <c>Nobody</c> for fire-and-forget.</param>
|
|
/// <param name="correlationId">Application-level correlation id threaded through the execution.</param>
|
|
/// <param name="completionTarget">The owning <see cref="ScriptActor"/>, which receives the completion message.</param>
|
|
/// <param name="siteCommunicationActor">Site communication actor (resolved on the actor thread) for <c>Notify.Status</c>.</param>
|
|
/// <param name="logger">Logger for script execution events.</param>
|
|
/// <param name="scope">Script scope controlling which APIs are available.</param>
|
|
/// <param name="runId">Per-script run counter, used only in log messages (replaces the former per-run actor name).</param>
|
|
/// <param name="healthCollector">Optional health collector for recording execution metrics.</param>
|
|
/// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
|
|
/// <param name="parentExecutionId">ExecutionId of the spawning execution for audit correlation; null for root runs.</param>
|
|
/// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global value.</param>
|
|
/// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
|
|
/// <param name="trace">Flow Debugger trace flag (design 2026-09-05 §5.2): when true the run emits <c>ScriptTrace</c> rows and returns them inline on the <see cref="ScriptCallResult"/>.</param>
|
|
/// <param name="triggerExtra">Flow Debugger: trigger detail for a traced value-change run (stamped into the <c>ScriptRun</c> row's <c>Extra</c>); null for calls.</param>
|
|
public static void LaunchScript(
|
|
string scriptName,
|
|
string instanceName,
|
|
Script<object?> compiledScript,
|
|
IReadOnlyDictionary<string, object?>? parameters,
|
|
int callDepth,
|
|
IActorRef instanceActor,
|
|
SharedScriptLibrary sharedScriptLibrary,
|
|
SiteRuntimeOptions options,
|
|
IActorRef replyTo,
|
|
string correlationId,
|
|
IActorRef completionTarget,
|
|
ICanTell? siteCommunicationActor,
|
|
ILogger logger,
|
|
ScriptScope scope,
|
|
long runId,
|
|
ISiteHealthCollector? healthCollector,
|
|
IServiceProvider? serviceProvider,
|
|
Guid? parentExecutionId,
|
|
int? executionTimeoutSeconds,
|
|
ScriptExecutionScheduler? scheduler,
|
|
bool trace = false,
|
|
ScriptTraceExtra? triggerExtra = null)
|
|
{
|
|
var timeout = ResolveTimeout(executionTimeoutSeconds, options);
|
|
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
|
|
|
|
// Armed HERE, on the caller's (actor) thread, not inside the queued body: queue wait
|
|
// must consume the script's own budget, otherwise a saturated pool silently grants
|
|
// every backlogged run a fresh full timeout.
|
|
var cts = new CancellationTokenSource(timeout);
|
|
try
|
|
{
|
|
_ = Task.Factory.StartNew(
|
|
() => RunScriptAsync(
|
|
scriptName, instanceName, compiledScript, parameters, callDepth,
|
|
instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
|
|
completionTarget, siteCommunicationActor, logger, scope, runId,
|
|
healthCollector, serviceProvider, parentExecutionId, timeout,
|
|
executionScheduler, cts, trace, triggerExtra),
|
|
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
|
|
.Unwrap();
|
|
}
|
|
catch
|
|
{
|
|
// The body never got queued, so nothing will ever dispose the CTS.
|
|
cts.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Launches an alarm on-trigger run. Same contract as <see cref="LaunchScript"/>, with the
|
|
/// firing alarm's level/priority/message exposed to the body through the <c>Alarm</c> global.
|
|
/// </summary>
|
|
/// <param name="alarmName">The canonical name of the alarm that triggered.</param>
|
|
/// <param name="instanceName">The name of the owning instance.</param>
|
|
/// <param name="level">The alarm severity level at the time of triggering.</param>
|
|
/// <param name="priority">The alarm priority value.</param>
|
|
/// <param name="message">The alarm message to pass to the script.</param>
|
|
/// <param name="compiledScript">The pre-compiled on-trigger script to execute.</param>
|
|
/// <param name="instanceActor">Reference to the instance actor for attribute/script calls.</param>
|
|
/// <param name="sharedScriptLibrary">Shared script library providing common utilities.</param>
|
|
/// <param name="options">Site runtime configuration options, including the execution timeout.</param>
|
|
/// <param name="completionTarget">The owning <see cref="AlarmActor"/>, which receives the completion message.</param>
|
|
/// <param name="logger">Logger for execution diagnostics.</param>
|
|
/// <param name="healthCollector">
|
|
/// Optional health collector for recording execution metrics (arch-review 2026-08-19
|
|
/// P1-3): the alarm on-trigger run shares the script-error counter with
|
|
/// <see cref="RunScriptAsync"/> so a wedged/failed alarm body is visible the same way a
|
|
/// wedged/failed instance script is.
|
|
/// </param>
|
|
/// <param name="serviceProvider">
|
|
/// Optional DI service provider (arch-review 2026-08-19 P1-3): resolves the singleton
|
|
/// <see cref="ISiteEventLogger"/> (stuck-watchdog reporting) and
|
|
/// <see cref="ScriptRunSummaryRecorder"/> (aggregate run counters), exactly like the
|
|
/// instance-script path.
|
|
/// </param>
|
|
/// <param name="runId">Per-alarm run counter, used only in log messages.</param>
|
|
/// <param name="executionTimeoutSeconds">The on-trigger script's per-script timeout in seconds. Null or non-positive falls back to the global value.</param>
|
|
/// <param name="parentExecutionId">
|
|
/// ParentExecutionId tag-cascade: the <c>ExecutionId</c> of the execution whose attribute
|
|
/// write fired this alarm. Null when the firing value came from the Data Connection Layer
|
|
/// (external data has no spawning execution) — that on-trigger run is a tree ROOT.
|
|
/// </param>
|
|
/// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
|
|
public static void LaunchAlarmScript(
|
|
string alarmName,
|
|
string instanceName,
|
|
AlarmLevel level,
|
|
int priority,
|
|
string message,
|
|
Script<object?> compiledScript,
|
|
IActorRef instanceActor,
|
|
SharedScriptLibrary sharedScriptLibrary,
|
|
SiteRuntimeOptions options,
|
|
IActorRef completionTarget,
|
|
ILogger logger,
|
|
ISiteHealthCollector? healthCollector,
|
|
IServiceProvider? serviceProvider,
|
|
long runId,
|
|
int? executionTimeoutSeconds,
|
|
Guid? parentExecutionId,
|
|
ScriptExecutionScheduler? scheduler)
|
|
{
|
|
var timeout = ResolveTimeout(executionTimeoutSeconds, options);
|
|
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
|
|
|
|
// Enqueue-anchored deadline; see LaunchScript.
|
|
var cts = new CancellationTokenSource(timeout);
|
|
try
|
|
{
|
|
_ = Task.Factory.StartNew(
|
|
() => RunAlarmScriptAsync(
|
|
alarmName, instanceName, level, priority, message, compiledScript,
|
|
instanceActor, sharedScriptLibrary, options, completionTarget, logger,
|
|
healthCollector, serviceProvider, runId, parentExecutionId, timeout,
|
|
executionScheduler, cts),
|
|
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
|
|
.Unwrap();
|
|
}
|
|
catch
|
|
{
|
|
cts.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Per-script timeout overrides the global default. A null or non-positive per-script
|
|
/// value (≤ 0) falls back to the global.
|
|
/// </summary>
|
|
private static TimeSpan ResolveTimeout(int? executionTimeoutSeconds, SiteRuntimeOptions options)
|
|
=> TimeSpan.FromSeconds(
|
|
executionTimeoutSeconds is { } perScript && perScript > 0
|
|
? perScript
|
|
: options.ScriptExecutionTimeoutSeconds);
|
|
|
|
private static async Task RunScriptAsync(
|
|
string scriptName,
|
|
string instanceName,
|
|
Script<object?> compiledScript,
|
|
IReadOnlyDictionary<string, object?>? parameters,
|
|
int callDepth,
|
|
IActorRef instanceActor,
|
|
SharedScriptLibrary sharedScriptLibrary,
|
|
SiteRuntimeOptions options,
|
|
IActorRef replyTo,
|
|
string correlationId,
|
|
IActorRef completionTarget,
|
|
ICanTell? siteCommunicationActor,
|
|
ILogger logger,
|
|
ScriptScope scope,
|
|
long runId,
|
|
ISiteHealthCollector? healthCollector,
|
|
IServiceProvider? serviceProvider,
|
|
Guid? parentExecutionId,
|
|
TimeSpan timeout,
|
|
ScriptExecutionScheduler executionScheduler,
|
|
CancellationTokenSource cts,
|
|
bool trace = false,
|
|
ScriptTraceExtra? triggerExtra = null)
|
|
{
|
|
IServiceScope? serviceScope = null;
|
|
// Flow Debugger (design §5.4): the inline sink exists only for a traced run that has a
|
|
// caller to reply to — a trace-armed trigger run (replyTo == Nobody) writes every row to the
|
|
// audit writer for PullTraceEvents and would only keep a never-read list alive. The
|
|
// ScriptRun row itself is emitted below on every exit path of a traced run.
|
|
// Arch-review F1: the inline copy is redacted at the collector, so the collector carries
|
|
// the host's IAuditRedactor. That is a SINGLETON (AddAuditLog), resolved here from the
|
|
// ROOT provider for the same reason siteEventLogger below is — the traced ScriptRun row
|
|
// is emitted from the catch blocks too, including the shed-at-dequeue path that throws
|
|
// before the DI scope exists. Null in tests / hosts without AddAuditLog: raw, still capped.
|
|
var collector = trace && !replyTo.IsNobody()
|
|
? new FlowTraceCollector(
|
|
redactor: serviceProvider?.GetService<ZB.MOM.WW.Audit.IAuditRedactor>())
|
|
: null;
|
|
var startedUtc = DateTime.UtcNow;
|
|
ScriptRuntimeContext? context = null;
|
|
IAuditWriter? auditWriter = null;
|
|
var siteId = string.Empty;
|
|
string? sourceNode = null;
|
|
// ISiteEventLogger is a singleton; resolve from the root provider so
|
|
// it is available to the catch blocks regardless of scope state.
|
|
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
|
|
|
|
// WP3.2 (site_events volume policy): both are singletons, resolved from the root
|
|
// provider up front so they too are available to the catch blocks. optionsMonitor is
|
|
// null in tests that pass no serviceProvider (SingleServiceProvider et al.) — the
|
|
// constructor-passed `options` is the fallback, exactly like siteEventLogger's null
|
|
// path degrades to "no site event log". Reading CurrentValue HERE (once per run, at
|
|
// the top) rather than re-reading it at each call site is what "hot-togglable" means
|
|
// in practice: an operator's live edit to PerRunScriptEvents/PerRunScriptEventScripts
|
|
// takes effect on the NEXT run, without a restart.
|
|
var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
|
|
var optionsMonitor = serviceProvider?.GetService<IOptionsMonitor<SiteRuntimeOptions>>();
|
|
var effectiveOptions = optionsMonitor?.CurrentValue ?? options;
|
|
var emitPerRunEvents = ScriptRunEventPolicy.ShouldEmitPerRun(effectiveOptions, instanceName, scriptName);
|
|
var runStopwatch = Stopwatch.StartNew();
|
|
|
|
// WP3.1: identify the worker this body is holding, so the watchdog can detach and
|
|
// replace it if the body wedges. Read in the FIRST synchronous segment — a script
|
|
// that later hops threads on an await no longer holds this worker, and the run-stamp
|
|
// guard in TryDetachWorker will (correctly) refuse to detach it.
|
|
var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
|
|
var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;
|
|
|
|
var completed = 0;
|
|
ArmStuckScriptWatchdog(
|
|
cts, options, logger, siteEventLogger, executionScheduler,
|
|
workerSlot, workerRunStamp, timeout,
|
|
() => Volatile.Read(ref completed) != 0,
|
|
$"Script '{scriptName}' on instance '{instanceName}'",
|
|
instanceName, $"ScriptActor:{scriptName}");
|
|
|
|
try
|
|
{
|
|
// WP3.1 shed-at-dequeue: the deadline was armed at ENQUEUE, so an already-cancelled
|
|
// token here means this run spent its entire budget queueing. Running it now would
|
|
// burn a scarce script thread on work that is already stale — throw straight into
|
|
// the existing timeout path instead, BEFORE the "started" event and before the body.
|
|
cts.Token.ThrowIfCancellationRequested();
|
|
|
|
// Resolve integration services from DI (scoped lifetime)
|
|
IExternalSystemClient? externalSystemClient = null;
|
|
IDatabaseGateway? databaseGateway = null;
|
|
// Notification Outbox: the S&F engine is a singleton; the site identity
|
|
// provider supplies the site id stamped on enqueued notifications.
|
|
StoreAndForwardService? storeAndForward = null;
|
|
// The writer is a singleton (FallbackAuditWriter
|
|
// composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
|
|
// that haven't called AddAuditLog, which the helper handles as a no-op.
|
|
// (siteId / auditWriter / sourceNode are declared above the try so the
|
|
// traced ScriptRun emission in the catch blocks can reach them.)
|
|
// Site-local tracking store
|
|
// backing Tracking.Status(id). Singleton; null in tests / hosts
|
|
// that haven't wired the store, which the helper handles by
|
|
// throwing on access.
|
|
IOperationTrackingStore? operationTrackingStore = null;
|
|
// Site-side cached-call
|
|
// telemetry forwarder. Singleton bound to the AuditLog
|
|
// composition root; null in tests / hosts that haven't called
|
|
// AddAuditLog, in which case the cached-call helpers degrade
|
|
// to the no-emission path (the underlying S&F handoff still
|
|
// happens and a TrackedOperationId is still returned).
|
|
ICachedCallTelemetryForwarder? cachedForwarder = null;
|
|
// SourceNode-stamping: the local node name
|
|
// resolved from INodeIdentityProvider — node-a/node-b on site
|
|
// hosts. Null in tests / hosts that haven't registered the
|
|
// provider, in which case NotificationSubmit.SourceNode and
|
|
// SiteCallOperational.SourceNode stay null and central
|
|
// persists the rows with SourceNode NULL.
|
|
|
|
if (serviceProvider != null)
|
|
{
|
|
serviceScope = serviceProvider.CreateScope();
|
|
externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
|
|
databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
|
|
storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
|
|
siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
|
|
?? string.Empty;
|
|
auditWriter = serviceScope.ServiceProvider.GetService<IAuditWriter>();
|
|
operationTrackingStore = serviceScope.ServiceProvider.GetService<IOperationTrackingStore>();
|
|
cachedForwarder = serviceScope.ServiceProvider.GetService<ICachedCallTelemetryForwarder>();
|
|
sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
|
|
}
|
|
|
|
context = new ScriptRuntimeContext(
|
|
instanceActor,
|
|
sharedScriptLibrary,
|
|
callDepth,
|
|
options.MaxScriptCallDepth,
|
|
timeout,
|
|
instanceName,
|
|
logger,
|
|
externalSystemClient,
|
|
databaseGateway,
|
|
storeAndForward,
|
|
siteCommunicationActor,
|
|
siteId,
|
|
// Notification Outbox (FU3): stamp the executing script onto outbound
|
|
// notifications using the Site Event Logging "Source" convention.
|
|
sourceScript: $"ScriptActor:{scriptName}",
|
|
// Emit one ApiOutbound/ApiCall row per
|
|
// ExternalSystem.Call. Writer is best-effort; failures are logged
|
|
// and swallowed inside the helper so the script's call path is
|
|
// never aborted by an audit failure.
|
|
auditWriter: auditWriter,
|
|
// Site-local tracking store
|
|
// backing Tracking.Status(id). Authoritative source of truth for
|
|
// cached-call status — read directly by the script API.
|
|
operationTrackingStore: operationTrackingStore,
|
|
// Cached-call telemetry
|
|
// forwarder for ExternalSystem.CachedCall / Database.CachedWrite
|
|
// CachedSubmit emission + the immediate-success terminal-row
|
|
// emission. Best-effort: null degrades the helpers to a
|
|
// no-emission path; the S&F handoff and TrackedOperationId
|
|
// return are unaffected.
|
|
cachedForwarder: cachedForwarder,
|
|
// The spawning execution's
|
|
// id for an inbound-API-routed call. The routed script still
|
|
// mints its own fresh ExecutionId — this records the spawner.
|
|
// Null for normal (tag-change / timer) runs.
|
|
parentExecutionId: parentExecutionId,
|
|
// SourceNode-stamping: the local node name
|
|
// (node-a/node-b on a site) — threaded down so Notify.Send
|
|
// and the four cached-call telemetry constructors can stamp
|
|
// it onto NotificationSubmit.SourceNode and
|
|
// SiteCallOperational.SourceNode respectively.
|
|
sourceNode: sourceNode,
|
|
// Thread the singleton site event logger so
|
|
// recursion-limit violations at CallScript/CallShared emit a
|
|
// script Error site event in addition to ILogger.LogError.
|
|
siteEventLogger: siteEventLogger,
|
|
// WaitForAttribute (spec §4.3/§4.4): thread the per-script
|
|
// execution-timeout token so Attributes.WaitAsync's Ask is
|
|
// bounded by the script's own ExecutionTimeoutSeconds — a
|
|
// shorter script deadline wins over the wait's own timeout.
|
|
scriptTimeoutToken: cts.Token,
|
|
// Flow Debugger (design §5.2/§5.4): the trace flag and the inline sink.
|
|
isTraced: trace,
|
|
traceCollector: collector);
|
|
|
|
var globals = new ScriptGlobals
|
|
{
|
|
Instance = context,
|
|
Parameters = new ScriptParameters(parameters ?? new Dictionary<string, object?>()),
|
|
CancellationToken = cts.Token,
|
|
Scope = scope
|
|
};
|
|
|
|
// WP3.2: the aggregate ALWAYS records this run (that's the point of the interval
|
|
// summary), but the per-run Info event itself is gated by ScriptRunEventPolicy —
|
|
// off by default, on globally via PerRunScriptEvents, or on for named scripts via
|
|
// PerRunScriptEventScripts. Fire-and-forget (the `_ =` discards the task) so the
|
|
// event log can never block or fault the script's own run; mirrors the existing
|
|
// Error-path emit, which stays unconditional regardless of this policy.
|
|
summaryRecorder?.RecordStarted(instanceName, scriptName);
|
|
if (emitPerRunEvents)
|
|
{
|
|
_ = siteEventLogger?.LogEventAsync(
|
|
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
|
$"Script '{scriptName}' on instance '{instanceName}' started");
|
|
}
|
|
|
|
var state = await compiledScript.RunAsync(globals, cts.Token);
|
|
|
|
// Flow Debugger: the ScriptRun row goes into the collector BEFORE the reply so the
|
|
// inline list the caller receives includes the run itself. Complete() (not Snapshot)
|
|
// closes the sink at the same instant — outbound rows finishing after this point
|
|
// (plan 2026-09-06 D5) go to the audit writer only.
|
|
EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
|
|
context.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
|
|
parameters, state.ReturnValue, error: null, triggerExtra);
|
|
|
|
// Send result to requester if this was an Ask-based call
|
|
if (!replyTo.IsNobody())
|
|
{
|
|
replyTo.Tell(new ScriptCallResult(
|
|
correlationId, true, state.ReturnValue, null,
|
|
context.ExecutionId, collector?.Complete()));
|
|
}
|
|
|
|
// Operational `script` event — execution completed successfully. Same gating as
|
|
// the started event above.
|
|
summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
|
if (emitPerRunEvents)
|
|
{
|
|
_ = siteEventLogger?.LogEventAsync(
|
|
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
|
$"Script '{scriptName}' on instance '{instanceName}' completed");
|
|
}
|
|
|
|
// Notify the owning ScriptActor of completion (also releases its in-flight slot).
|
|
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
healthCollector?.IncrementScriptError();
|
|
summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
|
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
|
|
logger.LogWarning("{Message} (run {RunId})", errorMsg, runId);
|
|
|
|
// Failures recorded to site event log; script NOT disabled after failure. WP3.2:
|
|
// Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
|
|
_ = siteEventLogger?.LogEventAsync(
|
|
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
|
|
|
|
EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
|
|
context?.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
|
|
parameters, null, errorMsg, triggerExtra);
|
|
|
|
if (!replyTo.IsNobody())
|
|
{
|
|
replyTo.Tell(new ScriptCallResult(
|
|
correlationId, false, null, errorMsg, context?.ExecutionId, collector?.Complete()));
|
|
}
|
|
|
|
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
healthCollector?.IncrementScriptError();
|
|
summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
|
// Failures recorded to site event log; script NOT disabled after failure. WP3.2:
|
|
// Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
|
|
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
|
|
logger.LogError(ex, "Script execution failed: {Script} on {Instance} (run {RunId})",
|
|
scriptName, instanceName, runId);
|
|
|
|
_ = siteEventLogger?.LogEventAsync(
|
|
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());
|
|
|
|
EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
|
|
context?.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
|
|
parameters, null, errorMsg, triggerExtra);
|
|
|
|
if (!replyTo.IsNobody())
|
|
{
|
|
replyTo.Tell(new ScriptCallResult(
|
|
correlationId, false, null, errorMsg, context?.ExecutionId, collector?.Complete()));
|
|
}
|
|
|
|
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
|
|
}
|
|
finally
|
|
{
|
|
// Mark the body finished so the stuck-script watchdog (registered on
|
|
// cts.Token above) treats a timely cancellation as NOT stuck.
|
|
Interlocked.Exchange(ref completed, 1);
|
|
// Dispose the DI scope (and scoped services) after script execution completes
|
|
serviceScope?.Dispose();
|
|
cts.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Arch-review F7: a traced run that never launched (shed at the in-flight cap, or
|
|
/// suppressed by <c>MinTimeBetweenRuns</c>) still leaves a <c>Failed</c>
|
|
/// <c>ScriptTrace.ScriptRun</c> row, so the Flow Debugger explains the gap with the guard
|
|
/// that refused the run instead of reporting <c>NO_TRIGGER_WITHIN_WINDOW</c> — which reads
|
|
/// as "no value-change trigger on this attribute, or no device echo" and sends the operator
|
|
/// to check the wrong things.
|
|
///
|
|
/// <para>There is no collector: a shed run has no reply to ride on (an Ask caller gets the
|
|
/// shed <c>ScriptCallResult</c> from <c>ScriptActor.ShedRun</c>, not a trace list), so the
|
|
/// row goes to the audit writer only and reaches the debugger via <c>PullTraceEvents</c> /
|
|
/// Replay. The writer and identity providers come from a fresh scope because this runs on
|
|
/// the ACTOR thread, outside any launch scope. <c>elapsedMs</c> is 0 — the run never ran.
|
|
/// Best-effort and never throws: a no-op when there is no provider or no writer.</para>
|
|
/// </summary>
|
|
internal static void EmitSuppressedRun(
|
|
IServiceProvider? serviceProvider,
|
|
ILogger logger,
|
|
string scriptName,
|
|
string instanceName,
|
|
Guid? parentExecutionId,
|
|
string reason,
|
|
ScriptTraceExtra? triggerExtra)
|
|
{
|
|
if (serviceProvider is null) return;
|
|
try
|
|
{
|
|
using var scope = serviceProvider.CreateScope();
|
|
var writer = scope.ServiceProvider.GetService<IAuditWriter>();
|
|
if (writer is null) return;
|
|
var siteId = scope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId ?? string.Empty;
|
|
var sourceNode = scope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
|
|
|
|
EmitScriptRun(
|
|
trace: true, collector: null, writer, logger, scriptName, instanceName, siteId, sourceNode,
|
|
executionId: Guid.NewGuid(), parentExecutionId, DateTime.UtcNow, elapsedMs: 0,
|
|
parameters: null, returnValue: null, error: reason, triggerExtra);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogWarning(ex, "Flow trace suppressed-run emission failed for {Script}", scriptName);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Flow Debugger (design 2026-09-05 §5.1): emits the <c>ScriptTrace.ScriptRun</c> row for a
|
|
/// traced run — collector first (so the inline reply carries it), then the best-effort
|
|
/// audit writer. <c>OccurredAtUtc</c> is the run's START so the row sorts before the
|
|
/// writes/waits the run made. No-op when untraced; never throws.
|
|
/// </summary>
|
|
private static void EmitScriptRun(
|
|
bool trace, FlowTraceCollector? collector, IAuditWriter? auditWriter, ILogger logger,
|
|
string scriptName, string instanceName, string siteId, string? sourceNode,
|
|
Guid? executionId, Guid? parentExecutionId, DateTime startedUtc, long elapsedMs,
|
|
IReadOnlyDictionary<string, object?>? parameters, object? returnValue, string? error,
|
|
ScriptTraceExtra? triggerExtra)
|
|
{
|
|
if (!trace) return;
|
|
try
|
|
{
|
|
var sourceScript = $"ScriptActor:{scriptName}";
|
|
var evt = ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.ScriptRun,
|
|
error is null ? AuditStatus.Delivered : AuditStatus.Failed,
|
|
occurredAtUtc: DateTime.SpecifyKind(startedUtc, DateTimeKind.Utc),
|
|
actor: sourceScript, target: scriptName, sourceNode: sourceNode,
|
|
executionId: executionId, parentExecutionId: parentExecutionId,
|
|
sourceSiteId: string.IsNullOrEmpty(siteId) ? null : siteId,
|
|
sourceInstanceId: instanceName, sourceScript: sourceScript,
|
|
durationMs: (int)Math.Min(elapsedMs, int.MaxValue),
|
|
errorMessage: error,
|
|
requestSummary: TraceJson(parameters),
|
|
responseSummary: TraceJson(returnValue),
|
|
extra: (triggerExtra ?? new ScriptTraceExtra { TriggerKind = "call" }).ToJson());
|
|
collector?.Add(evt);
|
|
if (auditWriter is null) return;
|
|
var t = auditWriter.WriteAsync(evt, CancellationToken.None);
|
|
if (!t.IsCompletedSuccessfully)
|
|
{
|
|
_ = t.ContinueWith(
|
|
x => logger.LogWarning(x.Exception, "Flow trace ScriptRun write failed for {Script}", scriptName),
|
|
CancellationToken.None,
|
|
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
|
|
TaskScheduler.Default);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogWarning(ex, "Flow trace ScriptRun emission failed for {Script}", scriptName);
|
|
}
|
|
}
|
|
|
|
private static string? TraceJson(object? value)
|
|
{
|
|
if (value is null) return null;
|
|
try { return JsonSerializer.Serialize(value, ScriptTraceExtra.JsonOptions); }
|
|
catch (Exception) { return null; }
|
|
}
|
|
|
|
private static async Task RunAlarmScriptAsync(
|
|
string alarmName,
|
|
string instanceName,
|
|
AlarmLevel level,
|
|
int priority,
|
|
string message,
|
|
Script<object?> compiledScript,
|
|
IActorRef instanceActor,
|
|
SharedScriptLibrary sharedScriptLibrary,
|
|
SiteRuntimeOptions options,
|
|
IActorRef completionTarget,
|
|
ILogger logger,
|
|
ISiteHealthCollector? healthCollector,
|
|
IServiceProvider? serviceProvider,
|
|
long runId,
|
|
Guid? parentExecutionId,
|
|
TimeSpan timeout,
|
|
ScriptExecutionScheduler executionScheduler,
|
|
CancellationTokenSource cts)
|
|
{
|
|
// arch-review 2026-08-19 P1-3: singletons resolved from the root provider up front,
|
|
// exactly like RunScriptAsync, so they are available to the catch blocks regardless
|
|
// of scope state. Null in tests that pass no serviceProvider — degrades the same way
|
|
// the instance-script path does (no site event log / no aggregate counters).
|
|
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
|
|
var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
|
|
|
|
// The on-trigger run has no per-script name of its own in the site-event/summary
|
|
// sense — key it under the alarm, mirroring the "AlarmActor:{name}" source
|
|
// convention already used for this alarm's other site events.
|
|
var scriptName = $"AlarmActor:{alarmName}";
|
|
var runStopwatch = Stopwatch.StartNew();
|
|
|
|
// arch-review 2026-08-19 P1-3: identify the worker this body is holding, so the
|
|
// stuck-script watchdog can detach and replace it if the body wedges — same
|
|
// mechanism, same gauge, as the instance-script path. Read in the FIRST synchronous
|
|
// segment, before any await.
|
|
var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
|
|
var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;
|
|
|
|
var completed = 0;
|
|
ArmStuckScriptWatchdog(
|
|
cts, options, logger, siteEventLogger, executionScheduler,
|
|
workerSlot, workerRunStamp, timeout,
|
|
() => Volatile.Read(ref completed) != 0,
|
|
$"Alarm on-trigger script for '{alarmName}' on instance '{instanceName}'",
|
|
instanceName, scriptName);
|
|
|
|
try
|
|
{
|
|
// Enqueue-anchored deadline: an already-cancelled token means this on-trigger run
|
|
// spent its whole budget queueing behind other script bodies. Shed it rather than
|
|
// run it late — the alarm it would react to is already stale.
|
|
cts.Token.ThrowIfCancellationRequested();
|
|
|
|
// An alarm on-trigger run can call Instance.CallScript()
|
|
// via the ScriptRuntimeContext injected into globals
|
|
var context = new ScriptRuntimeContext(
|
|
instanceActor,
|
|
sharedScriptLibrary,
|
|
currentCallDepth: 0,
|
|
options.MaxScriptCallDepth,
|
|
timeout,
|
|
instanceName,
|
|
logger,
|
|
// ParentExecutionId tag-cascade: the
|
|
// alarm on-trigger run mints its own fresh ExecutionId (the
|
|
// ctor's `?? NewGuid()` fallback) and records the firing
|
|
// execution's id as its ParentExecutionId — null (a root)
|
|
// only when the firing value came from the DCL.
|
|
parentExecutionId: parentExecutionId,
|
|
// WaitForAttribute (spec §4.4): thread the alarm on-trigger
|
|
// script's per-script execution-timeout token so a
|
|
// Attributes.WaitAsync inside an on-trigger script is bounded
|
|
// by the same script deadline.
|
|
scriptTimeoutToken: cts.Token);
|
|
|
|
var globals = new ScriptGlobals
|
|
{
|
|
Instance = context,
|
|
Parameters = new ScriptParameters(),
|
|
CancellationToken = cts.Token,
|
|
Alarm = new AlarmContext
|
|
{
|
|
Name = alarmName,
|
|
Level = level,
|
|
Priority = priority,
|
|
Message = message
|
|
}
|
|
};
|
|
|
|
summaryRecorder?.RecordStarted(instanceName, scriptName);
|
|
|
|
await compiledScript.RunAsync(globals, cts.Token);
|
|
|
|
summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
|
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
healthCollector?.IncrementScriptError();
|
|
summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
|
logger.LogWarning(
|
|
"Alarm on-trigger script for {Alarm} on {Instance} timed out (run {RunId})",
|
|
alarmName, instanceName, runId);
|
|
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
healthCollector?.IncrementScriptError();
|
|
summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
|
// Failures logged, alarm continues
|
|
logger.LogError(ex,
|
|
"Alarm on-trigger script for {Alarm} on {Instance} failed (run {RunId})",
|
|
alarmName, instanceName, runId);
|
|
completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
|
|
}
|
|
finally
|
|
{
|
|
// Mark the body finished so the stuck-script watchdog (registered on
|
|
// cts.Token above) treats a timely cancellation as NOT stuck.
|
|
Interlocked.Exchange(ref completed, 1);
|
|
cts.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Arms the stuck-script watchdog (S2, extended by WP3.1).
|
|
///
|
|
/// <para>The CTS firing only REQUESTS cooperative cancellation; it does NOT free a thread
|
|
/// blocked in synchronous I/O. When the deadline elapses, this waits
|
|
/// <see cref="SiteRuntimeOptions.StuckScriptGraceMs"/> and, if the body still has not
|
|
/// returned, (a) names the script loudly on the site event log and (b) DETACHES the worker
|
|
/// thread it is holding and starts a replacement — so a wedged script costs the pool one
|
|
/// thread only until the watchdog fires, not forever. At the detach cap it logs an Error
|
|
/// instead of growing threads without bound.</para>
|
|
///
|
|
/// <para>The <c>Task.Run</c>/<c>Task.Delay</c> run on the shared thread pool, never on the
|
|
/// (possibly saturated) script scheduler — deliberate, and the whole point of the watchdog.
|
|
/// <c>Register</c> fires on cancellation only; a normally-completing run disposes its CTS
|
|
/// with no callback. A run cancelled while still QUEUED flips its completed flag in the
|
|
/// body's <c>finally</c> long before the grace elapses, so it is correctly not reported.</para>
|
|
/// </summary>
|
|
private static void ArmStuckScriptWatchdog(
|
|
CancellationTokenSource cts,
|
|
SiteRuntimeOptions options,
|
|
ILogger logger,
|
|
ISiteEventLogger? siteEventLogger,
|
|
ScriptExecutionScheduler scheduler,
|
|
int? workerSlot,
|
|
long workerRunStamp,
|
|
TimeSpan timeout,
|
|
Func<bool> isCompleted,
|
|
string subject,
|
|
string instanceName,
|
|
string source)
|
|
{
|
|
var graceMs = options.StuckScriptGraceMs;
|
|
cts.Token.Register(() => _ = Task.Run(async () =>
|
|
{
|
|
await Task.Delay(graceMs);
|
|
if (isCompleted()) return;
|
|
|
|
var stuckMsg = $"{subject} exceeded its {timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — " +
|
|
"its dedicated script-execution thread is blocked (cooperative cancellation not observed).";
|
|
|
|
var outcome = workerSlot is { } slot
|
|
? scheduler.TryDetachWorker(slot, workerRunStamp)
|
|
: WorkerDetachOutcome.NotRunning;
|
|
|
|
switch (outcome)
|
|
{
|
|
case WorkerDetachOutcome.Detached:
|
|
stuckMsg += " The worker thread has been DETACHED and replaced; it will exit when the body returns.";
|
|
break;
|
|
case WorkerDetachOutcome.AtCap:
|
|
stuckMsg += $" The script-execution pool already holds {scheduler.DetachedThreadCount} detached " +
|
|
"stuck threads (at cap); NOT replacing this one.";
|
|
break;
|
|
case WorkerDetachOutcome.NotRunning:
|
|
// The body is not on a worker thread (it hopped threads on an await), so no
|
|
// dedicated thread is lost — report it, but there is nothing to replace.
|
|
break;
|
|
}
|
|
|
|
logger.LogError(stuckMsg);
|
|
_ = siteEventLogger?.LogEventAsync("script", "Error", instanceName, source, stuckMsg);
|
|
}));
|
|
}
|
|
}
|