using System.Diagnostics;
using Akka.Actor;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;

/// <summary>
/// WP3.1: launches one script or alarm on-trigger run directly from its owning coordinator
/// (<see cref="ScriptActor"/> / <see cref="AlarmActor"/>), replacing the short-lived
/// <c>ScriptExecutionActor</c> and <c>AlarmExecutionActor</c>.
///
/// <para>Those actors were already inert shells: neither declared a single <c>Receive</c>
/// handler (they executed from their constructor), neither had a <c>PostStop</c>, state, or
/// stash, and their <c>IActorRef</c> was never a message target — the whole lifecycle lived
/// inside a detached <see cref="Task"/> the actor never observed. What they cost was a real
/// actor cell, mailbox, and name registration per run, plus a per-spawn expression-tree
/// <c>Props.Create</c>. Removing them changes no semantics; the run body below is the former
/// <c>ExecuteScript</c>/<c>ExecuteAlarmScript</c> body, unified.</para>
///
/// <para>Two behaviours DID change, both deliberately:</para>
/// <list type="number">
///   <item>The deadline <see cref="CancellationTokenSource"/> is now armed by the CALLER,
///   before the body is queued to the <see cref="ScriptExecutionScheduler"/>, so queue wait
///   consumes the script's own budget. A body that dequeues past its deadline skips execution
///   entirely and takes the timeout path — a saturated pool sheds stale work instead of
///   running it late.</item>
///   <item>The stuck-script watchdog now DETACHES and replaces the worker thread a wedged
///   script is holding (<see cref="ScriptExecutionScheduler.TryDetachWorker"/>) instead of
///   only naming it, so the pool recovers its capacity.</item>
/// </list>
///
/// <para>Ordering, supervision, telemetry, DI scoping, Ask replies, completion messages, and
/// the audit <c>ExecutionId</c>/<c>ParentExecutionId</c> threading are all unchanged. The
/// launch call itself can throw only if the target scheduler is unusable (e.g. disposed); the
/// caller wraps it so that failure replies to the Ask caller and decrements the in-flight
/// counter rather than escalating to the coordinator's supervisor.</para>
/// </summary>
internal static class ScriptRunLauncher
{
    /// <summary>
    /// Launches an instance script run. Returns as soon as the body is queued; the run reports
    /// completion to <paramref name="completionTarget"/>.
    /// </summary>
    /// <param name="scriptName">Name of the script being executed.</param>
    /// <param name="instanceName">Name of the instance that owns the script.</param>
    /// <param name="compiledScript">Compiled Roslyn script to execute.</param>
    /// <param name="parameters">Optional named parameter values for the script.</param>
    /// <param name="callDepth">Current call-nesting depth (used to enforce the max-depth limit).</param>
    /// <param name="instanceActor">Instance actor reference for attribute access.</param>
    /// <param name="sharedScriptLibrary">Library of shared scripts available during execution.</param>
    /// <param name="options">Site runtime options applied during execution.</param>
    /// <param name="replyTo">Actor reference that receives the script result; <c>Nobody</c> for fire-and-forget.</param>
    /// <param name="correlationId">Application-level correlation id threaded through the execution.</param>
    /// <param name="completionTarget">The owning <see cref="ScriptActor"/>, which receives the completion message.</param>
    /// <param name="siteCommunicationActor">Site communication actor (resolved on the actor thread) for <c>Notify.Status</c>.</param>
    /// <param name="logger">Logger for script execution events.</param>
    /// <param name="scope">Script scope controlling which APIs are available.</param>
    /// <param name="runId">Per-script run counter, used only in log messages (replaces the former per-run actor name).</param>
    /// <param name="healthCollector">Optional health collector for recording execution metrics.</param>
    /// <param name="serviceProvider">Optional DI service provider for script execution services.</param>
    /// <param name="parentExecutionId">ExecutionId of the spawning execution for audit correlation; null for root runs.</param>
    /// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global value.</param>
    /// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
    /// <param name="trace">Flow Debugger trace flag (design 2026-09-05 §5.2): when true the run emits <c>ScriptTrace</c> rows and returns them inline on the <see cref="ScriptCallResult"/>.</param>
    /// <param name="triggerExtra">Flow Debugger: trigger detail for a traced value-change run (stamped into the <c>ScriptRun</c> row's <c>Extra</c>); null for calls.</param>
    public static void LaunchScript(
        string scriptName,
        string instanceName,
        Script<object?> compiledScript,
        IReadOnlyDictionary<string, object?>? parameters,
        int callDepth,
        IActorRef instanceActor,
        SharedScriptLibrary sharedScriptLibrary,
        SiteRuntimeOptions options,
        IActorRef replyTo,
        string correlationId,
        IActorRef completionTarget,
        ICanTell? siteCommunicationActor,
        ILogger logger,
        ScriptScope scope,
        long runId,
        ISiteHealthCollector? healthCollector,
        IServiceProvider? serviceProvider,
        Guid? parentExecutionId,
        int? executionTimeoutSeconds,
        ScriptExecutionScheduler? scheduler,
        bool trace = false,
        ScriptTraceExtra? triggerExtra = null)
    {
        var timeout = ResolveTimeout(executionTimeoutSeconds, options);
        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);

        // Armed HERE, on the caller's (actor) thread, not inside the queued body: queue wait
        // must consume the script's own budget, otherwise a saturated pool silently grants
        // every backlogged run a fresh full timeout.
        var cts = new CancellationTokenSource(timeout);
        try
        {
            _ = Task.Factory.StartNew(
                () => RunScriptAsync(
                    scriptName, instanceName, compiledScript, parameters, callDepth,
                    instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
                    completionTarget, siteCommunicationActor, logger, scope, runId,
                    healthCollector, serviceProvider, parentExecutionId, timeout,
                    executionScheduler, cts, trace, triggerExtra),
                CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
                .Unwrap();
        }
        catch
        {
            // The body never got queued, so nothing will ever dispose the CTS.
            cts.Dispose();
            throw;
        }
    }

    /// <summary>
    /// Launches an alarm on-trigger run. Same contract as <see cref="LaunchScript"/>, with the
    /// firing alarm's level/priority/message exposed to the body through the <c>Alarm</c> global.
    /// </summary>
    /// <param name="alarmName">The canonical name of the alarm that triggered.</param>
    /// <param name="instanceName">The name of the owning instance.</param>
    /// <param name="level">The alarm severity level at the time of triggering.</param>
    /// <param name="priority">The alarm priority value.</param>
    /// <param name="message">The alarm message to pass to the script.</param>
    /// <param name="compiledScript">The pre-compiled on-trigger script to execute.</param>
    /// <param name="instanceActor">Reference to the instance actor for attribute/script calls.</param>
    /// <param name="sharedScriptLibrary">Shared script library providing common utilities.</param>
    /// <param name="options">Site runtime configuration options, including the execution timeout.</param>
    /// <param name="completionTarget">The owning <see cref="AlarmActor"/>, which receives the completion message.</param>
    /// <param name="logger">Logger for execution diagnostics.</param>
    /// <param name="healthCollector">
    /// Optional health collector for recording execution metrics (arch-review 2026-08-19
    /// P1-3): the alarm on-trigger run shares the script-error counter with
    /// <see cref="RunScriptAsync"/> so a wedged/failed alarm body is visible the same way a
    /// wedged/failed instance script is.
    /// </param>
    /// <param name="serviceProvider">
    /// Optional DI service provider (arch-review 2026-08-19 P1-3): resolves the singleton
    /// <see cref="ISiteEventLogger"/> (stuck-watchdog reporting) and
    /// <see cref="ScriptRunSummaryRecorder"/> (aggregate run counters), exactly like the
    /// instance-script path.
    /// </param>
    /// <param name="runId">Per-alarm run counter, used only in log messages.</param>
    /// <param name="executionTimeoutSeconds">The on-trigger script's per-script timeout in seconds. Null or non-positive falls back to the global value.</param>
    /// <param name="parentExecutionId">
    /// ParentExecutionId tag-cascade: the <c>ExecutionId</c> of the execution whose attribute
    /// write fired this alarm. Null when the firing value came from the Data Connection Layer
    /// (external data has no spawning execution) — that on-trigger run is a tree ROOT.
    /// </param>
    /// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
    public static void LaunchAlarmScript(
        string alarmName,
        string instanceName,
        AlarmLevel level,
        int priority,
        string message,
        Script<object?> compiledScript,
        IActorRef instanceActor,
        SharedScriptLibrary sharedScriptLibrary,
        SiteRuntimeOptions options,
        IActorRef completionTarget,
        ILogger logger,
        ISiteHealthCollector? healthCollector,
        IServiceProvider? serviceProvider,
        long runId,
        int? executionTimeoutSeconds,
        Guid? parentExecutionId,
        ScriptExecutionScheduler? scheduler)
    {
        var timeout = ResolveTimeout(executionTimeoutSeconds, options);
        var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);

        // Enqueue-anchored deadline; see LaunchScript.
        var cts = new CancellationTokenSource(timeout);
        try
        {
            _ = Task.Factory.StartNew(
                () => RunAlarmScriptAsync(
                    alarmName, instanceName, level, priority, message, compiledScript,
                    instanceActor, sharedScriptLibrary, options, completionTarget, logger,
                    healthCollector, serviceProvider, runId, parentExecutionId, timeout,
                    executionScheduler, cts),
                CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
                .Unwrap();
        }
        catch
        {
            cts.Dispose();
            throw;
        }
    }

    /// <summary>
    /// Per-script timeout overrides the global default. A null or non-positive per-script
    /// value (≤ 0) falls back to the global.
    /// </summary>
    private static TimeSpan ResolveTimeout(int? executionTimeoutSeconds, SiteRuntimeOptions options)
        => TimeSpan.FromSeconds(
            executionTimeoutSeconds is { } perScript && perScript > 0
                ? perScript
                : options.ScriptExecutionTimeoutSeconds);

    private static async Task RunScriptAsync(
        string scriptName,
        string instanceName,
        Script<object?> compiledScript,
        IReadOnlyDictionary<string, object?>? parameters,
        int callDepth,
        IActorRef instanceActor,
        SharedScriptLibrary sharedScriptLibrary,
        SiteRuntimeOptions options,
        IActorRef replyTo,
        string correlationId,
        IActorRef completionTarget,
        ICanTell? siteCommunicationActor,
        ILogger logger,
        ScriptScope scope,
        long runId,
        ISiteHealthCollector? healthCollector,
        IServiceProvider? serviceProvider,
        Guid? parentExecutionId,
        TimeSpan timeout,
        ScriptExecutionScheduler executionScheduler,
        CancellationTokenSource cts,
        bool trace = false,
        ScriptTraceExtra? triggerExtra = null)
    {
        IServiceScope? serviceScope = null;
        // Flow Debugger (design §5.4): the inline sink exists only for a traced run that has a
        // caller to reply to — a trace-armed trigger run (replyTo == Nobody) writes every row to the
        // audit writer for PullTraceEvents and would only keep a never-read list alive. The
        // ScriptRun row itself is emitted below on every exit path of a traced run.
        // Arch-review F1: the inline copy is redacted at the collector, so the collector carries
        // the host's IAuditRedactor. That is a SINGLETON (AddAuditLog), resolved here from the
        // ROOT provider for the same reason siteEventLogger below is — the traced ScriptRun row
        // is emitted from the catch blocks too, including the shed-at-dequeue path that throws
        // before the DI scope exists. Null in tests / hosts without AddAuditLog: raw, still capped.
        var collector = trace && !replyTo.IsNobody()
            ? new FlowTraceCollector(
                redactor: serviceProvider?.GetService<ZB.MOM.WW.Audit.IAuditRedactor>())
            : null;
        var startedUtc = DateTime.UtcNow;
        ScriptRuntimeContext? context = null;
        IAuditWriter? auditWriter = null;
        var siteId = string.Empty;
        string? sourceNode = null;
        // ISiteEventLogger is a singleton; resolve from the root provider so
        // it is available to the catch blocks regardless of scope state.
        var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();

        // WP3.2 (site_events volume policy): both are singletons, resolved from the root
        // provider up front so they too are available to the catch blocks. optionsMonitor is
        // null in tests that pass no serviceProvider (SingleServiceProvider et al.) — the
        // constructor-passed `options` is the fallback, exactly like siteEventLogger's null
        // path degrades to "no site event log". Reading CurrentValue HERE (once per run, at
        // the top) rather than re-reading it at each call site is what "hot-togglable" means
        // in practice: an operator's live edit to PerRunScriptEvents/PerRunScriptEventScripts
        // takes effect on the NEXT run, without a restart.
        var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
        var optionsMonitor = serviceProvider?.GetService<IOptionsMonitor<SiteRuntimeOptions>>();
        var effectiveOptions = optionsMonitor?.CurrentValue ?? options;
        var emitPerRunEvents = ScriptRunEventPolicy.ShouldEmitPerRun(effectiveOptions, instanceName, scriptName);
        var runStopwatch = Stopwatch.StartNew();

        // WP3.1: identify the worker this body is holding, so the watchdog can detach and
        // replace it if the body wedges. Read in the FIRST synchronous segment — a script
        // that later hops threads on an await no longer holds this worker, and the run-stamp
        // guard in TryDetachWorker will (correctly) refuse to detach it.
        var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
        var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;

        var completed = 0;
        ArmStuckScriptWatchdog(
            cts, options, logger, siteEventLogger, executionScheduler,
            workerSlot, workerRunStamp, timeout,
            () => Volatile.Read(ref completed) != 0,
            $"Script '{scriptName}' on instance '{instanceName}'",
            instanceName, $"ScriptActor:{scriptName}");

        try
        {
            // WP3.1 shed-at-dequeue: the deadline was armed at ENQUEUE, so an already-cancelled
            // token here means this run spent its entire budget queueing. Running it now would
            // burn a scarce script thread on work that is already stale — throw straight into
            // the existing timeout path instead, BEFORE the "started" event and before the body.
            cts.Token.ThrowIfCancellationRequested();

            // Resolve integration services from DI (scoped lifetime)
            IExternalSystemClient? externalSystemClient = null;
            IDatabaseGateway? databaseGateway = null;
            // Notification Outbox: the S&F engine is a singleton; the site identity
            // provider supplies the site id stamped on enqueued notifications.
            StoreAndForwardService? storeAndForward = null;
            // The writer is a singleton (FallbackAuditWriter
            // composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
            // that haven't called AddAuditLog, which the helper handles as a no-op.
            // (siteId / auditWriter / sourceNode are declared above the try so the
            // traced ScriptRun emission in the catch blocks can reach them.)
            // Site-local tracking store
            // backing Tracking.Status(id). Singleton; null in tests / hosts
            // that haven't wired the store, which the helper handles by
            // throwing on access.
            IOperationTrackingStore? operationTrackingStore = null;
            // Site-side cached-call
            // telemetry forwarder. Singleton bound to the AuditLog
            // composition root; null in tests / hosts that haven't called
            // AddAuditLog, in which case the cached-call helpers degrade
            // to the no-emission path (the underlying S&F handoff still
            // happens and a TrackedOperationId is still returned).
            ICachedCallTelemetryForwarder? cachedForwarder = null;
            // SourceNode-stamping: the local node name
            // resolved from INodeIdentityProvider — node-a/node-b on site
            // hosts. Null in tests / hosts that haven't registered the
            // provider, in which case NotificationSubmit.SourceNode and
            // SiteCallOperational.SourceNode stay null and central
            // persists the rows with SourceNode NULL.

            if (serviceProvider != null)
            {
                serviceScope = serviceProvider.CreateScope();
                externalSystemClient = serviceScope.ServiceProvider.GetService<IExternalSystemClient>();
                databaseGateway = serviceScope.ServiceProvider.GetService<IDatabaseGateway>();
                storeAndForward = serviceScope.ServiceProvider.GetService<StoreAndForwardService>();
                siteId = serviceScope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId
                    ?? string.Empty;
                auditWriter = serviceScope.ServiceProvider.GetService<IAuditWriter>();
                operationTrackingStore = serviceScope.ServiceProvider.GetService<IOperationTrackingStore>();
                cachedForwarder = serviceScope.ServiceProvider.GetService<ICachedCallTelemetryForwarder>();
                sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
            }

            context = new ScriptRuntimeContext(
                instanceActor,
                sharedScriptLibrary,
                callDepth,
                options.MaxScriptCallDepth,
                timeout,
                instanceName,
                logger,
                externalSystemClient,
                databaseGateway,
                storeAndForward,
                siteCommunicationActor,
                siteId,
                // Notification Outbox (FU3): stamp the executing script onto outbound
                // notifications using the Site Event Logging "Source" convention.
                sourceScript: $"ScriptActor:{scriptName}",
                // Emit one ApiOutbound/ApiCall row per
                // ExternalSystem.Call. Writer is best-effort; failures are logged
                // and swallowed inside the helper so the script's call path is
                // never aborted by an audit failure.
                auditWriter: auditWriter,
                // Site-local tracking store
                // backing Tracking.Status(id). Authoritative source of truth for
                // cached-call status — read directly by the script API.
                operationTrackingStore: operationTrackingStore,
                // Cached-call telemetry
                // forwarder for ExternalSystem.CachedCall / Database.CachedWrite
                // CachedSubmit emission + the immediate-success terminal-row
                // emission. Best-effort: null degrades the helpers to a
                // no-emission path; the S&F handoff and TrackedOperationId
                // return are unaffected.
                cachedForwarder: cachedForwarder,
                // The spawning execution's
                // id for an inbound-API-routed call. The routed script still
                // mints its own fresh ExecutionId — this records the spawner.
                // Null for normal (tag-change / timer) runs.
                parentExecutionId: parentExecutionId,
                // SourceNode-stamping: the local node name
                // (node-a/node-b on a site) — threaded down so Notify.Send
                // and the four cached-call telemetry constructors can stamp
                // it onto NotificationSubmit.SourceNode and
                // SiteCallOperational.SourceNode respectively.
                sourceNode: sourceNode,
                // Thread the singleton site event logger so
                // recursion-limit violations at CallScript/CallShared emit a
                // script Error site event in addition to ILogger.LogError.
                siteEventLogger: siteEventLogger,
                // WaitForAttribute (spec §4.3/§4.4): thread the per-script
                // execution-timeout token so Attributes.WaitAsync's Ask is
                // bounded by the script's own ExecutionTimeoutSeconds — a
                // shorter script deadline wins over the wait's own timeout.
                scriptTimeoutToken: cts.Token,
                // Flow Debugger (design §5.2/§5.4): the trace flag and the inline sink.
                isTraced: trace,
                traceCollector: collector);

            var globals = new ScriptGlobals
            {
                Instance = context,
                Parameters = new ScriptParameters(parameters ?? new Dictionary<string, object?>()),
                CancellationToken = cts.Token,
                Scope = scope
            };

            // WP3.2: the aggregate ALWAYS records this run (that's the point of the interval
            // summary), but the per-run Info event itself is gated by ScriptRunEventPolicy —
            // off by default, on globally via PerRunScriptEvents, or on for named scripts via
            // PerRunScriptEventScripts. Fire-and-forget (the `_ =` discards the task) so the
            // event log can never block or fault the script's own run; mirrors the existing
            // Error-path emit, which stays unconditional regardless of this policy.
            summaryRecorder?.RecordStarted(instanceName, scriptName);
            if (emitPerRunEvents)
            {
                _ = siteEventLogger?.LogEventAsync(
                    "script", "Info", instanceName, $"ScriptActor:{scriptName}",
                    $"Script '{scriptName}' on instance '{instanceName}' started");
            }

            var state = await compiledScript.RunAsync(globals, cts.Token);

            // Flow Debugger: the ScriptRun row goes into the collector BEFORE the reply so the
            // inline list the caller receives includes the run itself. Complete() (not Snapshot)
            // closes the sink at the same instant — outbound rows finishing after this point
            // (plan 2026-09-06 D5) go to the audit writer only.
            EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
                context.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
                parameters, state.ReturnValue, error: null, triggerExtra);

            // Send result to requester if this was an Ask-based call
            if (!replyTo.IsNobody())
            {
                replyTo.Tell(new ScriptCallResult(
                    correlationId, true, state.ReturnValue, null,
                    context.ExecutionId, collector?.Complete()));
            }

            // Operational `script` event — execution completed successfully. Same gating as
            // the started event above.
            summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
            if (emitPerRunEvents)
            {
                _ = siteEventLogger?.LogEventAsync(
                    "script", "Info", instanceName, $"ScriptActor:{scriptName}",
                    $"Script '{scriptName}' on instance '{instanceName}' completed");
            }

            // Notify the owning ScriptActor of completion (also releases its in-flight slot).
            completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
        }
        catch (OperationCanceledException)
        {
            healthCollector?.IncrementScriptError();
            summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
            var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
            logger.LogWarning("{Message} (run {RunId})", errorMsg, runId);

            // Failures recorded to site event log; script NOT disabled after failure. WP3.2:
            // Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
            _ = siteEventLogger?.LogEventAsync(
                "script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);

            EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
                context?.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
                parameters, null, errorMsg, triggerExtra);

            if (!replyTo.IsNobody())
            {
                replyTo.Tell(new ScriptCallResult(
                    correlationId, false, null, errorMsg, context?.ExecutionId, collector?.Complete()));
            }

            completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
        }
        catch (Exception ex)
        {
            healthCollector?.IncrementScriptError();
            summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
            // Failures recorded to site event log; script NOT disabled after failure. WP3.2:
            // Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
            var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
            logger.LogError(ex, "Script execution failed: {Script} on {Instance} (run {RunId})",
                scriptName, instanceName, runId);

            _ = siteEventLogger?.LogEventAsync(
                "script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());

            EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
                context?.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
                parameters, null, errorMsg, triggerExtra);

            if (!replyTo.IsNobody())
            {
                replyTo.Tell(new ScriptCallResult(
                    correlationId, false, null, errorMsg, context?.ExecutionId, collector?.Complete()));
            }

            completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
        }
        finally
        {
            // Mark the body finished so the stuck-script watchdog (registered on
            // cts.Token above) treats a timely cancellation as NOT stuck.
            Interlocked.Exchange(ref completed, 1);
            // Dispose the DI scope (and scoped services) after script execution completes
            serviceScope?.Dispose();
            cts.Dispose();
        }
    }

    /// <summary>
    /// Arch-review F7: a traced run that never launched (shed at the in-flight cap, or
    /// suppressed by <c>MinTimeBetweenRuns</c>) still leaves a <c>Failed</c>
    /// <c>ScriptTrace.ScriptRun</c> row, so the Flow Debugger explains the gap with the guard
    /// that refused the run instead of reporting <c>NO_TRIGGER_WITHIN_WINDOW</c> — which reads
    /// as "no value-change trigger on this attribute, or no device echo" and sends the operator
    /// to check the wrong things.
    ///
    /// <para>There is no collector: a shed run has no reply to ride on (an Ask caller gets the
    /// shed <c>ScriptCallResult</c> from <c>ScriptActor.ShedRun</c>, not a trace list), so the
    /// row goes to the audit writer only and reaches the debugger via <c>PullTraceEvents</c> /
    /// Replay. The writer and identity providers come from a fresh scope because this runs on
    /// the ACTOR thread, outside any launch scope. <c>elapsedMs</c> is 0 — the run never ran.
    /// Best-effort and never throws: a no-op when there is no provider or no writer.</para>
    /// </summary>
    internal static void EmitSuppressedRun(
        IServiceProvider? serviceProvider,
        ILogger logger,
        string scriptName,
        string instanceName,
        Guid? parentExecutionId,
        string reason,
        ScriptTraceExtra? triggerExtra)
    {
        if (serviceProvider is null) return;
        try
        {
            using var scope = serviceProvider.CreateScope();
            var writer = scope.ServiceProvider.GetService<IAuditWriter>();
            if (writer is null) return;
            var siteId = scope.ServiceProvider.GetService<ISiteIdentityProvider>()?.SiteId ?? string.Empty;
            var sourceNode = scope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;

            EmitScriptRun(
                trace: true, collector: null, writer, logger, scriptName, instanceName, siteId, sourceNode,
                executionId: Guid.NewGuid(), parentExecutionId, DateTime.UtcNow, elapsedMs: 0,
                parameters: null, returnValue: null, error: reason, triggerExtra);
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "Flow trace suppressed-run emission failed for {Script}", scriptName);
        }
    }

    /// <summary>
    /// Flow Debugger (design 2026-09-05 §5.1): emits the <c>ScriptTrace.ScriptRun</c> row for a
    /// traced run — collector first (so the inline reply carries it), then the best-effort
    /// audit writer. <c>OccurredAtUtc</c> is the run's START so the row sorts before the
    /// writes/waits the run made. No-op when untraced; never throws.
    /// </summary>
    private static void EmitScriptRun(
        bool trace, FlowTraceCollector? collector, IAuditWriter? auditWriter, ILogger logger,
        string scriptName, string instanceName, string siteId, string? sourceNode,
        Guid? executionId, Guid? parentExecutionId, DateTime startedUtc, long elapsedMs,
        IReadOnlyDictionary<string, object?>? parameters, object? returnValue, string? error,
        ScriptTraceExtra? triggerExtra)
    {
        if (!trace) return;
        try
        {
            var sourceScript = $"ScriptActor:{scriptName}";
            var evt = ScadaBridgeAuditEventFactory.Create(
                AuditChannel.ScriptTrace, AuditKind.ScriptRun,
                error is null ? AuditStatus.Delivered : AuditStatus.Failed,
                occurredAtUtc: DateTime.SpecifyKind(startedUtc, DateTimeKind.Utc),
                actor: sourceScript, target: scriptName, sourceNode: sourceNode,
                executionId: executionId, parentExecutionId: parentExecutionId,
                sourceSiteId: string.IsNullOrEmpty(siteId) ? null : siteId,
                sourceInstanceId: instanceName, sourceScript: sourceScript,
                durationMs: (int)Math.Min(elapsedMs, int.MaxValue),
                errorMessage: error,
                requestSummary: TraceJson(parameters),
                responseSummary: TraceJson(returnValue),
                extra: (triggerExtra ?? new ScriptTraceExtra { TriggerKind = "call" }).ToJson());
            collector?.Add(evt);
            if (auditWriter is null) return;
            var t = auditWriter.WriteAsync(evt, CancellationToken.None);
            if (!t.IsCompletedSuccessfully)
            {
                _ = t.ContinueWith(
                    x => logger.LogWarning(x.Exception, "Flow trace ScriptRun write failed for {Script}", scriptName),
                    CancellationToken.None,
                    TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
                    TaskScheduler.Default);
            }
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "Flow trace ScriptRun emission failed for {Script}", scriptName);
        }
    }

    private static string? TraceJson(object? value)
    {
        if (value is null) return null;
        try { return JsonSerializer.Serialize(value, ScriptTraceExtra.JsonOptions); }
        catch (Exception) { return null; }
    }

    private static async Task RunAlarmScriptAsync(
        string alarmName,
        string instanceName,
        AlarmLevel level,
        int priority,
        string message,
        Script<object?> compiledScript,
        IActorRef instanceActor,
        SharedScriptLibrary sharedScriptLibrary,
        SiteRuntimeOptions options,
        IActorRef completionTarget,
        ILogger logger,
        ISiteHealthCollector? healthCollector,
        IServiceProvider? serviceProvider,
        long runId,
        Guid? parentExecutionId,
        TimeSpan timeout,
        ScriptExecutionScheduler executionScheduler,
        CancellationTokenSource cts)
    {
        // arch-review 2026-08-19 P1-3: singletons resolved from the root provider up front,
        // exactly like RunScriptAsync, so they are available to the catch blocks regardless
        // of scope state. Null in tests that pass no serviceProvider — degrades the same way
        // the instance-script path does (no site event log / no aggregate counters).
        var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
        var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();

        // The on-trigger run has no per-script name of its own in the site-event/summary
        // sense — key it under the alarm, mirroring the "AlarmActor:{name}" source
        // convention already used for this alarm's other site events.
        var scriptName = $"AlarmActor:{alarmName}";
        var runStopwatch = Stopwatch.StartNew();

        // arch-review 2026-08-19 P1-3: identify the worker this body is holding, so the
        // stuck-script watchdog can detach and replace it if the body wedges — same
        // mechanism, same gauge, as the instance-script path. Read in the FIRST synchronous
        // segment, before any await.
        var workerSlot = ScriptExecutionScheduler.CurrentWorkerSlot;
        var workerRunStamp = workerSlot is { } slot ? executionScheduler.CurrentRunStamp(slot) : 0L;

        var completed = 0;
        ArmStuckScriptWatchdog(
            cts, options, logger, siteEventLogger, executionScheduler,
            workerSlot, workerRunStamp, timeout,
            () => Volatile.Read(ref completed) != 0,
            $"Alarm on-trigger script for '{alarmName}' on instance '{instanceName}'",
            instanceName, scriptName);

        try
        {
            // Enqueue-anchored deadline: an already-cancelled token means this on-trigger run
            // spent its whole budget queueing behind other script bodies. Shed it rather than
            // run it late — the alarm it would react to is already stale.
            cts.Token.ThrowIfCancellationRequested();

            // An alarm on-trigger run can call Instance.CallScript()
            // via the ScriptRuntimeContext injected into globals
            var context = new ScriptRuntimeContext(
                instanceActor,
                sharedScriptLibrary,
                currentCallDepth: 0,
                options.MaxScriptCallDepth,
                timeout,
                instanceName,
                logger,
                // ParentExecutionId tag-cascade: the
                // alarm on-trigger run mints its own fresh ExecutionId (the
                // ctor's `?? NewGuid()` fallback) and records the firing
                // execution's id as its ParentExecutionId — null (a root)
                // only when the firing value came from the DCL.
                parentExecutionId: parentExecutionId,
                // WaitForAttribute (spec §4.4): thread the alarm on-trigger
                // script's per-script execution-timeout token so a
                // Attributes.WaitAsync inside an on-trigger script is bounded
                // by the same script deadline.
                scriptTimeoutToken: cts.Token);

            var globals = new ScriptGlobals
            {
                Instance = context,
                Parameters = new ScriptParameters(),
                CancellationToken = cts.Token,
                Alarm = new AlarmContext
                {
                    Name = alarmName,
                    Level = level,
                    Priority = priority,
                    Message = message
                }
            };

            summaryRecorder?.RecordStarted(instanceName, scriptName);

            await compiledScript.RunAsync(globals, cts.Token);

            summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
            completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, true));
        }
        catch (OperationCanceledException)
        {
            healthCollector?.IncrementScriptError();
            summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
            logger.LogWarning(
                "Alarm on-trigger script for {Alarm} on {Instance} timed out (run {RunId})",
                alarmName, instanceName, runId);
            completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
        }
        catch (Exception ex)
        {
            healthCollector?.IncrementScriptError();
            summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
            // Failures logged, alarm continues
            logger.LogError(ex,
                "Alarm on-trigger script for {Alarm} on {Instance} failed (run {RunId})",
                alarmName, instanceName, runId);
            completionTarget.Tell(new AlarmActor.AlarmExecutionCompleted(alarmName, false));
        }
        finally
        {
            // Mark the body finished so the stuck-script watchdog (registered on
            // cts.Token above) treats a timely cancellation as NOT stuck.
            Interlocked.Exchange(ref completed, 1);
            cts.Dispose();
        }
    }

    /// <summary>
    /// Arms the stuck-script watchdog (S2, extended by WP3.1).
    ///
    /// <para>The CTS firing only REQUESTS cooperative cancellation; it does NOT free a thread
    /// blocked in synchronous I/O. When the deadline elapses, this waits
    /// <see cref="SiteRuntimeOptions.StuckScriptGraceMs"/> and, if the body still has not
    /// returned, (a) names the script loudly on the site event log and (b) DETACHES the worker
    /// thread it is holding and starts a replacement — so a wedged script costs the pool one
    /// thread only until the watchdog fires, not forever. At the detach cap it logs an Error
    /// instead of growing threads without bound.</para>
    ///
    /// <para>The <c>Task.Run</c>/<c>Task.Delay</c> run on the shared thread pool, never on the
    /// (possibly saturated) script scheduler — deliberate, and the whole point of the watchdog.
    /// <c>Register</c> fires on cancellation only; a normally-completing run disposes its CTS
    /// with no callback. A run cancelled while still QUEUED flips its completed flag in the
    /// body's <c>finally</c> long before the grace elapses, so it is correctly not reported.</para>
    /// </summary>
    private static void ArmStuckScriptWatchdog(
        CancellationTokenSource cts,
        SiteRuntimeOptions options,
        ILogger logger,
        ISiteEventLogger? siteEventLogger,
        ScriptExecutionScheduler scheduler,
        int? workerSlot,
        long workerRunStamp,
        TimeSpan timeout,
        Func<bool> isCompleted,
        string subject,
        string instanceName,
        string source)
    {
        var graceMs = options.StuckScriptGraceMs;
        cts.Token.Register(() => _ = Task.Run(async () =>
        {
            await Task.Delay(graceMs);
            if (isCompleted()) return;

            var stuckMsg = $"{subject} exceeded its {timeout.TotalSeconds:F0}s timeout and is STILL EXECUTING — " +
                           "its dedicated script-execution thread is blocked (cooperative cancellation not observed).";

            var outcome = workerSlot is { } slot
                ? scheduler.TryDetachWorker(slot, workerRunStamp)
                : WorkerDetachOutcome.NotRunning;

            switch (outcome)
            {
                case WorkerDetachOutcome.Detached:
                    stuckMsg += " The worker thread has been DETACHED and replaced; it will exit when the body returns.";
                    break;
                case WorkerDetachOutcome.AtCap:
                    stuckMsg += $" The script-execution pool already holds {scheduler.DetachedThreadCount} detached " +
                                "stuck threads (at cap); NOT replacing this one.";
                    break;
                case WorkerDetachOutcome.NotRunning:
                    // The body is not on a worker thread (it hopped threads on an await), so no
                    // dedicated thread is lost — report it, but there is nothing to replace.
                    break;
            }

            logger.LogError(stuckMsg);
            _ = siteEventLogger?.LogEventAsync("script", "Error", instanceName, source, stuckMsg);
        }));
    }
}