ScadaBridge/src/ZB.MOM.WW.ScadaBridge.InboundAPI/InboundScriptExecutor.cs

using System.Collections.Concurrent;
using System.Text.Json;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.ScriptAnalysis;

namespace ZB.MOM.WW.ScadaBridge.InboundAPI;

/// <summary>
/// Executes the C# script associated with an inbound API method.
/// Compiles method scripts via Roslyn and caches compiled delegates.
/// </summary>
public class InboundScriptExecutor
{
    private readonly ILogger<InboundScriptExecutor> _logger;

    // A cached compiled handler, paired with the exact script text it was
    // compiled from. The DB row is authoritative: on every request the cached
    // Script is ordinal-compared against the freshly-fetched ApiMethod.Script; a
    // mismatch means the cache is stale (a standby node compiled its own copy at
    // startup, or the row was edited via direct SQL / bundle import) and the
    // handler is recompiled in place. A null Script means "no revision info,
    // serve as-is" — the RegisterHandler test seam, exempt from revision checks.
    private sealed record CachedHandler(string? Script, Func<InboundScriptContext, Task<object?>> Handler);

    // This executor is registered as a singleton and its handler cache
    // is read and written from concurrent ASP.NET request threads. A plain Dictionary is
    // not safe for concurrent read/write, so a ConcurrentDictionary is used throughout.
    private readonly ConcurrentDictionary<string, CachedHandler> _scriptHandlers = new();

    // In-flight compiles keyed by method name + script text (see CompileSingleFlight).
    private readonly ConcurrentDictionary<string, Lazy<(Func<InboundScriptContext, Task<object?>>? Handler, IReadOnlyList<string> Errors)>> _inFlightCompiles = new();

    // A script that fails to compile (or violates the trust model)
    // is recorded here — keyed by method name, valued by the exact script text
    // that failed — so it is compiled at most once. Without this, every subsequent
    // request for a broken method re-runs the expensive Roslyn compilation — a CPU
    // amplification vector since the inbound API has no rate limiting. A record only
    // short-circuits when the CURRENT script text is the one that failed: a changed
    // script always gets a fresh compile. The entry is cleared whenever the method
    // is (re)compiled successfully.
    //
    // Bound the cache so a spam attack of unique method names cannot
    // grow it without bound. Once the cap is reached new bad-method records are
    // dropped — the cache is just a fast-fail optimisation; the per-request DB
    // lookup remains the correctness path.
    private const int KnownBadMethodsCap = 1000;
    private readonly ConcurrentDictionary<string, string> _knownBadMethods = new();

    /// <summary>
    /// Diagnostic helper — returns the current size of the
    /// known-bad-methods cache so tests can assert the cap is honoured. Internal
    /// so the cache itself stays an implementation detail.
    /// </summary>
    internal int KnownBadMethodCount => _knownBadMethods.Count;

    /// <summary>
    /// Records <paramref name="methodName"/> → <paramref name="script"/> in the
    /// known-bad-methods cache so the CURRENT failing script text short-circuits the
    /// next request. An existing record for the method is overwritten so a newer bad
    /// script replaces an older one (the fast-fail always tracks the latest failure).
    /// A brand-new record is only added while the cache has not reached
    /// <see cref="KnownBadMethodsCap"/>; once full, new method names are dropped
    /// (paying the cheap recompile next time rather than leaking memory under a
    /// unique-name flood). Records are cleared on a successful (re)compile.
    /// </summary>
    private void TryRecordBadMethod(string methodName, string script)
    {
        // Overwrite an existing record (newer bad script wins) without counting
        // against the cap; only a genuinely new method name is gated by the cap.
        if (_knownBadMethods.ContainsKey(methodName))
        {
            _knownBadMethods[methodName] = script;
            return;
        }
        if (_knownBadMethods.Count >= KnownBadMethodsCap)
            return;
        _knownBadMethods.TryAdd(methodName, script);
    }

    private readonly IServiceProvider _serviceProvider;

    // Count of inbound executions whose handler outlived its request (timed out or
    // client-aborted) and is still running with its DI scope disposal deferred. A
    // non-cooperative script that ignores the cancellation token keeps running after
    // WaitAsync returns; the scope is disposed by the handler's own completion
    // continuation, not the request finally, and this gauge makes the pile-up visible.
    private long _abandonedExecutions;

    /// <summary>
    /// Diagnostic gauge — number of inbound executions currently orphaned (handler
    /// still running after a timeout/abort, with DI-scope disposal deferred to the
    /// handler's completion). Internal so the counter stays an implementation detail;
    /// promote to public if a health surface needs it later.
    /// </summary>
    internal long AbandonedExecutionCount => Interlocked.Read(ref _abandonedExecutions);

    /// <summary>
    /// Initializes a new instance of the InboundScriptExecutor.
    /// </summary>
    /// <param name="logger">Logger instance.</param>
    /// <param name="serviceProvider">Service provider for dependency resolution.</param>
    public InboundScriptExecutor(ILogger<InboundScriptExecutor> logger, IServiceProvider serviceProvider)
    {
        _logger = logger;
        _serviceProvider = serviceProvider;
    }

    /// <summary>
    /// Registers a compiled script handler for a method name. This is the script-less
    /// test seam: the handler is stored with no associated script text, so it is
    /// <b>exempt from the per-request revision check</b> and serves as-is regardless of
    /// the fetched row's script. Production compiles always flow through
    /// <see cref="CompileAndRegister(ApiMethod)"/>, which records the script text and is
    /// revision-checked.
    /// </summary>
    /// <param name="methodName">The method name to register.</param>
    /// <param name="handler">The compiled handler function.</param>
    public void RegisterHandler(string methodName, Func<InboundScriptContext, Task<object?>> handler)
    {
        _scriptHandlers[methodName] = new CachedHandler(null, handler);
    }

    /// <summary>
    /// Drops the compiled handler AND any known-bad record for the method. The
    /// consumption seam for the script-artifact invalidation contract (bundle import,
    /// out-of-band updates, failover activation). Safe to call for unknown names.
    /// The per-request revision check in ExecuteAsync remains the correctness
    /// fallback when an invalidation is missed.
    /// </summary>
    /// <param name="methodName">The method name to invalidate.</param>
    public void InvalidateMethod(string methodName)
    {
        _scriptHandlers.TryRemove(methodName, out _);
        _knownBadMethods.TryRemove(methodName, out _);
    }

    /// <summary>
    /// Removes a compiled script handler for a method name. Delegates to
    /// <see cref="InvalidateMethod"/> (public API preserved).
    /// </summary>
    /// <param name="methodName">The method name to remove.</param>
    public void RemoveHandler(string methodName) => InvalidateMethod(methodName);

    /// <summary>Checks compilation without publishing or invalidating a live handler.</summary>
    /// <param name="method">The API method whose script is compiled.</param>
    /// <param name="errors">The compilation diagnostics, empty on success.</param>
    /// <returns><c>true</c> when the script compiles; otherwise <c>false</c>.</returns>
    public bool ValidateCompilation(ApiMethod method, out IReadOnlyList<string> errors)
    {
        var (handler, diagnostics) = Compile(method);
        errors = diagnostics;
        return handler is not null;
    }

    /// <summary>
    /// Compiles and registers a single API method script. Returns <c>false</c> if the
    /// script is empty, fails Roslyn compilation, or violates the script trust model.
    /// </summary>
    /// <param name="method">The API method to compile and register.</param>
    /// <returns>True if successfully compiled and registered, false otherwise.</returns>
    public bool CompileAndRegister(ApiMethod method) => CompileAndRegister(method, out _);

    /// <summary>
    /// Compiles and registers a single API method script, additionally reporting the
    /// compilation diagnostics. Returns <c>false</c> together with the human-readable
    /// error list if the script is empty, fails Roslyn compilation, or violates the
    /// script trust model. On failure this method does not overwrite any previously
    /// registered handler, but the persisted DB row is authoritative: <see cref="ExecuteAsync"/>
    /// revision-checks the cached handler against the freshly-fetched
    /// <c>ApiMethod.Script</c> on every request, so once a broken script is saved the
    /// method returns a compilation error consistently on every node rather than the
    /// stale delegate silently serving. The management Create/Update path can therefore
    /// surface a "saved but did not compile" warning that reflects real, consistent
    /// behavior (see <c>HandleCreateApiMethod</c>/<c>HandleUpdateApiMethod</c>).
    /// </summary>
    /// <param name="method">The API method to compile and register.</param>
    /// <param name="errors">On failure, the compilation/trust-model error messages; empty on success.</param>
    /// <returns>True if successfully compiled and registered, false otherwise.</returns>
    public bool CompileAndRegister(ApiMethod method, out IReadOnlyList<string> errors)
    {
        var (handler, compileErrors) = CompileSingleFlight(method);
        if (handler == null)
        {
            // Record the failure so the lazy-compile path does not
            // keep recompiling a broken script on every request. Routed
            // through the capped TryRecordBadMethod helper so the cache
            // cannot grow without bound under a flood of unique method names.
            errors = compileErrors;
            TryRecordBadMethod(method.Name, method.Script ?? string.Empty);
            return false;
        }

        // The method definition was (re)compiled successfully — drop any stale
        // failure record so a fixed script is no longer treated as bad.
        errors = Array.Empty<string>();
        _knownBadMethods.TryRemove(method.Name, out _);
        return Register(method, handler);
    }

    /// <summary>
    /// Single-flight wrapper over <see cref="Compile"/>, keyed by method name + script
    /// revision: N concurrent first-callers of a new/changed method share ONE compile (and
    /// one non-collectible loaded assembly) instead of each running Roslyn. The entry is
    /// removed as soon as the compile finishes, so a later edit recompiles normally.
    /// </summary>
    private (Func<InboundScriptContext, Task<object?>>? Handler, IReadOnlyList<string> Errors) CompileSingleFlight(ApiMethod method)
    {
        var key = method.Name + "\0" + (method.Script ?? string.Empty);
        var lazy = _inFlightCompiles.GetOrAdd(key, _ =>
            new Lazy<(Func<InboundScriptContext, Task<object?>>?, IReadOnlyList<string>)>(
                () => Compile(method), LazyThreadSafetyMode.ExecutionAndPublication));
        try
        {
            return lazy.Value;
        }
        finally
        {
            _inFlightCompiles.TryRemove(new KeyValuePair<string, Lazy<(Func<InboundScriptContext, Task<object?>>?, IReadOnlyList<string>)>>(key, lazy));
        }
    }

    private bool Register(ApiMethod method, Func<InboundScriptContext, Task<object?>> handler)
    {
        _scriptHandlers[method.Name] = new CachedHandler(method.Script, handler);
        return true;
    }

    /// <summary>
    /// Compiles a single API method script into an executable handler. Returns
    /// <c>null</c> when the script is missing, fails to compile, or violates the
    /// script trust model. Does not mutate the handler cache.
    /// </summary>
    /// <summary>
    /// Roslyn scripting options for every inbound-API method compile.
    ///
    /// <para>
    /// <b>Built ONCE, deliberately. Do not inline this back into <see cref="Compile"/>.</b>
    /// <c>WithReferences(Assembly[])</c> resolves each assembly through
    /// <c>MetadataReference.CreateFromFile</c>, which does not cache — every call mints a
    /// fresh <c>AssemblyMetadata</c> → <c>PEReader</c> → <c>NativeHeapMemoryBlock</c> holding
    /// an unmanaged copy of the assembly metadata that nothing disposes. Per-compile options
    /// therefore leak native memory for the life of the process, invisibly to the GC and to
    /// gcdump. Method compiles are not one-shot: every method re-registration and every
    /// revision change recompiles, so the growth is unbounded on a long-lived central node.
    /// </para>
    ///
    /// <para>
    /// Same defect, same shape as the Site-side one confirmed from a live dump on 2026-08-12
    /// (<c>SiteRuntime.Scripts.ScriptCompilationService.SharedScriptOptions</c>).
    /// </para>
    ///
    /// <para>
    /// Static options alone are NOT enough: every method compile binds the
    /// transitive closure of these references, and Roslyn's default resolver
    /// re-resolves that closure through <c>MetadataReference.CreateFromFile</c> on
    /// EVERY compile (confirmed live post-5a781c70 on 2026-08-12: 6 inbound
    /// methods held 2,699 AssemblyMetadata/PEReader/MetadataImageReference
    /// objects, ~150 per compile — and method revisions recompile, so growth was
    /// unbounded). The shared <see cref="CachingScriptMetadataResolver"/>
    /// memoizes those resolutions process-wide.
    /// </para>
    /// </summary>
    private static readonly ScriptOptions SharedScriptOptions = ScriptOptions.Default
        .WithReferences(
            typeof(object).Assembly,
            typeof(Enumerable).Assembly,
            typeof(Dictionary<,>).Assembly,
            typeof(RouteHelper).Assembly,
            typeof(ScriptParameters).Assembly,
            typeof(Microsoft.CSharp.RuntimeBinder.CSharpArgumentInfo).Assembly)
        .WithMetadataResolver(CachingScriptMetadataResolver.Instance)
        .WithImports(
            "System",
            "System.Collections.Generic",
            "System.Linq",
            "System.Threading.Tasks");

    private (Func<InboundScriptContext, Task<object?>>? Handler, IReadOnlyList<string> Errors) Compile(ApiMethod method)
    {
        if (string.IsNullOrWhiteSpace(method.Script))
        {
            _logger.LogWarning("API method {Method} has no script code", method.Name);
            return (null, new[] { "Method has no script code." });
        }

        // Enforce the script trust model before compiling. Roslyn
        // scripting performs no API allow/deny-listing, so forbidden namespaces must
        // be rejected statically or the script could reach the host process.
        var violations = ForbiddenApiChecker.FindViolations(method.Script);
        if (violations.Count > 0)
        {
            _logger.LogWarning(
                "API method {Method} script rejected — trust model violation(s): {Violations}",
                method.Name, string.Join("; ", violations));
            return (null, violations.Select(v => "Trust model violation: " + v).ToList());
        }

        try
        {
            var compiled = CSharpScript.Create<object?>(
                method.Script,
                SharedScriptOptions,
                globalsType: typeof(InboundScriptContext));

            var diagnostics = compiled.Compile();
            var errors = diagnostics
                .Where(d => d.Severity == Microsoft.CodeAnalysis.DiagnosticSeverity.Error)
                .Select(d => d.GetMessage())
                .ToList();

            if (errors.Count > 0)
            {
                _logger.LogWarning(
                    "API method {Method} script compilation failed: {Errors}",
                    method.Name, string.Join("; ", errors));
                return (null, errors);
            }

            _logger.LogInformation("API method {Method} script compiled", method.Name);
            return (async ctx =>
            {
                var state = await compiled.RunAsync(ctx, ctx.CancellationToken);
                return state.ReturnValue;
            }, Array.Empty<string>());
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to compile API method {Method} script", method.Name);
            return (null, new[] { "Compilation error: " + ex.Message });
        }
    }

    /// <summary>
    /// Executes the script for the given method with the provided context.
    /// </summary>
    /// <param name="method">The API method containing the script to execute.</param>
    /// <param name="parameters">Input parameters for the script.</param>
    /// <param name="route">Route helper for cross-site routing.</param>
    /// <param name="timeout">Timeout duration for script execution.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <param name="parentExecutionId">
    /// ParentExecutionId: the inbound API request's per-request
    /// <c>ExecutionId</c> (minted early by <c>AuditWriteMiddleware</c> and stashed
    /// on <c>HttpContext.Items</c>). When supplied, a routed
    /// <c>Route.To(...).Call(...)</c> inside the script carries it as
    /// <see cref="RouteToCallRequest.ParentExecutionId"/> so the spawned site
    /// script execution points back at this inbound request. Null when the script
    /// runs outside an inbound API request flow.
    /// </param>
    /// <param name="trace">When true, the run is traced for the Flow Debugger.</param>
    /// <returns>The result of executing the script.</returns>
    public async Task<InboundScriptResult> ExecuteAsync(
        ApiMethod method,
        IReadOnlyDictionary<string, object?> parameters,
        RouteHelper route,
        TimeSpan timeout,
        CancellationToken cancellationToken = default,
        // Deliberate ordering: this optional parameter trails the CancellationToken
        // because it was appended additively for non-breaking contract evolution.
        // Every call site passes it by named argument (parentExecutionId:).
        Guid? parentExecutionId = null,
        // Flow Debugger (design 2026-09-05 §5.2): trace the routed site runs this method makes.
        // Additive; the endpoint passes ApiMethod.TraceAllCalls, the Flow Debugger passes true.
        bool trace = false)
    {
        // Keep the timeout source and the request-abort source
        // separable. A single linked CTS makes a genuine client disconnect
        // indistinguishable from a method timeout, so a normal disconnect would be
        // logged and reported as "Script execution timed out". Use a dedicated
        // timeout CTS, linked with the request token, so the two can be told apart.
        using var timeoutCts = new CancellationTokenSource(timeout);
        using var cts = CancellationTokenSource.CreateLinkedTokenSource(
            cancellationToken, timeoutCts.Token);

        // IpsenMES MoveIn: expose the scoped, parameterized Database helper to the script
        // (reads + writes). Resolve the gateway from a fresh DI scope
        // (declared outside the try so it lives until after the script handler runs and is
        // disposed here) so a scoped IDatabaseGateway is honoured. GetService (not
        // GetRequiredService) so a method that never touches Database still runs even when
        // no IDatabaseGateway is registered; the helper throws on first use if built
        // without a gateway.
        //
        // A service provider that cannot produce a scope (e.g. a bare test double with
        // no IServiceScopeFactory) is tolerated: the gateway is simply unavailable, so
        // a script not using Database still runs, while one that does fails on use.
        IServiceScope? scope = null;
        // Hoisted out of the try so the finally can tell whether the handler task
        // outlived the request. If it did (non-cooperative script after a timeout or
        // client abort), disposing the scope underneath it would fault the still-running
        // task with ObjectDisposedException — so the finally defers disposal instead.
        Task<object?>? handlerTask = null;
        // Flow Debugger step logs (design 2026-09-09 D7): hoisted so the finally can flush the
        // "lines dropped" closing row on EVERY exit path — success, timeout, or a thrown script.
        InboundScriptLogger? scriptLogger = null;
        try
        {
            scope = _serviceProvider.CreateScope();
        }
        catch (InvalidOperationException ex)
        {
            // No scope factory available (e.g. a non-scoping test-double provider).
            // In production this should never happen; log so a genuine container
            // misconfiguration is visible rather than silently disabling Database.
            _logger.LogWarning(ex,
                "Could not create a DI scope for method {Method}; Database will be unavailable to its script",
                method.Name);
        }

        try
        {
            var gateway = scope?.ServiceProvider.GetService<IDatabaseGateway>();
            // Pass the method timeout so the helper derives a
            // CommandTimeout backstop and forwards cts.Token to every async DB call —
            // a slow query is then bounded by the method deadline.
            //
            // Flow Debugger outbound calls (plan 2026-09-06, D7): the helper audits every
            // Database.* call as a DbOutbound/DbWrite row. The inbound request's own
            // execution id (the parentExecutionId argument — see its doc above) is the
            // row's ExecutionId, exactly as the RouteCall row stamps it; an inbound request
            // is a root, so the row's ParentExecutionId is null. The writer + node identity
            // come from the same scope as the gateway (both null-tolerant: an unregistered
            // writer means no row, never a failed script), and the trace sink is the one the
            // caller attached to the RouteHelper so a Live run collects both row kinds.
            // Shared by the Database helper's outbound rows and the script's Log.* lines, so both
            // land in the same store and carry the same SourceNode.
            var auditWriter = scope?.ServiceProvider.GetService<ICentralAuditWriter>();
            var nodeIdentity = scope?.ServiceProvider.GetService<INodeIdentityProvider>();
            var dbHelper = new InboundDatabaseHelper(
                gateway, cts.Token, timeout,
                auditWriter: auditWriter,
                nodeIdentity: nodeIdentity,
                executionId: parentExecutionId,
                parentExecutionId: null,
                methodName: method.Name,
                traceSink: route.TraceSink,
                logger: _logger);

            // Bind the route helper to the method deadline so a
            // routed Route.To(...).Call(...) inherits the method-level timeout
            // without the script having to thread the context token by hand.
            //
            // Also pass the raw request-abort token separately so
            // Route.To(...).WaitForAttribute(...) can be bounded by its WAIT timeout
            // (not the generic method deadline) while still being cancelled by a
            // client disconnect — see RouteTarget.WaitForAttribute.
            //
            // Also bind the inbound request's ExecutionId (ParentExecutionId) so a
            // routed call carries it as ParentExecutionId — the
            // spawned site script execution points back at this inbound request.
            // Arch-review F11: the always-on RouteCall row must honour the method's
            // SkipBodyCapture override — the same per-target lookup AuditWriteMiddleware
            // performs for the inbound row (AuditWriteMiddleware.cs:183-187) — so an
            // operator who suppressed a method's body from the audit log doesn't get it
            // back via the routed call's arguments (derived from that same body).
            var auditOptions = scope?.ServiceProvider.GetService<IOptionsMonitor<AuditLogOptions>>()?.CurrentValue;
            var skipBodyCapture = auditOptions != null
                && auditOptions.PerTargetOverrides.TryGetValue(method.Name, out var perTargetOverride)
                && perTargetOverride.SkipBodyCapture;

            // Flow Debugger step logs (design 2026-09-09 D4/D5/D7): the script's Log global.
            // Lines ALWAYS reach this node's logger; only a TRACED execution also builds
            // ScriptTrace/ScriptLog rows, which are written best-effort to the central store for
            // Replay and then handed to the Live run's inline sink — exactly as
            // RouteHelper.EmitRouteCall orders its RouteCall row. An untraced request builds no
            // delegate and resolves no extra service.
            var methodName = method.Name;
            var maxLogLines = InboundApiOptions.DefaultMaxLogLinesPerRun;
            Action<AuditEvent>? logEmit = null;
            if (trace)
            {
                // The row cap comes from options when they are available (a sandbox/test provider
                // that registers none falls back to the documented default).
                maxLogLines = scope?.ServiceProvider
                    .GetService<IOptionsMonitor<InboundApiOptions>>()?.CurrentValue.MaxLogLinesPerRun
                    ?? InboundApiOptions.DefaultMaxLogLinesPerRun;
                var logTraceSink = route.TraceSink;
                logEmit = evt =>
                {
                    // Durable first: the Replay row must never depend on the Live sink succeeding.
                    WriteScriptLogBestEffort(auditWriter, evt, methodName);
                    try
                    {
                        logTraceSink?.Invoke(evt);
                    }
                    catch (Exception)
                    {
                        // A debug observer must never fail the request: a misbehaving Live sink is
                        // swallowed here so it cannot mask the durable write that already happened.
                    }
                };
            }

            scriptLogger = new InboundScriptLogger(
                _logger,
                emit: logEmit,
                maxLines: maxLogLines,
                methodName: methodName,
                executionId: parentExecutionId,
                sourceNode: nodeIdentity?.NodeName);

            var context = new InboundScriptContext(
                parameters,
                route.WithDeadline(cts.Token)
                     .WithRequestAborted(cancellationToken)
                     .WithParentExecutionId(parentExecutionId)
                     .WithTrace(trace)
                     .WithBodyCapture(!skipBodyCapture),
                dbHelper,
                cts.Token,
                scriptLogger);

            _scriptHandlers.TryGetValue(method.Name, out var cached);
            // Revision check: a cached handler compiled from different script text is stale
            // (standby-after-failover, direct SQL edit, bundle import). Null Script = test seam, exempt.
            if (cached is { Script: not null }
                && !string.Equals(cached.Script, method.Script, StringComparison.Ordinal))
            {
                _logger.LogInformation(
                    "API method {Method} script changed since it was cached; recompiling", method.Name);
                cached = null;
            }

            if (cached == null)
            {
                // Fast-fail only when the CURRENT script text is the one already known bad.
                if (_knownBadMethods.TryGetValue(method.Name, out var badScript)
                    && string.Equals(badScript, method.Script, StringComparison.Ordinal))
                    return new InboundScriptResult(false, null, "Script compilation failed for this method", "SCRIPT_COMPILE_FAILED");

                // Lazy compile (handles methods created after startup, and stale/known-bad
                // recompiles). Compile outside the cache so a failed compile is not stored,
                // then add so concurrent first-callers share a single handler instance.
                var (compiled, _) = CompileSingleFlight(method);
                if (compiled == null)
                {
                    // Cache the failure so the next request short-circuits above.
                    // Routed through TryRecordBadMethod so the
                    // cache is bounded under a flood of unique method names.
                    TryRecordBadMethod(method.Name, method.Script ?? string.Empty);
                    return new InboundScriptResult(false, null, "Script compilation failed for this method", "SCRIPT_COMPILE_FAILED");
                }

                _knownBadMethods.TryRemove(method.Name, out _);
                cached = new CachedHandler(method.Script, compiled);
                _scriptHandlers[method.Name] = cached; // last-write-wins; concurrent first-callers race benignly
            }

            handlerTask = cached.Handler(context);
            var result = await handlerTask.WaitAsync(cts.Token);

            var resultJson = result != null
                ? JsonSerializer.Serialize(result)
                : null;

            // Thread the JSON-Schema $ref resolution seam into return
            // validation so a method whose ReturnDefinition uses a {"$ref":"lib:Name"}
            // resolves the reference at RUNTIME (not just at deploy time). The
            // shared-schema library is pre-loaded ONCE from the per-execution DI scope
            // (the seam the validator consumes is synchronous), and ONLY when the
            // definition actually uses a $ref — a $ref-free return definition skips the
            // repository round-trip entirely. A dangling ref surfaces as a descriptive
            // validation failure naming the missing reference, not an opaque parse error.
            var sharedSchemaRepo =
                scope?.ServiceProvider.GetService<ISharedSchemaRepository>();
            var resolveRef = await SchemaRefResolver.BuildAsync(
                sharedSchemaRepo, [method.ReturnDefinition], cts.Token);

            // Validate the script's return value against the
            // method's declared ReturnDefinition. A method whose script returns a
            // shape inconsistent with its definition must not silently emit a
            // malformed 200 — surface it as a script failure (500) and log.
            var returnValidation = ReturnValueValidator.Validate(resultJson, method.ReturnDefinition, resolveRef);
            if (!returnValidation.IsValid)
            {
                _logger.LogWarning(
                    "API method {Method} return value rejected: {Error}",
                    method.Name, returnValidation.ErrorMessage);
                return new InboundScriptResult(false, null, "Method return value did not match its return definition");
            }

            return new InboundScriptResult(true, resultJson, null);
        }
        catch (OperationCanceledException)
        {
            // Distinguish a genuine method timeout from a client
            // abort. Only the timeout CTS firing is a real timeout; if the caller's
            // request token fired, the client disconnected — do not pollute the
            // timeout log (reserved for genuine script-execution timeouts).
            if (timeoutCts.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
            {
                _logger.LogWarning("Script execution timed out for method {Method}", method.Name);
                return new InboundScriptResult(false, null, "Script execution timed out", "TIMEOUT");
            }

            _logger.LogDebug("Inbound API request for method {Method} cancelled by client", method.Name);
            return new InboundScriptResult(false, null, "Request cancelled by client");
        }
        catch (RouteSiteScopeException ex)
        {
            // Arch-review F2: the script routed to a site outside the caller's site
            // scope. RouteTarget.ResolveSiteAsync refused BEFORE any site traffic, so
            // the script never saw a routed reply — there is nothing to roll back.
            // The message is the operator-facing denial, safe to surface verbatim
            // (it names only the instance and its site), unlike the generic
            // SCRIPT_ERROR path which deliberately hides internals.
            //
            // Ordered before the SiteUnreachableException catch purely for
            // readability: the two types are siblings under InvalidOperationException,
            // so neither shadows the other.
            _logger.LogWarning(
                "API method {Method} refused a routed call outside the caller's site scope: {Message}",
                method.Name, ex.Message);
            return new InboundScriptResult(false, null, ex.Message, "SITE_SCOPE_DENIED");
        }
        catch (SiteUnreachableException)
        {
            // C3: a routed Route.To(...).Call(...) reached a site that could not be
            // contacted. Surface the machine-readable SITE_UNREACHABLE code (spec
            // Component-InboundAPI failure body) instead of the generic SCRIPT_ERROR.
            _logger.LogWarning("Routed site unreachable for method {Method}", method.Name);
            return new InboundScriptResult(false, null, "Site unreachable", "SITE_UNREACHABLE");
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Script execution failed for method {Method}", method.Name);
            // Safe error message, no internal details — the caller only ever sees this.
            // Flow Debugger step logs (design 2026-09-09 D3): the exception itself rides the
            // additive ErrorDetail so the endpoint can put it on the inbound AUDIT row, where
            // the Flow Debugger's inbound node reads it. It must never reach the HTTP body.
            return new InboundScriptResult(
                false, null, "Internal script error", "SCRIPT_ERROR", ErrorDetail: ex.ToString());
        }
        finally
        {
            // The script body has finished (or been abandoned): close out its log lines before the
            // caller completes its trace collector. Idempotent and a no-op when nothing was dropped.
            try
            {
                scriptLogger?.FlushDropped();
            }
            catch (Exception)
            {
                // A debug observer must never fail the request.
            }

            if (handlerTask is null || handlerTask.IsCompleted)
            {
                scope?.Dispose();
            }
            else
            {
                // The handler outlived the request (non-cooperative script after a
                // timeout/client abort). Do NOT dispose the scope underneath it —
                // defer cleanup to the task's own completion, and count the orphan
                // so repeated timeouts of a slow method are visible.
                Interlocked.Increment(ref _abandonedExecutions);
                ScadaBridgeTelemetry.RecordInboundAbandonedExecution(method.Name);
                _logger.LogWarning(
                    "Abandoned inbound execution for method {Method} is still running; DI scope disposal deferred", method.Name);
                var abandonedScope = scope;
                _ = handlerTask.ContinueWith(t =>
                {
                    _ = t.Exception; // observe the fault so it never surfaces as unobserved
                    abandonedScope?.Dispose();
                    Interlocked.Decrement(ref _abandonedExecutions);
                }, TaskScheduler.Default);
            }
        }
    }

    /// <summary>
    /// Writes one <c>ScriptLog</c> row to the central audit store without ever blocking or failing
    /// the script (design 2026-09-09 D5) — the same fire-and-observe pattern
    /// <c>RouteHelper.EmitRouteCall</c> uses for its always-on <c>RouteCall</c> row. Never awaited on
    /// a script path: an audit write completes on the writer's own flush cadence.
    /// </summary>
    private void WriteScriptLogBestEffort(ICentralAuditWriter? writer, AuditEvent evt, string methodName)
    {
        if (writer is null) return;
        try
        {
            var t = writer.WriteAsync(evt, CancellationToken.None);
            if (!t.IsCompletedSuccessfully)
            {
                _ = t.ContinueWith(
                    x => _logger.LogWarning(x.Exception, "Flow trace ScriptLog write failed for method {Method}", methodName),
                    CancellationToken.None,
                    TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
                    TaskScheduler.Default);
            }
        }
        catch (Exception)
        {
            // A debug observer must never fail the request: an audit failure is not a script failure.
        }
    }
}

/// <summary>
/// Context provided to inbound API scripts.
/// </summary>
public class InboundScriptContext
{
    /// <summary>
    /// The script parameters.
    /// </summary>
    public ScriptParameters Parameters { get; }

    /// <summary>
    /// The route helper for cross-site routing.
    /// </summary>
    public RouteHelper Route { get; }

    /// <summary>
    /// Scoped, parameterized database access for the script (named connections; bound
    /// parameters; reads and writes — see <see cref="InboundDatabaseHelper"/>).
    /// </summary>
    public InboundDatabaseHelper Database { get; }

    /// <summary>
    /// The cancellation token for script execution.
    /// </summary>
    public CancellationToken CancellationToken { get; }

    /// <summary>
    /// Flow Debugger step logs (design 2026-09-09 D4): the script's leveled logger —
    /// <c>Log.Debug/Info/Warn/Error(message, exception?)</c>. Always reaches the node's log; a
    /// traced execution additionally records each line as a <c>ScriptTrace/ScriptLog</c> audit row
    /// the Flow Debugger folds onto the step that produced it.
    /// </summary>
    public InboundScriptLogger Log { get; }

    /// <summary>
    /// Initializes a new instance of the InboundScriptContext.
    /// </summary>
    /// <param name="parameters">The input parameters for the script.</param>
    /// <param name="route">The route helper for cross-site routing.</param>
    /// <param name="database">The scoped, parameterized database helper for the script.</param>
    /// <param name="cancellationToken">The cancellation token.</param>
    /// <param name="log">
    /// The script's <c>Log</c> global. Optional so existing constructions (and the Central UI
    /// sandbox host) keep compiling: when omitted the context gets an inert logger that emits no
    /// audit rows and writes nowhere.
    /// </param>
    public InboundScriptContext(
        IReadOnlyDictionary<string, object?> parameters,
        RouteHelper route,
        InboundDatabaseHelper database,
        CancellationToken cancellationToken = default,
        InboundScriptLogger? log = null)
    {
        Parameters = new ScriptParameters(parameters);
        Route = route;
        Database = database;
        CancellationToken = cancellationToken;
        Log = log ?? new InboundScriptLogger(
            NullLogger.Instance, emit: null, InboundApiOptions.DefaultMaxLogLinesPerRun,
            methodName: string.Empty, executionId: null);
    }
}

/// <summary>
/// Result of executing an inbound API script.
/// </summary>
/// <param name="Success">Whether the script executed successfully.</param>
/// <param name="ResultJson">The serialized script return value on success; null otherwise.</param>
/// <param name="ErrorMessage">A sanitized, caller-safe error message on failure; null on success.</param>
/// <param name="ErrorCode">
/// C3: an additive, machine-readable error code the endpoint surfaces in the failure
/// body's <c>code</c> field (e.g. <c>TIMEOUT</c>, <c>SITE_UNREACHABLE</c>,
/// <c>SCRIPT_COMPILE_FAILED</c>, <c>SCRIPT_ERROR</c>). Null when the executor does not
/// classify the failure; the endpoint then falls back to <c>SCRIPT_ERROR</c>.
/// </param>
/// <param name="ErrorDetail">
/// Flow Debugger step logs (design 2026-09-09 D3): verbose, operator-only failure detail —
/// the method script's exception <c>ToString()</c>. <b>Audit-row only.</b> The endpoint stashes
/// it on <c>HttpContext.Items[AuditWriteMiddleware.ErrorDetailItemKey]</c> so it lands on the
/// inbound audit row (and therefore on the Flow Debugger's inbound node); it is NEVER written
/// to the HTTP response, which keeps carrying the sanitised <see cref="ErrorMessage"/>. Null
/// for failures whose message already says everything (timeouts, client aborts, site-scope
/// denials, compile failures).
/// </param>
public record InboundScriptResult(
    bool Success,
    string? ResultJson,
    string? ErrorMessage,
    string? ErrorCode = null,
    string? ErrorDetail = null);