lmxopcua/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs

using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Serilog;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;

namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation;

/// <summary>
///     The <c>Calculation</c> pseudo-driver. Its tags are ordinary raw tags whose value is <b>computed</b>
///     by a C# script over other tags' live values — there is no backend, no endpoint, no discovery, no
///     writes. It implements:
///     <list type="bullet">
///       <item><see cref="IDriver"/> — lifecycle + always-Connected health.</item>
///       <item><see cref="IDependencyConsumer"/> — the host feeds it the other tags' live values.</item>
///       <item><see cref="ISubscribable"/> — computed values flow out via <see cref="OnDataChange"/>
///       exactly like any driver, so the ordinary node fan-out + the mux re-entry (calc-of-calc) apply.</item>
///       <item><see cref="IReadable"/> — on-demand read returns the last computed snapshot
///       (<c>BadWaitingForInitialData</c> before the first evaluation).</item>
///     </list>
///
///     <para>
///         <b>Triggers.</b> A tag re-evaluates on any of its declared dependencies changing (change-trigger,
///         gated exactly like <c>VirtualTagActor</c>: nothing publishes until every declared dep has arrived
///         at least once, and equal results are deduped) and/or on a timer (one timer per interval group).
///     </para>
///     <para>
///         <b>Where scripts run.</b> <see cref="OnDependencyValue"/> is called
///         directly and synchronously by <c>DependencyConsumerMuxAdapter</c> — on an Akka actor's own
///         dispatcher thread — so it does no evaluation itself: it records the value, <i>prepares</i> the
///         affected tags (definition + dependency snapshot, taken at trigger time) and hands them to a
///         single serialized <b>evaluation pump</b>. Script execution therefore never runs on the mux
///         callback thread and never runs under <see cref="_lock"/>; the pump and the timer ticks are
///         serialized against each other by <see cref="_evalGate"/> instead, which preserves the
///         one-evaluation-at-a-time ordering the state lock used to provide.
///     </para>
///     <para>
///         <b>Error semantics (mini-design §6).</b> An evaluator <c>Failure</c> publishes <b>Bad</b> quality
///         carrying the last-known value, once per Good→Bad transition and only after all deps have arrived,
///         and emits a <c>ScriptLogEntry</c> on the <c>script-logs</c> topic (via the injected
///         <see cref="ScriptRootLogger"/>'s <c>ScriptLogTopicSink</c>). Recovery to Good is force-published.
///         A historized Bad records <c>BadInternalError</c>.
///     </para>
/// </summary>
public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscribable, IReadable, ISkippedTagReporter, IDisposable
{
    /// <summary>Authored tags this driver could not build into its calc table. A raw tag with
    /// no <c>scriptId</c> is not a calc tag and computes nothing.</summary>
    private readonly SkippedTagLedger _skippedTags = new();

    /// <inheritdoc />
    public SkippedTagReport? GetSkippedTags() => _skippedTags.Snapshot();

    // OPC UA status codes surfaced by the calc driver.
    private const uint StatusGood = 0u;
    private const uint StatusBadWaitingForInitialData = 0x80320000u;
    private const uint StatusBadInternalError = 0x80020000u;

    private readonly string _driverInstanceId;
    private readonly CalculationDriverOptions _options;
    private readonly ILogger<CalculationDriver> _logger;
    private readonly ScriptRootLogger _scriptRoot;
    private readonly CalculationEvaluator _evaluator;

    // Authored calc tags, keyed by RawPath (identity + evaluation id). Built at construction so
    // DependencyRefs is available BEFORE InitializeAsync (the host reads it when spawning the mux-adapter),
    // and REBUILT on every ReinitializeAsync / body-carrying InitializeAsync (an in-place re-bind of an edited calc-tag set).
    private readonly Dictionary<string, CalculationTagDefinition> _tagsByRawPath = new(StringComparer.Ordinal);

    // rawPath (a dependency) → the calc tags that read it AND are change-triggered. Drives OnDependencyValue.
    private readonly Dictionary<string, List<string>> _changeDependents = new(StringComparer.Ordinal);

    // Union of every tag's declared dependency refs — the interest set the host registers on the mux.
    // Reassigned by RebuildTagTable so a redeploy that adds/removes dependencies re-registers on the mux.
    private IReadOnlyCollection<string> _dependencyRefs = Array.Empty<string>();

    // Live evaluation state. Guarded by _lock: OnDependencyValue (the mux actor's thread), timer ticks
    // and the evaluation pump (pool threads) all touch it, so state mutation is serialized. _lock is a
    // SHORT-HELD state lock — it is taken for dictionary work only and is NEVER held across a script run.
    private readonly object _lock = new();
    // Dependency SNAPSHOTS, not bare values: the status code is what tells a
    // comms-lost input apart from a genuine Good null, and storing values only is what let this driver
    // compute a Good output from a dead source.
    private readonly Dictionary<string, DataValueSnapshot> _depValues = new(StringComparer.Ordinal);
    private readonly HashSet<string> _arrivedDeps = new(StringComparer.Ordinal);
    private readonly Dictionary<string, TagState> _stateByRawPath = new(StringComparer.Ordinal);

    // Serializes script EXECUTION (and the result-apply that follows it) across the evaluation pump and
    // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus
    // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync,
    // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse.
    // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async —
    // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's
    // budget clock starts only once the pool has picked the script up, so pool starvation is never
    // reported as a script timeout).
    private readonly SemaphoreSlim _evalGate = new(1, 1);

    // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight
    // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false).
    private readonly CancellationTokenSource _evalCts = new();

    // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single
    // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an
    // empty queue under the lock, so an enqueue can never be lost against a pump that is exiting.
    private readonly Queue<PendingEvaluation> _pending = new();
    private bool _pumpScheduled;

    private CalculationTimerScheduler? _timers;
    private volatile bool _running;
    private DateTime? _lastComputeUtc;
    private volatile bool _disposed;

    /// <summary>Occurs when a calc tag's computed value changes (or transitions Good↔Bad).</summary>
    public event EventHandler<DataChangeEventArgs>? OnDataChange;

    /// <summary>Initializes a new <see cref="CalculationDriver"/>.</summary>
    /// <param name="options">Bound options carrying the authored raw calc tags.</param>
    /// <param name="driverInstanceId">Stable logical id of this driver instance.</param>
    /// <param name="scriptRoot">Root script logger — user <c>ctx.Logger</c> output + failure entries fan out
    /// to the <c>script-logs</c> topic through its <c>ScriptLogTopicSink</c>. When null a no-op logger is used.</param>
    /// <param name="logger">Host-side diagnostics logger; defaults to the null logger.</param>
    public CalculationDriver(
        CalculationDriverOptions options,
        string driverInstanceId,
        ScriptRootLogger? scriptRoot = null,
        ILogger<CalculationDriver>? logger = null)
    {
        ArgumentNullException.ThrowIfNull(options);
        ArgumentException.ThrowIfNullOrWhiteSpace(driverInstanceId);
        _options = options;
        _driverInstanceId = driverInstanceId;
        _logger = logger ?? NullLogger<CalculationDriver>.Instance;
        _scriptRoot = scriptRoot ?? new ScriptRootLogger(new LoggerConfiguration().CreateLogger());
        // _logger (ILogger<CalculationDriver>) implements the non-generic ILogger, so it is passed straight
        // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution
        // throws) land on the SAME logger the driver itself was constructed with, not a separate one.
        _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting);

        RebuildTagTable(_options);
    }

    /// <summary>
    ///     (Re)build the authored tag table (<see cref="_tagsByRawPath"/>), the change-trigger fan-out
    ///     (<see cref="_changeDependents"/>), the dependency-ref interest set (<see cref="_dependencyRefs"/>),
    ///     and the per-tag state map (<see cref="_stateByRawPath"/>) from <paramref name="config"/>. Called
    ///     once at construction AND on every <see cref="ReinitializeAsync"/> so an in-place config change — a
    ///     calc-tag add/remove/edit, or a script-source edit — actually takes effect. Extracted so the ctor
    ///     and reinit paths share ONE construction routine and cannot drift.
    ///     <para>Per-tag <see cref="TagState"/> is preserved for a tag that survives the edit (cheap last-known
    ///     value carry-over); a brand-new tag starts fresh; a removed tag's state is dropped. MUST be called
    ///     under <see cref="_lock"/> when invoked post-construction (the ctor runs single-threaded).</para>
    /// </summary>
    /// <param name="config">The (new) bound options carrying the authored raw calc tags.</param>
    private void RebuildTagTable(CalculationDriverOptions config)
    {
        _tagsByRawPath.Clear();
        _changeDependents.Clear();
        // Retain surviving tags' state; anything not re-authored is dropped when we swap this in below.
        var retainedState = new Dictionary<string, TagState>(StringComparer.Ordinal);

        _skippedTags.BeginRebuild(config.RawTags.Count);
        foreach (var entry in config.RawTags)
        {
            if (!CalculationTagDefinition.TryFromTagConfig(entry.TagConfig, entry.RawPath, out var def))
            {
                _skippedTags.Skip(entry.RawPath, "no scriptId — not a calc tag");
                _logger.LogWarning(
                    "Calculation {Driver}: raw tag {RawPath} has no scriptId; skipping (not a calc tag)",
                    _driverInstanceId, entry.RawPath);
                continue;
            }
            if (string.IsNullOrWhiteSpace(def.ScriptSource))
            {
                _logger.LogWarning(
                    "Calculation {Driver}: calc tag {RawPath} (script {Script}) has no resolved scriptSource; " +
                    "it will not compute until the deploy artifact injects the script source",
                    _driverInstanceId, entry.RawPath, def.ScriptId);
            }
            _tagsByRawPath[entry.RawPath] = def;
            // Carry the last-known state forward for a surviving tag; a brand-new tag starts fresh.
            retainedState[entry.RawPath] =
                _stateByRawPath.TryGetValue(entry.RawPath, out var prior) ? prior : new TagState();
            if (def.ChangeTriggered)
            {
                foreach (var dep in def.DependencyRefs)
                {
                    if (!_changeDependents.TryGetValue(dep, out var list))
                        _changeDependents[dep] = list = new List<string>();
                    if (!list.Contains(entry.RawPath, StringComparer.Ordinal)) list.Add(entry.RawPath);
                }
            }
        }

        _skippedTags.Commit();

        // Swap in the retained/new state map — a removed tag's state is dropped so it stops publishing.
        _stateByRawPath.Clear();
        foreach (var (rawPath, state) in retainedState) _stateByRawPath[rawPath] = state;

        _dependencyRefs = _tagsByRawPath.Values
            .SelectMany(t => t.DependencyRefs)
            .Distinct(StringComparer.Ordinal)
            .ToArray();
    }

    // Reinit config-bind options — mirrors the factory's bind (case-insensitive, enum-string, trailing-comma
    // tolerant) so an in-place config payload re-binds identically to the spawn payload.
    private static readonly JsonSerializerOptions ReinitJsonOptions = new()
    {
        PropertyNameCaseInsensitive = true,
        ReadCommentHandling = JsonCommentHandling.Skip,
        AllowTrailingCommas = true,
        Converters = { new JsonStringEnumConverter() },
    };

    /// <summary>Re-bind the authored raw calc tags from a new merged DriverConfig JSON (an in-place
    /// config payload). The ctor-fixed <see cref="CalculationEvaluator"/> timeout is kept — <c>RunTimeout</c> is bound
    /// at spawn (a change to it is out of this in-place path's scope). A parse failure logs and keeps the
    /// current tags so a malformed delta can never blank the driver.</summary>
    private CalculationDriverOptions ParseOptions(string driverConfigJson)
    {
        if (!string.IsNullOrWhiteSpace(driverConfigJson))
        {
            try
            {
                var dto = JsonSerializer.Deserialize<ReinitConfigDto>(driverConfigJson, ReinitJsonOptions);
                if (dto is not null)
                {
                    return new CalculationDriverOptions
                    {
                        RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [],
                        RunTimeout = _options.RunTimeout,
                        Scripting = _options.Scripting,
                    };
                }
            }
            catch (JsonException ex)
            {
                _logger.LogWarning(ex,
                    "Calculation {Driver}: reinit config parse failed; keeping the current tag table",
                    _driverInstanceId);
            }
        }
        return new CalculationDriverOptions
        {
            RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting,
        };
    }

    /// <summary>Re-bind the tag table from <paramref name="driverConfigJson"/> — shared by
    /// <see cref="ReinitializeAsync"/> and (for a config with a body) <see cref="InitializeAsync"/> so the two
    /// adoption paths cannot drift. Timers must already be stopped.</summary>
    private void AdoptConfig(string driverConfigJson)
    {
        var newOptions = ParseOptions(driverConfigJson);
        lock (_lock)
        {
            // Queued evaluations were prepared against the OUTGOING tag table (old script source, old
            // dependency set), so they are stale the moment the rebuild lands — drop them rather than
            // publish one last pre-edit value after the delta was applied.
            _pending.Clear();
            RebuildTagTable(newOptions);
        }
        // Drop compiled scripts (IScriptCacheOwner contract — stops collectible ALCs accreting AND forces a
        // recompile from each tag's NEW ScriptSource).
        _evaluator.ClearCompiledScripts();
    }

    /// <summary>True when the supplied DriverConfig JSON carries a real body. The bootstrapper always passes a
    /// populated document; unit tests pass <c>"{}"</c> to exercise lifecycle shape — those keep the
    /// constructor-supplied tags (an empty document would otherwise bind to an EMPTY tag table).</summary>
    private static bool HasConfigBody(string? driverConfigJson)
    {
        if (string.IsNullOrWhiteSpace(driverConfigJson)) return false;
        var trimmed = driverConfigJson.Trim();
        return trimmed is not "{}" and not "[]";
    }

    /// <summary>Minimal reinit-bind DTO — the calc driver has no backend config, only the injected raw tags
    /// (+ an optional eval-timeout, ignored on reinit since the evaluator is ctor-fixed). Mirrors the factory's
    /// <c>CalculationDriverConfigDto</c>.</summary>
    private sealed class ReinitConfigDto
    {
        /// <summary>The raw tag entries carried by the re-initialisation config.</summary>
        public List<RawTagEntry>? RawTags { get; init; }
        /// <summary>The optional evaluation timeout in milliseconds; ignored on re-initialisation.</summary>
        public int? RunTimeoutMs { get; init; }
    }

    // ---- IDriver ----

    /// <inheritdoc />
    public string DriverInstanceId => _driverInstanceId;

    /// <inheritdoc />
    public string DriverType => DriverTypeNames.Calculation;

    // InitializeAsync takes a config document and the IDriver contract is that the driver serves it
    // (the runtime's mid-connect adoption path was deleted) - so it adopts a config carrying a real
    // body (the convention every other driver follows), and disposes any running timer scheduler before
    // arming a fresh one (re-initialise without teardown must not leak). A "{}" / empty document keeps
    // the constructor-supplied tags, as unit tests rely on.
    /// <inheritdoc />
    public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
    {
        StopTimers();
        if (HasConfigBody(driverConfigJson)) AdoptConfig(driverConfigJson);
        _running = true;
        StartTimers();
        _logger.LogInformation(
            "Calculation {Driver}: initialized with {TagCount} calc tag(s), {DepCount} distinct dependency ref(s)",
            _driverInstanceId, _tagsByRawPath.Count, _dependencyRefs.Count);
        return Task.CompletedTask;
    }

    /// <inheritdoc />
    public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
    {
        // An in-place config change: re-bind the new config and REBUILD the tag table + dependency-ref set, so
        // added tags compute, removed tags stop publishing and edited scripts serve the new result. The runtime
        // itself applies a DriverConfig change by respawning, where a fresh driver registers its
        // new dependency refs on construction; this path keeps the IDriver contract honest for any caller.
        StopTimers();
        AdoptConfig(driverConfigJson);
        // Re-arm timers off the rebuilt tag set.
        _running = true;
        StartTimers();
        return Task.CompletedTask;
    }

    /// <inheritdoc />
    public Task ShutdownAsync(CancellationToken cancellationToken)
    {
        _running = false;
        StopTimers();
        // The pump exits at its next iteration; an evaluation already in flight is bounded by RunTimeout.
        lock (_lock) _pending.Clear();
        // R07:L2 — nothing in the runtime disposes a driver, and every config change is a respawn, so
        // the compiled scripts' collectible ALCs survived each generation until GC got round to the whole
        // driver graph. Drop them here (idempotent; a later InitializeAsync recompiles on first use). An
        // in-flight run that races this is already handled by the evaluator's retry-once path.
        _evaluator.ClearCompiledScripts();
        return Task.CompletedTask;
    }

    // Current health, derived from the driver's own lifecycle state: Healthy while running; otherwise
    // Unknown, retaining the last compute time. Health is never a hard-coded Healthy - a driver whose
    // timers are stopped (before InitializeAsync, after ShutdownAsync, after Dispose) must not report
    // Healthy, matching every other driver, whose _health field lands on Unknown at shutdown.
    //
    // There is no transport to be Degraded about - this driver computes from the dependency mux, so its
    // only meaningful states are "running" and "not running". _running is the flag the compute paths
    // themselves gate on, which is what makes it the honest input here.
    /// <inheritdoc />
    public DriverHealth GetHealth() => _running
        ? new DriverHealth(DriverState.Healthy, _lastComputeUtc, null)
        : new DriverHealth(DriverState.Unknown, _lastComputeUtc, null);


    // ---- IDependencyConsumer ----

    /// <inheritdoc />
    public IReadOnlyCollection<string> DependencyRefs => _dependencyRefs;

    /// <inheritdoc />
    public void OnDependencyValue(string rawPath, object? value, uint statusCode, DateTime timestampUtc)
    {
        lock (_lock)
        {
            // Recorded REGARDLESS of _running (R07:H6): the mux adapter registers interest at construction,
            // so inputs — the mux's retained-value replay in particular — can land before InitializeAsync.
            // Dropping them left every calc tag waiting for each input to change again; keeping them means
            // the first trigger after init evaluates from current inputs. Only the EVALUATION is gated.
            _depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc);
            _arrivedDeps.Add(rawPath);
            if (!_running) return;
            if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return;

            // QUEUE each change-triggered dependent — never evaluate here. This
            // runs on the caller's thread, which for the live path is the Akka dispatcher thread of
            // Runtime's DependencyConsumerMuxAdapter, and a script run is bounded only by RunTimeout.
            // Everything under this lock is dictionary work; the script runs on the pump.
            var queued = false;
            foreach (var calcRawPath in dependents)
            {
                if (PrepareEvaluation(calcRawPath, timestampUtc) is not { } prepared) continue;
                _pending.Enqueue(prepared);
                queued = true;
            }
            if (queued) SchedulePump();
        }
    }

    // ---- ISubscribable ----

    /// <inheritdoc />
    public Task<ISubscriptionHandle> SubscribeAsync(
        IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(fullReferences);
        var handle = new CalculationSubscriptionHandle($"calc-{Guid.NewGuid():N}");
        // Initial-data callback (OPC UA convention): surface the current snapshot for each subscribed calc
        // tag so a freshly-materialised node shows BadWaitingForInitialData until its first evaluation lands.
        foreach (var reference in fullReferences)
        {
            var snapshot = ReadOne(reference);
            OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, reference, snapshot));
        }
        return Task.FromResult<ISubscriptionHandle>(handle);
    }

    /// <inheritdoc />
    public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken)
        => Task.CompletedTask;

    // ---- IReadable ----

    /// <inheritdoc />
    public Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(
        IReadOnlyList<string> fullReferences, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(fullReferences);
        var results = new DataValueSnapshot[fullReferences.Count];
        for (var i = 0; i < fullReferences.Count; i++)
            results[i] = ReadOne(fullReferences[i]);
        return Task.FromResult<IReadOnlyList<DataValueSnapshot>>(results);
    }

    private DataValueSnapshot ReadOne(string reference)
    {
        var now = DateTime.UtcNow;
        lock (_lock)
        {
            // Gated on HAS PUBLISHED, not on "has a value": a tag whose only publish was an inputs-Bad
            // report has no value of its own, and answering BadWaitingForInitialData for it would tell a
            // polled reader "never observed" while the subscriber on the same node had been told
            // BadNoCommunication. Something HAS been observed about such a tag — that its input is dead.
            if (!_stateByRawPath.TryGetValue(reference, out var st) || !st.HasPublished)
                return new DataValueSnapshot(null, StatusBadWaitingForInitialData, null, now);
            // Exactly what the subscriber was last told — including a propagated input sub-code. Deriving
            // it from a two-state flag here would make a read disagree with the subscription over the
            // same node.
            var status = st.LastPublishedStatus;
            return new DataValueSnapshot(st.LastValue, status, st.LastTimestampUtc, st.LastTimestampUtc);
        }
    }

    // ---- evaluation ----

    /// <summary>One prepared evaluation: the calc tag's definition, the dependency snapshot taken when its
    /// trigger fired, and that trigger's timestamp. Built under <see cref="_lock"/> by
    /// <see cref="PrepareEvaluation"/> and executed outside it, so the script itself needs no shared state.</summary>
    /// <param name="Definition">The calc tag being evaluated.</param>
    /// <param name="Dependencies">Its declared reads' snapshots — value, status and timestamps — as of the trigger.</param>
    /// <param name="TimestampUtc">The trigger's timestamp, stamped on whatever is published.</param>
    /// <param name="WorstInputStatus">The worst status among <paramref name="Dependencies"/>
    /// (<see cref="InputQuality.WorstOf"/>). Bad ⇒ the script is HELD and this status is published;
    /// Good or Uncertain ⇒ the script runs and its result is stamped with it.</param>
    private sealed record PendingEvaluation(
        CalculationTagDefinition Definition,
        IReadOnlyDictionary<string, DataValueSnapshot> Dependencies,
        DateTime TimestampUtc,
        uint WorstInputStatus);

    /// <summary>
    ///     Prepare one calc tag's evaluation: resolve the definition, apply the change-gate, and snapshot
    ///     the tag's declared dependency values. MUST be called under <see cref="_lock"/>. Returns
    ///     <see langword="null"/> when the tag must not evaluate — unknown, no resolved script source
    ///     (logged at init), or still warming up.
    ///     <para>Change-gate (mini-design §4): publish nothing until every declared dep has arrived at
    ///     least once. This MUST hold BEFORE the evaluator runs, not just before the publish: a
    ///     partially-warmed-up snapshot is missing entries for the not-yet-arrived deps, and
    ///     <c>VirtualTagContext.GetTag</c> returns a Bad snapshot (<c>.Value == null</c>) for those — so the
    ///     common <c>(double)ctx.GetTag("x").Value</c> idiom throws <c>NullReferenceException</c> on every
    ///     warm-up tick for any multi-dependency script whose deps don't all arrive in the same tick.
    ///     Evaluating anyway (and discarding the result afterwards) used to turn ordinary warm-up into a
    ///     spurious "script threw" script-log Warning on every partial arrival, plus a wasted
    ///     Roslyn/compiled-cache invocation. Applying it here rather than on the pump is equivalent:
    ///     <see cref="_arrivedDeps"/> only ever grows, and the arrival that completes the set queues its own
    ///     evaluation.</para>
    ///     <para>Snapshotting the dependency values HERE — at trigger time — is what keeps the queued path
    ///     semantically identical to the old inline one: a burst of arrivals evaluates against the values
    ///     that triggered it rather than collapsing onto whatever happened to be current when the pump
    ///     caught up.</para>
    /// </summary>
    /// <param name="rawPath">The calc tag to prepare.</param>
    /// <param name="timestampUtc">The trigger's timestamp.</param>
    /// <returns>The prepared evaluation, or null when the tag must not evaluate.</returns>
    private PendingEvaluation? PrepareEvaluation(string rawPath, DateTime timestampUtc)
    {
        if (!_tagsByRawPath.TryGetValue(rawPath, out var def)) return null;
        if (string.IsNullOrWhiteSpace(def.ScriptSource)) return null; // unresolved script — never computes
        if (!def.DependencyRefs.All(_arrivedDeps.Contains)) return null; // still warming up — do not evaluate

        var deps = new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal);
        foreach (var dep in def.DependencyRefs)
            if (_depValues.TryGetValue(dep, out var v)) deps[dep] = v;

        // Worst-of-input, computed from the SAME snapshot the evaluation will use, so the status
        // published can never disagree with the values it was derived from.
        var worst = InputQuality.WorstOf(def.DependencyRefs.Select(r => deps.GetValueOrDefault(r)));
        return new PendingEvaluation(def, deps, timestampUtc, worst);
    }

    /// <summary>Start the evaluation pump if it is not already running. MUST be called under
    /// <see cref="_lock"/> — the flag and the queue are checked together, which is what makes "an enqueue
    /// either starts the pump or is seen by a pump that has not yet exited" hold.</summary>
    private void SchedulePump()
    {
        if (_pumpScheduled) return;
        _pumpScheduled = true;
        _ = Task.Run(RunEvaluationPumpAsync);
    }

    /// <summary>Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a
    /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated
    /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread
    /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured
    /// false timeouts under pool starvation).</summary>
    private async Task RunEvaluationPumpAsync()
    {
        while (true)
        {
            PendingEvaluation next;
            lock (_lock)
            {
                if (!_running) _pending.Clear();
                if (_pending.Count == 0)
                {
                    _pumpScheduled = false;
                    return;
                }
                next = _pending.Dequeue();
            }

            // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still
            // evaluate one at a time while no host thread can block behind a script.
            if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed
            try
            {
                await RunEvaluationAsync(next).ConfigureAwait(false);
            }
            catch (Exception ex)
            {
                // The evaluator never throws for script faults, so this is a defect path only — it
                // must not kill the pump and strand every queued evaluation behind it.
                _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed",
                    _driverInstanceId, next.Definition.RawPath);
            }
            finally
            {
                _evalGate.Release();
            }
        }
    }

    /// <summary>Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's
    /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task
    /// completes).</summary>
    private async Task EvaluateTagFromTimerAsync(string rawPath)
    {
        if (!_running) return;
        PendingEvaluation? prepared;
        lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow);
        if (prepared is null) return;
        if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed
        try
        {
            await RunEvaluationAsync(prepared).ConfigureAwait(false);
        }
        finally
        {
            _evalGate.Release();
        }
    }

    /// <summary>Acquire <see cref="_evalGate"/>; false once <see cref="Dispose"/> has disposed it (a pump
    /// iteration or timer tick that raced the teardown simply stops).</summary>
    private async Task<bool> TryEnterEvalGateAsync()
    {
        if (_disposed) return false;
        try
        {
            await _evalGate.WaitAsync().ConfigureAwait(false);
        }
        catch (ObjectDisposedException)
        {
            return false;
        }
        if (!_disposed) return true;
        // Dispose landed while we waited (it gave up the gate without disposing it): back out.
        _evalGate.Release();
        return false;
    }

    /// <summary>Run one prepared evaluation and publish the outcome. MUST be called under
    /// <see cref="_evalGate"/> and NOT under <see cref="_lock"/>: the script run is the whole point of the
    /// split, and the script log + <see cref="OnDataChange"/> fan-out both invoke subscriber code.</summary>
    /// <param name="pending">The prepared evaluation.</param>
    private async Task RunEvaluationAsync(PendingEvaluation pending)
    {
        var def = pending.Definition;

        // HOLD ON BAD (matching ScriptedAlarmEngine's AreInputsReady): a Bad input is not
        // usable, so the script does NOT run — which removes the "(double)ctx.GetTag(x).Value threw"
        // storm at its source rather than catching it downstream — and the tag reports the input's own
        // sub-code. Uncertain IS usable: the script runs and the output inherits the Uncertain code.
        if (!InputQuality.IsUsable(pending.WorstInputStatus))
        {
            (object? Value, uint StatusCode)? badPublish;
            lock (_lock)
            {
                if (!_stateByRawPath.TryGetValue(def.RawPath, out var badState)) return;
                badPublish = MarkInputsBad(badState, pending.WorstInputStatus, pending.TimestampUtc);
            }

            // No script log: an input dying is not a script fault, and the script log is the per-failure
            // SCRIPT diagnostic. Here the status code is the diagnostic.
            if (badPublish is { } bad) Publish(def.RawPath, bad.Value, bad.StatusCode, pending.TimestampUtc);
            return;
        }

        // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are
        // then the input's real ones rather than a fabricated Good stamped at evaluation time.
        var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
            .ConfigureAwait(false);

        string? failureToLog = null;
        (object? Value, uint StatusCode)? toPublish;
        lock (_lock)
        {
            // A ReinitializeAsync can drop the tag while its script is running; its result is then stale
            // by definition and there is no state left to record it against.
            if (!_stateByRawPath.TryGetValue(def.RawPath, out var st)) return;

            if (!result.Success)
            {
                var reason = result.Reason ?? "evaluator failure";
                toPublish = MarkEvaluationFailed(st, pending.TimestampUtc);
                // The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes
                // (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which
                // flooded the script log at the input's cadence. The Bad publish was already once per
                // transition; the log now matches it.
                if (toPublish is not null || !string.Equals(st.LastFailureReason, reason, StringComparison.Ordinal))
                    failureToLog = reason;
                st.LastFailureReason = reason;
            }
            else
            {
                st.LastFailureReason = null;
                toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc);
            }
        }

        if (failureToLog is not null) PublishScriptLog(def, failureToLog);
        if (toPublish is { } p) Publish(def.RawPath, p.Value, p.StatusCode, pending.TimestampUtc);
    }

    /// <summary>Record a successful evaluation. MUST be called under <see cref="_lock"/>; returns what to
    /// publish, or null when the result is deduped.
    /// <para><b>The dedup key is (value, status), not value alone</b> — generalising the
    /// Bad-recovery bypass 02/S13 introduced. Recovery is a publish, never an assumption: the node is
    /// currently carrying the previous status, so a result whose VALUE is unchanged but whose STATUS
    /// differs — the shape of every recovery, since a device that comes back usually comes back holding
    /// the same reading — must still land, or the node stays Bad forever.</para></summary>
    /// <param name="st">The tag's published state.</param>
    /// <param name="value">The computed value.</param>
    /// <param name="statusCode">The status to stamp: the worst input status (Good, or an Uncertain sub-code).</param>
    /// <param name="timestampUtc">The trigger's timestamp.</param>
    /// <returns>The snapshot to publish, or null when deduped.</returns>
    private (object? Value, uint StatusCode)? MarkEvaluationSucceeded(
        TagState st, object? value, uint statusCode, DateTime timestampUtc)
    {
        if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && ValuesEqual(st.LastValue, value))
            return null;

        st.HasValue = true;
        st.LastValue = value;
        st.HasPublished = true;
        st.LastPublishedStatus = statusCode;
        st.LastTimestampUtc = timestampUtc;
        _lastComputeUtc = DateTime.UtcNow;
        return (value, statusCode);
    }

    /// <summary>Value equality for the dedup: structural for arrays (R07:H8 — <c>object.Equals</c> is reference
    /// equality for a <c>double[]</c>, so an array-returning timer script republished every tick through
    /// the dual-namespace fan-out and the historian), ordinary <see cref="object.Equals(object?, object?)"/>
    /// otherwise.</summary>
    private static bool ValuesEqual(object? a, object? b)
    {
        if (a is Array x && b is Array y)
            return System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(x, y);
        return Equals(a, b);
    }

    /// <summary>Record that this tag's INPUTS are Bad, so no evaluation ran. Publishes the
    /// worst input's status — carrying the last known value — <b>once per change of that status</b>: a
    /// repeat of the same Bad is silent, while <c>BadNoCommunication</c> arriving over an earlier
    /// <c>BadInternalError</c> (or the reverse) is not masked, because the sub-code is what names which
    /// failure the operator is looking at. MUST be called under <see cref="_lock"/>.</summary>
    /// <param name="st">The tag's published state.</param>
    /// <param name="worstInputStatus">The worst status among the tag's declared inputs.</param>
    /// <param name="timestampUtc">The trigger's timestamp.</param>
    /// <returns>The snapshot to publish, or null when that status was already published.</returns>
    private static (object? Value, uint StatusCode)? MarkInputsBad(
        TagState st, uint worstInputStatus, DateTime timestampUtc)
    {
        if (st.HasPublished && st.LastPublishedStatus == worstInputStatus) return null;

        st.HasPublished = true;
        st.LastPublishedStatus = worstInputStatus;
        st.LastTimestampUtc = timestampUtc;
        // Carry the last-known value (null if none) with the input's Bad status.
        return (st.HasValue ? st.LastValue : null, worstInputStatus);
    }

    /// <summary>Failure tail (mirrors <c>VirtualTagActor</c> 02/S13): the caller always emits the
    /// per-failure script-log; this additionally returns a Bad snapshot — carrying the last-known value —
    /// once per Good→Bad transition, and null on a repeat failure. MUST be called under
    /// <see cref="_lock"/>. Only reached once every declared dependency has arrived (the warm-up gate
    /// returns before the evaluator ever runs), so there is no separate "inputs ready" condition here.</summary>
    /// <param name="st">The tag's published state.</param>
    /// <param name="timestampUtc">The trigger's timestamp.</param>
    /// <returns>The Bad snapshot to publish on a fresh Good→Bad transition, else null.</returns>
    private static (object? Value, uint StatusCode)? MarkEvaluationFailed(TagState st, DateTime timestampUtc)
    {
        if (st.HasPublished && st.LastPublishedStatus == StatusBadInternalError) return null;

        st.HasPublished = true;
        st.LastPublishedStatus = StatusBadInternalError;
        st.LastTimestampUtc = timestampUtc;
        // Carry the last-known value (null if none) with Bad quality.
        return (st.HasValue ? st.LastValue : null, StatusBadInternalError);
    }

    private void Publish(string rawPath, object? value, uint statusCode, DateTime timestampUtc)
    {
        var snapshot = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc);
        // Fired outside both locks, from the single evaluation worker — so publish order still follows
        // evaluation order, and a subscriber can neither deadlock against the state lock nor re-enter the
        // driver behind it (DriverInstanceActor marshals the event onto its own thread regardless).
        OnDataChange?.Invoke(this, new DataChangeEventArgs(SharedHandle, rawPath, snapshot));
    }

    private void PublishScriptLog(CalculationTagDefinition def, string reason)
    {
        // Route through the script logger bound with ScriptId + the calc tag's RawPath (in the VirtualTagId
        // slot the Script-log page attributes on). The ScriptRootLogger's ScriptLogTopicSink converts this
        // Warning event into a ScriptLogEntry on the script-logs DPS topic — the driver needs no Akka access.
        _scriptRoot.Logger
            .ForContext(ScriptLoggerFactory.ScriptIdProperty, def.ScriptId)
            .ForContext(ScriptLoggerFactory.VirtualTagIdProperty, def.RawPath)
            .Warning("{Reason}", reason);
    }

    private void StartTimers()
    {
        var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray();
        if (timed.Length == 0) return;
        _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger);
        _timers.Start(timed);
    }

    private void StopTimers()
    {
        _timers?.Dispose();
        _timers = null;
    }

    /// <summary>Test/diagnostic accessor: number of timer ticks skipped for in-flight groups.</summary>
    internal long TimerSkippedTickCount => _timers?.SkippedTickCount ?? 0;

    /// <summary>Test/diagnostic accessor: sources currently remembered as failing to compile.</summary>
    internal int CompileFailureCountForTest => _evaluator.CompileFailureCount;

    /// <summary>Test/diagnostic accessor: compiled scripts currently held by the evaluator's cache.</summary>
    internal int CompiledScriptCountForTest => _evaluator.CompiledScriptCount;

    /// <summary>Test/diagnostic accessor: <see langword="true"/> when no change-triggered evaluation is
    /// queued or in flight. A test that used to assert straight after <see cref="OnDependencyValue"/> —
    /// which evaluated inline — waits on this instead, so the assertion still runs against a settled
    /// driver rather than a sleep.</summary>
    internal bool IsEvaluationIdle
    {
        get { lock (_lock) return _pending.Count == 0 && !_pumpScheduled; }
    }

    // A single shared handle for host-pushed publishes (change + timer). SubscribeAsync mints its own
    // per-call handle for the initial-data callback; steady-state publishes reuse this so OnDataChange
    // always carries a non-null handle.
    private static readonly CalculationSubscriptionHandle SharedHandle = new("calc-shared");

    /// <summary>Stops the timers and disposes the script evaluator.</summary>
    public void Dispose()
    {
        if (_disposed) return;
        _disposed = true;
        _running = false;
        StopTimers();
        lock (_lock) _pending.Clear();
        // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop
        // the compiled scripts.
        try { _evalCts.Cancel(); } catch (ObjectDisposedException) { }
        _evaluator.Dispose();

        // Then take the gate: once Dispose holds it no evaluation is mid-run (the token — read only under
        // the gate — has been observed), so the gate and the CTS can be disposed. A late pump iteration or
        // timer tick sees _disposed / an ObjectDisposedException in TryEnterEvalGateAsync and stops. Bounded:
        // a run that ignores the cancel cannot hold Dispose; in that case both are left to the GC.
        if (_evalGate.Wait(DisposeGateTimeout))
        {
            _evalCts.Dispose();
            _evalGate.Dispose();
            _evalResourcesDisposed = true;
        }
        else
        {
            _logger.LogWarning("Calculation {Driver}: an evaluation did not release the gate within {Timeout} of Dispose; leaving it to the GC",
                _driverInstanceId, DisposeGateTimeout);
        }
    }

    private static readonly TimeSpan DisposeGateTimeout = TimeSpan.FromSeconds(2);
    private volatile bool _evalResourcesDisposed;

    /// <summary>Test hook — whether <see cref="Dispose"/> disposed the evaluation gate and its CTS.</summary>
    internal bool EvalResourcesDisposedForTest => _evalResourcesDisposed;

    /// <summary>Per-tag published state (guarded by <see cref="_lock"/>).</summary>
    private sealed class TagState
    {
        public bool HasValue;
        public object? LastValue;

        /// <summary>Whether anything has been published for this tag yet. Distinct from
        /// <see cref="HasValue"/>: an inputs-Bad publish carries no value of its own.</summary>
        public bool HasPublished;

        /// <summary>The status code last published for this tag. Replaces the old two-state
        /// <c>LastPublishedBad</c> flag: the dedup key is (value, status), and a Bad status
        /// now carries the failing INPUT's sub-code rather than a fixed <c>BadInternalError</c>.</summary>
        public uint LastPublishedStatus;
        public DateTime LastTimestampUtc;

        /// <summary>The reason of the last failed evaluation (null after a success / inputs-Bad hold) — the
        /// gate that keeps the script-log entry to the transition and to a CHANGE of reason.</summary>
        public string? LastFailureReason;
    }

    private sealed record CalculationSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle;
}