using System.Collections.Concurrent;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;

/// <summary>
/// The outcome of a <see cref="ScriptExecutionScheduler.TryDetachWorker"/> attempt.
/// </summary>
public enum WorkerDetachOutcome
{
    /// <summary>
    /// The recorded slot/run pair no longer identifies a running task (the script finished,
    /// or it never held a worker because it hopped threads on an <c>await</c>). Nothing was
    /// detached and nothing needed to be — the worker is not lost.
    /// </summary>
    NotRunning,

    /// <summary>The worker was marked detached and a replacement thread was started.</summary>
    Detached,

    /// <summary>
    /// The number of live detached threads already equals the pool size, so no replacement was
    /// started. Bounded starvation is preferable to unbounded thread growth when scripts wedge
    /// en masse; the caller is expected to log this loudly.
    /// </summary>
    AtCap
}

/// <summary>
/// A dedicated, grow-only <see cref="TaskScheduler"/> for running script
/// and alarm on-trigger bodies.
///
/// Script bodies may perform synchronous blocking I/O (a database connection, a
/// synchronous external-system call). Running them on the shared .NET
/// <see cref="ThreadPool"/> lets a burst of blocking scripts starve the pool and stall
/// unrelated Akka dispatchers and HTTP request handling. This scheduler owns a set
/// of dedicated threads, so script blocking is contained to those threads and cannot
/// exhaust the global pool.
///
/// <para>WP3.1 changed three things about that pool:</para>
/// <list type="number">
///   <item>It is no longer fixed-size. <see cref="EnsureCapacity"/> grows it towards
///   <see cref="ComputeTargetThreads"/> (instance-scaled, clamped between the configured
///   floor and ceiling). It is deliberately <em>grow-only</em>: undeploying instances
///   leaves idle threads, which cost nothing measurable and avoid drain/steal complexity.</item>
///   <item>A worker whose task has wedged in uninterruptible blocking I/O can be
///   <see cref="TryDetachWorker">detached</see> and replaced, so a stuck script no longer
///   permanently costs the pool a thread. Live detached threads are capped at the pool size.</item>
///   <item>Trigger-expression evaluations no longer run here at all — they are non-blocking
///   by construction and now run on the shared thread pool behind
///   <see cref="TriggerEvalGate"/>, so an alarm's Expression trigger can never queue behind
///   eight blocking script bodies (finding #4).</item>
/// </list>
///
/// The scheduler is process-wide (one set of threads for all instances) and is sized
/// from <see cref="SiteRuntimeOptions"/> the first time it is configured.
/// </summary>
public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
{
    /// <summary>
    /// Deployed instances per script-execution thread used by <see cref="ComputeTargetThreads"/>.
    /// A named constant rather than an option: no deployment needs to tune the ratio
    /// independently of the floor (<see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/>)
    /// and the ceiling (<see cref="SiteRuntimeOptions.ScriptExecutionMaxThreadCount"/>).
    /// </summary>
    internal const int InstancesPerScriptThread = 8;

    /// <summary>Thread-name prefix; also the inlining guard in <see cref="TryExecuteTaskInline"/>.</summary>
    private const string ThreadNamePrefix = "script-execution-";

    private readonly BlockingCollection<Task> _queue = new();

    /// <summary>
    /// Immutable-on-read snapshot array of worker slots. Growth (EnsureCapacity, detach
    /// replacement) publishes a NEW longer array under <see cref="_growLock"/>; existing
    /// slot objects are carried over by reference so a worker's index stays stable for its
    /// whole life. Readers take one volatile read and then work on that snapshot.
    /// </summary>
    private volatile WorkerSlot[] _slots;

    /// <summary>Guards every mutation of <see cref="_slots"/>, <see cref="_configuredCount"/>, and the detach bookkeeping.</summary>
    private readonly object _growLock = new();

    /// <summary>
    /// Number of NON-detached workers — i.e. the pool's nominal size. Unchanged by a
    /// detach (each detach starts a replacement), grown only by <see cref="EnsureCapacity"/>.
    /// </summary>
    private int _configuredCount;

    /// <summary>Detached workers that have not yet finished their wedged task and exited.</summary>
    private int _detachedLive;

    /// <summary>Monotonic run identity; see <see cref="WorkerSlot.RunStamp"/>.</summary>
    private long _runStampSeed;

    private int _disposed;

    /// <summary>
    /// The slot index of the script-execution worker running the current thread, or null on
    /// any other thread. Captured by the first synchronous segment of a script body so the
    /// stuck-script watchdog can identify — and replace — the exact worker a wedged script
    /// is holding. A script that hopped threads on an <c>await</c> is no longer on its worker,
    /// which is correct: it holds no thread and must not cause a detach.
    /// </summary>
    [ThreadStatic]
    internal static int? CurrentWorkerSlot;

    private static volatile ScriptExecutionScheduler? _shared;
    private static readonly object SharedLock = new();

    /// <summary>
    /// The process-wide script-execution scheduler, used as the default when no scheduler
    /// is injected. Lazily created on first use with the thread count from
    /// <see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/> (the floor — the Deployment
    /// Manager grows it from there as instances deploy); the first caller wins, subsequent
    /// calls reuse the existing instance.
    ///
    /// If the cached instance has been disposed it is recreated rather than handed back:
    /// a disposed scheduler can execute no work, so returning it would silently poison
    /// every subsequent script and alarm execution for the rest of the process. Nothing
    /// disposes the shared instance today, but the guard keeps that latent failure mode
    /// closed regardless of a future shutdown path or a multi-site host.
    /// </summary>
    /// <param name="options">Site runtime options supplying the thread count for the scheduler.</param>
    /// <returns>A live process-wide <see cref="ScriptExecutionScheduler"/> instance, creating it on first call.</returns>
    public static ScriptExecutionScheduler Shared(SiteRuntimeOptions options)
    {
        var existing = _shared;
        if (existing is { IsDisposed: false })
            return existing;

        lock (SharedLock)
        {
            if (_shared is null || _shared.IsDisposed)
                _shared = new ScriptExecutionScheduler(options.ScriptExecutionThreadCount);
            return _shared;
        }
    }

    /// <summary>
    /// Pure sizing function for the blocking script-execution pool: the configured floor,
    /// raised to one thread per <see cref="InstancesPerScriptThread"/> enabled instances,
    /// clamped to the configured ceiling. Static and side-effect-free so the policy is
    /// unit-testable without starting a single thread.
    /// </summary>
    /// <param name="enabledInstances">Number of currently-running (enabled) Instance Actors.</param>
    /// <param name="options">Site runtime options supplying the floor and ceiling.</param>
    /// <returns>The target worker-thread count, always at least 1.</returns>
    public static int ComputeTargetThreads(int enabledInstances, SiteRuntimeOptions options)
    {
        var floor = Math.Max(1, options.ScriptExecutionThreadCount);
        // A ceiling below the floor is rejected by SiteRuntimeOptionsValidator; clamp here
        // too so a directly-constructed options object can never invert the range.
        var ceiling = Math.Max(floor, options.ScriptExecutionMaxThreadCount);
        var scaled = enabledInstances <= 0
            ? 0
            : (int)Math.Ceiling(enabledInstances / (double)InstancesPerScriptThread);
        return Math.Clamp(Math.Max(floor, scaled), 1, ceiling);
    }

    /// <summary>
    /// Creates a scheduler backed by <paramref name="threadCount"/> dedicated threads.
    /// </summary>
    /// <param name="threadCount">Initial number of dedicated worker threads to create.</param>
    public ScriptExecutionScheduler(int threadCount)
    {
        if (threadCount < 1)
            threadCount = 1;

        _slots = [];
        lock (_growLock)
        {
            GrowTo(threadCount);
        }
    }

    /// <summary><see langword="true"/> once <see cref="Dispose"/> has run; a disposed scheduler
    /// can execute no further work and must not be handed out by <see cref="Shared"/>.</summary>
    public bool IsDisposed => Volatile.Read(ref _disposed) != 0;

    /// <inheritdoc />
    public override int MaximumConcurrencyLevel => Volatile.Read(ref _configuredCount);

    /// <summary>Number of tasks waiting in the queue (not counting those currently executing).</summary>
    public int QueueDepth => _queue.Count;

    /// <summary>
    /// Workers whose task wedged past its deadline plus the stuck-script grace, which have been
    /// detached and replaced but have not yet returned. Surfaced on the site health report as
    /// <c>DetachedScriptThreads</c>: a non-zero, non-draining value means script bodies are
    /// permanently blocking threads.
    /// </summary>
    public int DetachedThreadCount => Volatile.Read(ref _detachedLive);

    /// <summary>Number of worker threads currently executing a task (detached workers included — they really are busy).</summary>
    public int BusyThreadCount
    {
        get
        {
            var slots = _slots;
            var count = 0;
            foreach (var slot in slots)
                if (Volatile.Read(ref slot.BusySinceTicks) != 0) count++;
            return count;
        }
    }

    /// <summary>
    /// Age of the longest-running in-flight task across all workers, or null when
    /// the pool is idle. A large value with a full <see cref="BusyThreadCount"/>
    /// indicates a stuck/blocked script holding a dedicated thread.
    /// </summary>
    public TimeSpan? OldestBusyAge
    {
        get
        {
            var now = Environment.TickCount64;
            var slots = _slots;
            long oldestSince = 0;
            foreach (var slot in slots)
            {
                var since = Volatile.Read(ref slot.BusySinceTicks);
                if (since != 0 && (oldestSince == 0 || since < oldestSince))
                    oldestSince = since;
            }
            return oldestSince == 0 ? null : TimeSpan.FromMilliseconds(Math.Max(0, now - oldestSince));
        }
    }

    /// <summary>
    /// Grows the pool to <paramref name="target"/> non-detached workers. Idempotent and
    /// never shrinking: a target at or below the current size is a no-op. Called from
    /// <c>DeploymentManagerActor.UpdateInstanceCounts</c> on every deploy/undeploy/enable/
    /// disable and per staggered startup batch.
    /// </summary>
    /// <param name="target">Desired number of non-detached worker threads.</param>
    /// <returns>The pool's non-detached worker count after the call.</returns>
    public int EnsureCapacity(int target)
    {
        if (IsDisposed) return Volatile.Read(ref _configuredCount);
        if (target <= Volatile.Read(ref _configuredCount)) return Volatile.Read(ref _configuredCount);

        lock (_growLock)
        {
            if (IsDisposed || target <= _configuredCount) return _configuredCount;
            GrowTo(target - _configuredCount);
            return _configuredCount;
        }
    }

    /// <summary>
    /// The identity stamp of the task currently running on <paramref name="slot"/>, or 0 when
    /// that worker is idle. Captured alongside the slot index at script-body start and handed
    /// back to <see cref="TryDetachWorker"/>, which detaches only if the SAME task is still
    /// running. A monotonic counter rather than a timestamp: two runs on one worker inside the
    /// same <see cref="Environment.TickCount64"/> tick would otherwise be indistinguishable.
    /// </summary>
    /// <param name="slot">The worker slot index.</param>
    /// <returns>The current run stamp, or 0 when the slot is idle or out of range.</returns>
    internal long CurrentRunStamp(int slot)
    {
        var slots = _slots;
        return slot >= 0 && slot < slots.Length ? Volatile.Read(ref slots[slot].RunStamp) : 0L;
    }

    /// <summary>
    /// Detaches the worker at <paramref name="slot"/> — if it is still running the task
    /// identified by <paramref name="observedRunStamp"/> — and starts a replacement thread, so
    /// a script wedged in uninterruptible blocking I/O no longer costs the pool a thread
    /// permanently. The detached worker exits (instead of pulling more work) as soon as its
    /// wedged task finally returns, so capacity never silently doubles-and-drains.
    /// </summary>
    /// <param name="slot">The worker slot recorded when the script body started.</param>
    /// <param name="observedRunStamp">The run stamp recorded at the same moment.</param>
    /// <returns>What was done; see <see cref="WorkerDetachOutcome"/>.</returns>
    internal WorkerDetachOutcome TryDetachWorker(int slot, long observedRunStamp)
    {
        if (observedRunStamp == 0 || IsDisposed) return WorkerDetachOutcome.NotRunning;

        lock (_growLock)
        {
            if (IsDisposed) return WorkerDetachOutcome.NotRunning;

            var slots = _slots;
            if (slot < 0 || slot >= slots.Length) return WorkerDetachOutcome.NotRunning;

            var worker = slots[slot];
            // Same task still running on that exact worker? If the stamp moved (or went to 0)
            // the script finished or never held this thread — there is nothing lost to replace.
            if (Volatile.Read(ref worker.RunStamp) != observedRunStamp) return WorkerDetachOutcome.NotRunning;
            if (Volatile.Read(ref worker.Detached) != 0) return WorkerDetachOutcome.NotRunning;

            // Bound: never hold more than 2x threads (N wedged + N live). Volatile read —
            // the decrement side runs outside _growLock (see below).
            if (Volatile.Read(ref _detachedLive) >= _configuredCount) return WorkerDetachOutcome.AtCap;

            Volatile.Write(ref worker.Detached, 1);
            // Interlocked, NOT ++: the matching decrement in WorkerLoop is lock-free (the
            // exiting worker never touches _growLock), so a plain read-modify-write here can
            // lose that concurrent decrement. The drift is upward-only and permanent — the
            // gauge over-reports and TryDetachWorker starts returning AtCap while real
            // capacity is available. The surrounding sequence keeps the lock; only this one
            // field is shared with a lock-free writer.
            Interlocked.Increment(ref _detachedLive);
            _configuredCount--;   // the detached worker no longer counts towards the pool …
            GrowTo(1);            // … and GrowTo puts the count back by starting its replacement.
            return WorkerDetachOutcome.Detached;
        }
    }

    /// <summary>Appends <paramref name="count"/> new worker slots + threads. Caller holds <see cref="_growLock"/>.</summary>
    private void GrowTo(int count)
    {
        var existing = _slots;
        var grown = new WorkerSlot[existing.Length + count];
        Array.Copy(existing, grown, existing.Length);

        for (var i = 0; i < count; i++)
        {
            var index = existing.Length + i;
            grown[index] = new WorkerSlot();
        }

        // Publish the array BEFORE starting the threads: a worker's very first action is to
        // index its own slot, which must already be visible on the published snapshot.
        _slots = grown;
        _configuredCount += count;

        for (var i = 0; i < count; i++)
        {
            var index = existing.Length + i;
            var thread = new Thread(() => WorkerLoop(index))
            {
                IsBackground = true,
                Name = ThreadNamePrefix + index
            };
            grown[index].Thread = thread;
            thread.Start();
        }
    }

    private void WorkerLoop(int index)
    {
        CurrentWorkerSlot = index;
        var slot = _slots[index];
        try
        {
            foreach (var task in _queue.GetConsumingEnumerable())
            {
                Volatile.Write(ref slot.RunStamp, Interlocked.Increment(ref _runStampSeed));
                Volatile.Write(ref slot.BusySinceTicks, Environment.TickCount64);
                try
                {
                    TryExecuteTask(task);
                }
                finally
                {
                    Volatile.Write(ref slot.BusySinceTicks, 0);
                    Volatile.Write(ref slot.RunStamp, 0);
                }

                // Detached while this task ran: a replacement worker is already live, so
                // exit rather than pulling more work — otherwise capacity would silently
                // double once the wedged script finally returned.
                if (Volatile.Read(ref slot.Detached) != 0)
                {
                    Interlocked.Decrement(ref _detachedLive);
                    return;
                }
            }
        }
        catch (ObjectDisposedException)
        {
            // Scheduler disposed — worker exits.
        }
        finally
        {
            CurrentWorkerSlot = null;
        }
    }

    /// <inheritdoc />
    protected override void QueueTask(Task task) => _queue.Add(task);

    /// <inheritdoc />
    protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued)
    {
        // Only inline if we are already on one of this scheduler's worker threads,
        // so script work never escapes onto a thread-pool thread.
        if (Thread.CurrentThread.Name?.StartsWith(ThreadNamePrefix, StringComparison.Ordinal) != true)
            return false;

        return TryExecuteTask(task);
    }

    /// <inheritdoc />
    protected override IEnumerable<Task> GetScheduledTasks() => _queue.ToArray();

    /// <summary>Signals the worker threads to stop and waits for them to drain, then disposes the queue.</summary>
    public void Dispose()
    {
        if (Interlocked.Exchange(ref _disposed, 1) != 0)
            return;

        _queue.CompleteAdding();
        foreach (var slot in _slots)
            slot.Thread?.Join(TimeSpan.FromSeconds(5));
        _queue.Dispose();
    }

    /// <summary>
    /// Per-worker bookkeeping. One instance per slot index, created once and carried by
    /// reference across every <see cref="_slots"/> growth so a worker's state survives
    /// the array being replaced.
    /// </summary>
    private sealed class WorkerSlot
    {
        /// <summary>
        /// <see cref="Environment.TickCount64"/> at which the worker picked up its current
        /// task, 0 when idle. Written by the worker, read lock-free by the gauges.
        /// </summary>
        public long BusySinceTicks;

        /// <summary>
        /// Monotonic identity of the task currently running on this worker, 0 when idle.
        /// Distinguishes two runs that start inside the same clock tick, which
        /// <see cref="BusySinceTicks"/> alone cannot.
        /// </summary>
        public long RunStamp;

        /// <summary>Non-zero once the worker has been detached; it exits after its current task.</summary>
        public int Detached;

        /// <summary>The worker thread, for <see cref="Dispose"/> to join.</summary>
        public Thread? Thread;
    }
}
