using Akka.Actor;
using Akka.TestKit;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// Test group 6 — the per-script in-flight cap and its shed policy.
///
/// <para>Previously every trigger spawned another run unconditionally: a trigger firing
/// faster than its script completes produced unbounded fan-out onto a bounded thread pool.
/// The cap (<see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>, default 4) sheds the
/// NEWEST run instead. Keeping the four already queued/running — which are closest to their
/// own deadlines and already charged against them — is the policy that never reorders runs
/// and needs no queue at all: the scheduler's FIFO already IS the queue.</para>
///
/// <para>A shed is always counted on the health collector, emits a site event rate-limited to
/// one per script per minute (so a hot trigger cannot flood <c>site_events</c>), and — for an
/// Ask-based <c>CallScript</c> — replies with an explicit error rather than letting a nested
/// call or inbound-API route hang to its Ask timeout.</para>
/// </summary>
public class ScriptRunShedTests : TestKit, IDisposable
{
    private readonly SharedScriptLibrary _sharedLibrary;
    private readonly ScriptExecutionScheduler _scheduler = new(8);

    /// <summary>Creates the shared script library and the gate that blocks scripts so runs stay in flight.</summary>
    public ScriptRunShedTests()
    {
        var compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedLibrary = new SharedScriptLibrary(
            compilationService, NullLogger<SharedScriptLibrary>.Instance);
        ShedHooks.Gate = new SemaphoreSlim(0);
    }

    /// <inheritdoc />
    void IDisposable.Dispose()
    {
        ShedHooks.Gate.Release(64);
        Shutdown();
        _scheduler.Dispose();
    }

    private static Script<object?> BlockingScript() => CompileRaw(
        "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.ShedHooks.Gate.Wait(); return null;");

    private static Script<object?> CompileRaw(string code)
    {
        var options = ScriptOptions.Default
            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
                typeof(ShedHooks).Assembly)
            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
        var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
        script.Compile();
        return script;
    }

    private static SiteRuntimeOptions Options() => new()
    {
        MaxConcurrentRunsPerScript = 4,
        // Long enough that nothing times out inside the test window — the cap, not the
        // deadline, must be what refuses the fifth run.
        ScriptExecutionTimeoutSeconds = 120,
        StuckScriptGraceMs = 120_000
    };

    /// <summary>Verifies that a fifth concurrent run is shed, counted and reported as a site event once, and the caller is answered with an error.</summary>
    [Fact]
    public void FifthConcurrentRun_IsShed_Counted_EventedOnce_AndAnsweredWithAnError()
    {
        var siteLog = new FakeSiteEventLogger();
        var health = new SiteHealthCollector();
        var instance = CreateTestProbe().Ref;
        var options = Options();

        var actor = ActorOfAsTestActorRef<ScriptActor>(
            Props.Create(() => new ScriptActor(
                "Hot", "Inst1", instance, BlockingScript(),
                new ResolvedScript { CanonicalName = "Hot", TriggerType = "Call" },
                _sharedLibrary, options, NullLogger<ScriptActor>.Instance,
                null, null, health, new SingleServiceProvider(siteLog), _scheduler, null)),
            "shed-" + Guid.NewGuid().ToString("N"));

        // Fill the cap: four runs, all blocked in their bodies.
        for (var i = 0; i < 4; i++)
            actor.Tell(new ScriptCallRequest("Hot", null, 0, $"corr-{i}"), ActorRefs.NoSender);

        AwaitAssert(() =>
        {
            Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
            Assert.Equal(4, _scheduler.BusyThreadCount);
        }, TimeSpan.FromSeconds(15));

        // Fifth: shed. The Ask caller is answered explicitly instead of hanging.
        var caller = CreateTestProbe();
        actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-shed-1"), caller.Ref);

        var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
        Assert.False(result.Success);
        Assert.Equal("corr-shed-1", result.CorrelationId);
        Assert.Contains("shed", result.ErrorMessage, StringComparison.OrdinalIgnoreCase);
        Assert.Contains("in flight", result.ErrorMessage, StringComparison.OrdinalIgnoreCase);

        // Still exactly four in flight — the shed run was never launched.
        Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);

        // Sixth: counted again, but the Warning site event is rate-limited to one per minute.
        actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-shed-2"), caller.Ref);
        var second = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
        Assert.False(second.Success);

        AwaitAssert(() =>
        {
            var shedEvents = siteLog.OfType("script")
                .Where(r => r.Severity == "Warning" && r.Message.Contains("shed"))
                .ToArray();
            Assert.Single(shedEvents);
            Assert.Equal("ScriptActor:Hot", shedEvents[0].Source);
            Assert.Equal("Inst1", shedEvents[0].InstanceId);
        }, TimeSpan.FromSeconds(5));

        // Both sheds were counted on the health report even though only one was evented.
        Assert.Equal(2, health.CollectReport("site-1").ScriptRunShedCount);

        // One completion frees a slot, and the next trigger launches again.
        ShedHooks.Gate.Release();
        AwaitAssert(() => Assert.Equal(3, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));

        actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-after"), ActorRefs.NoSender);
        AwaitAssert(() => Assert.Equal(4, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
    }

    /// <summary>Verifies that alarm on-trigger runs are shed at the same cap and produce an alarm-scoped site event.</summary>
    [Fact]
    public void AlarmOnTriggerRuns_ShedAtTheSameCap_WithAnAlarmScopedSiteEvent()
    {
        var siteLog = new FakeSiteEventLogger();
        var health = new SiteHealthCollector();
        var instanceProbe = CreateTestProbe();
        var options = Options();

        var alarm = ActorOfAsTestActorRef<AlarmActor>(
            Props.Create(() => new AlarmActor(
                "Flapper", "Inst1", instanceProbe.Ref,
                new ResolvedAlarm
                {
                    CanonicalName = "Flapper",
                    TriggerType = "ValueMatch",
                    TriggerConfiguration = "{\"attributeName\":\"Status\",\"matchValue\":\"Critical\"}",
                    PriorityLevel = 100
                },
                BlockingScript(), _sharedLibrary, options, NullLogger<AlarmActor>.Instance,
                null, null, health, new SingleServiceProvider(siteLog), null, _scheduler, null)),
            "alarm-shed-" + Guid.NewGuid().ToString("N"));

        // Each raise edge spawns one on-trigger run; clear between raises to re-arm the edge.
        void Flap(int cycle)
        {
            alarm.Tell(new AttributeValueChanged(
                "Inst1", "Status", "Status", "Critical", "Good", DateTimeOffset.UtcNow.AddSeconds(cycle)));
            alarm.Tell(new AttributeValueChanged(
                "Inst1", "Status", "Status", "Normal", "Good", DateTimeOffset.UtcNow.AddSeconds(cycle)));
        }

        for (var i = 0; i < 4; i++) Flap(i);
        AwaitAssert(() => Assert.Equal(4, alarm.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));

        // Fifth raise is shed — there is no Ask caller on this path, so it surfaces purely as
        // a counter plus the rate-limited Warning event.
        Flap(4);
        Flap(5);

        // The shed COUNT assertion lives inside the awaited block, not after it.
        // ShedAlarmRun increments the counter and only then emits the (rate-limited)
        // site event, and that event fires on the FIRST shed only — so an awaited
        // gate on the event observes Flap(4)'s shed and orders nothing whatsoever
        // with respect to Flap(5)'s, which is a separate mailbox message with no
        // observable of its own on this path (an alarm on-trigger run has no Ask
        // caller to reply to — AlarmActor.ShedAlarmRun). Asserting the count bare
        // after the gate assumed both sheds had been dequeued by the time the
        // first one's event landed; under load the second can still be pending and
        // the count reads 1. Reproduced deterministically by deferring Flap(5) by
        // 2 s: the bare form failed with "Expected: 2 / Actual: 1", while the
        // ScriptActor sibling above — which IS ordered, because ShedRun replies
        // ScriptCallResult after incrementing — kept passing.
        //
        // The count is ACCUMULATED across polls rather than re-read, because
        // SiteHealthCollector.CollectReport DRAINS the interval counters
        // (Interlocked.Exchange(ref _scriptRunShedCount, 0)) — a poll loop that
        // simply re-read it would consume the first shed and never see 2.
        var shedCounted = 0;
        AwaitAssert(() =>
        {
            shedCounted += health.CollectReport("site-1").ScriptRunShedCount;

            var shedEvents = siteLog.OfType("script")
                .Where(r => r.Severity == "Warning" && r.Message.Contains("shed"))
                .ToArray();

            // Both sheds counted, even though the event is rate-limited to one.
            Assert.Equal(2, shedCounted);
            Assert.Single(shedEvents);
            Assert.Equal("AlarmActor:Flapper", shedEvents[0].Source);

            // ...and neither shed run was ever launched: still exactly four in flight.
            Assert.Equal(4, alarm.UnderlyingActor.RunsInFlight);
        }, TimeSpan.FromSeconds(10));
    }

    // ---------------------------------------------------------------------------------------
    // Arch-review F7: a TRACED run refused by either guard (the in-flight cap, or
    // MinTimeBetweenRuns) used to emit nothing at all, so the Flow Debugger reported
    // NO_TRIGGER_WITHIN_WINDOW — "no value-change trigger on this attribute, or no device echo"
    // — which is wrong in both cases and sends the operator to check the wrong things. Each
    // guard now leaves a Failed ScriptTrace/ScriptRun row naming the guard that refused it.
    // ---------------------------------------------------------------------------------------

    /// <summary>Audit writer fake: records every emitted row and can wait for one matching a predicate.</summary>
    private sealed class CapturingAuditWriter : IAuditWriter
    {
        private readonly List<AuditEvent> _events = new();

        /// <inheritdoc />
        public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
        {
            lock (_events) _events.Add(evt);
            return Task.CompletedTask;
        }

        /// <summary>Takes a copy of the rows captured so far.</summary>
        /// <returns>An array of the captured audit events.</returns>
        public AuditEvent[] Snapshot()
        {
            lock (_events) return _events.ToArray();
        }

        /// <summary>Polls until a captured row matches the predicate or the timeout expires.</summary>
        /// <param name="predicate">The condition a row must satisfy.</param>
        /// <param name="timeout">The maximum time to wait.</param>
        /// <returns>The first matching audit event, or null if none matched in time.</returns>
        public AuditEvent? WaitFor(Func<AuditEvent, bool> predicate, TimeSpan timeout)
        {
            var deadline = DateTime.UtcNow + timeout;
            while (DateTime.UtcNow < deadline)
            {
                lock (_events)
                {
                    var hit = _events.FirstOrDefault(predicate);
                    if (hit is not null) return hit;
                }
                Thread.Sleep(25);
            }
            return null;
        }
    }

    private static IServiceProvider AuditServices(ISiteEventLogger siteLog, IAuditWriter writer)
    {
        var services = new ServiceCollection();
        services.AddSingleton(siteLog);
        services.AddSingleton(writer);
        return services.BuildServiceProvider();
    }

    private static ResolvedScript ValueChangeScript(string name, TimeSpan? minTimeBetweenRuns = null) => new()
    {
        CanonicalName = name,
        TriggerType = "valuechange",
        TriggerConfiguration = "{\"attribute\":\"Status\"}",
        MinTimeBetweenRuns = minTimeBetweenRuns
    };

    private static bool IsFailedScriptRun(AuditEvent evt) =>
        evt.Action == "ScriptTrace.ScriptRun" && evt.AsRow().Status.ToString() == "Failed";

    /// <summary>Verifies that a traced run shed at the cap emits a failed script-run row parented to the writer.</summary>
    [Fact]
    public void TracedRun_ShedAtTheCap_EmitsAFailedScriptRunRow_ParentedToTheWriter()
    {
        var siteLog = new FakeSiteEventLogger();
        var writer = new CapturingAuditWriter();
        var health = new SiteHealthCollector();
        var instance = CreateTestProbe().Ref;
        var options = Options();

        var actor = ActorOfAsTestActorRef<ScriptActor>(
            Props.Create(() => new ScriptActor(
                "Hot", "Inst1", instance, BlockingScript(),
                ValueChangeScript("Hot"),
                _sharedLibrary, options, NullLogger<ScriptActor>.Instance,
                null, null, health, AuditServices(siteLog, writer), _scheduler, null)),
            "shed-traced-" + Guid.NewGuid().ToString("N"));

        // Fill the cap with four untraced runs, all blocked in their bodies.
        for (var i = 0; i < 4; i++)
            actor.Tell(new ScriptCallRequest("Hot", null, 0, $"corr-{i}"), ActorRefs.NoSender);

        AwaitAssert(() => Assert.Equal(4, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));

        // The fifth trigger is a TRACED value change stamped by its writing execution.
        var parent = Guid.NewGuid();
        actor.Tell(new AttributeValueChanged(
            "Inst1", "Status", "Status", "Running", "Good", DateTimeOffset.UtcNow,
            SourceExecutionId: parent, Trace: true));

        var evt = writer.WaitFor(IsFailedScriptRun, TimeSpan.FromSeconds(10));
        Assert.NotNull(evt);
        var row = evt!.AsRow();
        Assert.Equal(parent, row.ParentExecutionId);
        Assert.NotNull(row.ExecutionId);
        Assert.StartsWith("shed: MaxConcurrentRunsPerScript", row.ErrorMessage);
        Assert.Equal("Hot", evt.Target);
        Assert.Equal("Inst1", row.SourceInstanceId);
        // The trigger detail survives, so the debugger can still name the attribute.
        var extra = ScriptTraceExtra.TryParse(row.Extra)!;
        Assert.Equal("valuechange", extra.TriggerKind);
        Assert.Equal("Status", extra.TriggerAttribute);

        // Still exactly four in flight — the shed run was never launched.
        Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
    }

    /// <summary>Verifies that a traced run suppressed by the minimum time between runs emits a failed script-run row.</summary>
    [Fact]
    public void TracedRun_SuppressedByMinTimeBetweenRuns_EmitsAFailedScriptRunRow()
    {
        var siteLog = new FakeSiteEventLogger();
        var writer = new CapturingAuditWriter();
        var health = new SiteHealthCollector();
        var instance = CreateTestProbe().Ref;
        var options = Options();

        var actor = ActorOfAsTestActorRef<ScriptActor>(
            Props.Create(() => new ScriptActor(
                "Throttled", "Inst1", instance, CompileRaw("return null;"),
                ValueChangeScript("Throttled", TimeSpan.FromMinutes(10)),
                _sharedLibrary, options, NullLogger<ScriptActor>.Instance,
                null, null, health, AuditServices(siteLog, writer), _scheduler, null)),
            "throttled-traced-" + Guid.NewGuid().ToString("N"));

        // First traced change runs (and emits a Delivered row); the second, back to back,
        // is inside MinTimeBetweenRuns and never launches.
        var parent = Guid.NewGuid();
        for (var i = 0; i < 2; i++)
        {
            actor.Tell(new AttributeValueChanged(
                "Inst1", "Status", "Status", $"Run{i}", "Good", DateTimeOffset.UtcNow,
                SourceExecutionId: parent, Trace: true));
        }

        var evt = writer.WaitFor(IsFailedScriptRun, TimeSpan.FromSeconds(10));
        Assert.NotNull(evt);
        var row = evt!.AsRow();
        Assert.Equal(parent, row.ParentExecutionId);
        Assert.StartsWith("suppressed: MinTimeBetweenRuns", row.ErrorMessage);

        // The first run still produced its ordinary Delivered row — the guard row is additive.
        AwaitAssert(() => Assert.Contains(
            writer.Snapshot(),
            e => e.Action == "ScriptTrace.ScriptRun" && e.AsRow().Status.ToString() == "Delivered"),
            TimeSpan.FromSeconds(10));
    }
}

/// <summary>Test hook used to hold script runs in flight while the cap is exercised.</summary>
public static class ShedHooks
{
    /// <summary>Gate the blocking test scripts wait on; reset per test.</summary>
    public static SemaphoreSlim Gate = new(0);
}