lmxopcua/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs

using Serilog;
using Serilog.Core;
using Serilog.Events;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;

namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;

/// <summary>
///     <see cref="ScriptRunGuard"/> — the per-script run quarantine shared by the virtual-tag evaluator, the
///     Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so
///     before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard
///     refuses to start a run while the previous one is still executing, quarantines a script after three
///     consecutive timeouts (a refusal is not a strike), and quarantines a script whose single run allocates past the budget.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptRunGuardTests
{
    /// <summary>
    ///     A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool
    ///     thread it leaves behind frees itself after the test). <c>System.DateTime</c> is outside the sandbox
    ///     deny-list.
    /// </summary>
    private const string SpinScript =
        """
        var end = System.DateTime.UtcNow.AddSeconds(6);
        while (System.DateTime.UtcNow < end) { }
        return 1;
        """;

    private const string FastScript = "return 42;";

    /// <summary>Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts.</summary>
    private const string BriefSpinScript =
        """
        var end = System.DateTime.UtcNow.AddMilliseconds(250);
        while (System.DateTime.UtcNow < end) { }
        return 1;
        """;

    private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60);

    private static ScriptEvaluator<FakeScriptContext, int> Compile(string source) =>
        ScriptEvaluator<FakeScriptContext, int>.Compile(source);

    private static Task<int> Run(ScriptRunGuard guard, string key, string source,
        ScriptEvaluator<FakeScriptContext, int> evaluator, TimeSpan? timeout = null, CancellationToken ct = default) =>
        guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct);

    [Fact]
    public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot()
    {
        var guard = new ScriptRunGuard();
        using var fast = Compile(FastScript);

        var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken);

        value.ShouldBe(42);
        guard.InFlightCount.ShouldBe(0);
        guard.StartedRunCount.ShouldBe(1);
        guard.IsQuarantined("vt-1", out _).ShouldBeFalse();
    }

    [Fact]
    public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan()
    {
        var guard = new ScriptRunGuard();
        using var spin = Compile(SpinScript);

        await Should.ThrowAsync<ScriptTimeoutException>(() =>
            Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));

        // The abandoned script is still executing on its pool thread — the guard knows, the caller does not.
        guard.InFlightCount.ShouldBe(1);
        guard.StartedRunCount.ShouldBe(1);
    }

    [Fact]
    public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another()
    {
        var guard = new ScriptRunGuard();
        using var spin = Compile(SpinScript);
        await Should.ThrowAsync<ScriptTimeoutException>(() =>
            Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));

        var ex = await Should.ThrowAsync<ScriptRunRefusedException>(() =>
            Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));

        ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
        guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run");
        guard.InFlightCount.ShouldBe(1);
        guard.RefusedWhileRunningCount.ShouldBe(1);
    }

    [Fact]
    public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating()
    {
        var guard = new ScriptRunGuard();
        using var spin = Compile(BriefSpinScript);
        using var fast = Compile(FastScript);
        var ct = TestContext.Current.CancellationToken;

        // Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run,
        // not a refusal (a refusal is not a strike; see the next test).
        await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
        await WaitForIdle(guard);
        await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
        await WaitForIdle(guard);
        var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
        await WaitForIdle(guard);

        quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
        quarantined.Message.ShouldContain("3 consecutive");
        guard.IsQuarantined("bad", out var reason).ShouldBeTrue();
        reason.ShouldNotBeNull();
        guard.QuarantinedCount.ShouldBe(1);

        // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run.
        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
        guard.StartedRunCount.ShouldBe(3);

        // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key.
        (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42);
        guard.IsQuarantined("good", out _).ShouldBeFalse();
        guard.StartedRunCount.ShouldBe(4);
    }

    /// <summary>
    ///     A refusal is NOT a strike (review fix round 1, I1): on a 1 Hz trigger with a 2 s budget a single
    ///     4 s overrun is one timeout followed by two refusals, and counting those would quarantine a script
    ///     that merely ran long once. A refusal is a Bad publish plus its counter, and it leaves the timeout
    ///     streak untouched — neither adding to it nor resetting it.
    /// </summary>
    [Fact]
    public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
        using var spin = Compile(BriefSpinScript);
        var ct = TestContext.Current.CancellationToken;

        // timeout #1, then refusals while the orphan is still running — however many, no quarantine.
        await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
        for (var i = 0; i < 5; i++)
            await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
        guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes");
        guard.RefusedWhileRunningCount.ShouldBe(5);
        await WaitForIdle(guard);

        // …and they did not RESET the streak either: the second real timeout reaches the threshold of 2.
        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
        guard.IsQuarantined("k", out _).ShouldBeTrue();
        await WaitForIdle(guard);
    }

    /// <summary>
    ///     The thread-leak regression, and the stuck-orphan quarantine (fix round 2). Pre-guard, a
    ///     <c>while(true)</c> script on a 1 Hz trigger leaked one pool thread per trigger — 30 orphans after 30
    ///     triggers, trending toward pool starvation. Now the first trigger's orphan is the only one; later
    ///     triggers are refused while it is young, and once it has run longer than
    ///     <see cref="ScriptingOptions.StuckOrphanQuarantineFactor"/> × the timeout (10 × 60 ms here — the
    ///     trigger cadence is scaled down from 1 Hz to keep the test short) the next refusal quarantines it.
    ///     A healthy sibling keeps evaluating throughout (the positive control an absence assertion needs).
    /// </summary>
    /// <remarks>
    ///     The load-bearing assertions are on the guard's own bookkeeping (<see cref="ScriptRunGuard.InFlightCount"/>
    ///     / <see cref="ScriptRunGuard.StartedRunCount"/>): <see cref="ThreadPool.ThreadCount"/> moves with
    ///     every parallel test class on a loaded machine, so it is kept as a coarse secondary check only —
    ///     it must stay far below the 30 the leak would have produced.
    /// </remarks>
    [Fact]
    public async Task Runaway_script_leaves_one_orphan_is_quarantined_once_stuck_and_a_sibling_keeps_evaluating()
    {
        var guard = new ScriptRunGuard();
        using var spin = Compile(SpinScript);
        using var fast = Compile(FastScript);
        var ct = TestContext.Current.CancellationToken;
        var threadsBefore = ThreadPool.ThreadCount;
        var stuckAfter = ShortTimeout * ScriptingOptions.DefaultStuckOrphanQuarantineFactor; // 600 ms

        var healthyResults = new List<int>();
        var outcomes = new List<string>();
        var sw = System.Diagnostics.Stopwatch.StartNew();
        for (var trigger = 0; trigger < 30; trigger++)
        {
            try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); outcomes.Add("value"); }
            catch (ScriptTimeoutException) { outcomes.Add("timeout"); }
            catch (ScriptRunRefusedException) { outcomes.Add(sw.Elapsed < stuckAfter ? "refused-young" : "refused"); }
            catch (ScriptQuarantinedException) { outcomes.Add("quarantined"); }

            healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct));
            await Task.Delay(50, ct);
        }

        guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing");
        guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling");
        outcomes[0].ShouldBe("timeout");
        outcomes.ShouldContain("refused-young", "a young orphan is only refused — a short overrun is not quarantined");
        guard.IsQuarantined("runaway", out var reason).ShouldBeTrue("the orphan outlived 10x its budget: it is stuck");
        reason!.ShouldContain("stuck");
        outcomes[^1].ShouldBe("quarantined");
        guard.QuarantinedCount.ShouldBe(1);
        healthyResults.ShouldAllBe(v => v == 42);
        healthyResults.Count.ShouldBe(30);

        var threadDelta = ThreadPool.ThreadCount - threadsBefore;
        threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads");
    }

    /// <summary>
    ///     Positive control for the stuck rule: a 2x overrun (one timeout, refusals while the orphan is young)
    ///     is NOT quarantined, and once the orphan completes the script evaluates normally again.
    /// </summary>
    [Fact]
    public async Task A_2x_overrun_is_not_quarantined_and_resumes_evaluating_once_the_orphan_completes()
    {
        var guard = new ScriptRunGuard();
        const string source =
            """
            var ms = (int)(ctx.GetTag("spinMs").Value ?? 0);
            var end = System.DateTime.UtcNow.AddMilliseconds(ms);
            while (System.DateTime.UtcNow < end) { }
            return 7;
            """;
        using var eval = Compile(source);
        var ct = TestContext.Current.CancellationToken;
        var timeout = TimeSpan.FromMilliseconds(100);

        FakeScriptContext Ctx(int spinMs)
        {
            var c = new FakeScriptContext();
            c.Tags["spinMs"] = new Core.Abstractions.DataValueSnapshot(spinMs, 0u, c.Now, c.Now);
            return c;
        }

        // Warm the JIT so the first measured run's overrun is the script's, not the compiler's.
        (await guard.RunAsync("k", source, eval, Ctx(0), TimeSpan.FromSeconds(5), ct)).ShouldBe(7);

        // 2x overrun: 200 ms against a 100 ms budget — times out, then is refused while it finishes.
        await Should.ThrowAsync<ScriptTimeoutException>(() => guard.RunAsync("k", source, eval, Ctx(200), timeout, ct));
        await Should.ThrowAsync<ScriptRunRefusedException>(() => guard.RunAsync("k", source, eval, Ctx(0), timeout, ct));
        await WaitForIdle(guard);

        guard.IsQuarantined("k", out _).ShouldBeFalse("a 2x overrun is far below the 10x stuck threshold");
        (await guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)).ShouldBe(7, "the script resumes evaluating");
        guard.QuarantinedCount.ShouldBe(0);
    }

    /// <summary>The stuck rule is a knob: with <see cref="ScriptingOptions.StuckOrphanQuarantineFactor"/> ≤ 0 a
    /// stuck orphan is only ever refused.</summary>
    [Fact]
    public async Task StuckOrphanQuarantineFactor_zero_disables_the_stuck_rule()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = 0 });
        using var spin = Compile(SpinScript);
        var ct = TestContext.Current.CancellationToken;

        await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct));
        await Task.Delay(ShortTimeout * 12, ct);
        await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct));
        guard.IsQuarantined("k", out _).ShouldBeFalse();
    }

    [Fact]
    public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
        using var spin = Compile(SpinScript);
        using var fast = Compile(FastScript);
        var ct = TestContext.Current.CancellationToken;

        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
        guard.IsQuarantined("vt", out _).ShouldBeTrue();

        // The operator fixed the script: a different source identity under the same key starts clean. The
        // old orphan is a sunk cost — it keeps running, but it must not block the FIXED script.
        (await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42);
        guard.IsQuarantined("vt", out _).ShouldBeFalse();
        guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes");
    }

    [Fact]
    public async Task Reset_and_ResetAll_clear_a_quarantine()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
        using var spin = Compile(SpinScript);
        using var fast = Compile(FastScript);
        var ct = TestContext.Current.CancellationToken;

        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct));
        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct));
        guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true);

        guard.Reset("a");
        guard.IsQuarantined("a", out _).ShouldBeFalse();
        guard.IsQuarantined("b", out _).ShouldBeTrue();

        guard.ResetAll();
        guard.IsQuarantined("b", out _).ShouldBeFalse();
        // The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it).
        (await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42);
    }

    [Fact]
    public async Task A_run_that_allocates_past_the_budget_is_quarantined()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 });
        // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the
        // allocation as dead code (which is exactly what it does to the naive one-liner).
        const string hungry =
            "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;";
        using var evaluator = Compile(hungry);
        using var fast = Compile(FastScript);
        var ct = TestContext.Current.CancellationToken;

        var ex = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "hog", hungry, evaluator, ct: ct));
        ex.Message.ShouldContain("allocat");
        guard.IsQuarantined("hog", out var reason).ShouldBeTrue();
        reason.ShouldNotBeNull();
        reason.ShouldContain("allocat");

        // Never dispatched again, and the slot was released (the run DID complete).
        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "hog", hungry, evaluator, ct: ct));
        guard.StartedRunCount.ShouldBe(1);
        guard.InFlightCount.ShouldBe(0);

        // Positive control: a modest allocation under the budget is fine.
        (await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42);
    }

    /// <summary>
    ///     lmxopcua#784 — the quarantine Error reaches the main log through <see cref="ScriptLogCompanionSink"/>.
    ///     The guard's logger is a seam's script root logger, which carries no ScriptName, so the mirror read
    ///     "[Script] unknown emitted Error ..." and every quarantine on the node shared one "unknown" rate-limit
    ///     key. The guard now binds the script instance key: the mirror names each script, and two scripts are
    ///     rate-limited apart (with a budget of 1 per window, the second quarantine is still mirrored).
    /// </summary>
    [Fact]
    public async Task The_quarantine_line_mirrored_to_the_main_log_names_the_script_and_is_rate_limited_per_script()
    {
        var mainSink = new CapturingSink();
        var mainLogger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(mainSink).CreateLogger();
        var scriptRoot = new LoggerConfiguration()
            .MinimumLevel.Verbose()
            .WriteTo.Sink(new ScriptLogCompanionSink(mainLogger, limiter: new ScriptLogRateLimiter(1, TimeSpan.FromMinutes(5))))
            .CreateLogger();
        var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }, scriptRoot);
        using var spin = Compile(SpinScript);
        var ct = TestContext.Current.CancellationToken;

        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "vt-spin-a", SpinScript, spin, ShortTimeout, ct));
        await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "vt-spin-b", SpinScript, spin, ShortTimeout, ct));

        var mirrored = mainSink.Events.Select(e => e.RenderMessage()).ToList();
        mirrored.Count.ShouldBe(2, string.Join(" | ", mirrored));
        mirrored.ShouldAllBe(m => !m.Contains("unknown"));
        mirrored.ShouldContain(m => m.Contains("vt-spin-a") && m.StartsWith("[Script] \"vt-spin-a\" emitted Error"));
        mirrored.ShouldContain(m => m.Contains("vt-spin-b") && m.StartsWith("[Script] \"vt-spin-b\" emitted Error"));
    }

    private sealed class CapturingSink : ILogEventSink
    {
        public List<LogEvent> Events { get; } = [];

        public void Emit(LogEvent logEvent)
        {
            lock (Events) Events.Add(logEvent);
        }
    }

    [Fact]
    public async Task A_disabled_allocation_budget_never_quarantines()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 });
        // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the
        // allocation as dead code (which is exactly what it does to the naive one-liner).
        const string hungry =
            "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;";
        using var evaluator = Compile(hungry);

        (await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1));
        guard.IsQuarantined("hog", out _).ShouldBeFalse();
    }

    [Fact]
    public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
        const string throwing = "throw new System.InvalidOperationException(\"boom\");";
        using var thrower = Compile(throwing);
        // A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts.
        const string briefSpin =
            """
            var end = System.DateTime.UtcNow.AddMilliseconds(250);
            while (System.DateTime.UtcNow < end) { }
            return 1;
            """;
        using var spin = Compile(briefSpin);
        var ct = TestContext.Current.CancellationToken;

        // timeout #1
        await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
        await WaitForIdle(guard);

        // A throw is a completion — the script returned control — so the consecutive counter resets…
        var ex = await Should.ThrowAsync<InvalidOperationException>(() => Run(guard, "k", throwing, thrower, ct: ct));
        ex.Message.ShouldBe("boom");
        guard.InFlightCount.ShouldBe(0);

        // …which is why this second timeout does NOT reach the threshold of 2.
        await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
        guard.IsQuarantined("k", out _).ShouldBeFalse();
        await WaitForIdle(guard);
    }

    [Fact]
    public async Task Caller_cancellation_wins_over_the_timeout()
    {
        var guard = new ScriptRunGuard();
        using var spin = Compile(SpinScript);
        using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80));

        await Should.ThrowAsync<OperationCanceledException>(() =>
            Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token));
        guard.IsQuarantined("k", out _).ShouldBeFalse();
    }

    [Fact]
    public async Task Quarantine_is_observable_through_the_event_and_the_counter()
    {
        var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
        using var spin = Compile(SpinScript);
        var seen = new List<(string Key, string Reason)>();
        guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason));

        await Should.ThrowAsync<ScriptQuarantinedException>(() =>
            Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));

        seen.ShouldHaveSingleItem();
        seen[0].Key.ShouldBe("k");
        seen[0].Reason.ShouldContain("consecutive");
        guard.QuarantinedCount.ShouldBe(1);
    }

    private static async Task WaitForIdle(ScriptRunGuard guard)
    {
        var deadline = DateTime.UtcNow.AddSeconds(5);
        while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline)
            await Task.Delay(10);
        guard.InFlightCount.ShouldBe(0, "the brief spin must have finished");
    }
}