ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ScriptDeadlineAtEnqueueTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// A script's execution deadline is armed when the run is ENQUEUED, not
/// when it is dequeued.
///
/// <para>Previously the deadline <see cref="CancellationTokenSource"/> was constructed inside
/// the queued body, so a run that spent ten minutes waiting behind blocked scripts still got a
/// fresh full 30 s budget when it finally started — and then ran work whose triggering
/// condition was long stale. Now queue wait consumes the run's own budget, and a body that
/// dequeues past its deadline is SHED at dequeue: it never executes, and takes the existing
/// timeout path (site event, script-error counter, error reply, completion message).</para>
/// </summary>
public class ScriptDeadlineAtEnqueueTests : TestKit, IDisposable
{
    private readonly SharedScriptLibrary _sharedLibrary;

    /// <summary>Initializes the test with an isolated script execution scheduler.</summary>
    public ScriptDeadlineAtEnqueueTests()
    {
        var compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedLibrary = new SharedScriptLibrary(
            compilationService, NullLogger<SharedScriptLibrary>.Instance);
        DeadlineHooks.Gate = new SemaphoreSlim(0);
        DeadlineHooks.SecondScriptRan = false;
    }

    /// <inheritdoc />
    void IDisposable.Dispose()
    {
        DeadlineHooks.Gate.Release(8);
        Shutdown();
    }

    private static Script<object?> CompileRaw(string code)
    {
        var options = ScriptOptions.Default
            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
                typeof(DeadlineHooks).Assembly)
            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
        var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
        script.Compile();
        return script;
    }

    private IActorRef BuildScriptActor(
        string name, Script<object?> compiled, SiteRuntimeOptions options,
        ScriptExecutionScheduler scheduler, IServiceProvider? serviceProvider)
    {
        var instance = CreateTestProbe().Ref;
        var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
        return ActorOf(Props.Create(() => new ScriptActor(
            name, "Inst1", instance, compiled, config, _sharedLibrary, options,
            NullLogger<ScriptActor>.Instance, null, null, null, serviceProvider, scheduler, null)));
    }

    /// <summary>Verifies that a run that dequeues past its deadline is shed without executing its body.</summary>
    [Fact]
    public void ARunThatDequeuesPastItsDeadline_IsShedWithoutExecutingItsBody()
    {
        using var scheduler = new ScriptExecutionScheduler(1);
        var siteLog = new FakeSiteEventLogger();

        // The single worker is held by a first script for longer than the second script's
        // entire timeout.
        var blocker = BuildScriptActor(
            "Blocker",
            CompileRaw("ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.DeadlineHooks.Gate.Wait(); return null;"),
            new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 120, StuckScriptGraceMs = 120_000 },
            scheduler, null);
        blocker.Tell(new ScriptCallRequest("Blocker", null, 0, "corr-blocker"), ActorRefs.NoSender);
        AwaitAssert(() => Assert.Equal(1, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15));

        // A 1 s script queued behind it. Its grace is generous so that if the watchdog DID
        // fire it would have ample opportunity to emit — the assertion below is that it does not.
        var lateOptions = new SiteRuntimeOptions
        {
            ScriptExecutionTimeoutSeconds = 1,
            StuckScriptGraceMs = 1000,
            // The defaults leave per-run Started/Completed events OFF, which would make the
            // "no started event" assertion below vacuously true whether the body ran or not.
            // Opt back in so that assertion actually discriminates: a body that reached the
            // run loop WOULD emit "started" here, and its absence is therefore evidence.
            PerRunScriptEvents = true
        };
        var late = BuildScriptActor(
            "Late",
            CompileRaw(
                "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.DeadlineHooks.SecondScriptRan = true; return 1;"),
            lateOptions, scheduler, new SingleServiceProvider(siteLog));

        var caller = CreateTestProbe();
        late.Tell(new ScriptCallRequest("Late", null, 0, "corr-late"), caller.Ref);

        // Register row 40 — edge 1 (replaces a bare Thread.Sleep(2s)): the late run must
        // actually be SITTING IN THE SCHEDULER QUEUE. The old test only slept and hoped;
        // a blocker that failed to occupy the worker would have let the late run execute
        // normally, and the shed-at-dequeue path under test would never have been reached.
        AwaitCondition(() => scheduler.QueueDepth >= 1, TimeSpan.FromSeconds(15),
            TimeSpan.FromMilliseconds(25));

        // Register row 40 — edge 2: hold until the late run's 1 s budget has provably
        // elapsed, re-reading the queue on every poll and bailing out early the instant the
        // run leaves the queue. The post-condition below then proves the budget burned
        // *while queued* — the actual design claim ("queue wait consumes the run's budget"),
        // which a fixed sleep asserted nothing about.
        var queuedFor = System.Diagnostics.Stopwatch.StartNew();
        AwaitCondition(
            () => queuedFor.Elapsed >= TimeSpan.FromSeconds(1.5) || scheduler.QueueDepth == 0,
            TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(25));
        Assert.True(scheduler.QueueDepth >= 1,
            "the late run left the queue before its budget elapsed, so this run never " +
            "exercised the shed-at-dequeue path");

        DeadlineHooks.Gate.Release();

        var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
        Assert.False(result.Success);
        Assert.Contains("timed out", result.ErrorMessage);

        // The body never ran: stale work is shed at dequeue rather than executed late.
        Assert.False(DeadlineHooks.SecondScriptRan,
            "the queued body executed even though its deadline had already passed");

        AwaitAssert(() =>
        {
            var rows = siteLog.OfType("script");
            // Timeout path only — no "started" Info event, because the body was skipped.
            // Per-run events are ON for this script (see lateOptions), so this absence is a
            // real signal rather than the global default.
            Assert.Contains(rows, r => r.Severity == "Error" && r.Message.Contains("timed out"));
            Assert.DoesNotContain(rows, r => r.Message.Contains("started", StringComparison.OrdinalIgnoreCase));
        }, TimeSpan.FromSeconds(5));

        // ── Register row 40 — the watchdog absence, proved by a positive control ──
        //
        // The claim is that the shed run was NOT reported as a stuck thread: it was cancelled
        // while queued, so it never held a worker. The old test slept 2 s and asserted the
        // absence, which false-passes whenever the watchdog simply had not got around to
        // evaluating anything yet — the assertion could not distinguish "the watchdog looked
        // and correctly stayed silent" from "the watchdog never looked".
        //
        // So: enqueue a script that genuinely wedges a worker, on the SAME scheduler and with
        // the SAME 1 s timeout + 1 s grace, and armed strictly LATER than the late run's
        // watchdog. Its stuck report is the observable edge — once it lands, every watchdog
        // armed at or before it has provably run its grace delay and made its decision.
        //
        // The body blocks its worker SYNCHRONOUSLY (so the watchdog can detach the thread it
        // holds) but for a BOUNDED 2.5 s that comfortably clears 1 s timeout + 1 s grace and
        // then self-releases. A gate held until test teardown would work too, but it leaves a
        // blocked OS thread alive past the test and perturbs timing-sensitive tests elsewhere
        // in this assembly.
        var wedged = BuildScriptActor(
            "Wedged",
            CompileRaw("System.Threading.Thread.Sleep(2500); return null;"),
            new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 1000 },
            scheduler, new SingleServiceProvider(siteLog));
        wedged.Tell(new ScriptCallRequest("Wedged", null, 0, "corr-wedged"), ActorRefs.NoSender);

        // Positive event, bounded wait: the watchdog fires, names the wedged script and
        // detaches its worker.
        AwaitAssert(() =>
        {
            Assert.Contains(siteLog.OfType("script"),
                r => r.Message.Contains("'Wedged'", StringComparison.Ordinal) &&
                     r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase));
        }, TimeSpan.FromSeconds(30), TimeSpan.FromMilliseconds(50));

        // Negative control, evaluated only now that a later-armed watchdog has demonstrably
        // completed its grace delay: the shed run was never reported stuck.
        Assert.DoesNotContain(siteLog.OfType("script"),
            r => r.Message.Contains("'Late'", StringComparison.Ordinal) &&
                 r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase));

        // And the detach counter moved for exactly one run — the wedged one. The old
        // `Assert.Equal(0, DetachedThreadCount)` was satisfied by a counter that never moves
        // for any reason; this pins that it does move, and that the shed run did not move it.
        Assert.Equal(1, scheduler.DetachedThreadCount);

        // Wait for the positive control's body to unwind and its detached worker to exit
        // BEFORE the scheduler is disposed at end of scope, so the test leaves no threads
        // behind for the rest of the assembly to contend with.
        AwaitCondition(() => scheduler.BusyThreadCount == 0 && scheduler.DetachedThreadCount == 0,
            TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(25));
    }
}

/// <summary>Test hooks for the enqueue-anchored deadline test.</summary>
public static class DeadlineHooks
{
    /// <summary>Gate the blocking first script waits on.</summary>
    public static SemaphoreSlim Gate = new(0);

    /// <summary>Set by the second script's body — must stay false when the run is shed at dequeue.</summary>
    public static bool SecondScriptRan;
}