ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ScriptDeadlineAtEnqueueTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// WP3.1 test group 3 — a script's execution deadline is armed when the run is ENQUEUED, not
/// when it is dequeued.
///
/// <para>Before WP3.1 the deadline <see cref="CancellationTokenSource"/> was constructed inside
/// the queued body, so a run that spent ten minutes waiting behind blocked scripts still got a
/// fresh full 30 s budget when it finally started — and then ran work whose triggering
/// condition was long stale. Now queue wait consumes the run's own budget, and a body that
/// dequeues past its deadline is SHED at dequeue: it never executes, and takes the existing
/// timeout path (site event, script-error counter, error reply, completion message).</para>
/// </summary>
public class ScriptDeadlineAtEnqueueTests : TestKit, IDisposable
{
    private readonly SharedScriptLibrary _sharedLibrary;

    public ScriptDeadlineAtEnqueueTests()
    {
        var compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedLibrary = new SharedScriptLibrary(
            compilationService, NullLogger<SharedScriptLibrary>.Instance);
        DeadlineHooks.Gate = new SemaphoreSlim(0);
        DeadlineHooks.SecondScriptRan = false;
    }

    void IDisposable.Dispose()
    {
        DeadlineHooks.Gate.Release(8);
        Shutdown();
    }

    private static Script<object?> CompileRaw(string code)
    {
        var options = ScriptOptions.Default
            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
                typeof(DeadlineHooks).Assembly)
            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
        var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
        script.Compile();
        return script;
    }

    private IActorRef BuildScriptActor(
        string name, Script<object?> compiled, SiteRuntimeOptions options,
        ScriptExecutionScheduler scheduler, IServiceProvider? serviceProvider)
    {
        var instance = CreateTestProbe().Ref;
        var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
        return ActorOf(Props.Create(() => new ScriptActor(
            name, "Inst1", instance, compiled, config, _sharedLibrary, options,
            NullLogger<ScriptActor>.Instance, null, null, null, serviceProvider, scheduler, null)));
    }

    [Fact]
    public void ARunThatDequeuesPastItsDeadline_IsShedWithoutExecutingItsBody()
    {
        using var scheduler = new ScriptExecutionScheduler(1);
        var siteLog = new FakeSiteEventLogger();

        // The single worker is held by a first script for longer than the second script's
        // entire timeout.
        var blocker = BuildScriptActor(
            "Blocker",
            CompileRaw("ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.DeadlineHooks.Gate.Wait(); return null;"),
            new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 120, StuckScriptGraceMs = 120_000 },
            scheduler, null);
        blocker.Tell(new ScriptCallRequest("Blocker", null, 0, "corr-blocker"), ActorRefs.NoSender);
        AwaitAssert(() => Assert.Equal(1, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15));

        // A 1 s script queued behind it. Its grace is generous so that if the watchdog DID
        // fire it would have ample opportunity to emit — the assertion below is that it does not.
        var lateOptions = new SiteRuntimeOptions
        {
            ScriptExecutionTimeoutSeconds = 1,
            StuckScriptGraceMs = 1000
        };
        var late = BuildScriptActor(
            "Late",
            CompileRaw(
                "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.DeadlineHooks.SecondScriptRan = true; return 1;"),
            lateOptions, scheduler, new SingleServiceProvider(siteLog));

        var caller = CreateTestProbe();
        late.Tell(new ScriptCallRequest("Late", null, 0, "corr-late"), caller.Ref);

        // Let its whole budget elapse while it is still queued, then free the worker.
        Thread.Sleep(TimeSpan.FromSeconds(2));
        DeadlineHooks.Gate.Release();

        var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
        Assert.False(result.Success);
        Assert.Contains("timed out", result.ErrorMessage);

        // The body never ran: stale work is shed at dequeue rather than executed late.
        Assert.False(DeadlineHooks.SecondScriptRan,
            "the queued body executed even though its deadline had already passed");

        AwaitAssert(() =>
        {
            var rows = siteLog.OfType("script");
            // Timeout path only — no "started" Info event, because the body was skipped.
            Assert.Contains(rows, r => r.Severity == "Error" && r.Message.Contains("timed out"));
            Assert.DoesNotContain(rows, r => r.Message.Contains("started", StringComparison.OrdinalIgnoreCase));
        }, TimeSpan.FromSeconds(5));

        // And the watchdog did NOT report it as a stuck thread: it was cancelled while queued,
        // so it never held a worker.
        Thread.Sleep(TimeSpan.FromSeconds(2)); // well past the 1 s grace
        Assert.DoesNotContain(siteLog.OfType("script"),
            r => r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase));
        Assert.Equal(0, scheduler.DetachedThreadCount);
    }
}

/// <summary>Test hooks for the enqueue-anchored deadline test.</summary>
public static class DeadlineHooks
{
    /// <summary>Gate the blocking first script waits on.</summary>
    public static SemaphoreSlim Gate = new(0);

    /// <summary>Set by the second script's body — must stay false when the run is shed at dequeue.</summary>
    public static bool SecondScriptRan;
}