ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/TriggerEvalStarvationTests.cs

using System.Diagnostics;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;

/// <summary>
/// WP3.1 test groups 1 and 2 — the regression pin for arch-review finding #4 (High).
///
/// <para><b>The finding.</b> Trigger-expression evaluation used to be queued onto the same
/// fixed 8-thread <see cref="ScriptExecutionScheduler"/> that runs blocking script bodies. Eight
/// scripts blocked in synchronous I/O therefore stalled EVERY Expression trigger on the node —
/// scripts and alarms alike — for an unbounded time. Worse, the evaluation's 2 s timeout was
/// constructed INSIDE the queued body, so it did not start until the evaluation was dequeued:
/// the operator saw neither a raise nor a timeout, just silence.</para>
///
/// <para><b>The fix these tests pin.</b> Evaluations are non-blocking by construction
/// (<see cref="TriggerExpressionGlobals"/> exposes only reads over an in-memory snapshot, and
/// the trust gate has already denied I/O), so they run as plain async work on the shared .NET
/// thread pool behind <see cref="TriggerEvalGate"/> — never on the blocking pool. And their
/// deadline is armed at ENQUEUE, so gate-wait time burns the same budget.</para>
/// </summary>
public class TriggerEvalStarvationTests : TestKit, IDisposable
{
    private readonly SharedScriptLibrary _sharedLibrary;

    public TriggerEvalStarvationTests()
    {
        var compilationService = new ScriptCompilationService(
            NullLogger<ScriptCompilationService>.Instance);
        _sharedLibrary = new SharedScriptLibrary(
            compilationService, NullLogger<SharedScriptLibrary>.Instance);
        StarvationHooks.Gate = new SemaphoreSlim(0);
    }

    void IDisposable.Dispose()
    {
        StarvationHooks.Gate.Release(32);
        Shutdown();
    }

    private static Script<object?> CompileRaw(string code)
    {
        var options = ScriptOptions.Default
            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
                typeof(StarvationHooks).Assembly)
            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
        var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
        script.Compile();
        return script;
    }

    private static Script<object?> CompileTriggerExpression(string expression)
    {
        var options = ScriptOptions.Default
            .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly)
            .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
        var script = CSharpScript.Create<object?>(expression, options, typeof(TriggerExpressionGlobals));
        script.Compile();
        return script;
    }

    /// <summary>
    /// GROUP 1 — the finding-#4 regression pin. Eight script bodies blocked on a semaphore
    /// occupy every thread of an 8-thread pool; an Expression-triggered alarm must still raise
    /// in well under 2 s.
    ///
    /// <para>On the pre-WP3.1 wiring this test does not merely fail slowly — it never
    /// completes: the evaluation sits in the same FIFO behind eight bodies that only unblock
    /// after the assertion window, and its 2 s timeout has not even started ticking.</para>
    /// </summary>
    [Fact]
    public void EightBlockedScripts_DoNotDelayAnAlarmsExpressionEvaluation()
    {
        using var scheduler = new ScriptExecutionScheduler(8);
        using var evalGate = new TriggerEvalGate(4);

        // Occupy every thread of the blocking pool with a real script run.
        var wedge = CompileRaw(
            "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StarvationHooks.Gate.Wait(); return null;");
        for (var i = 0; i < 8; i++)
        {
            var name = $"Blocker{i}";
            var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
            var instance = CreateTestProbe().Ref;
            var actor = ActorOf(Props.Create(() => new ScriptActor(
                name, "Inst1", instance, wedge, config, _sharedLibrary, new SiteRuntimeOptions(),
                NullLogger<ScriptActor>.Instance, null, null, null, null, scheduler, evalGate)));
            actor.Tell(new ScriptCallRequest(name, null, 0, $"corr-block-{i}"), ActorRefs.NoSender);
        }

        AwaitAssert(
            () => Assert.Equal(8, scheduler.BusyThreadCount),
            TimeSpan.FromSeconds(15));

        // Now fire an Expression-triggered alarm through the SAME scheduler seam.
        var instanceProbe = CreateTestProbe();
        var alarm = ActorOf(Props.Create(() => new AlarmActor(
            "ExprAlarm", "Inst1", instanceProbe.Ref,
            new ResolvedAlarm
            {
                CanonicalName = "ExprAlarm",
                TriggerType = "Expression",
                TriggerConfiguration = "{\"expression\":\"true\"}",
                PriorityLevel = 900
            },
            null, _sharedLibrary, new SiteRuntimeOptions(), NullLogger<AlarmActor>.Instance,
            CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));

        var stopwatch = Stopwatch.StartNew();
        alarm.Tell(new AttributeValueChanged(
            "Inst1", "Temp", "Temp", 99.0, "Good", DateTimeOffset.UtcNow));

        var raised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(2));
        stopwatch.Stop();

        Assert.Equal(AlarmState.Active, raised.State);
        Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(2),
            $"alarm raised only after {stopwatch.Elapsed} — the evaluation queued behind blocked script bodies");

        // The pool really was saturated for the whole window; the eval simply never used it.
        Assert.Equal(8, scheduler.BusyThreadCount);
    }

    /// <summary>
    /// GROUP 2 — the evaluation deadline is measured from ENQUEUE, so time spent waiting on a
    /// saturated <see cref="TriggerEvalGate"/> burns the same budget. With the gate fully
    /// occupied for the whole assertion window, a queued evaluation must still resolve (as
    /// false) at its timeout rather than stalling indefinitely — and the trigger must drain
    /// rather than park, so the next change still evaluates.
    /// </summary>
    [Fact]
    public async Task SaturatedEvalGate_ResolvesTheQueuedEvaluationAtItsEnqueueAnchoredDeadline()
    {
        using var scheduler = new ScriptExecutionScheduler(1);
        using var evalGate = new TriggerEvalGate(1);
        var options = new SiteRuntimeOptions { TriggerEvalTimeoutSeconds = 1 };

        var instanceProbe = CreateTestProbe();
        var alarm = ActorOf(Props.Create(() => new AlarmActor(
            "ExprAlarm", "Inst1", instanceProbe.Ref,
            new ResolvedAlarm
            {
                CanonicalName = "ExprAlarm",
                TriggerType = "Expression",
                TriggerConfiguration = "{\"expression\":\"true\"}",
                PriorityLevel = 500
            },
            null, _sharedLibrary, options, NullLogger<AlarmActor>.Instance,
            CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));

        // 1. Free gate: the expression evaluates true and the alarm raises.
        alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 1, "Good", DateTimeOffset.UtcNow));
        var raised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
        Assert.Equal(AlarmState.Active, raised.State);

        // 2. Occupy the only permit for the whole of the next step.
        await evalGate.WaitAsync(CancellationToken.None);
        Assert.Equal(0, evalGate.AvailablePermits);

        // 3. The next evaluation can never acquire the gate. Its deadline was armed at
        //    enqueue, so it cancels at ~1 s, is treated as false, and clears the alarm.
        //    Pre-WP3.1 the clock started at dequeue, so this would hang forever.
        var stopwatch = Stopwatch.StartNew();
        alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 2, "Good", DateTimeOffset.UtcNow));
        var cleared = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
        stopwatch.Stop();

        Assert.Equal(AlarmState.Normal, cleared.State);
        Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(6),
            $"queued evaluation took {stopwatch.Elapsed} to resolve — the deadline is not enqueue-anchored");
        Assert.Equal(0, evalGate.AvailablePermits); // still held: it genuinely never ran

        // 4. Not parked: releasing the gate and sending another change evaluates again.
        evalGate.Release();
        alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 3, "Good", DateTimeOffset.UtcNow));
        var reRaised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
        Assert.Equal(AlarmState.Active, reRaised.State);
    }
}

/// <summary>Test hook used to block script-execution worker threads deterministically.</summary>
public static class StarvationHooks
{
    /// <summary>Gate the blocking test scripts wait on; reset per test.</summary>
    public static SemaphoreSlim Gate = new(0);
}