using System.Diagnostics; using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.CodeAnalysis.CSharp.Scripting; using Microsoft.CodeAnalysis.Scripting; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// WP3.1 test groups 1 and 2 — the regression pin for arch-review finding #4 (High). /// /// The finding. Trigger-expression evaluation used to be queued onto the same /// fixed 8-thread that runs blocking script bodies. Eight /// scripts blocked in synchronous I/O therefore stalled EVERY Expression trigger on the node — /// scripts and alarms alike — for an unbounded time. Worse, the evaluation's 2 s timeout was /// constructed INSIDE the queued body, so it did not start until the evaluation was dequeued: /// the operator saw neither a raise nor a timeout, just silence. /// /// The fix these tests pin. Evaluations are non-blocking by construction /// ( exposes only reads over an in-memory snapshot, and /// the trust gate has already denied I/O), so they run as plain async work on the shared .NET /// thread pool behind — never on the blocking pool. And their /// deadline is armed at ENQUEUE, so gate-wait time burns the same budget. /// public class TriggerEvalStarvationTests : TestKit, IDisposable { private readonly SharedScriptLibrary _sharedLibrary; public TriggerEvalStarvationTests() { var compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedLibrary = new SharedScriptLibrary( compilationService, NullLogger.Instance); StarvationHooks.Gate = new SemaphoreSlim(0); } void IDisposable.Dispose() { StarvationHooks.Gate.Release(32); Shutdown(); } private static Script CompileRaw(string code) { var options = ScriptOptions.Default .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(StarvationHooks).Assembly) .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks"); var script = CSharpScript.Create(code, options, typeof(ScriptGlobals)); script.Compile(); return script; } private static Script CompileTriggerExpression(string expression) { var options = ScriptOptions.Default .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly) .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks"); var script = CSharpScript.Create(expression, options, typeof(TriggerExpressionGlobals)); script.Compile(); return script; } /// /// GROUP 1 — the finding-#4 regression pin. Eight script bodies blocked on a semaphore /// occupy every thread of an 8-thread pool; an Expression-triggered alarm must still raise /// in well under 2 s. /// /// On the pre-WP3.1 wiring this test does not merely fail slowly — it never /// completes: the evaluation sits in the same FIFO behind eight bodies that only unblock /// after the assertion window, and its 2 s timeout has not even started ticking. /// [Fact] public void EightBlockedScripts_DoNotDelayAnAlarmsExpressionEvaluation() { using var scheduler = new ScriptExecutionScheduler(8); using var evalGate = new TriggerEvalGate(4); // Occupy every thread of the blocking pool with a real script run. var wedge = CompileRaw( "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StarvationHooks.Gate.Wait(); return null;"); for (var i = 0; i < 8; i++) { var name = $"Blocker{i}"; var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" }; var instance = CreateTestProbe().Ref; var actor = ActorOf(Props.Create(() => new ScriptActor( name, "Inst1", instance, wedge, config, _sharedLibrary, new SiteRuntimeOptions(), NullLogger.Instance, null, null, null, null, scheduler, evalGate))); actor.Tell(new ScriptCallRequest(name, null, 0, $"corr-block-{i}"), ActorRefs.NoSender); } AwaitAssert( () => Assert.Equal(8, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15)); // Now fire an Expression-triggered alarm through the SAME scheduler seam. var instanceProbe = CreateTestProbe(); var alarm = ActorOf(Props.Create(() => new AlarmActor( "ExprAlarm", "Inst1", instanceProbe.Ref, new ResolvedAlarm { CanonicalName = "ExprAlarm", TriggerType = "Expression", TriggerConfiguration = "{\"expression\":\"true\"}", PriorityLevel = 900 }, null, _sharedLibrary, new SiteRuntimeOptions(), NullLogger.Instance, CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate))); var stopwatch = Stopwatch.StartNew(); alarm.Tell(new AttributeValueChanged( "Inst1", "Temp", "Temp", 99.0, "Good", DateTimeOffset.UtcNow)); var raised = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(2)); stopwatch.Stop(); Assert.Equal(AlarmState.Active, raised.State); Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(2), $"alarm raised only after {stopwatch.Elapsed} — the evaluation queued behind blocked script bodies"); // The pool really was saturated for the whole window; the eval simply never used it. Assert.Equal(8, scheduler.BusyThreadCount); } /// /// GROUP 2 — the evaluation deadline is measured from ENQUEUE, so time spent waiting on a /// saturated burns the same budget. With the gate fully /// occupied for the whole assertion window, a queued evaluation must still resolve (as /// false) at its timeout rather than stalling indefinitely — and the trigger must drain /// rather than park, so the next change still evaluates. /// [Fact] public async Task SaturatedEvalGate_ResolvesTheQueuedEvaluationAtItsEnqueueAnchoredDeadline() { using var scheduler = new ScriptExecutionScheduler(1); using var evalGate = new TriggerEvalGate(1); var options = new SiteRuntimeOptions { TriggerEvalTimeoutSeconds = 1 }; var instanceProbe = CreateTestProbe(); var alarm = ActorOf(Props.Create(() => new AlarmActor( "ExprAlarm", "Inst1", instanceProbe.Ref, new ResolvedAlarm { CanonicalName = "ExprAlarm", TriggerType = "Expression", TriggerConfiguration = "{\"expression\":\"true\"}", PriorityLevel = 500 }, null, _sharedLibrary, options, NullLogger.Instance, CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate))); // 1. Free gate: the expression evaluates true and the alarm raises. alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 1, "Good", DateTimeOffset.UtcNow)); var raised = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.Equal(AlarmState.Active, raised.State); // 2. Occupy the only permit for the whole of the next step. await evalGate.WaitAsync(CancellationToken.None); Assert.Equal(0, evalGate.AvailablePermits); // 3. The next evaluation can never acquire the gate. Its deadline was armed at // enqueue, so it cancels at ~1 s, is treated as false, and clears the alarm. // Pre-WP3.1 the clock started at dequeue, so this would hang forever. var stopwatch = Stopwatch.StartNew(); alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 2, "Good", DateTimeOffset.UtcNow)); var cleared = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10)); stopwatch.Stop(); Assert.Equal(AlarmState.Normal, cleared.State); Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(6), $"queued evaluation took {stopwatch.Elapsed} to resolve — the deadline is not enqueue-anchored"); Assert.Equal(0, evalGate.AvailablePermits); // still held: it genuinely never ran // 4. Not parked: releasing the gate and sending another change evaluates again. evalGate.Release(); alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 3, "Good", DateTimeOffset.UtcNow)); var reRaised = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.Equal(AlarmState.Active, reRaised.State); } } /// Test hook used to block script-execution worker threads deterministically. public static class StarvationHooks { /// Gate the blocking test scripts wait on; reset per test. public static SemaphoreSlim Gate = new(0); }