using Akka.Actor; using Akka.Event; using Akka.TestKit; using Akka.TestKit.Xunit2; using Microsoft.CodeAnalysis.CSharp.Scripting; using Microsoft.CodeAnalysis.Scripting; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors; /// /// WP3.1 supervision-parity set (design memo §4, seven pins) — the reworked successor to /// ExecutionActorTests. /// /// WP3.1 eliminated the short-lived ScriptExecutionActor and /// AlarmExecutionActor: neither had a Receive handler, a PostStop, or any /// state, and neither's IActorRef was ever a message target — the entire lifecycle lived /// in a detached task. Runs are now launched directly by the coordinator via /// . These tests pin every behaviour the removed actors /// provided onto its replacement: exception and timeout containment, the Ask reply, the /// completion notification, one DI scope per run, the audit ParentExecutionId threading, and /// the supervision outcome (coordinator unaffected — no stop, no restart). /// public class ScriptRunLauncherParityTests : TestKit, IDisposable { private readonly SharedScriptLibrary _sharedLibrary; private readonly ScriptCompilationService _compilationService; /// Own pool per test class (#18 seam), so a wedged body cannot strand the process-wide one. private readonly ScriptExecutionScheduler _scheduler = new(4); public ScriptRunLauncherParityTests() { _compilationService = new ScriptCompilationService( NullLogger.Instance); _sharedLibrary = new SharedScriptLibrary( _compilationService, NullLogger.Instance); } void IDisposable.Dispose() { Shutdown(); _scheduler.Dispose(); } // ── helpers ────────────────────────────────────────────────────────────────── private static Script CompileScript(string code) { var scriptOptions = ScriptOptions.Default .WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(RunLauncherHooks).Assembly) .WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks"); var script = CSharpScript.Create(code, scriptOptions, typeof(ScriptGlobals)); script.Compile(); return script; } private static SiteRuntimeOptions Options( int timeoutSeconds = 30, int graceMs = 30000, bool perRunScriptEvents = false) => new() { MaxScriptCallDepth = 10, ScriptExecutionTimeoutSeconds = timeoutSeconds, StuckScriptGraceMs = graceMs, // WP3.2: per-run Started/Completed Info events default OFF; tests that pin the // legacy per-run event shape opt back in explicitly via this flag. PerRunScriptEvents = perRunScriptEvents }; private static ResolvedScript CallScript(int? timeoutSeconds = null) => new() { CanonicalName = "Runner", TriggerType = "Call", ExecutionTimeoutSeconds = timeoutSeconds }; private TestActorRef BuildScriptActor( Script? compiled, SiteRuntimeOptions options, IServiceProvider? serviceProvider = null, ISiteHealthCollector? healthCollector = null, ScriptExecutionScheduler? scheduler = null, int? perScriptTimeoutSeconds = null, IActorRef? instanceActor = null) { var instance = instanceActor ?? CreateTestProbe().Ref; return ActorOfAsTestActorRef( Props.Create(() => new ScriptActor( "Runner", "Inst1", instance, compiled, CallScript(perScriptTimeoutSeconds), _sharedLibrary, options, NullLogger.Instance, null, null, healthCollector, serviceProvider, scheduler ?? _scheduler, null)), "script-" + Guid.NewGuid().ToString("N")); } // ── Pin 1: throwing body — coordinator survives, everything is reported ─────── [Fact] public void ThrowingScriptBody_LeavesScriptActorAliveAndReportsEverything() { var siteLog = new FakeSiteEventLogger(); var health = new SiteHealthCollector(); var actor = BuildScriptActor( CompileScript("throw new InvalidOperationException(\"boom\");"), Options(), new SingleServiceProvider(siteLog), health); Watch(actor); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-throw"), caller.Ref); var result = caller.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Equal("corr-throw", result.CorrelationId); Assert.Contains("boom", result.ErrorMessage); AwaitAssert(() => { // Error site event + script-error counter, exactly as the execution actor emitted. Assert.Contains(siteLog.OfType("script"), r => r.Severity == "Error" && r.Message.Contains("failed", StringComparison.OrdinalIgnoreCase)); // The in-flight slot is released, so the script can run again. Assert.Equal(0, actor.UnderlyingActor.RunsInFlight); }, TimeSpan.FromSeconds(5)); Assert.Equal(1, health.CollectReport("site-1").ScriptErrorCount); // The coordinator is neither stopped nor restarted — a throwing body was always // contained inside the run's own try/catch, and still is. Watch() above means a stop // would deliver Terminated to the TestActor; a still-answering call proves it is live. ExpectNoMsg(TimeSpan.FromMilliseconds(300)); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-still-alive"), caller.Ref); Assert.Equal("corr-still-alive", caller.ExpectMsg(TimeSpan.FromSeconds(10)).CorrelationId); } // ── Pin 2: launch-path throw — improved over the old silent hang ────────────── /// /// The only failure the removed per-run child could surface was a constructor throw — /// e.g. queueing onto a disposed . The old /// OneForOneStrategy logged it and stopped the child, leaving the Ask caller to /// hang with no reply at all. WP3.1 folds that into a launch-path catch that replies, an /// intentional improvement pinned here so it is explicit rather than accidental. /// [Fact] public void LaunchPathThrow_RepliesToTheCallerAndLeavesTheCoordinatorAlive() { var dead = new ScriptExecutionScheduler(1); dead.Dispose(); var actor = BuildScriptActor( CompileScript("return 1;"), Options(), scheduler: dead); Watch(actor); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-launch"), caller.Ref); var result = caller.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Equal("corr-launch", result.CorrelationId); Assert.Contains("could not be launched", result.ErrorMessage); // Watch() above means a stop would deliver Terminated to the TestActor; none arrives, // so the coordinator neither died nor restarted on a launch failure. ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // The in-flight counter is incremented before the launch, so the catch MUST balance // it — otherwise a run of launch failures would permanently wedge the script at cap. AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5)); } // ── Pin 3: exactly one DI scope per run, disposed on every path ─────────────── [Theory] [InlineData("return 1;", 30)] // success [InlineData("throw new InvalidOperationException(\"boom\");", 30)] // failure [InlineData("while (true) { await Task.Delay(25, CancellationToken); }", 1)] // timeout public void EachRun_CreatesOneDiScope_AndDisposesItExactlyOnce(string code, int timeoutSeconds) { var spy = new ScopeSpyServiceProvider(); var actor = BuildScriptActor( CompileScript(code), Options(timeoutSeconds), spy); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-scope"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(15)); AwaitAssert(() => { Assert.Equal(1, spy.ScopesCreated); Assert.Equal(1, spy.ScopesDisposed); }, TimeSpan.FromSeconds(5)); } // ── Pin 4: audit correlation threading survives the actor removal ───────────── /// /// A routed must still reach the run's /// — this is the inbound-API leg of the audit execution /// tree, and it used to be threaded through the execution actor's constructor. /// [Fact] public void RoutedParentExecutionId_ReachesTheRunsScriptRuntimeContext() { RunLauncherHooks.CapturedContext = null; var parent = Guid.NewGuid(); var actor = BuildScriptActor( CompileScript( "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.CapturedContext = Instance; return 1;"), Options()); var caller = CreateTestProbe(); actor.Tell( new ScriptCallRequest("Runner", null, 0, "corr-parent", ParentExecutionId: parent), caller.Ref); var result = caller.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); AwaitAssert(() => { Assert.NotNull(RunLauncherHooks.CapturedContext); Assert.Equal(parent, RunLauncherHooks.CapturedContext!.ParentExecutionId); // The routed run still mints its OWN ExecutionId — the parent is a pointer, not a copy. Assert.NotEqual(parent, RunLauncherHooks.CapturedContext.ExecutionId); }, TimeSpan.FromSeconds(5)); } // ── Pin 5: timeout resolution parity (perScript ?? global, <= 0 => global) ──── [Theory] [InlineData(300, 1)] // per-script override wins over a much longer global [InlineData(1, null)] // null per-script falls back to the global [InlineData(1, 0)] // non-positive per-script is treated as "use global" public void TimeoutResolution_MatchesTheRemovedExecutionActor(int globalSeconds, int? perScriptSeconds) { var siteLog = new FakeSiteEventLogger(); var actor = BuildScriptActor( CompileScript("while (true) { await Task.Delay(25, CancellationToken); }"), Options(globalSeconds), new SingleServiceProvider(siteLog), perScriptTimeoutSeconds: perScriptSeconds); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-timeout"), caller.Ref); // If the effective timeout were the 300 s global (case 1) or ignored (cases 2/3) this // would not answer inside the window. var result = caller.ExpectMsg(TimeSpan.FromSeconds(15)); Assert.False(result.Success); Assert.Contains("timed out", result.ErrorMessage); AwaitAssert( () => Assert.Contains(siteLog.OfType("script"), r => r.Severity == "Error" && r.Message.Contains("timed out")), TimeSpan.FromSeconds(5)); } // ── Pin 6: stop-during-run parity ──────────────────────────────────────────── /// /// Stopping a ScriptActor mid-run must NOT cancel the in-flight run (redeploy/undeploy /// semantics: running scripts are allowed to finish). The run completes normally and its /// completion message dead-letters, exactly as the old per-run child's /// parent.Tell did once the subtree was stopped — dead letters are a health metric, /// not an error. /// [Fact] public void StoppingTheScriptActorMidRun_LetsTheRunFinishAndDeadLettersItsCompletion() { RunLauncherHooks.Gate = new SemaphoreSlim(0); RunLauncherHooks.Finished = new ManualResetEventSlim(false); RunLauncherHooks.ObservedCancellation = null; var actor = BuildScriptActor( CompileScript( "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.Gate.Wait();" + "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.ObservedCancellation = CancellationToken.IsCancellationRequested;" + "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.Finished.Set();" + "return 1;"), Options()); var deadLetters = CreateTestProbe(); Sys.EventStream.Subscribe(deadLetters.Ref, typeof(DeadLetter)); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-stop"), caller.Ref); // Wait until the body is actually on a worker thread, then stop the coordinator. AwaitAssert(() => Assert.Equal(1, _scheduler.BusyThreadCount), TimeSpan.FromSeconds(10)); Watch(actor); Sys.Stop(actor); ExpectTerminated(actor, TimeSpan.FromSeconds(10)); RunLauncherHooks.Gate.Release(); // The run ran to completion and was never cancelled by the stop. Assert.True(RunLauncherHooks.Finished.Wait(TimeSpan.FromSeconds(10))); Assert.False(RunLauncherHooks.ObservedCancellation); // The Ask caller still gets its result (the reply target is not the stopped actor)… var result = caller.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); // …and the completion notification aimed at the now-stopped coordinator dead-letters. deadLetters.FishForMessage( d => d.Message is ScriptActor.ScriptExecutionCompleted, TimeSpan.FromSeconds(10)); } // ── Pin 7: alarm side ──────────────────────────────────────────────────────── /// /// An alarm on-trigger run still receives the Alarm globals (name/level/priority/ /// message) and still reports AlarmExecutionCompleted back to its AlarmActor — /// observable here through the in-flight counter returning to zero, which only the /// completion message can do. /// [Fact] public void AlarmOnTriggerRun_GetsAlarmGlobals_AndCompletesBackToTheAlarmActor() { RunLauncherHooks.CapturedAlarm = null; var onTrigger = CompileScript( "ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.CapturedAlarm = Alarm; return null;"); var instanceProbe = CreateTestProbe(); var alarm = ActorOfAsTestActorRef( Props.Create(() => new AlarmActor( "TempBand", "Inst1", instanceProbe.Ref, new ResolvedAlarm { CanonicalName = "TempBand", TriggerType = "HiLo", TriggerConfiguration = "{\"attributeName\":\"Temp\",\"hi\":80,\"hiHi\":95,\"hiMessage\":\"too hot\"}", PriorityLevel = 42 }, onTrigger, _sharedLibrary, Options(), NullLogger.Instance, null, null, null, null, null, _scheduler, null)), "alarm-" + Guid.NewGuid().ToString("N")); alarm.Tell(new Commons.Messages.Streaming.AttributeValueChanged( "Inst1", "Temp", "Temp", 90.0, "Good", DateTimeOffset.UtcNow)); instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => { Assert.NotNull(RunLauncherHooks.CapturedAlarm); Assert.Equal("TempBand", RunLauncherHooks.CapturedAlarm!.Name); Assert.Equal(AlarmLevel.High, RunLauncherHooks.CapturedAlarm.Level); Assert.Equal("too hot", RunLauncherHooks.CapturedAlarm.Message); // Only AlarmExecutionCompleted releases the slot. Assert.Equal(0, alarm.UnderlyingActor.RunsInFlight); }, TimeSpan.FromSeconds(10)); } // ── Retargeted from ExecutionActorTests: success path + operational events ──── [Fact] public void SuccessfulRun_RepliesWithTheReturnValue() { var actor = BuildScriptActor(CompileScript("return 7 * 6;"), Options()); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-ok"), caller.Ref); var result = caller.ExpectMsg(TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.Equal("corr-ok", result.CorrelationId); Assert.Equal(42, result.ReturnValue); } [Fact] public void SuccessfulRun_EmitsStartedThenCompletedInfoEvents() { var siteLog = new FakeSiteEventLogger(); // WP3.2: per-run Info events are opt-in now (default off, interval summary instead) — // this test pins the legacy per-run shape, so it opts back in explicitly. var actor = BuildScriptActor( CompileScript("return 7 * 6;"), Options(perRunScriptEvents: true), new SingleServiceProvider(siteLog)); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => { var rows = siteLog.OfType("script"); Assert.Equal(2, rows.Count); Assert.All(rows, r => { Assert.Equal("Info", r.Severity); Assert.Equal("Inst1", r.InstanceId); Assert.Equal("ScriptActor:Runner", r.Source); }); Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase); Assert.Contains("completed", rows[1].Message, StringComparison.OrdinalIgnoreCase); }, TimeSpan.FromSeconds(5)); } [Fact] public void FailingRun_EmitsStartedInfoThenErrorEvent() { var siteLog = new FakeSiteEventLogger(); // WP3.2: the started Info event is opt-in (default off); the Error event on failure // stays unconditional regardless. This test pins the opted-in shape explicitly. var actor = BuildScriptActor( CompileScript("throw new InvalidOperationException(\"boom\");"), Options(perRunScriptEvents: true), new SingleServiceProvider(siteLog)); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt-err"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => { var rows = siteLog.OfType("script"); Assert.Equal(2, rows.Count); Assert.Equal("Info", rows[0].Severity); Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase); Assert.Equal("Error", rows[1].Severity); }, TimeSpan.FromSeconds(5)); } [Fact] public void FireAndForgetRun_NeedsNoReplyTarget() { var siteLog = new FakeSiteEventLogger(); // WP3.2: opted in so the completed Info event (the observable proof the // fire-and-forget run actually ran to completion) is emitted; the policy itself is // covered separately by ScriptRunEventPolicyTests. var actor = BuildScriptActor( CompileScript("return 1;"), Options(perRunScriptEvents: true), new SingleServiceProvider(siteLog)); // Trigger-driven spawns pass ActorRefs.NoSender as replyTo; drive that path via an // interval-free Call script by telling the actor to run with no sender. actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-nobody"), ActorRefs.NoSender); AwaitAssert( () => Assert.Contains(siteLog.OfType("script"), r => r.Message.Contains("completed", StringComparison.OrdinalIgnoreCase)), TimeSpan.FromSeconds(10)); } // ── WP3.2: site_events volume policy — actor-level integration ──────────────── // // The pure ScriptRunEventPolicy matching rules (exact/wildcard/global/non-matching) are // unit-tested directly in ScriptRunEventPolicyTests; these prove the policy is actually // wired through the real ScriptActor -> ScriptRunLauncher path: the default is silent but // the aggregate still sees the run, per-script opt-in works end to end, and the // IOptionsMonitor read makes the switch hot-togglable without an actor restart. [Fact] public async Task DefaultOptions_SuccessfulRun_EmitsNoPerRunEvents_ButTheSummaryRecorderSeesIt() { var siteLog = new FakeSiteEventLogger(); var recorder = new ScriptRunSummaryRecorder(); var actor = BuildScriptActor( CompileScript("return 1;"), Options(), new SingleServiceProvider(siteLog, summaryRecorder: recorder)); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-default-off"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); // RecordCompleted happens-before the ScriptExecutionCompleted Tell that releases the // in-flight slot (see RunScriptAsync), so waiting on RunsInFlight == 0 makes the // subsequent flush deterministic — no sleep, no retry-flush race. AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5)); Assert.Empty(siteLog.OfType("script")); var flushLog = new FakeSiteEventLogger(); var emitted = await recorder.FlushAsync(flushLog); Assert.True(emitted); Assert.Equal( "1 runs: 1 completed, 0 failed, 0 timed out across 1 scripts", flushLog.OfType("script").Single().Message); } [Fact] public void PerScriptOptIn_ExactMatch_RestoresPerRunEventsForThatScriptOnly() { var siteLog = new FakeSiteEventLogger(); var options = Options(); options.PerRunScriptEventScripts = ["Inst1/Runner"]; var actor = BuildScriptActor( CompileScript("return 1;"), options, new SingleServiceProvider(siteLog)); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-exact-optin"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => Assert.Equal(2, siteLog.OfType("script").Count), TimeSpan.FromSeconds(5)); } [Fact] public void PerScriptOptIn_InstanceWildcard_RestoresPerRunEvents() { var siteLog = new FakeSiteEventLogger(); var options = Options(); options.PerRunScriptEventScripts = ["Inst1/*"]; var actor = BuildScriptActor( CompileScript("return 1;"), options, new SingleServiceProvider(siteLog)); var caller = CreateTestProbe(); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-wildcard-optin"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => Assert.Equal(2, siteLog.OfType("script").Count), TimeSpan.FromSeconds(5)); } [Fact] public void HotToggle_FlippingTheOptionsMonitor_ChangesTheNextRunWithNoActorRestart() { var siteLog = new FakeSiteEventLogger(); var monitor = new TestOptionsMonitor(Options()); var actor = BuildScriptActor( CompileScript("return 1;"), Options(), new SingleServiceProvider(siteLog, monitor)); var caller = CreateTestProbe(); // Off: the monitor's initial value has PerRunScriptEvents=false. actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-hot-1"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5)); Assert.Empty(siteLog.OfType("script")); // Flip live — same actor, no restart, no redeploy. monitor.Set(Options(perRunScriptEvents: true)); actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-hot-2"), caller.Ref); caller.ExpectMsg(TimeSpan.FromSeconds(10)); AwaitAssert(() => Assert.Equal(2, siteLog.OfType("script").Count), TimeSpan.FromSeconds(5)); } } /// /// Hooks a compiled test script can reach from inside a script body: a gate to block a /// worker thread deterministically, and capture slots for the run's /// and (both of which the run /// otherwise exposes to nobody). /// public static class RunLauncherHooks { /// Gate a blocking test script waits on; reset per test. public static SemaphoreSlim Gate = new(0); /// Set by a test script once its body has run to completion. public static ManualResetEventSlim Finished = new(false); /// Whether the script observed a cancellation request at the end of its body. public static bool? ObservedCancellation; /// The runtime context handed to the last captured run. public static ScriptRuntimeContext? CapturedContext; /// The Alarm global handed to the last captured on-trigger run. public static AlarmContext? CapturedAlarm; }