Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ScriptRunLauncherParityTests.cs
T

470 lines
21 KiB
C#

using Akka.Actor;
using Akka.Event;
using Akka.TestKit;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
/// <summary>
/// WP3.1 supervision-parity set (design memo §4, seven pins) — the reworked successor to
/// <c>ExecutionActorTests</c>.
///
/// <para>WP3.1 eliminated the short-lived <c>ScriptExecutionActor</c> and
/// <c>AlarmExecutionActor</c>: neither had a <c>Receive</c> handler, a <c>PostStop</c>, or any
/// state, and neither's <c>IActorRef</c> was ever a message target — the entire lifecycle lived
/// in a detached task. Runs are now launched directly by the coordinator via
/// <see cref="ScriptRunLauncher"/>. These tests pin every behaviour the removed actors
/// provided onto its replacement: exception and timeout containment, the Ask reply, the
/// completion notification, one DI scope per run, the audit ParentExecutionId threading, and
/// the supervision outcome (coordinator unaffected — no stop, no restart).</para>
/// </summary>
public class ScriptRunLauncherParityTests : TestKit, IDisposable
{
private readonly SharedScriptLibrary _sharedLibrary;
private readonly ScriptCompilationService _compilationService;
/// <summary>Own pool per test class (#18 seam), so a wedged body cannot strand the process-wide one.</summary>
private readonly ScriptExecutionScheduler _scheduler = new(4);
public ScriptRunLauncherParityTests()
{
_compilationService = new ScriptCompilationService(
NullLogger<ScriptCompilationService>.Instance);
_sharedLibrary = new SharedScriptLibrary(
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
}
void IDisposable.Dispose()
{
Shutdown();
_scheduler.Dispose();
}
// ── helpers ──────────────────────────────────────────────────────────────────
private static Script<object?> CompileScript(string code)
{
var scriptOptions = ScriptOptions.Default
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
typeof(RunLauncherHooks).Assembly)
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
script.Compile();
return script;
}
private static SiteRuntimeOptions Options(int timeoutSeconds = 30, int graceMs = 30000)
=> new()
{
MaxScriptCallDepth = 10,
ScriptExecutionTimeoutSeconds = timeoutSeconds,
StuckScriptGraceMs = graceMs
};
private static ResolvedScript CallScript(int? timeoutSeconds = null) => new()
{
CanonicalName = "Runner",
TriggerType = "Call",
ExecutionTimeoutSeconds = timeoutSeconds
};
private TestActorRef<ScriptActor> BuildScriptActor(
Script<object?>? compiled,
SiteRuntimeOptions options,
IServiceProvider? serviceProvider = null,
ISiteHealthCollector? healthCollector = null,
ScriptExecutionScheduler? scheduler = null,
int? perScriptTimeoutSeconds = null,
IActorRef? instanceActor = null)
{
var instance = instanceActor ?? CreateTestProbe().Ref;
return ActorOfAsTestActorRef<ScriptActor>(
Props.Create(() => new ScriptActor(
"Runner", "Inst1", instance, compiled, CallScript(perScriptTimeoutSeconds),
_sharedLibrary, options, NullLogger<ScriptActor>.Instance,
null, null, healthCollector, serviceProvider, scheduler ?? _scheduler, null)),
"script-" + Guid.NewGuid().ToString("N"));
}
// ── Pin 1: throwing body — coordinator survives, everything is reported ───────
[Fact]
public void ThrowingScriptBody_LeavesScriptActorAliveAndReportsEverything()
{
var siteLog = new FakeSiteEventLogger();
var health = new SiteHealthCollector();
var actor = BuildScriptActor(
CompileScript("throw new InvalidOperationException(\"boom\");"),
Options(),
new SingleServiceProvider(siteLog),
health);
Watch(actor);
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-throw"), caller.Ref);
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
Assert.False(result.Success);
Assert.Equal("corr-throw", result.CorrelationId);
Assert.Contains("boom", result.ErrorMessage);
AwaitAssert(() =>
{
// Error site event + script-error counter, exactly as the execution actor emitted.
Assert.Contains(siteLog.OfType("script"),
r => r.Severity == "Error" && r.Message.Contains("failed", StringComparison.OrdinalIgnoreCase));
// The in-flight slot is released, so the script can run again.
Assert.Equal(0, actor.UnderlyingActor.RunsInFlight);
}, TimeSpan.FromSeconds(5));
Assert.Equal(1, health.CollectReport("site-1").ScriptErrorCount);
// The coordinator is neither stopped nor restarted — a throwing body was always
// contained inside the run's own try/catch, and still is. Watch() above means a stop
// would deliver Terminated to the TestActor; a still-answering call proves it is live.
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-still-alive"), caller.Ref);
Assert.Equal("corr-still-alive",
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10)).CorrelationId);
}
// ── Pin 2: launch-path throw — improved over the old silent hang ──────────────
/// <summary>
/// The only failure the removed per-run child could surface was a constructor throw —
/// e.g. queueing onto a disposed <see cref="ScriptExecutionScheduler"/>. The old
/// <c>OneForOneStrategy</c> logged it and stopped the child, leaving the Ask caller to
/// hang with no reply at all. WP3.1 folds that into a launch-path catch that replies, an
/// intentional improvement pinned here so it is explicit rather than accidental.
/// </summary>
[Fact]
public void LaunchPathThrow_RepliesToTheCallerAndLeavesTheCoordinatorAlive()
{
var dead = new ScriptExecutionScheduler(1);
dead.Dispose();
var actor = BuildScriptActor(
CompileScript("return 1;"), Options(), scheduler: dead);
Watch(actor);
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-launch"), caller.Ref);
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
Assert.False(result.Success);
Assert.Equal("corr-launch", result.CorrelationId);
Assert.Contains("could not be launched", result.ErrorMessage);
// Watch() above means a stop would deliver Terminated to the TestActor; none arrives,
// so the coordinator neither died nor restarted on a launch failure.
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
// The in-flight counter is incremented before the launch, so the catch MUST balance
// it — otherwise a run of launch failures would permanently wedge the script at cap.
AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5));
}
// ── Pin 3: exactly one DI scope per run, disposed on every path ───────────────
[Theory]
[InlineData("return 1;", 30)] // success
[InlineData("throw new InvalidOperationException(\"boom\");", 30)] // failure
[InlineData("while (true) { await Task.Delay(25, CancellationToken); }", 1)] // timeout
public void EachRun_CreatesOneDiScope_AndDisposesItExactlyOnce(string code, int timeoutSeconds)
{
var spy = new ScopeSpyServiceProvider();
var actor = BuildScriptActor(
CompileScript(code), Options(timeoutSeconds), spy);
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-scope"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
AwaitAssert(() =>
{
Assert.Equal(1, spy.ScopesCreated);
Assert.Equal(1, spy.ScopesDisposed);
}, TimeSpan.FromSeconds(5));
}
// ── Pin 4: audit correlation threading survives the actor removal ─────────────
/// <summary>
/// A routed <see cref="ScriptCallRequest.ParentExecutionId"/> must still reach the run's
/// <see cref="ScriptRuntimeContext"/> — this is the inbound-API leg of the audit execution
/// tree, and it used to be threaded through the execution actor's constructor.
/// </summary>
[Fact]
public void RoutedParentExecutionId_ReachesTheRunsScriptRuntimeContext()
{
RunLauncherHooks.CapturedContext = null;
var parent = Guid.NewGuid();
var actor = BuildScriptActor(
CompileScript(
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.CapturedContext = Instance; return 1;"),
Options());
var caller = CreateTestProbe();
actor.Tell(
new ScriptCallRequest("Runner", null, 0, "corr-parent", ParentExecutionId: parent),
caller.Ref);
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
Assert.True(result.Success, result.ErrorMessage);
AwaitAssert(() =>
{
Assert.NotNull(RunLauncherHooks.CapturedContext);
Assert.Equal(parent, RunLauncherHooks.CapturedContext!.ParentExecutionId);
// The routed run still mints its OWN ExecutionId — the parent is a pointer, not a copy.
Assert.NotEqual(parent, RunLauncherHooks.CapturedContext.ExecutionId);
}, TimeSpan.FromSeconds(5));
}
// ── Pin 5: timeout resolution parity (perScript ?? global, <= 0 => global) ────
[Theory]
[InlineData(300, 1)] // per-script override wins over a much longer global
[InlineData(1, null)] // null per-script falls back to the global
[InlineData(1, 0)] // non-positive per-script is treated as "use global"
public void TimeoutResolution_MatchesTheRemovedExecutionActor(int globalSeconds, int? perScriptSeconds)
{
var siteLog = new FakeSiteEventLogger();
var actor = BuildScriptActor(
CompileScript("while (true) { await Task.Delay(25, CancellationToken); }"),
Options(globalSeconds),
new SingleServiceProvider(siteLog),
perScriptTimeoutSeconds: perScriptSeconds);
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-timeout"), caller.Ref);
// If the effective timeout were the 300 s global (case 1) or ignored (cases 2/3) this
// would not answer inside the window.
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
Assert.False(result.Success);
Assert.Contains("timed out", result.ErrorMessage);
AwaitAssert(
() => Assert.Contains(siteLog.OfType("script"),
r => r.Severity == "Error" && r.Message.Contains("timed out")),
TimeSpan.FromSeconds(5));
}
// ── Pin 6: stop-during-run parity ────────────────────────────────────────────
/// <summary>
/// Stopping a ScriptActor mid-run must NOT cancel the in-flight run (redeploy/undeploy
/// semantics: running scripts are allowed to finish). The run completes normally and its
/// completion message dead-letters, exactly as the old per-run child's
/// <c>parent.Tell</c> did once the subtree was stopped — dead letters are a health metric,
/// not an error.
/// </summary>
[Fact]
public void StoppingTheScriptActorMidRun_LetsTheRunFinishAndDeadLettersItsCompletion()
{
RunLauncherHooks.Gate = new SemaphoreSlim(0);
RunLauncherHooks.Finished = new ManualResetEventSlim(false);
RunLauncherHooks.ObservedCancellation = null;
var actor = BuildScriptActor(
CompileScript(
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.Gate.Wait();" +
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.ObservedCancellation = CancellationToken.IsCancellationRequested;" +
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.Finished.Set();" +
"return 1;"),
Options());
var deadLetters = CreateTestProbe();
Sys.EventStream.Subscribe(deadLetters.Ref, typeof(DeadLetter));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-stop"), caller.Ref);
// Wait until the body is actually on a worker thread, then stop the coordinator.
AwaitAssert(() => Assert.Equal(1, _scheduler.BusyThreadCount), TimeSpan.FromSeconds(10));
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, TimeSpan.FromSeconds(10));
RunLauncherHooks.Gate.Release();
// The run ran to completion and was never cancelled by the stop.
Assert.True(RunLauncherHooks.Finished.Wait(TimeSpan.FromSeconds(10)));
Assert.False(RunLauncherHooks.ObservedCancellation);
// The Ask caller still gets its result (the reply target is not the stopped actor)…
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
Assert.True(result.Success, result.ErrorMessage);
// …and the completion notification aimed at the now-stopped coordinator dead-letters.
deadLetters.FishForMessage<DeadLetter>(
d => d.Message is ScriptActor.ScriptExecutionCompleted,
TimeSpan.FromSeconds(10));
}
// ── Pin 7: alarm side ────────────────────────────────────────────────────────
/// <summary>
/// An alarm on-trigger run still receives the <c>Alarm</c> globals (name/level/priority/
/// message) and still reports <c>AlarmExecutionCompleted</c> back to its AlarmActor —
/// observable here through the in-flight counter returning to zero, which only the
/// completion message can do.
/// </summary>
[Fact]
public void AlarmOnTriggerRun_GetsAlarmGlobals_AndCompletesBackToTheAlarmActor()
{
RunLauncherHooks.CapturedAlarm = null;
var onTrigger = CompileScript(
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.CapturedAlarm = Alarm; return null;");
var instanceProbe = CreateTestProbe();
var alarm = ActorOfAsTestActorRef<AlarmActor>(
Props.Create(() => new AlarmActor(
"TempBand", "Inst1", instanceProbe.Ref,
new ResolvedAlarm
{
CanonicalName = "TempBand",
TriggerType = "HiLo",
TriggerConfiguration = "{\"attributeName\":\"Temp\",\"hi\":80,\"hiHi\":95,\"hiMessage\":\"too hot\"}",
PriorityLevel = 42
},
onTrigger, _sharedLibrary, Options(), NullLogger<AlarmActor>.Instance,
null, null, null, null, null, _scheduler, null)),
"alarm-" + Guid.NewGuid().ToString("N"));
alarm.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
"Inst1", "Temp", "Temp", 90.0, "Good", DateTimeOffset.UtcNow));
instanceProbe.ExpectMsg<Commons.Messages.Streaming.AlarmStateChanged>(TimeSpan.FromSeconds(10));
AwaitAssert(() =>
{
Assert.NotNull(RunLauncherHooks.CapturedAlarm);
Assert.Equal("TempBand", RunLauncherHooks.CapturedAlarm!.Name);
Assert.Equal(AlarmLevel.High, RunLauncherHooks.CapturedAlarm.Level);
Assert.Equal("too hot", RunLauncherHooks.CapturedAlarm.Message);
// Only AlarmExecutionCompleted releases the slot.
Assert.Equal(0, alarm.UnderlyingActor.RunsInFlight);
}, TimeSpan.FromSeconds(10));
}
// ── Retargeted from ExecutionActorTests: success path + operational events ────
[Fact]
public void SuccessfulRun_RepliesWithTheReturnValue()
{
var actor = BuildScriptActor(CompileScript("return 7 * 6;"), Options());
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-ok"), caller.Ref);
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
Assert.True(result.Success, result.ErrorMessage);
Assert.Equal("corr-ok", result.CorrelationId);
Assert.Equal(42, result.ReturnValue);
}
[Fact]
public void SuccessfulRun_EmitsStartedThenCompletedInfoEvents()
{
var siteLog = new FakeSiteEventLogger();
var actor = BuildScriptActor(
CompileScript("return 7 * 6;"), Options(), new SingleServiceProvider(siteLog));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
AwaitAssert(() =>
{
var rows = siteLog.OfType("script");
Assert.Equal(2, rows.Count);
Assert.All(rows, r =>
{
Assert.Equal("Info", r.Severity);
Assert.Equal("Inst1", r.InstanceId);
Assert.Equal("ScriptActor:Runner", r.Source);
});
Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase);
Assert.Contains("completed", rows[1].Message, StringComparison.OrdinalIgnoreCase);
}, TimeSpan.FromSeconds(5));
}
[Fact]
public void FailingRun_EmitsStartedInfoThenErrorEvent()
{
var siteLog = new FakeSiteEventLogger();
var actor = BuildScriptActor(
CompileScript("throw new InvalidOperationException(\"boom\");"),
Options(), new SingleServiceProvider(siteLog));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt-err"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
AwaitAssert(() =>
{
var rows = siteLog.OfType("script");
Assert.Equal(2, rows.Count);
Assert.Equal("Info", rows[0].Severity);
Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase);
Assert.Equal("Error", rows[1].Severity);
}, TimeSpan.FromSeconds(5));
}
[Fact]
public void FireAndForgetRun_NeedsNoReplyTarget()
{
var siteLog = new FakeSiteEventLogger();
var actor = BuildScriptActor(
CompileScript("return 1;"), Options(), new SingleServiceProvider(siteLog));
// Trigger-driven spawns pass ActorRefs.NoSender as replyTo; drive that path via an
// interval-free Call script by telling the actor to run with no sender.
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-nobody"), ActorRefs.NoSender);
AwaitAssert(
() => Assert.Contains(siteLog.OfType("script"),
r => r.Message.Contains("completed", StringComparison.OrdinalIgnoreCase)),
TimeSpan.FromSeconds(10));
}
}
/// <summary>
/// Hooks a compiled test script can reach from inside a script body: a gate to block a
/// worker thread deterministically, and capture slots for the run's
/// <see cref="ScriptRuntimeContext"/> and <see cref="AlarmContext"/> (both of which the run
/// otherwise exposes to nobody).
/// </summary>
public static class RunLauncherHooks
{
/// <summary>Gate a blocking test script waits on; reset per test.</summary>
public static SemaphoreSlim Gate = new(0);
/// <summary>Set by a test script once its body has run to completion.</summary>
public static ManualResetEventSlim Finished = new(false);
/// <summary>Whether the script observed a cancellation request at the end of its body.</summary>
public static bool? ObservedCancellation;
/// <summary>The runtime context handed to the last captured run.</summary>
public static ScriptRuntimeContext? CapturedContext;
/// <summary>The <c>Alarm</c> global handed to the last captured on-trigger run.</summary>
public static AlarmContext? CapturedAlarm;
}