perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution
This commit is contained in:
Joseph Doherty
@@ -1028,14 +1028,22 @@ public class AlarmActorTests : TestKit, IDisposable
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExpressionAlarm_EvaluatesOnSchedulerThread_AndActivates()
|
||||
public void ExpressionAlarm_EvaluatesOffTheBlockingScriptPool_AndActivates()
|
||||
{
|
||||
// TRUE only when evaluated on a script-execution thread. Before P1 the
|
||||
// expression ran on the actor dispatcher (name is NOT "script-execution-*")
|
||||
// → false → alarm never activates. After P1 it runs on the script scheduler.
|
||||
// WP3.1 retarget of the former P1 assertion, whose sense is deliberately INVERTED.
|
||||
//
|
||||
// P1 moved the evaluation off the actor's dispatcher and onto the dedicated
|
||||
// script-execution scheduler, and this test asserted exactly that. WP3.1 (finding #4)
|
||||
// proved that destination wrong for alarms in particular: sharing the blocking pool
|
||||
// meant an alarm that should raise in milliseconds queued behind blocked script bodies
|
||||
// for an unbounded time, and its 2 s evaluation timeout did not even start ticking
|
||||
// until it was dequeued. Evaluations now run on the shared .NET thread pool behind
|
||||
// TriggerEvalGate.
|
||||
//
|
||||
// TRUE only when evaluated OFF a script-execution thread.
|
||||
var expr = CompileRawTriggerExpression(
|
||||
"System.Threading.Thread.CurrentThread.Name != null && " +
|
||||
"System.Threading.Thread.CurrentThread.Name.StartsWith(\"script-execution-\")");
|
||||
"System.Threading.Thread.CurrentThread.Name == null || " +
|
||||
"!System.Threading.Thread.CurrentThread.Name.StartsWith(\"script-execution-\")");
|
||||
var alarmConfig = new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "ExprAlarm",
|
||||
|
||||
-1
@@ -217,7 +217,6 @@ public class AlarmCascadeParentExecutionTests : TestKit, IDisposable
|
||||
var executionId = Guid.NewGuid();
|
||||
var context = new ScriptRuntimeContext(
|
||||
probe.Ref,
|
||||
ActorRefs.Nobody,
|
||||
_sharedLibrary,
|
||||
currentCallDepth: 0,
|
||||
maxCallDepth: 10,
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test group 9 — warm-then-gate deploys and startup batch pre-warm.
|
||||
///
|
||||
/// <para>The site-side compile gate (S3) must stay synchronous on the Deployment Manager's
|
||||
/// thread, because redeploy-supersede and delete-during-redeploy both depend on strict mailbox
|
||||
/// FIFO. But the Roslyn compile it performs used to hold the singleton for the whole
|
||||
/// compilation, stalling every OTHER instance's commands behind one instance's scripts. WP3.1
|
||||
/// warms the compile off-thread first and then re-runs the gate as pure cache hits, with a
|
||||
/// per-instance in-flight guard preserving same-instance ordering.</para>
|
||||
///
|
||||
/// <para>These tests pin the ordering contract, not the timing: a command for the SAME instance
|
||||
/// arriving during a warm must be queued and applied after the deploy, a superseded deploy must
|
||||
/// answer its deployer instead of leaving it to Ask-timeout, and commands for DIFFERENT
|
||||
/// instances must not block each other.</para>
|
||||
///
|
||||
/// <para>Shares the <c>SiteScriptCompileCache</c> collection because the batch pre-warm test
|
||||
/// asserts on that process-wide cache's hit counter.</para>
|
||||
/// </summary>
|
||||
[Collection("SiteScriptCompileCache")]
|
||||
public class DeploymentWarmThenGateTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly ScriptCompilationService _compilationService;
|
||||
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
||||
private readonly TestLocalDb _localDb;
|
||||
|
||||
public DeploymentWarmThenGateTests()
|
||||
{
|
||||
_localDb = TestLocalDb.CreateTemp("dm-warm-gate-test");
|
||||
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
_compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedScriptLibrary = new SharedScriptLibrary(
|
||||
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
Shutdown();
|
||||
var path = _localDb.Path;
|
||||
_localDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(path);
|
||||
}
|
||||
|
||||
private IActorRef CreateDeploymentManager(ISiteHealthCollector? healthCollector = null) =>
|
||||
ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage, _compilationService, _sharedScriptLibrary, null,
|
||||
new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance, null,
|
||||
healthCollector)));
|
||||
|
||||
/// <summary>
|
||||
/// Captures the deployed-instance count the Deployment Manager reports. The count is
|
||||
/// mutated only on the actor thread — <c>HandleDeploy</c> adds the instance name,
|
||||
/// <c>HandleDelete</c> removes it — so it is an exact, storage-race-free record of the
|
||||
/// order in which the two commands were APPLIED.
|
||||
/// </summary>
|
||||
private sealed class DeployedCountCollector : ISiteHealthCollector
|
||||
{
|
||||
public int LastDeployedCount { get; private set; }
|
||||
public void IncrementScriptError() { }
|
||||
public void IncrementAlarmError() { }
|
||||
public void IncrementDeadLetter() { }
|
||||
public void IncrementSiteAuditWriteFailures() { }
|
||||
public void IncrementAuditRedactionFailure() { }
|
||||
public void UpdateSiteAuditBacklog(Commons.Types.SiteAuditBacklogSnapshot snapshot) { }
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { }
|
||||
public void RemoveConnection(string connectionName) { }
|
||||
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }
|
||||
public void UpdateConnectionEndpoint(string connectionName, string endpoint) { }
|
||||
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { }
|
||||
public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths) { }
|
||||
public void SetInstanceCounts(int deployed, int enabled, int disabled) => LastDeployedCount = deployed;
|
||||
public void SetParkedMessageCount(int count) { }
|
||||
public void SetNodeHostname(string hostname) { }
|
||||
public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) { }
|
||||
public void SetActiveNode(bool isActive) { }
|
||||
public bool IsActiveNode => true;
|
||||
public Commons.Messages.Health.SiteHealthReport CollectReport(string siteId)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
private static string ConfigJson(string instanceName, string? scriptCode = null) =>
|
||||
JsonSerializer.Serialize(new FlattenedConfiguration
|
||||
{
|
||||
InstanceUniqueName = instanceName,
|
||||
Attributes =
|
||||
[
|
||||
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
|
||||
],
|
||||
Scripts = scriptCode is null
|
||||
? []
|
||||
: [new ResolvedScript { CanonicalName = "Worker", Code = scriptCode, TriggerType = "Call" }]
|
||||
});
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteArrivingDuringTheCompileWarm_IsQueuedAndAppliedAfterTheDeploy()
|
||||
{
|
||||
var health = new DeployedCountCollector();
|
||||
var dm = CreateDeploymentManager(health);
|
||||
await Task.Delay(500); // empty startup
|
||||
Assert.Equal(0, health.LastDeployedCount);
|
||||
|
||||
var deployProbe = CreateTestProbe();
|
||||
var deleteProbe = CreateTestProbe();
|
||||
|
||||
// Back-to-back on the mailbox: the delete lands while the deploy's compile warm is
|
||||
// still in flight, so it must be queued rather than racing ahead of the deploy.
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-1", "WarmPump", "h1", ConfigJson("WarmPump", "return 1;"), "admin", DateTimeOffset.UtcNow),
|
||||
deployProbe.Ref);
|
||||
dm.Tell(new DeleteInstanceCommand("del-1", "WarmPump", DateTimeOffset.UtcNow), deleteProbe.Ref);
|
||||
|
||||
var deploy = deployProbe.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.Equal(DeploymentStatus.Success, deploy.Status);
|
||||
|
||||
var delete = deleteProbe.ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.True(delete.Success);
|
||||
|
||||
// Terminal in-memory state: the deploy applied FIRST (adding the instance) and the
|
||||
// delete applied SECOND (removing it), leaving the count at 0. Had the delete raced
|
||||
// ahead of the warm it would have removed nothing and the deploy would have left the
|
||||
// count at 1. This is the ordering signal rather than the SQLite row, because the
|
||||
// deploy's store and the delete's remove are independent background tasks whose
|
||||
// completion order the actor has never guaranteed (true before WP3.1 as well).
|
||||
AwaitAssert(() => Assert.Equal(0, health.LastDeployedCount), TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecondDeployDuringTheWarm_SupersedesTheFirst_AndAnswersItsDeployer()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
var first = CreateTestProbe();
|
||||
var second = CreateTestProbe();
|
||||
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-a", "SupersedePump", "h1", ConfigJson("SupersedePump", "return 1;"), "admin", DateTimeOffset.UtcNow),
|
||||
first.Ref);
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-b", "SupersedePump", "h2", ConfigJson("SupersedePump", "return 2;"), "admin", DateTimeOffset.UtcNow),
|
||||
second.Ref);
|
||||
|
||||
// The displaced deployer is answered rather than left to time out its Ask.
|
||||
var superseded = first.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.Equal("dep-a", superseded.DeploymentId);
|
||||
Assert.Equal(DeploymentStatus.Failed, superseded.Status);
|
||||
Assert.Contains("superseded", superseded.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var winner = second.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
||||
Assert.Equal("dep-b", winner.DeploymentId);
|
||||
Assert.Equal(DeploymentStatus.Success, winner.Status);
|
||||
|
||||
// Exactly one row, carrying the winning revision hash.
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "SupersedePump");
|
||||
Assert.Equal("h2", row.RevisionHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeploysForDifferentInstances_DoNotBlockEachOther()
|
||||
{
|
||||
var dm = CreateDeploymentManager();
|
||||
await Task.Delay(500);
|
||||
|
||||
var probeX = CreateTestProbe();
|
||||
var probeY = CreateTestProbe();
|
||||
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-x", "PumpX", "hx", ConfigJson("PumpX", "return 1;"), "admin", DateTimeOffset.UtcNow),
|
||||
probeX.Ref);
|
||||
dm.Tell(new DeployInstanceCommand(
|
||||
"dep-y", "PumpY", "hy", ConfigJson("PumpY", "return 2;"), "admin", DateTimeOffset.UtcNow),
|
||||
probeY.Ref);
|
||||
|
||||
// Both apply; the per-instance warm guard scopes to the instance, so a warm for X
|
||||
// never queues a command for Y.
|
||||
Assert.Equal(DeploymentStatus.Success,
|
||||
probeX.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
|
||||
Assert.Equal(DeploymentStatus.Success,
|
||||
probeY.ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15)).Status);
|
||||
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Contains(configs, c => c.InstanceUniqueName == "PumpX");
|
||||
Assert.Contains(configs, c => c.InstanceUniqueName == "PumpY");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StaggeredStartup_PreWarmsEachBatchSoInstanceActorPreStartCompilesAreCacheHits()
|
||||
{
|
||||
// Two instances sharing one script body. The batch pre-warm compiles it once; the
|
||||
// second config's warm and BOTH Instance Actors' PreStart compiles are then hits.
|
||||
// Before WP3.1 every Instance Actor Roslyn-compiled its own scripts inside PreStart,
|
||||
// serialising a site's whole failover recovery behind compilation.
|
||||
const string sharedCode = "return 41 + 1;";
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"BatchOne", ConfigJson("BatchOne", sharedCode), "d1", "h1", true);
|
||||
await _storage.StoreDeployedConfigAsync(
|
||||
"BatchTwo", ConfigJson("BatchTwo", sharedCode), "d2", "h2", true);
|
||||
|
||||
SiteScriptCompileCache.Clear();
|
||||
Assert.Equal(0, SiteScriptCompileCache.Hits);
|
||||
|
||||
CreateDeploymentManager();
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
// One compile, then repeated hits: the second pre-warm plus both PreStarts.
|
||||
Assert.True(SiteScriptCompileCache.Hits >= 3,
|
||||
$"expected the pre-warmed body to be served from cache, saw {SiteScriptCompileCache.Hits} hits");
|
||||
}, TimeSpan.FromSeconds(20));
|
||||
}
|
||||
}
|
||||
@@ -1,440 +0,0 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Regression coverage for SiteRuntime-016 — the short-lived execution actors
|
||||
/// (<see cref="ScriptExecutionActor"/>, <see cref="AlarmExecutionActor"/>) were
|
||||
/// previously untested. Covers success, exception, timeout, Ask-reply, and the
|
||||
/// PoisonPill self-stop after completion.
|
||||
/// </summary>
|
||||
public class ExecutionActorTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
private readonly ScriptCompilationService _compilationService;
|
||||
|
||||
public ExecutionActorTests()
|
||||
{
|
||||
_compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose() => Shutdown();
|
||||
|
||||
private static Script<object?> CompileScript(string code)
|
||||
{
|
||||
var scriptOptions = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
private static SiteRuntimeOptions Options(int timeoutSeconds = 30)
|
||||
=> new() { MaxScriptCallDepth = 10, ScriptExecutionTimeoutSeconds = timeoutSeconds };
|
||||
|
||||
// ── ScriptExecutionActor ──
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_Success_RepliesWithResultAndStops()
|
||||
{
|
||||
var compiled = CompileScript("return 7 * 6;");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Answer", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(),
|
||||
replyTo.Ref, "corr-1", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null)));
|
||||
|
||||
Watch(exec);
|
||||
|
||||
var result = replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal("corr-1", result.CorrelationId);
|
||||
Assert.Equal(42, result.ReturnValue);
|
||||
|
||||
// The actor must PoisonPill itself once execution completes.
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── M1.8: site event log `script` started/completed ────────────────────
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_Success_EmitsScriptStartedAndCompletedInfoEvents()
|
||||
{
|
||||
var compiled = CompileScript("return 7 * 6;");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Answer", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(),
|
||||
replyTo.Ref, "corr-evt-1", NullLogger.Instance,
|
||||
ScriptScope.Root, null, new SingleServiceProvider(siteLog))));
|
||||
|
||||
Watch(exec);
|
||||
replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var rows = siteLog.OfType("script");
|
||||
// started + completed, both Info, in order.
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.All(rows, r =>
|
||||
{
|
||||
Assert.Equal("Info", r.Severity);
|
||||
Assert.Equal("Inst1", r.InstanceId);
|
||||
Assert.Equal("ScriptActor:Answer", r.Source);
|
||||
});
|
||||
Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Contains("completed", rows[1].Message, StringComparison.OrdinalIgnoreCase);
|
||||
}, TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_Failure_EmitsStartedInfoThenErrorEvent()
|
||||
{
|
||||
var compiled = CompileScript("throw new InvalidOperationException(\"boom\");");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Bad", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(),
|
||||
replyTo.Ref, "corr-evt-2", NullLogger.Instance,
|
||||
ScriptScope.Root, null, new SingleServiceProvider(siteLog))));
|
||||
|
||||
Watch(exec);
|
||||
replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var rows = siteLog.OfType("script");
|
||||
// started (Info) + failed (Error) — no completed.
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.Equal("Info", rows[0].Severity);
|
||||
Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Equal("Error", rows[1].Severity);
|
||||
}, TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_ScriptThrows_RepliesFailureAndStops()
|
||||
{
|
||||
var compiled = CompileScript("throw new InvalidOperationException(\"boom\");");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Bad", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(),
|
||||
replyTo.Ref, "corr-2", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null)));
|
||||
|
||||
Watch(exec);
|
||||
|
||||
var result = replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal("corr-2", result.CorrelationId);
|
||||
Assert.Contains("boom", result.ErrorMessage);
|
||||
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_Timeout_RepliesFailureAndStops()
|
||||
{
|
||||
// A long busy loop that observes the cancellation token so the
|
||||
// 1-second timeout fires cooperatively.
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Slow", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 1),
|
||||
replyTo.Ref, "corr-3", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null)));
|
||||
|
||||
Watch(exec);
|
||||
|
||||
var result = replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("timed out", result.ErrorMessage);
|
||||
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_PerScriptTimeout_OverridesLongerGlobal()
|
||||
{
|
||||
// M2.5 (#9): a short per-script timeout (1s) must win over a long global
|
||||
// (300s), so the busy loop is cancelled at the per-script value.
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Slow", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 300),
|
||||
replyTo.Ref, "corr-perscript", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null, null,
|
||||
/* executionTimeoutSeconds */ 1)));
|
||||
|
||||
Watch(exec);
|
||||
|
||||
var result = replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("timed out", result.ErrorMessage);
|
||||
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_NullPerScriptTimeout_FallsBackToGlobal()
|
||||
{
|
||||
// M2.5 (#9): a null per-script timeout falls back to the global (1s here),
|
||||
// so the busy loop is still cancelled at the global value.
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Slow", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 1),
|
||||
replyTo.Ref, "corr-fallback", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null, null,
|
||||
/* executionTimeoutSeconds */ null)));
|
||||
|
||||
Watch(exec);
|
||||
|
||||
var result = replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("timed out", result.ErrorMessage);
|
||||
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_NonPositivePerScriptTimeout_FallsBackToGlobal()
|
||||
{
|
||||
// M2.5 (#9): a non-positive per-script value (<= 0) is treated as "use
|
||||
// global", so the busy loop is cancelled at the global (1s) value.
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"Slow", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 1),
|
||||
replyTo.Ref, "corr-clamp", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null, null,
|
||||
/* executionTimeoutSeconds */ 0)));
|
||||
|
||||
Watch(exec);
|
||||
|
||||
var result = replyTo.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("timed out", result.ErrorMessage);
|
||||
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptExecutionActor_NoReplyTo_StillStopsAfterCompletion()
|
||||
{
|
||||
var compiled = CompileScript("return 1;");
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
// ActorRefs.Nobody as replyTo — fire-and-forget execution.
|
||||
var exec = ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"FireForget", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, Options(),
|
||||
ActorRefs.Nobody, "corr-4", NullLogger.Instance,
|
||||
ScriptScope.Root, null, null)));
|
||||
|
||||
Watch(exec);
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── AlarmExecutionActor ──
|
||||
|
||||
[Fact]
|
||||
public void AlarmExecutionActor_Success_StopsAfterCompletion()
|
||||
{
|
||||
var compiled = CompileScript("return 0;");
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new AlarmExecutionActor(
|
||||
"HiTemp", "Inst1", AlarmLevel.High, 5, "High temperature",
|
||||
compiled, instanceActor.Ref, _sharedLibrary, Options(),
|
||||
NullLogger.Instance)));
|
||||
|
||||
Watch(exec);
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmExecutionActor_ScriptThrows_StillStops()
|
||||
{
|
||||
var compiled = CompileScript("throw new System.Exception(\"alarm-boom\");");
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new AlarmExecutionActor(
|
||||
"HiTemp", "Inst1", AlarmLevel.High, 5, "High temperature",
|
||||
compiled, instanceActor.Ref, _sharedLibrary, Options(),
|
||||
NullLogger.Instance)));
|
||||
|
||||
Watch(exec);
|
||||
// Even on a throwing on-trigger body, the actor must self-stop.
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmExecutionActor_PerScriptTimeout_OverridesLongerGlobal()
|
||||
{
|
||||
// M2.5 (#9): the alarm on-trigger script's per-script timeout (1s) wins
|
||||
// over a long global (300s). The busy loop is cancelled and the actor
|
||||
// self-stops (the timeout is logged, alarm continues).
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new AlarmExecutionActor(
|
||||
"HiTemp", "Inst1", AlarmLevel.High, 5, "High temperature",
|
||||
compiled, instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 300),
|
||||
NullLogger.Instance, /* executionTimeoutSeconds */ 1)));
|
||||
|
||||
Watch(exec);
|
||||
// If the per-script timeout were ignored it would block ~300s and this
|
||||
// ExpectTerminated would fail; with the override it stops within ~1s.
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmExecutionActor_NullPerScriptTimeout_FallsBackToGlobal()
|
||||
{
|
||||
// M2.5 (#9): a null per-script timeout falls back to the global (1s here),
|
||||
// so the busy loop is cancelled at the global value and the actor self-stops.
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new AlarmExecutionActor(
|
||||
"HiTemp", "Inst1", AlarmLevel.High, 5, "High temperature",
|
||||
compiled, instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 1),
|
||||
NullLogger.Instance, /* executionTimeoutSeconds */ null)));
|
||||
|
||||
Watch(exec);
|
||||
// Global timeout (1s) must fire even when per-script is null.
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmExecutionActor_NonPositivePerScriptTimeout_FallsBackToGlobal()
|
||||
{
|
||||
// M2.5 (#9): a non-positive per-script value (<= 0) is treated as "use
|
||||
// global", so the busy loop is cancelled at the global (1s) value.
|
||||
var compiled = CompileScript(
|
||||
"while (true) { await System.Threading.Tasks.Task.Delay(50, CancellationToken); }");
|
||||
var instanceActor = CreateTestProbe();
|
||||
|
||||
var exec = ActorOf(Props.Create(() => new AlarmExecutionActor(
|
||||
"HiTemp", "Inst1", AlarmLevel.High, 5, "High temperature",
|
||||
compiled, instanceActor.Ref, _sharedLibrary, Options(timeoutSeconds: 1),
|
||||
NullLogger.Instance, /* executionTimeoutSeconds */ 0)));
|
||||
|
||||
Watch(exec);
|
||||
// Non-positive per-script timeout must be ignored; global (1s) must fire.
|
||||
ExpectTerminated(exec, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
// ── S2: stuck-script watchdog names the script holding a scheduler thread ──
|
||||
|
||||
/// <summary>
|
||||
/// Compiles a raw script that can reach the test-assembly <see cref="StuckTestHooks"/>
|
||||
/// (so a script body can block on a gate). Bypasses the trust validator, like
|
||||
/// <see cref="CompileScript"/>.
|
||||
/// </summary>
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var scriptOptions = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly, typeof(StuckTestHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimedOutScript_WithBlockedThread_EmitsStuckThreadSiteEvent()
|
||||
{
|
||||
// A script that blocks synchronously on a gate NEVER observes cooperative
|
||||
// cancellation, so the CTS firing at the 1s timeout does not free its
|
||||
// scheduler thread. After the 200ms grace the watchdog must name it loudly.
|
||||
StuckTestHooks.Gate = new SemaphoreSlim(0);
|
||||
var compiled = CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StuckTestHooks.Gate.Wait(); return null;");
|
||||
var replyTo = CreateTestProbe();
|
||||
var instanceActor = CreateTestProbe();
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var options = new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 1, StuckScriptGraceMs = 200 };
|
||||
|
||||
ActorOf(Props.Create(() => new ScriptExecutionActor(
|
||||
"StuckScript", "Inst1", compiled, null, 0,
|
||||
instanceActor.Ref, _sharedLibrary, options,
|
||||
replyTo.Ref, "corr-stuck", NullLogger.Instance,
|
||||
ScriptScope.Root, null, new SingleServiceProvider(siteLog))));
|
||||
|
||||
try
|
||||
{
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var rows = siteLog.OfType("script");
|
||||
Assert.Contains(rows, r =>
|
||||
r.Severity == "Error" &&
|
||||
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) &&
|
||||
r.Message.Contains("StuckScript"));
|
||||
}, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Free the blocked scheduler thread so the test run stays clean.
|
||||
StuckTestHooks.Gate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test hook the stuck-script watchdog test uses to block a script-execution
|
||||
/// thread deterministically: the compiled body waits on <see cref="Gate"/>, which
|
||||
/// the test releases once it has observed the stuck-thread site event.
|
||||
/// </summary>
|
||||
public static class StuckTestHooks
|
||||
{
|
||||
/// <summary>Gate a blocking test script waits on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
@@ -403,13 +403,15 @@ public class ScriptActorTests : TestKit, IDisposable
|
||||
"ExprFault", "Expression", "{\"expression\":\"true\",\"mode\":\"OnTrue\"}", null, expr);
|
||||
try
|
||||
{
|
||||
actor.Tell(Change("A", "1")); // eval starts on the scheduler and BLOCKS → _evalInFlight = true
|
||||
actor.Tell(Change("A", "1")); // eval starts off-thread and BLOCKS → _evalInFlight = true
|
||||
AwaitAssert(() => Assert.Equal(1, EvalGate.Entries), TimeSpan.FromSeconds(10));
|
||||
|
||||
// #18 seam: the blocked evaluation is running on THIS class's injected
|
||||
// scheduler — not the process-wide singleton — so a worker it strands can
|
||||
// never starve another test class.
|
||||
Assert.Equal(1, _scheduler.BusyThreadCount);
|
||||
// WP3.1: the blocked evaluation no longer occupies a script-execution worker at
|
||||
// all — it runs on the shared thread pool behind TriggerEvalGate, which is the
|
||||
// whole point of finding #4. The blocking pool must be completely idle here; the
|
||||
// pre-WP3.1 assertion was the opposite (BusyThreadCount == 1).
|
||||
Assert.Equal(0, _scheduler.BusyThreadCount);
|
||||
Assert.Equal(0, _scheduler.QueueDepth);
|
||||
|
||||
actor.Tell(Change("A", "2")); // coalesces → _evalPending = true
|
||||
|
||||
@@ -432,15 +434,24 @@ public class ScriptActorTests : TestKit, IDisposable
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExpressionTrigger_EvaluatesOnScriptSchedulerThread_AndStillFires()
|
||||
public void ExpressionTrigger_EvaluatesOffTheBlockingScriptPool_AndStillFires()
|
||||
{
|
||||
// The expression is TRUE only when evaluated on a script-execution thread.
|
||||
// Before P1 it ran synchronously on the actor's dispatcher thread (name is
|
||||
// NOT "script-execution-*") → false → no fire. After P1 it runs on the
|
||||
// script scheduler → true → fire.
|
||||
// WP3.1 retarget of the former P1 assertion, whose sense is deliberately INVERTED.
|
||||
//
|
||||
// P1 moved trigger-expression evaluation off the actor's dispatcher thread and onto
|
||||
// the dedicated script-execution scheduler, and this test asserted exactly that
|
||||
// ("evaluated on a script-execution-* thread"). WP3.1 (finding #4) proved that
|
||||
// destination wrong: sharing the blocking pool meant N blocked script bodies stalled
|
||||
// every Expression trigger on the node indefinitely. Evaluations are non-blocking by
|
||||
// construction, so they now run as plain async work on the shared .NET thread pool
|
||||
// behind TriggerEvalGate.
|
||||
//
|
||||
// The expression is therefore TRUE only when evaluated OFF a script-execution thread —
|
||||
// still off the dispatcher (the P1 property, covered by the coalescing/PipeTo tests),
|
||||
// and now provably off the blocking pool too.
|
||||
var expr = CompileRawTriggerExpression(
|
||||
"System.Threading.Thread.CurrentThread.Name != null && " +
|
||||
"System.Threading.Thread.CurrentThread.Name.StartsWith(\"script-execution-\")");
|
||||
"System.Threading.Thread.CurrentThread.Name == null || " +
|
||||
"!System.Threading.Thread.CurrentThread.Name.StartsWith(\"script-execution-\")");
|
||||
var (actor, instance) = CreateTriggeredActor(
|
||||
"ExprThread",
|
||||
"Expression",
|
||||
@@ -449,7 +460,7 @@ public class ScriptActorTests : TestKit, IDisposable
|
||||
expr);
|
||||
|
||||
actor.Tell(Change("Any", "1"));
|
||||
instance.ExpectMsg<SetStaticAttributeCommand>(TimeSpan.FromSeconds(10)); // fired ⇒ evaluated off-dispatcher
|
||||
instance.ExpectMsg<SetStaticAttributeCommand>(TimeSpan.FromSeconds(10)); // fired ⇒ evaluated off the blocking pool
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test group 3 — a script's execution deadline is armed when the run is ENQUEUED, not
|
||||
/// when it is dequeued.
|
||||
///
|
||||
/// <para>Before WP3.1 the deadline <see cref="CancellationTokenSource"/> was constructed inside
|
||||
/// the queued body, so a run that spent ten minutes waiting behind blocked scripts still got a
|
||||
/// fresh full 30 s budget when it finally started — and then ran work whose triggering
|
||||
/// condition was long stale. Now queue wait consumes the run's own budget, and a body that
|
||||
/// dequeues past its deadline is SHED at dequeue: it never executes, and takes the existing
|
||||
/// timeout path (site event, script-error counter, error reply, completion message).</para>
|
||||
/// </summary>
|
||||
public class ScriptDeadlineAtEnqueueTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
|
||||
public ScriptDeadlineAtEnqueueTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
DeadlineHooks.Gate = new SemaphoreSlim(0);
|
||||
DeadlineHooks.SecondScriptRan = false;
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
DeadlineHooks.Gate.Release(8);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var options = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(DeadlineHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
private IActorRef BuildScriptActor(
|
||||
string name, Script<object?> compiled, SiteRuntimeOptions options,
|
||||
ScriptExecutionScheduler scheduler, IServiceProvider? serviceProvider)
|
||||
{
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
|
||||
return ActorOf(Props.Create(() => new ScriptActor(
|
||||
name, "Inst1", instance, compiled, config, _sharedLibrary, options,
|
||||
NullLogger<ScriptActor>.Instance, null, null, null, serviceProvider, scheduler, null)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ARunThatDequeuesPastItsDeadline_IsShedWithoutExecutingItsBody()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
|
||||
// The single worker is held by a first script for longer than the second script's
|
||||
// entire timeout.
|
||||
var blocker = BuildScriptActor(
|
||||
"Blocker",
|
||||
CompileRaw("ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.DeadlineHooks.Gate.Wait(); return null;"),
|
||||
new SiteRuntimeOptions { ScriptExecutionTimeoutSeconds = 120, StuckScriptGraceMs = 120_000 },
|
||||
scheduler, null);
|
||||
blocker.Tell(new ScriptCallRequest("Blocker", null, 0, "corr-blocker"), ActorRefs.NoSender);
|
||||
AwaitAssert(() => Assert.Equal(1, scheduler.BusyThreadCount), TimeSpan.FromSeconds(15));
|
||||
|
||||
// A 1 s script queued behind it. Its grace is generous so that if the watchdog DID
|
||||
// fire it would have ample opportunity to emit — the assertion below is that it does not.
|
||||
var lateOptions = new SiteRuntimeOptions
|
||||
{
|
||||
ScriptExecutionTimeoutSeconds = 1,
|
||||
StuckScriptGraceMs = 1000
|
||||
};
|
||||
var late = BuildScriptActor(
|
||||
"Late",
|
||||
CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.DeadlineHooks.SecondScriptRan = true; return 1;"),
|
||||
lateOptions, scheduler, new SingleServiceProvider(siteLog));
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
late.Tell(new ScriptCallRequest("Late", null, 0, "corr-late"), caller.Ref);
|
||||
|
||||
// Let its whole budget elapse while it is still queued, then free the worker.
|
||||
Thread.Sleep(TimeSpan.FromSeconds(2));
|
||||
DeadlineHooks.Gate.Release();
|
||||
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("timed out", result.ErrorMessage);
|
||||
|
||||
// The body never ran: stale work is shed at dequeue rather than executed late.
|
||||
Assert.False(DeadlineHooks.SecondScriptRan,
|
||||
"the queued body executed even though its deadline had already passed");
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var rows = siteLog.OfType("script");
|
||||
// Timeout path only — no "started" Info event, because the body was skipped.
|
||||
Assert.Contains(rows, r => r.Severity == "Error" && r.Message.Contains("timed out"));
|
||||
Assert.DoesNotContain(rows, r => r.Message.Contains("started", StringComparison.OrdinalIgnoreCase));
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// And the watchdog did NOT report it as a stuck thread: it was cancelled while queued,
|
||||
// so it never held a worker.
|
||||
Thread.Sleep(TimeSpan.FromSeconds(2)); // well past the 1 s grace
|
||||
Assert.DoesNotContain(siteLog.OfType("script"),
|
||||
r => r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase));
|
||||
Assert.Equal(0, scheduler.DetachedThreadCount);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test hooks for the enqueue-anchored deadline test.</summary>
|
||||
public static class DeadlineHooks
|
||||
{
|
||||
/// <summary>Gate the blocking first script waits on.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
|
||||
/// <summary>Set by the second script's body — must stay false when the run is shed at dequeue.</summary>
|
||||
public static bool SecondScriptRan;
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using Akka.TestKit;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 supervision-parity set (design memo §4, seven pins) — the reworked successor to
|
||||
/// <c>ExecutionActorTests</c>.
|
||||
///
|
||||
/// <para>WP3.1 eliminated the short-lived <c>ScriptExecutionActor</c> and
|
||||
/// <c>AlarmExecutionActor</c>: neither had a <c>Receive</c> handler, a <c>PostStop</c>, or any
|
||||
/// state, and neither's <c>IActorRef</c> was ever a message target — the entire lifecycle lived
|
||||
/// in a detached task. Runs are now launched directly by the coordinator via
|
||||
/// <see cref="ScriptRunLauncher"/>. These tests pin every behaviour the removed actors
|
||||
/// provided onto its replacement: exception and timeout containment, the Ask reply, the
|
||||
/// completion notification, one DI scope per run, the audit ParentExecutionId threading, and
|
||||
/// the supervision outcome (coordinator unaffected — no stop, no restart).</para>
|
||||
/// </summary>
|
||||
public class ScriptRunLauncherParityTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
private readonly ScriptCompilationService _compilationService;
|
||||
|
||||
/// <summary>Own pool per test class (#18 seam), so a wedged body cannot strand the process-wide one.</summary>
|
||||
private readonly ScriptExecutionScheduler _scheduler = new(4);
|
||||
|
||||
public ScriptRunLauncherParityTests()
|
||||
{
|
||||
_compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
Shutdown();
|
||||
_scheduler.Dispose();
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
private static Script<object?> CompileScript(string code)
|
||||
{
|
||||
var scriptOptions = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(RunLauncherHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
private static SiteRuntimeOptions Options(int timeoutSeconds = 30, int graceMs = 30000)
|
||||
=> new()
|
||||
{
|
||||
MaxScriptCallDepth = 10,
|
||||
ScriptExecutionTimeoutSeconds = timeoutSeconds,
|
||||
StuckScriptGraceMs = graceMs
|
||||
};
|
||||
|
||||
private static ResolvedScript CallScript(int? timeoutSeconds = null) => new()
|
||||
{
|
||||
CanonicalName = "Runner",
|
||||
TriggerType = "Call",
|
||||
ExecutionTimeoutSeconds = timeoutSeconds
|
||||
};
|
||||
|
||||
private TestActorRef<ScriptActor> BuildScriptActor(
|
||||
Script<object?>? compiled,
|
||||
SiteRuntimeOptions options,
|
||||
IServiceProvider? serviceProvider = null,
|
||||
ISiteHealthCollector? healthCollector = null,
|
||||
ScriptExecutionScheduler? scheduler = null,
|
||||
int? perScriptTimeoutSeconds = null,
|
||||
IActorRef? instanceActor = null)
|
||||
{
|
||||
var instance = instanceActor ?? CreateTestProbe().Ref;
|
||||
return ActorOfAsTestActorRef<ScriptActor>(
|
||||
Props.Create(() => new ScriptActor(
|
||||
"Runner", "Inst1", instance, compiled, CallScript(perScriptTimeoutSeconds),
|
||||
_sharedLibrary, options, NullLogger<ScriptActor>.Instance,
|
||||
null, null, healthCollector, serviceProvider, scheduler ?? _scheduler, null)),
|
||||
"script-" + Guid.NewGuid().ToString("N"));
|
||||
}
|
||||
|
||||
// ── Pin 1: throwing body — coordinator survives, everything is reported ───────
|
||||
|
||||
[Fact]
|
||||
public void ThrowingScriptBody_LeavesScriptActorAliveAndReportsEverything()
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var health = new SiteHealthCollector();
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript("throw new InvalidOperationException(\"boom\");"),
|
||||
Options(),
|
||||
new SingleServiceProvider(siteLog),
|
||||
health);
|
||||
|
||||
Watch(actor);
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-throw"), caller.Ref);
|
||||
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal("corr-throw", result.CorrelationId);
|
||||
Assert.Contains("boom", result.ErrorMessage);
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
// Error site event + script-error counter, exactly as the execution actor emitted.
|
||||
Assert.Contains(siteLog.OfType("script"),
|
||||
r => r.Severity == "Error" && r.Message.Contains("failed", StringComparison.OrdinalIgnoreCase));
|
||||
// The in-flight slot is released, so the script can run again.
|
||||
Assert.Equal(0, actor.UnderlyingActor.RunsInFlight);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Equal(1, health.CollectReport("site-1").ScriptErrorCount);
|
||||
|
||||
// The coordinator is neither stopped nor restarted — a throwing body was always
|
||||
// contained inside the run's own try/catch, and still is. Watch() above means a stop
|
||||
// would deliver Terminated to the TestActor; a still-answering call proves it is live.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-still-alive"), caller.Ref);
|
||||
Assert.Equal("corr-still-alive",
|
||||
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10)).CorrelationId);
|
||||
}
|
||||
|
||||
// ── Pin 2: launch-path throw — improved over the old silent hang ──────────────
|
||||
|
||||
/// <summary>
|
||||
/// The only failure the removed per-run child could surface was a constructor throw —
|
||||
/// e.g. queueing onto a disposed <see cref="ScriptExecutionScheduler"/>. The old
|
||||
/// <c>OneForOneStrategy</c> logged it and stopped the child, leaving the Ask caller to
|
||||
/// hang with no reply at all. WP3.1 folds that into a launch-path catch that replies, an
|
||||
/// intentional improvement pinned here so it is explicit rather than accidental.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void LaunchPathThrow_RepliesToTheCallerAndLeavesTheCoordinatorAlive()
|
||||
{
|
||||
var dead = new ScriptExecutionScheduler(1);
|
||||
dead.Dispose();
|
||||
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript("return 1;"), Options(), scheduler: dead);
|
||||
|
||||
Watch(actor);
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-launch"), caller.Ref);
|
||||
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal("corr-launch", result.CorrelationId);
|
||||
Assert.Contains("could not be launched", result.ErrorMessage);
|
||||
|
||||
// Watch() above means a stop would deliver Terminated to the TestActor; none arrives,
|
||||
// so the coordinator neither died nor restarted on a launch failure.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
|
||||
// The in-flight counter is incremented before the launch, so the catch MUST balance
|
||||
// it — otherwise a run of launch failures would permanently wedge the script at cap.
|
||||
AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── Pin 3: exactly one DI scope per run, disposed on every path ───────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("return 1;", 30)] // success
|
||||
[InlineData("throw new InvalidOperationException(\"boom\");", 30)] // failure
|
||||
[InlineData("while (true) { await Task.Delay(25, CancellationToken); }", 1)] // timeout
|
||||
public void EachRun_CreatesOneDiScope_AndDisposesItExactlyOnce(string code, int timeoutSeconds)
|
||||
{
|
||||
var spy = new ScopeSpyServiceProvider();
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript(code), Options(timeoutSeconds), spy);
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-scope"), caller.Ref);
|
||||
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
Assert.Equal(1, spy.ScopesCreated);
|
||||
Assert.Equal(1, spy.ScopesDisposed);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── Pin 4: audit correlation threading survives the actor removal ─────────────
|
||||
|
||||
/// <summary>
|
||||
/// A routed <see cref="ScriptCallRequest.ParentExecutionId"/> must still reach the run's
|
||||
/// <see cref="ScriptRuntimeContext"/> — this is the inbound-API leg of the audit execution
|
||||
/// tree, and it used to be threaded through the execution actor's constructor.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void RoutedParentExecutionId_ReachesTheRunsScriptRuntimeContext()
|
||||
{
|
||||
RunLauncherHooks.CapturedContext = null;
|
||||
var parent = Guid.NewGuid();
|
||||
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.CapturedContext = Instance; return 1;"),
|
||||
Options());
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(
|
||||
new ScriptCallRequest("Runner", null, 0, "corr-parent", ParentExecutionId: parent),
|
||||
caller.Ref);
|
||||
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.True(result.Success, result.ErrorMessage);
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
Assert.NotNull(RunLauncherHooks.CapturedContext);
|
||||
Assert.Equal(parent, RunLauncherHooks.CapturedContext!.ParentExecutionId);
|
||||
// The routed run still mints its OWN ExecutionId — the parent is a pointer, not a copy.
|
||||
Assert.NotEqual(parent, RunLauncherHooks.CapturedContext.ExecutionId);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── Pin 5: timeout resolution parity (perScript ?? global, <= 0 => global) ────
|
||||
|
||||
[Theory]
|
||||
[InlineData(300, 1)] // per-script override wins over a much longer global
|
||||
[InlineData(1, null)] // null per-script falls back to the global
|
||||
[InlineData(1, 0)] // non-positive per-script is treated as "use global"
|
||||
public void TimeoutResolution_MatchesTheRemovedExecutionActor(int globalSeconds, int? perScriptSeconds)
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript("while (true) { await Task.Delay(25, CancellationToken); }"),
|
||||
Options(globalSeconds),
|
||||
new SingleServiceProvider(siteLog),
|
||||
perScriptTimeoutSeconds: perScriptSeconds);
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-timeout"), caller.Ref);
|
||||
|
||||
// If the effective timeout were the 300 s global (case 1) or ignored (cases 2/3) this
|
||||
// would not answer inside the window.
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("timed out", result.ErrorMessage);
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"),
|
||||
r => r.Severity == "Error" && r.Message.Contains("timed out")),
|
||||
TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── Pin 6: stop-during-run parity ────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Stopping a ScriptActor mid-run must NOT cancel the in-flight run (redeploy/undeploy
|
||||
/// semantics: running scripts are allowed to finish). The run completes normally and its
|
||||
/// completion message dead-letters, exactly as the old per-run child's
|
||||
/// <c>parent.Tell</c> did once the subtree was stopped — dead letters are a health metric,
|
||||
/// not an error.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void StoppingTheScriptActorMidRun_LetsTheRunFinishAndDeadLettersItsCompletion()
|
||||
{
|
||||
RunLauncherHooks.Gate = new SemaphoreSlim(0);
|
||||
RunLauncherHooks.Finished = new ManualResetEventSlim(false);
|
||||
RunLauncherHooks.ObservedCancellation = null;
|
||||
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.Gate.Wait();" +
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.ObservedCancellation = CancellationToken.IsCancellationRequested;" +
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.Finished.Set();" +
|
||||
"return 1;"),
|
||||
Options());
|
||||
|
||||
var deadLetters = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(deadLetters.Ref, typeof(DeadLetter));
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-stop"), caller.Ref);
|
||||
|
||||
// Wait until the body is actually on a worker thread, then stop the coordinator.
|
||||
AwaitAssert(() => Assert.Equal(1, _scheduler.BusyThreadCount), TimeSpan.FromSeconds(10));
|
||||
Watch(actor);
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, TimeSpan.FromSeconds(10));
|
||||
|
||||
RunLauncherHooks.Gate.Release();
|
||||
|
||||
// The run ran to completion and was never cancelled by the stop.
|
||||
Assert.True(RunLauncherHooks.Finished.Wait(TimeSpan.FromSeconds(10)));
|
||||
Assert.False(RunLauncherHooks.ObservedCancellation);
|
||||
|
||||
// The Ask caller still gets its result (the reply target is not the stopped actor)…
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.True(result.Success, result.ErrorMessage);
|
||||
|
||||
// …and the completion notification aimed at the now-stopped coordinator dead-letters.
|
||||
deadLetters.FishForMessage<DeadLetter>(
|
||||
d => d.Message is ScriptActor.ScriptExecutionCompleted,
|
||||
TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
// ── Pin 7: alarm side ────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// An alarm on-trigger run still receives the <c>Alarm</c> globals (name/level/priority/
|
||||
/// message) and still reports <c>AlarmExecutionCompleted</c> back to its AlarmActor —
|
||||
/// observable here through the in-flight counter returning to zero, which only the
|
||||
/// completion message can do.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AlarmOnTriggerRun_GetsAlarmGlobals_AndCompletesBackToTheAlarmActor()
|
||||
{
|
||||
RunLauncherHooks.CapturedAlarm = null;
|
||||
|
||||
var onTrigger = CompileScript(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.RunLauncherHooks.CapturedAlarm = Alarm; return null;");
|
||||
|
||||
var instanceProbe = CreateTestProbe();
|
||||
var alarm = ActorOfAsTestActorRef<AlarmActor>(
|
||||
Props.Create(() => new AlarmActor(
|
||||
"TempBand", "Inst1", instanceProbe.Ref,
|
||||
new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "TempBand",
|
||||
TriggerType = "HiLo",
|
||||
TriggerConfiguration = "{\"attributeName\":\"Temp\",\"hi\":80,\"hiHi\":95,\"hiMessage\":\"too hot\"}",
|
||||
PriorityLevel = 42
|
||||
},
|
||||
onTrigger, _sharedLibrary, Options(), NullLogger<AlarmActor>.Instance,
|
||||
null, null, null, null, null, _scheduler, null)),
|
||||
"alarm-" + Guid.NewGuid().ToString("N"));
|
||||
|
||||
alarm.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Inst1", "Temp", "Temp", 90.0, "Good", DateTimeOffset.UtcNow));
|
||||
|
||||
instanceProbe.ExpectMsg<Commons.Messages.Streaming.AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
Assert.NotNull(RunLauncherHooks.CapturedAlarm);
|
||||
Assert.Equal("TempBand", RunLauncherHooks.CapturedAlarm!.Name);
|
||||
Assert.Equal(AlarmLevel.High, RunLauncherHooks.CapturedAlarm.Level);
|
||||
Assert.Equal("too hot", RunLauncherHooks.CapturedAlarm.Message);
|
||||
// Only AlarmExecutionCompleted releases the slot.
|
||||
Assert.Equal(0, alarm.UnderlyingActor.RunsInFlight);
|
||||
}, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
// ── Retargeted from ExecutionActorTests: success path + operational events ────
|
||||
|
||||
[Fact]
|
||||
public void SuccessfulRun_RepliesWithTheReturnValue()
|
||||
{
|
||||
var actor = BuildScriptActor(CompileScript("return 7 * 6;"), Options());
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-ok"), caller.Ref);
|
||||
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.True(result.Success, result.ErrorMessage);
|
||||
Assert.Equal("corr-ok", result.CorrelationId);
|
||||
Assert.Equal(42, result.ReturnValue);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessfulRun_EmitsStartedThenCompletedInfoEvents()
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript("return 7 * 6;"), Options(), new SingleServiceProvider(siteLog));
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt"), caller.Ref);
|
||||
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var rows = siteLog.OfType("script");
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.All(rows, r =>
|
||||
{
|
||||
Assert.Equal("Info", r.Severity);
|
||||
Assert.Equal("Inst1", r.InstanceId);
|
||||
Assert.Equal("ScriptActor:Runner", r.Source);
|
||||
});
|
||||
Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Contains("completed", rows[1].Message, StringComparison.OrdinalIgnoreCase);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailingRun_EmitsStartedInfoThenErrorEvent()
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript("throw new InvalidOperationException(\"boom\");"),
|
||||
Options(), new SingleServiceProvider(siteLog));
|
||||
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt-err"), caller.Ref);
|
||||
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var rows = siteLog.OfType("script");
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.Equal("Info", rows[0].Severity);
|
||||
Assert.Contains("started", rows[0].Message, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Equal("Error", rows[1].Severity);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FireAndForgetRun_NeedsNoReplyTarget()
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var actor = BuildScriptActor(
|
||||
CompileScript("return 1;"), Options(), new SingleServiceProvider(siteLog));
|
||||
|
||||
// Trigger-driven spawns pass ActorRefs.NoSender as replyTo; drive that path via an
|
||||
// interval-free Call script by telling the actor to run with no sender.
|
||||
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-nobody"), ActorRefs.NoSender);
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"),
|
||||
r => r.Message.Contains("completed", StringComparison.OrdinalIgnoreCase)),
|
||||
TimeSpan.FromSeconds(10));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hooks a compiled test script can reach from inside a script body: a gate to block a
|
||||
/// worker thread deterministically, and capture slots for the run's
|
||||
/// <see cref="ScriptRuntimeContext"/> and <see cref="AlarmContext"/> (both of which the run
|
||||
/// otherwise exposes to nobody).
|
||||
/// </summary>
|
||||
public static class RunLauncherHooks
|
||||
{
|
||||
/// <summary>Gate a blocking test script waits on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
|
||||
/// <summary>Set by a test script once its body has run to completion.</summary>
|
||||
public static ManualResetEventSlim Finished = new(false);
|
||||
|
||||
/// <summary>Whether the script observed a cancellation request at the end of its body.</summary>
|
||||
public static bool? ObservedCancellation;
|
||||
|
||||
/// <summary>The runtime context handed to the last captured run.</summary>
|
||||
public static ScriptRuntimeContext? CapturedContext;
|
||||
|
||||
/// <summary>The <c>Alarm</c> global handed to the last captured on-trigger run.</summary>
|
||||
public static AlarmContext? CapturedAlarm;
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test group 6 — the per-script in-flight cap and its shed policy.
|
||||
///
|
||||
/// <para>Before WP3.1 every trigger spawned another run unconditionally: a trigger firing
|
||||
/// faster than its script completes produced unbounded fan-out onto a bounded thread pool.
|
||||
/// The cap (<see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>, default 4) sheds the
|
||||
/// NEWEST run instead. Keeping the four already queued/running — which are closest to their
|
||||
/// own deadlines and already charged against them — is the policy that never reorders runs
|
||||
/// and needs no queue at all: the scheduler's FIFO already IS the queue.</para>
|
||||
///
|
||||
/// <para>A shed is always counted on the health collector, emits a site event rate-limited to
|
||||
/// one per script per minute (so a hot trigger cannot flood <c>site_events</c>), and — for an
|
||||
/// Ask-based <c>CallScript</c> — replies with an explicit error rather than letting a nested
|
||||
/// call or inbound-API route hang to its Ask timeout.</para>
|
||||
/// </summary>
|
||||
public class ScriptRunShedTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
private readonly ScriptExecutionScheduler _scheduler = new(8);
|
||||
|
||||
public ScriptRunShedTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
ShedHooks.Gate = new SemaphoreSlim(0);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
ShedHooks.Gate.Release(64);
|
||||
Shutdown();
|
||||
_scheduler.Dispose();
|
||||
}
|
||||
|
||||
private static Script<object?> BlockingScript() => CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.ShedHooks.Gate.Wait(); return null;");
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var options = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(ShedHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
private static SiteRuntimeOptions Options() => new()
|
||||
{
|
||||
MaxConcurrentRunsPerScript = 4,
|
||||
// Long enough that nothing times out inside the test window — the cap, not the
|
||||
// deadline, must be what refuses the fifth run.
|
||||
ScriptExecutionTimeoutSeconds = 120,
|
||||
StuckScriptGraceMs = 120_000
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void FifthConcurrentRun_IsShed_Counted_EventedOnce_AndAnsweredWithAnError()
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var health = new SiteHealthCollector();
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var options = Options();
|
||||
|
||||
var actor = ActorOfAsTestActorRef<ScriptActor>(
|
||||
Props.Create(() => new ScriptActor(
|
||||
"Hot", "Inst1", instance, BlockingScript(),
|
||||
new ResolvedScript { CanonicalName = "Hot", TriggerType = "Call" },
|
||||
_sharedLibrary, options, NullLogger<ScriptActor>.Instance,
|
||||
null, null, health, new SingleServiceProvider(siteLog), _scheduler, null)),
|
||||
"shed-" + Guid.NewGuid().ToString("N"));
|
||||
|
||||
// Fill the cap: four runs, all blocked in their bodies.
|
||||
for (var i = 0; i < 4; i++)
|
||||
actor.Tell(new ScriptCallRequest("Hot", null, 0, $"corr-{i}"), ActorRefs.NoSender);
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
|
||||
Assert.Equal(4, _scheduler.BusyThreadCount);
|
||||
}, TimeSpan.FromSeconds(15));
|
||||
|
||||
// Fifth: shed. The Ask caller is answered explicitly instead of hanging.
|
||||
var caller = CreateTestProbe();
|
||||
actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-shed-1"), caller.Ref);
|
||||
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal("corr-shed-1", result.CorrelationId);
|
||||
Assert.Contains("shed", result.ErrorMessage, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Contains("in flight", result.ErrorMessage, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// Still exactly four in flight — the shed run was never launched.
|
||||
Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
|
||||
|
||||
// Sixth: counted again, but the Warning site event is rate-limited to one per minute.
|
||||
actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-shed-2"), caller.Ref);
|
||||
var second = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
||||
Assert.False(second.Success);
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var shedEvents = siteLog.OfType("script")
|
||||
.Where(r => r.Severity == "Warning" && r.Message.Contains("shed"))
|
||||
.ToArray();
|
||||
Assert.Single(shedEvents);
|
||||
Assert.Equal("ScriptActor:Hot", shedEvents[0].Source);
|
||||
Assert.Equal("Inst1", shedEvents[0].InstanceId);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Both sheds were counted on the health report even though only one was evented.
|
||||
Assert.Equal(2, health.CollectReport("site-1").ScriptRunShedCount);
|
||||
|
||||
// One completion frees a slot, and the next trigger launches again.
|
||||
ShedHooks.Gate.Release();
|
||||
AwaitAssert(() => Assert.Equal(3, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
||||
|
||||
actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-after"), ActorRefs.NoSender);
|
||||
AwaitAssert(() => Assert.Equal(4, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmOnTriggerRuns_ShedAtTheSameCap_WithAnAlarmScopedSiteEvent()
|
||||
{
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var health = new SiteHealthCollector();
|
||||
var instanceProbe = CreateTestProbe();
|
||||
var options = Options();
|
||||
|
||||
var alarm = ActorOfAsTestActorRef<AlarmActor>(
|
||||
Props.Create(() => new AlarmActor(
|
||||
"Flapper", "Inst1", instanceProbe.Ref,
|
||||
new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "Flapper",
|
||||
TriggerType = "ValueMatch",
|
||||
TriggerConfiguration = "{\"attributeName\":\"Status\",\"matchValue\":\"Critical\"}",
|
||||
PriorityLevel = 100
|
||||
},
|
||||
BlockingScript(), _sharedLibrary, options, NullLogger<AlarmActor>.Instance,
|
||||
null, null, health, new SingleServiceProvider(siteLog), null, _scheduler, null)),
|
||||
"alarm-shed-" + Guid.NewGuid().ToString("N"));
|
||||
|
||||
// Each raise edge spawns one on-trigger run; clear between raises to re-arm the edge.
|
||||
void Flap(int cycle)
|
||||
{
|
||||
alarm.Tell(new AttributeValueChanged(
|
||||
"Inst1", "Status", "Status", "Critical", "Good", DateTimeOffset.UtcNow.AddSeconds(cycle)));
|
||||
alarm.Tell(new AttributeValueChanged(
|
||||
"Inst1", "Status", "Status", "Normal", "Good", DateTimeOffset.UtcNow.AddSeconds(cycle)));
|
||||
}
|
||||
|
||||
for (var i = 0; i < 4; i++) Flap(i);
|
||||
AwaitAssert(() => Assert.Equal(4, alarm.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
||||
|
||||
// Fifth raise is shed — there is no Ask caller on this path, so it surfaces purely as
|
||||
// a counter plus the rate-limited Warning event.
|
||||
Flap(4);
|
||||
Flap(5);
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var shedEvents = siteLog.OfType("script")
|
||||
.Where(r => r.Severity == "Warning" && r.Message.Contains("shed"))
|
||||
.ToArray();
|
||||
Assert.Single(shedEvents);
|
||||
Assert.Equal("AlarmActor:Flapper", shedEvents[0].Source);
|
||||
}, TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.Equal(4, alarm.UnderlyingActor.RunsInFlight);
|
||||
Assert.Equal(2, health.CollectReport("site-1").ScriptRunShedCount);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test hook used to hold script runs in flight while the cap is exercised.</summary>
|
||||
public static class ShedHooks
|
||||
{
|
||||
/// <summary>Gate the blocking test scripts wait on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test group 4 — the stuck-script watchdog now REPLACES the thread a wedged script is
|
||||
/// holding, not just names it.
|
||||
///
|
||||
/// <para>Before WP3.1 the watchdog was observability only: a script blocked in synchronous,
|
||||
/// uninterruptible I/O never observes the cooperative cancellation the timeout requests, so
|
||||
/// the pool simply lost that dedicated thread — permanently, and silently apart from one log
|
||||
/// line. Eight such scripts left the site with no script execution at all. The watchdog now
|
||||
/// detaches the worker (it exits when its body finally returns) and starts a replacement, and
|
||||
/// the count of live detached workers is surfaced on the site health report as
|
||||
/// <c>DetachedScriptThreads</c>. Replacement is capped at the pool size, because bounded
|
||||
/// starvation beats unbounded thread growth when scripts wedge en masse.</para>
|
||||
/// </summary>
|
||||
public class StuckScriptWatchdogTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
|
||||
public StuckScriptWatchdogTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
WatchdogHooks.Gate = new SemaphoreSlim(0);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
// Free any still-wedged worker before tearing down, so no test leaves a blocked thread.
|
||||
WatchdogHooks.Gate.Release(8);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var scriptOptions = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(WatchdogHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>A body that blocks its worker thread outright — cooperative cancellation is never observed.</summary>
|
||||
private static Script<object?> WedgeScript() => CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.WatchdogHooks.Gate.Wait(); return null;");
|
||||
|
||||
private IActorRef BuildScriptActor(
|
||||
string name,
|
||||
Script<object?> compiled,
|
||||
SiteRuntimeOptions options,
|
||||
ScriptExecutionScheduler scheduler,
|
||||
IServiceProvider? serviceProvider)
|
||||
{
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
|
||||
return ActorOf(Props.Create(() => new ScriptActor(
|
||||
name, "Inst1", instance, compiled, config, _sharedLibrary, options,
|
||||
NullLogger<ScriptActor>.Instance, null, null, null, serviceProvider, scheduler, null)));
|
||||
}
|
||||
|
||||
private static SiteRuntimeOptions WatchdogOptions() => new()
|
||||
{
|
||||
ScriptExecutionTimeoutSeconds = 1,
|
||||
StuckScriptGraceMs = 200,
|
||||
MaxScriptCallDepth = 10
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void WedgedScript_DetachesItsWorker_StartsAReplacement_AndDrainsWhenFreed()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var options = WatchdogOptions();
|
||||
|
||||
var wedged = BuildScriptActor("Wedged", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
wedged.Tell(new ScriptCallRequest("Wedged", null, 0, "corr-wedge"), ActorRefs.NoSender);
|
||||
|
||||
// Timeout (1 s) + grace (200 ms) later the watchdog fires: the worker is detached and
|
||||
// a replacement thread starts.
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, scheduler.DetachedThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"), r =>
|
||||
r.Severity == "Error" &&
|
||||
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) &&
|
||||
r.Message.Contains("Wedged") &&
|
||||
r.Message.Contains("DETACHED")),
|
||||
TimeSpan.FromSeconds(5));
|
||||
|
||||
// The replacement worker is live: a fresh script runs even though the pool's only
|
||||
// original thread is still blocked. Before WP3.1 this reply never came.
|
||||
var healthy = BuildScriptActor("Healthy", CompileRaw("return 7;"), options, scheduler, null);
|
||||
var caller = CreateTestProbe();
|
||||
healthy.Tell(new ScriptCallRequest("Healthy", null, 0, "corr-healthy"), caller.Ref);
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
|
||||
Assert.True(result.Success, result.ErrorMessage);
|
||||
Assert.Equal(7, result.ReturnValue);
|
||||
|
||||
// Freeing the wedged body lets the detached worker exit rather than pull more work,
|
||||
// so the pool does not silently end up double-sized.
|
||||
WatchdogHooks.Gate.Release();
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(0, scheduler.DetachedThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
Assert.Equal(1, scheduler.MaximumConcurrencyLevel);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AtTheDetachCap_NoFurtherReplacementIsStarted_AndAnErrorEventIsEmitted()
|
||||
{
|
||||
// A one-thread pool caps live detached workers at one, so the SECOND wedge cannot be
|
||||
// replaced — the deliberate "bounded starvation beats unbounded thread growth" trade.
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var options = WatchdogOptions();
|
||||
|
||||
var first = BuildScriptActor("WedgeOne", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
first.Tell(new ScriptCallRequest("WedgeOne", null, 0, "corr-w1"), ActorRefs.NoSender);
|
||||
AwaitAssert(() => Assert.Equal(1, scheduler.DetachedThreadCount), TimeSpan.FromSeconds(15));
|
||||
|
||||
var second = BuildScriptActor("WedgeTwo", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
second.Tell(new ScriptCallRequest("WedgeTwo", null, 0, "corr-w2"), ActorRefs.NoSender);
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"), r =>
|
||||
r.Severity == "Error" &&
|
||||
r.Message.Contains("WedgeTwo") &&
|
||||
r.Message.Contains("at cap", StringComparison.OrdinalIgnoreCase)),
|
||||
TimeSpan.FromSeconds(20));
|
||||
|
||||
// Still exactly one detached worker: the second wedge was NOT replaced.
|
||||
Assert.Equal(1, scheduler.DetachedThreadCount);
|
||||
|
||||
WatchdogHooks.Gate.Release(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DetachedThreadCount_IsSurfacedOnTheSiteHealthReport()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
var wedged = BuildScriptActor("GaugeWedge", WedgeScript(), WatchdogOptions(), scheduler, null);
|
||||
wedged.Tell(new ScriptCallRequest("GaugeWedge", null, 0, "corr-gauge"), ActorRefs.NoSender);
|
||||
AwaitAssert(() => Assert.Equal(1, scheduler.DetachedThreadCount), TimeSpan.FromSeconds(15));
|
||||
|
||||
using var reporter = new ScriptSchedulerStatsReporter(
|
||||
collector, WatchdogOptions(), NullLogger<ScriptSchedulerStatsReporter>.Instance,
|
||||
pollInterval: TimeSpan.FromMilliseconds(50), scheduler: scheduler);
|
||||
await reporter.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, collector.CollectReport("site-1").DetachedScriptThreads),
|
||||
TimeSpan.FromSeconds(10));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await reporter.StopAsync(CancellationToken.None);
|
||||
WatchdogHooks.Gate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test hook the stuck-script watchdog tests use to block a script-execution thread
|
||||
/// deterministically: the compiled body waits on <see cref="Gate"/>, which the test releases
|
||||
/// once it has observed the detach.
|
||||
/// </summary>
|
||||
public static class WatchdogHooks
|
||||
{
|
||||
/// <summary>Gate a blocking test script waits on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test groups 1 and 2 — the regression pin for arch-review finding #4 (High).
|
||||
///
|
||||
/// <para><b>The finding.</b> Trigger-expression evaluation used to be queued onto the same
|
||||
/// fixed 8-thread <see cref="ScriptExecutionScheduler"/> that runs blocking script bodies. Eight
|
||||
/// scripts blocked in synchronous I/O therefore stalled EVERY Expression trigger on the node —
|
||||
/// scripts and alarms alike — for an unbounded time. Worse, the evaluation's 2 s timeout was
|
||||
/// constructed INSIDE the queued body, so it did not start until the evaluation was dequeued:
|
||||
/// the operator saw neither a raise nor a timeout, just silence.</para>
|
||||
///
|
||||
/// <para><b>The fix these tests pin.</b> Evaluations are non-blocking by construction
|
||||
/// (<see cref="TriggerExpressionGlobals"/> exposes only reads over an in-memory snapshot, and
|
||||
/// the trust gate has already denied I/O), so they run as plain async work on the shared .NET
|
||||
/// thread pool behind <see cref="TriggerEvalGate"/> — never on the blocking pool. And their
|
||||
/// deadline is armed at ENQUEUE, so gate-wait time burns the same budget.</para>
|
||||
/// </summary>
|
||||
public class TriggerEvalStarvationTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
|
||||
public TriggerEvalStarvationTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
StarvationHooks.Gate = new SemaphoreSlim(0);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
StarvationHooks.Gate.Release(32);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var options = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(StarvationHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
private static Script<object?> CompileTriggerExpression(string expression)
|
||||
{
|
||||
var options = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(expression, options, typeof(TriggerExpressionGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GROUP 1 — the finding-#4 regression pin. Eight script bodies blocked on a semaphore
|
||||
/// occupy every thread of an 8-thread pool; an Expression-triggered alarm must still raise
|
||||
/// in well under 2 s.
|
||||
///
|
||||
/// <para>On the pre-WP3.1 wiring this test does not merely fail slowly — it never
|
||||
/// completes: the evaluation sits in the same FIFO behind eight bodies that only unblock
|
||||
/// after the assertion window, and its 2 s timeout has not even started ticking.</para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void EightBlockedScripts_DoNotDelayAnAlarmsExpressionEvaluation()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(8);
|
||||
using var evalGate = new TriggerEvalGate(4);
|
||||
|
||||
// Occupy every thread of the blocking pool with a real script run.
|
||||
var wedge = CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StarvationHooks.Gate.Wait(); return null;");
|
||||
for (var i = 0; i < 8; i++)
|
||||
{
|
||||
var name = $"Blocker{i}";
|
||||
var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var actor = ActorOf(Props.Create(() => new ScriptActor(
|
||||
name, "Inst1", instance, wedge, config, _sharedLibrary, new SiteRuntimeOptions(),
|
||||
NullLogger<ScriptActor>.Instance, null, null, null, null, scheduler, evalGate)));
|
||||
actor.Tell(new ScriptCallRequest(name, null, 0, $"corr-block-{i}"), ActorRefs.NoSender);
|
||||
}
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(8, scheduler.BusyThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
// Now fire an Expression-triggered alarm through the SAME scheduler seam.
|
||||
var instanceProbe = CreateTestProbe();
|
||||
var alarm = ActorOf(Props.Create(() => new AlarmActor(
|
||||
"ExprAlarm", "Inst1", instanceProbe.Ref,
|
||||
new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "ExprAlarm",
|
||||
TriggerType = "Expression",
|
||||
TriggerConfiguration = "{\"expression\":\"true\"}",
|
||||
PriorityLevel = 900
|
||||
},
|
||||
null, _sharedLibrary, new SiteRuntimeOptions(), NullLogger<AlarmActor>.Instance,
|
||||
CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
alarm.Tell(new AttributeValueChanged(
|
||||
"Inst1", "Temp", "Temp", 99.0, "Good", DateTimeOffset.UtcNow));
|
||||
|
||||
var raised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(2));
|
||||
stopwatch.Stop();
|
||||
|
||||
Assert.Equal(AlarmState.Active, raised.State);
|
||||
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(2),
|
||||
$"alarm raised only after {stopwatch.Elapsed} — the evaluation queued behind blocked script bodies");
|
||||
|
||||
// The pool really was saturated for the whole window; the eval simply never used it.
|
||||
Assert.Equal(8, scheduler.BusyThreadCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GROUP 2 — the evaluation deadline is measured from ENQUEUE, so time spent waiting on a
|
||||
/// saturated <see cref="TriggerEvalGate"/> burns the same budget. With the gate fully
|
||||
/// occupied for the whole assertion window, a queued evaluation must still resolve (as
|
||||
/// false) at its timeout rather than stalling indefinitely — and the trigger must drain
|
||||
/// rather than park, so the next change still evaluates.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SaturatedEvalGate_ResolvesTheQueuedEvaluationAtItsEnqueueAnchoredDeadline()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
using var evalGate = new TriggerEvalGate(1);
|
||||
var options = new SiteRuntimeOptions { TriggerEvalTimeoutSeconds = 1 };
|
||||
|
||||
var instanceProbe = CreateTestProbe();
|
||||
var alarm = ActorOf(Props.Create(() => new AlarmActor(
|
||||
"ExprAlarm", "Inst1", instanceProbe.Ref,
|
||||
new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "ExprAlarm",
|
||||
TriggerType = "Expression",
|
||||
TriggerConfiguration = "{\"expression\":\"true\"}",
|
||||
PriorityLevel = 500
|
||||
},
|
||||
null, _sharedLibrary, options, NullLogger<AlarmActor>.Instance,
|
||||
CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));
|
||||
|
||||
// 1. Free gate: the expression evaluates true and the alarm raises.
|
||||
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 1, "Good", DateTimeOffset.UtcNow));
|
||||
var raised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(AlarmState.Active, raised.State);
|
||||
|
||||
// 2. Occupy the only permit for the whole of the next step.
|
||||
await evalGate.WaitAsync(CancellationToken.None);
|
||||
Assert.Equal(0, evalGate.AvailablePermits);
|
||||
|
||||
// 3. The next evaluation can never acquire the gate. Its deadline was armed at
|
||||
// enqueue, so it cancels at ~1 s, is treated as false, and clears the alarm.
|
||||
// Pre-WP3.1 the clock started at dequeue, so this would hang forever.
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 2, "Good", DateTimeOffset.UtcNow));
|
||||
var cleared = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
stopwatch.Stop();
|
||||
|
||||
Assert.Equal(AlarmState.Normal, cleared.State);
|
||||
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(6),
|
||||
$"queued evaluation took {stopwatch.Elapsed} to resolve — the deadline is not enqueue-anchored");
|
||||
Assert.Equal(0, evalGate.AvailablePermits); // still held: it genuinely never ran
|
||||
|
||||
// 4. Not parked: releasing the gate and sending another change evaluates again.
|
||||
evalGate.Release();
|
||||
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 3, "Good", DateTimeOffset.UtcNow));
|
||||
var reRaised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(AlarmState.Active, reRaised.State);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test hook used to block script-execution worker threads deterministically.</summary>
|
||||
public static class StarvationHooks
|
||||
{
|
||||
/// <summary>Gate the blocking test scripts wait on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
Reference in New Issue
Block a user