perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,196 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test groups 1 and 2 — the regression pin for arch-review finding #4 (High).
|
||||
///
|
||||
/// <para><b>The finding.</b> Trigger-expression evaluation used to be queued onto the same
|
||||
/// fixed 8-thread <see cref="ScriptExecutionScheduler"/> that runs blocking script bodies. Eight
|
||||
/// scripts blocked in synchronous I/O therefore stalled EVERY Expression trigger on the node —
|
||||
/// scripts and alarms alike — for an unbounded time. Worse, the evaluation's 2 s timeout was
|
||||
/// constructed INSIDE the queued body, so it did not start until the evaluation was dequeued:
|
||||
/// the operator saw neither a raise nor a timeout, just silence.</para>
|
||||
///
|
||||
/// <para><b>The fix these tests pin.</b> Evaluations are non-blocking by construction
|
||||
/// (<see cref="TriggerExpressionGlobals"/> exposes only reads over an in-memory snapshot, and
|
||||
/// the trust gate has already denied I/O), so they run as plain async work on the shared .NET
|
||||
/// thread pool behind <see cref="TriggerEvalGate"/> — never on the blocking pool. And their
|
||||
/// deadline is armed at ENQUEUE, so gate-wait time burns the same budget.</para>
|
||||
/// </summary>
|
||||
public class TriggerEvalStarvationTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
|
||||
public TriggerEvalStarvationTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
StarvationHooks.Gate = new SemaphoreSlim(0);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
StarvationHooks.Gate.Release(32);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var options = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(StarvationHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
private static Script<object?> CompileTriggerExpression(string expression)
|
||||
{
|
||||
var options = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(expression, options, typeof(TriggerExpressionGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GROUP 1 — the finding-#4 regression pin. Eight script bodies blocked on a semaphore
|
||||
/// occupy every thread of an 8-thread pool; an Expression-triggered alarm must still raise
|
||||
/// in well under 2 s.
|
||||
///
|
||||
/// <para>On the pre-WP3.1 wiring this test does not merely fail slowly — it never
|
||||
/// completes: the evaluation sits in the same FIFO behind eight bodies that only unblock
|
||||
/// after the assertion window, and its 2 s timeout has not even started ticking.</para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void EightBlockedScripts_DoNotDelayAnAlarmsExpressionEvaluation()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(8);
|
||||
using var evalGate = new TriggerEvalGate(4);
|
||||
|
||||
// Occupy every thread of the blocking pool with a real script run.
|
||||
var wedge = CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StarvationHooks.Gate.Wait(); return null;");
|
||||
for (var i = 0; i < 8; i++)
|
||||
{
|
||||
var name = $"Blocker{i}";
|
||||
var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var actor = ActorOf(Props.Create(() => new ScriptActor(
|
||||
name, "Inst1", instance, wedge, config, _sharedLibrary, new SiteRuntimeOptions(),
|
||||
NullLogger<ScriptActor>.Instance, null, null, null, null, scheduler, evalGate)));
|
||||
actor.Tell(new ScriptCallRequest(name, null, 0, $"corr-block-{i}"), ActorRefs.NoSender);
|
||||
}
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(8, scheduler.BusyThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
// Now fire an Expression-triggered alarm through the SAME scheduler seam.
|
||||
var instanceProbe = CreateTestProbe();
|
||||
var alarm = ActorOf(Props.Create(() => new AlarmActor(
|
||||
"ExprAlarm", "Inst1", instanceProbe.Ref,
|
||||
new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "ExprAlarm",
|
||||
TriggerType = "Expression",
|
||||
TriggerConfiguration = "{\"expression\":\"true\"}",
|
||||
PriorityLevel = 900
|
||||
},
|
||||
null, _sharedLibrary, new SiteRuntimeOptions(), NullLogger<AlarmActor>.Instance,
|
||||
CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
alarm.Tell(new AttributeValueChanged(
|
||||
"Inst1", "Temp", "Temp", 99.0, "Good", DateTimeOffset.UtcNow));
|
||||
|
||||
var raised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(2));
|
||||
stopwatch.Stop();
|
||||
|
||||
Assert.Equal(AlarmState.Active, raised.State);
|
||||
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(2),
|
||||
$"alarm raised only after {stopwatch.Elapsed} — the evaluation queued behind blocked script bodies");
|
||||
|
||||
// The pool really was saturated for the whole window; the eval simply never used it.
|
||||
Assert.Equal(8, scheduler.BusyThreadCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GROUP 2 — the evaluation deadline is measured from ENQUEUE, so time spent waiting on a
|
||||
/// saturated <see cref="TriggerEvalGate"/> burns the same budget. With the gate fully
|
||||
/// occupied for the whole assertion window, a queued evaluation must still resolve (as
|
||||
/// false) at its timeout rather than stalling indefinitely — and the trigger must drain
|
||||
/// rather than park, so the next change still evaluates.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SaturatedEvalGate_ResolvesTheQueuedEvaluationAtItsEnqueueAnchoredDeadline()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
using var evalGate = new TriggerEvalGate(1);
|
||||
var options = new SiteRuntimeOptions { TriggerEvalTimeoutSeconds = 1 };
|
||||
|
||||
var instanceProbe = CreateTestProbe();
|
||||
var alarm = ActorOf(Props.Create(() => new AlarmActor(
|
||||
"ExprAlarm", "Inst1", instanceProbe.Ref,
|
||||
new ResolvedAlarm
|
||||
{
|
||||
CanonicalName = "ExprAlarm",
|
||||
TriggerType = "Expression",
|
||||
TriggerConfiguration = "{\"expression\":\"true\"}",
|
||||
PriorityLevel = 500
|
||||
},
|
||||
null, _sharedLibrary, options, NullLogger<AlarmActor>.Instance,
|
||||
CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));
|
||||
|
||||
// 1. Free gate: the expression evaluates true and the alarm raises.
|
||||
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 1, "Good", DateTimeOffset.UtcNow));
|
||||
var raised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(AlarmState.Active, raised.State);
|
||||
|
||||
// 2. Occupy the only permit for the whole of the next step.
|
||||
await evalGate.WaitAsync(CancellationToken.None);
|
||||
Assert.Equal(0, evalGate.AvailablePermits);
|
||||
|
||||
// 3. The next evaluation can never acquire the gate. Its deadline was armed at
|
||||
// enqueue, so it cancels at ~1 s, is treated as false, and clears the alarm.
|
||||
// Pre-WP3.1 the clock started at dequeue, so this would hang forever.
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 2, "Good", DateTimeOffset.UtcNow));
|
||||
var cleared = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
stopwatch.Stop();
|
||||
|
||||
Assert.Equal(AlarmState.Normal, cleared.State);
|
||||
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(6),
|
||||
$"queued evaluation took {stopwatch.Elapsed} to resolve — the deadline is not enqueue-anchored");
|
||||
Assert.Equal(0, evalGate.AvailablePermits); // still held: it genuinely never ran
|
||||
|
||||
// 4. Not parked: releasing the gate and sending another change evaluates again.
|
||||
evalGate.Release();
|
||||
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 3, "Good", DateTimeOffset.UtcNow));
|
||||
var reRaised = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(AlarmState.Active, reRaised.State);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test hook used to block script-execution worker threads deterministically.</summary>
|
||||
public static class StarvationHooks
|
||||
{
|
||||
/// <summary>Gate the blocking test scripts wait on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
Reference in New Issue
Block a user