using System.Diagnostics;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.CodeAnalysis.CSharp.Scripting;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// WP3.1 test groups 1 and 2 — the regression pin for arch-review finding #4 (High).
///
/// The finding. Trigger-expression evaluation used to be queued onto the same
/// fixed 8-thread that runs blocking script bodies. Eight
/// scripts blocked in synchronous I/O therefore stalled EVERY Expression trigger on the node —
/// scripts and alarms alike — for an unbounded time. Worse, the evaluation's 2 s timeout was
/// constructed INSIDE the queued body, so it did not start until the evaluation was dequeued:
/// the operator saw neither a raise nor a timeout, just silence.
///
/// The fix these tests pin. Evaluations are non-blocking by construction
/// ( exposes only reads over an in-memory snapshot, and
/// the trust gate has already denied I/O), so they run as plain async work on the shared .NET
/// thread pool behind — never on the blocking pool. And their
/// deadline is armed at ENQUEUE, so gate-wait time burns the same budget.
///
public class TriggerEvalStarvationTests : TestKit, IDisposable
{
private readonly SharedScriptLibrary _sharedLibrary;
public TriggerEvalStarvationTests()
{
var compilationService = new ScriptCompilationService(
NullLogger.Instance);
_sharedLibrary = new SharedScriptLibrary(
compilationService, NullLogger.Instance);
StarvationHooks.Gate = new SemaphoreSlim(0);
}
void IDisposable.Dispose()
{
StarvationHooks.Gate.Release(32);
Shutdown();
}
private static Script CompileRaw(string code)
{
var options = ScriptOptions.Default
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
typeof(StarvationHooks).Assembly)
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
var script = CSharpScript.Create(code, options, typeof(ScriptGlobals));
script.Compile();
return script;
}
private static Script CompileTriggerExpression(string expression)
{
var options = ScriptOptions.Default
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly)
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
var script = CSharpScript.Create(expression, options, typeof(TriggerExpressionGlobals));
script.Compile();
return script;
}
///
/// GROUP 1 — the finding-#4 regression pin. Eight script bodies blocked on a semaphore
/// occupy every thread of an 8-thread pool; an Expression-triggered alarm must still raise
/// in well under 2 s.
///
/// On the pre-WP3.1 wiring this test does not merely fail slowly — it never
/// completes: the evaluation sits in the same FIFO behind eight bodies that only unblock
/// after the assertion window, and its 2 s timeout has not even started ticking.
///
[Fact]
public void EightBlockedScripts_DoNotDelayAnAlarmsExpressionEvaluation()
{
using var scheduler = new ScriptExecutionScheduler(8);
using var evalGate = new TriggerEvalGate(4);
// Occupy every thread of the blocking pool with a real script run.
var wedge = CompileRaw(
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.StarvationHooks.Gate.Wait(); return null;");
for (var i = 0; i < 8; i++)
{
var name = $"Blocker{i}";
var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
var instance = CreateTestProbe().Ref;
var actor = ActorOf(Props.Create(() => new ScriptActor(
name, "Inst1", instance, wedge, config, _sharedLibrary, new SiteRuntimeOptions(),
NullLogger.Instance, null, null, null, null, scheduler, evalGate)));
actor.Tell(new ScriptCallRequest(name, null, 0, $"corr-block-{i}"), ActorRefs.NoSender);
}
AwaitAssert(
() => Assert.Equal(8, scheduler.BusyThreadCount),
TimeSpan.FromSeconds(15));
// Now fire an Expression-triggered alarm through the SAME scheduler seam.
var instanceProbe = CreateTestProbe();
var alarm = ActorOf(Props.Create(() => new AlarmActor(
"ExprAlarm", "Inst1", instanceProbe.Ref,
new ResolvedAlarm
{
CanonicalName = "ExprAlarm",
TriggerType = "Expression",
TriggerConfiguration = "{\"expression\":\"true\"}",
PriorityLevel = 900
},
null, _sharedLibrary, new SiteRuntimeOptions(), NullLogger.Instance,
CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));
var stopwatch = Stopwatch.StartNew();
alarm.Tell(new AttributeValueChanged(
"Inst1", "Temp", "Temp", 99.0, "Good", DateTimeOffset.UtcNow));
var raised = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(2));
stopwatch.Stop();
Assert.Equal(AlarmState.Active, raised.State);
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(2),
$"alarm raised only after {stopwatch.Elapsed} — the evaluation queued behind blocked script bodies");
// The pool really was saturated for the whole window; the eval simply never used it.
Assert.Equal(8, scheduler.BusyThreadCount);
}
///
/// GROUP 2 — the evaluation deadline is measured from ENQUEUE, so time spent waiting on a
/// saturated burns the same budget. With the gate fully
/// occupied for the whole assertion window, a queued evaluation must still resolve (as
/// false) at its timeout rather than stalling indefinitely — and the trigger must drain
/// rather than park, so the next change still evaluates.
///
[Fact]
public async Task SaturatedEvalGate_ResolvesTheQueuedEvaluationAtItsEnqueueAnchoredDeadline()
{
using var scheduler = new ScriptExecutionScheduler(1);
using var evalGate = new TriggerEvalGate(1);
var options = new SiteRuntimeOptions { TriggerEvalTimeoutSeconds = 1 };
var instanceProbe = CreateTestProbe();
var alarm = ActorOf(Props.Create(() => new AlarmActor(
"ExprAlarm", "Inst1", instanceProbe.Ref,
new ResolvedAlarm
{
CanonicalName = "ExprAlarm",
TriggerType = "Expression",
TriggerConfiguration = "{\"expression\":\"true\"}",
PriorityLevel = 500
},
null, _sharedLibrary, options, NullLogger.Instance,
CompileTriggerExpression("true"), null, null, null, null, scheduler, evalGate)));
// 1. Free gate: the expression evaluates true and the alarm raises.
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 1, "Good", DateTimeOffset.UtcNow));
var raised = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10));
Assert.Equal(AlarmState.Active, raised.State);
// 2. Occupy the only permit for the whole of the next step.
await evalGate.WaitAsync(CancellationToken.None);
Assert.Equal(0, evalGate.AvailablePermits);
// 3. The next evaluation can never acquire the gate. Its deadline was armed at
// enqueue, so it cancels at ~1 s, is treated as false, and clears the alarm.
// Pre-WP3.1 the clock started at dequeue, so this would hang forever.
var stopwatch = Stopwatch.StartNew();
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 2, "Good", DateTimeOffset.UtcNow));
var cleared = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10));
stopwatch.Stop();
Assert.Equal(AlarmState.Normal, cleared.State);
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(6),
$"queued evaluation took {stopwatch.Elapsed} to resolve — the deadline is not enqueue-anchored");
Assert.Equal(0, evalGate.AvailablePermits); // still held: it genuinely never ran
// 4. Not parked: releasing the gate and sending another change evaluates again.
evalGate.Release();
alarm.Tell(new AttributeValueChanged("Inst1", "A", "A", 3, "Good", DateTimeOffset.UtcNow));
var reRaised = instanceProbe.ExpectMsg(TimeSpan.FromSeconds(10));
Assert.Equal(AlarmState.Active, reRaised.State);
}
}
/// Test hook used to block script-execution worker threads deterministically.
public static class StarvationHooks
{
/// Gate the blocking test scripts wait on; reset per test.
public static SemaphoreSlim Gate = new(0);
}