Joseph Doherty
105541b655
A traced run refused before launch — by the per-script in-flight cap
(MaxConcurrentRunsPerScript) or by MinTimeBetweenRuns — emitted no ScriptTrace row at
all, so the Flow Debugger reported NO_TRIGGER_WITHIN_WINDOW ("no value-change trigger on
this attribute, or no device echo"), which is wrong in both cases and sends the operator
to check the wrong things.
Both guards now leave a Failed ScriptTrace/ScriptRun row naming the guard that refused
the run, parented to the writing execution and carrying the trigger detail. The new
ScriptRunLauncher.EmitSuppressedRun reuses EmitScriptRun with elapsedMs 0 and no
collector (a shed run has no reply to ride on — the row reaches the debugger via
PullTraceEvents / Replay), resolving the writer and identity providers from a fresh
scope because it runs on the actor thread. Best-effort, never throws, no-op untraced.
Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29
375 lines
17 KiB
C#
375 lines
17 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit;
|
|
using Akka.TestKit.Xunit2;
|
|
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
|
using Microsoft.CodeAnalysis.Scripting;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
|
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
|
using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
|
|
|
/// <summary>
|
|
/// WP3.1 test group 6 — the per-script in-flight cap and its shed policy.
|
|
///
|
|
/// <para>Before WP3.1 every trigger spawned another run unconditionally: a trigger firing
|
|
/// faster than its script completes produced unbounded fan-out onto a bounded thread pool.
|
|
/// The cap (<see cref="SiteRuntimeOptions.MaxConcurrentRunsPerScript"/>, default 4) sheds the
|
|
/// NEWEST run instead. Keeping the four already queued/running — which are closest to their
|
|
/// own deadlines and already charged against them — is the policy that never reorders runs
|
|
/// and needs no queue at all: the scheduler's FIFO already IS the queue.</para>
|
|
///
|
|
/// <para>A shed is always counted on the health collector, emits a site event rate-limited to
|
|
/// one per script per minute (so a hot trigger cannot flood <c>site_events</c>), and — for an
|
|
/// Ask-based <c>CallScript</c> — replies with an explicit error rather than letting a nested
|
|
/// call or inbound-API route hang to its Ask timeout.</para>
|
|
/// </summary>
|
|
public class ScriptRunShedTests : TestKit, IDisposable
|
|
{
|
|
private readonly SharedScriptLibrary _sharedLibrary;
|
|
private readonly ScriptExecutionScheduler _scheduler = new(8);
|
|
|
|
public ScriptRunShedTests()
|
|
{
|
|
var compilationService = new ScriptCompilationService(
|
|
NullLogger<ScriptCompilationService>.Instance);
|
|
_sharedLibrary = new SharedScriptLibrary(
|
|
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
|
ShedHooks.Gate = new SemaphoreSlim(0);
|
|
}
|
|
|
|
void IDisposable.Dispose()
|
|
{
|
|
ShedHooks.Gate.Release(64);
|
|
Shutdown();
|
|
_scheduler.Dispose();
|
|
}
|
|
|
|
private static Script<object?> BlockingScript() => CompileRaw(
|
|
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.ShedHooks.Gate.Wait(); return null;");
|
|
|
|
private static Script<object?> CompileRaw(string code)
|
|
{
|
|
var options = ScriptOptions.Default
|
|
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
|
typeof(ShedHooks).Assembly)
|
|
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
|
var script = CSharpScript.Create<object?>(code, options, typeof(ScriptGlobals));
|
|
script.Compile();
|
|
return script;
|
|
}
|
|
|
|
private static SiteRuntimeOptions Options() => new()
|
|
{
|
|
MaxConcurrentRunsPerScript = 4,
|
|
// Long enough that nothing times out inside the test window — the cap, not the
|
|
// deadline, must be what refuses the fifth run.
|
|
ScriptExecutionTimeoutSeconds = 120,
|
|
StuckScriptGraceMs = 120_000
|
|
};
|
|
|
|
[Fact]
|
|
public void FifthConcurrentRun_IsShed_Counted_EventedOnce_AndAnsweredWithAnError()
|
|
{
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var health = new SiteHealthCollector();
|
|
var instance = CreateTestProbe().Ref;
|
|
var options = Options();
|
|
|
|
var actor = ActorOfAsTestActorRef<ScriptActor>(
|
|
Props.Create(() => new ScriptActor(
|
|
"Hot", "Inst1", instance, BlockingScript(),
|
|
new ResolvedScript { CanonicalName = "Hot", TriggerType = "Call" },
|
|
_sharedLibrary, options, NullLogger<ScriptActor>.Instance,
|
|
null, null, health, new SingleServiceProvider(siteLog), _scheduler, null)),
|
|
"shed-" + Guid.NewGuid().ToString("N"));
|
|
|
|
// Fill the cap: four runs, all blocked in their bodies.
|
|
for (var i = 0; i < 4; i++)
|
|
actor.Tell(new ScriptCallRequest("Hot", null, 0, $"corr-{i}"), ActorRefs.NoSender);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
|
|
Assert.Equal(4, _scheduler.BusyThreadCount);
|
|
}, TimeSpan.FromSeconds(15));
|
|
|
|
// Fifth: shed. The Ask caller is answered explicitly instead of hanging.
|
|
var caller = CreateTestProbe();
|
|
actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-shed-1"), caller.Ref);
|
|
|
|
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
|
Assert.False(result.Success);
|
|
Assert.Equal("corr-shed-1", result.CorrelationId);
|
|
Assert.Contains("shed", result.ErrorMessage, StringComparison.OrdinalIgnoreCase);
|
|
Assert.Contains("in flight", result.ErrorMessage, StringComparison.OrdinalIgnoreCase);
|
|
|
|
// Still exactly four in flight — the shed run was never launched.
|
|
Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
|
|
|
|
// Sixth: counted again, but the Warning site event is rate-limited to one per minute.
|
|
actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-shed-2"), caller.Ref);
|
|
var second = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
|
|
Assert.False(second.Success);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
var shedEvents = siteLog.OfType("script")
|
|
.Where(r => r.Severity == "Warning" && r.Message.Contains("shed"))
|
|
.ToArray();
|
|
Assert.Single(shedEvents);
|
|
Assert.Equal("ScriptActor:Hot", shedEvents[0].Source);
|
|
Assert.Equal("Inst1", shedEvents[0].InstanceId);
|
|
}, TimeSpan.FromSeconds(5));
|
|
|
|
// Both sheds were counted on the health report even though only one was evented.
|
|
Assert.Equal(2, health.CollectReport("site-1").ScriptRunShedCount);
|
|
|
|
// One completion frees a slot, and the next trigger launches again.
|
|
ShedHooks.Gate.Release();
|
|
AwaitAssert(() => Assert.Equal(3, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
|
|
|
actor.Tell(new ScriptCallRequest("Hot", null, 0, "corr-after"), ActorRefs.NoSender);
|
|
AwaitAssert(() => Assert.Equal(4, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
|
}
|
|
|
|
[Fact]
|
|
public void AlarmOnTriggerRuns_ShedAtTheSameCap_WithAnAlarmScopedSiteEvent()
|
|
{
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var health = new SiteHealthCollector();
|
|
var instanceProbe = CreateTestProbe();
|
|
var options = Options();
|
|
|
|
var alarm = ActorOfAsTestActorRef<AlarmActor>(
|
|
Props.Create(() => new AlarmActor(
|
|
"Flapper", "Inst1", instanceProbe.Ref,
|
|
new ResolvedAlarm
|
|
{
|
|
CanonicalName = "Flapper",
|
|
TriggerType = "ValueMatch",
|
|
TriggerConfiguration = "{\"attributeName\":\"Status\",\"matchValue\":\"Critical\"}",
|
|
PriorityLevel = 100
|
|
},
|
|
BlockingScript(), _sharedLibrary, options, NullLogger<AlarmActor>.Instance,
|
|
null, null, health, new SingleServiceProvider(siteLog), null, _scheduler, null)),
|
|
"alarm-shed-" + Guid.NewGuid().ToString("N"));
|
|
|
|
// Each raise edge spawns one on-trigger run; clear between raises to re-arm the edge.
|
|
void Flap(int cycle)
|
|
{
|
|
alarm.Tell(new AttributeValueChanged(
|
|
"Inst1", "Status", "Status", "Critical", "Good", DateTimeOffset.UtcNow.AddSeconds(cycle)));
|
|
alarm.Tell(new AttributeValueChanged(
|
|
"Inst1", "Status", "Status", "Normal", "Good", DateTimeOffset.UtcNow.AddSeconds(cycle)));
|
|
}
|
|
|
|
for (var i = 0; i < 4; i++) Flap(i);
|
|
AwaitAssert(() => Assert.Equal(4, alarm.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
|
|
|
// Fifth raise is shed — there is no Ask caller on this path, so it surfaces purely as
|
|
// a counter plus the rate-limited Warning event.
|
|
Flap(4);
|
|
Flap(5);
|
|
|
|
// The shed COUNT assertion lives inside the awaited block, not after it.
|
|
// ShedAlarmRun increments the counter and only then emits the (rate-limited)
|
|
// site event, and that event fires on the FIRST shed only — so an awaited
|
|
// gate on the event observes Flap(4)'s shed and orders nothing whatsoever
|
|
// with respect to Flap(5)'s, which is a separate mailbox message with no
|
|
// observable of its own on this path (an alarm on-trigger run has no Ask
|
|
// caller to reply to — AlarmActor.ShedAlarmRun). Asserting the count bare
|
|
// after the gate assumed both sheds had been dequeued by the time the
|
|
// first one's event landed; under load the second can still be pending and
|
|
// the count reads 1. Reproduced deterministically by deferring Flap(5) by
|
|
// 2 s: the bare form failed with "Expected: 2 / Actual: 1", while the
|
|
// ScriptActor sibling above — which IS ordered, because ShedRun replies
|
|
// ScriptCallResult after incrementing — kept passing.
|
|
//
|
|
// The count is ACCUMULATED across polls rather than re-read, because
|
|
// SiteHealthCollector.CollectReport DRAINS the interval counters
|
|
// (Interlocked.Exchange(ref _scriptRunShedCount, 0)) — a poll loop that
|
|
// simply re-read it would consume the first shed and never see 2.
|
|
var shedCounted = 0;
|
|
AwaitAssert(() =>
|
|
{
|
|
shedCounted += health.CollectReport("site-1").ScriptRunShedCount;
|
|
|
|
var shedEvents = siteLog.OfType("script")
|
|
.Where(r => r.Severity == "Warning" && r.Message.Contains("shed"))
|
|
.ToArray();
|
|
|
|
// Both sheds counted, even though the event is rate-limited to one.
|
|
Assert.Equal(2, shedCounted);
|
|
Assert.Single(shedEvents);
|
|
Assert.Equal("AlarmActor:Flapper", shedEvents[0].Source);
|
|
|
|
// ...and neither shed run was ever launched: still exactly four in flight.
|
|
Assert.Equal(4, alarm.UnderlyingActor.RunsInFlight);
|
|
}, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------
|
|
// Arch-review F7: a TRACED run refused by either guard (the in-flight cap, or
|
|
// MinTimeBetweenRuns) used to emit nothing at all, so the Flow Debugger reported
|
|
// NO_TRIGGER_WITHIN_WINDOW — "no value-change trigger on this attribute, or no device echo"
|
|
// — which is wrong in both cases and sends the operator to check the wrong things. Each
|
|
// guard now leaves a Failed ScriptTrace/ScriptRun row naming the guard that refused it.
|
|
// ---------------------------------------------------------------------------------------
|
|
|
|
/// <summary>Audit writer fake: records every emitted row and can wait for one matching a predicate.</summary>
|
|
private sealed class CapturingAuditWriter : IAuditWriter
|
|
{
|
|
private readonly List<AuditEvent> _events = new();
|
|
|
|
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
|
{
|
|
lock (_events) _events.Add(evt);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public AuditEvent[] Snapshot()
|
|
{
|
|
lock (_events) return _events.ToArray();
|
|
}
|
|
|
|
public AuditEvent? WaitFor(Func<AuditEvent, bool> predicate, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
lock (_events)
|
|
{
|
|
var hit = _events.FirstOrDefault(predicate);
|
|
if (hit is not null) return hit;
|
|
}
|
|
Thread.Sleep(25);
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static IServiceProvider AuditServices(ISiteEventLogger siteLog, IAuditWriter writer)
|
|
{
|
|
var services = new ServiceCollection();
|
|
services.AddSingleton(siteLog);
|
|
services.AddSingleton(writer);
|
|
return services.BuildServiceProvider();
|
|
}
|
|
|
|
private static ResolvedScript ValueChangeScript(string name, TimeSpan? minTimeBetweenRuns = null) => new()
|
|
{
|
|
CanonicalName = name,
|
|
TriggerType = "valuechange",
|
|
TriggerConfiguration = "{\"attribute\":\"Status\"}",
|
|
MinTimeBetweenRuns = minTimeBetweenRuns
|
|
};
|
|
|
|
private static bool IsFailedScriptRun(AuditEvent evt) =>
|
|
evt.Action == "ScriptTrace.ScriptRun" && evt.AsRow().Status.ToString() == "Failed";
|
|
|
|
[Fact]
|
|
public void TracedRun_ShedAtTheCap_EmitsAFailedScriptRunRow_ParentedToTheWriter()
|
|
{
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var writer = new CapturingAuditWriter();
|
|
var health = new SiteHealthCollector();
|
|
var instance = CreateTestProbe().Ref;
|
|
var options = Options();
|
|
|
|
var actor = ActorOfAsTestActorRef<ScriptActor>(
|
|
Props.Create(() => new ScriptActor(
|
|
"Hot", "Inst1", instance, BlockingScript(),
|
|
ValueChangeScript("Hot"),
|
|
_sharedLibrary, options, NullLogger<ScriptActor>.Instance,
|
|
null, null, health, AuditServices(siteLog, writer), _scheduler, null)),
|
|
"shed-traced-" + Guid.NewGuid().ToString("N"));
|
|
|
|
// Fill the cap with four untraced runs, all blocked in their bodies.
|
|
for (var i = 0; i < 4; i++)
|
|
actor.Tell(new ScriptCallRequest("Hot", null, 0, $"corr-{i}"), ActorRefs.NoSender);
|
|
|
|
AwaitAssert(() => Assert.Equal(4, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(15));
|
|
|
|
// The fifth trigger is a TRACED value change stamped by its writing execution.
|
|
var parent = Guid.NewGuid();
|
|
actor.Tell(new AttributeValueChanged(
|
|
"Inst1", "Status", "Status", "Running", "Good", DateTimeOffset.UtcNow,
|
|
SourceExecutionId: parent, Trace: true));
|
|
|
|
var evt = writer.WaitFor(IsFailedScriptRun, TimeSpan.FromSeconds(10));
|
|
Assert.NotNull(evt);
|
|
var row = evt!.AsRow();
|
|
Assert.Equal(parent, row.ParentExecutionId);
|
|
Assert.NotNull(row.ExecutionId);
|
|
Assert.StartsWith("shed: MaxConcurrentRunsPerScript", row.ErrorMessage);
|
|
Assert.Equal("Hot", evt.Target);
|
|
Assert.Equal("Inst1", row.SourceInstanceId);
|
|
// The trigger detail survives, so the debugger can still name the attribute.
|
|
var extra = ScriptTraceExtra.TryParse(row.Extra)!;
|
|
Assert.Equal("valuechange", extra.TriggerKind);
|
|
Assert.Equal("Status", extra.TriggerAttribute);
|
|
|
|
// Still exactly four in flight — the shed run was never launched.
|
|
Assert.Equal(4, actor.UnderlyingActor.RunsInFlight);
|
|
}
|
|
|
|
[Fact]
|
|
public void TracedRun_SuppressedByMinTimeBetweenRuns_EmitsAFailedScriptRunRow()
|
|
{
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var writer = new CapturingAuditWriter();
|
|
var health = new SiteHealthCollector();
|
|
var instance = CreateTestProbe().Ref;
|
|
var options = Options();
|
|
|
|
var actor = ActorOfAsTestActorRef<ScriptActor>(
|
|
Props.Create(() => new ScriptActor(
|
|
"Throttled", "Inst1", instance, CompileRaw("return null;"),
|
|
ValueChangeScript("Throttled", TimeSpan.FromMinutes(10)),
|
|
_sharedLibrary, options, NullLogger<ScriptActor>.Instance,
|
|
null, null, health, AuditServices(siteLog, writer), _scheduler, null)),
|
|
"throttled-traced-" + Guid.NewGuid().ToString("N"));
|
|
|
|
// First traced change runs (and emits a Delivered row); the second, back to back,
|
|
// is inside MinTimeBetweenRuns and never launches.
|
|
var parent = Guid.NewGuid();
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
actor.Tell(new AttributeValueChanged(
|
|
"Inst1", "Status", "Status", $"Run{i}", "Good", DateTimeOffset.UtcNow,
|
|
SourceExecutionId: parent, Trace: true));
|
|
}
|
|
|
|
var evt = writer.WaitFor(IsFailedScriptRun, TimeSpan.FromSeconds(10));
|
|
Assert.NotNull(evt);
|
|
var row = evt!.AsRow();
|
|
Assert.Equal(parent, row.ParentExecutionId);
|
|
Assert.StartsWith("suppressed: MinTimeBetweenRuns", row.ErrorMessage);
|
|
|
|
// The first run still produced its ordinary Delivered row — the guard row is additive.
|
|
AwaitAssert(() => Assert.Contains(
|
|
writer.Snapshot(),
|
|
e => e.Action == "ScriptTrace.ScriptRun" && e.AsRow().Status.ToString() == "Delivered"),
|
|
TimeSpan.FromSeconds(10));
|
|
}
|
|
}
|
|
|
|
/// <summary>Test hook used to hold script runs in flight while the cap is exercised.</summary>
|
|
public static class ShedHooks
|
|
{
|
|
/// <summary>Gate the blocking test scripts wait on; reset per test.</summary>
|
|
public static SemaphoreSlim Gate = new(0);
|
|
}
|