6aa5edaf30
- I1 ScriptRunGuard: a run refused because the previous one is still executing is a Bad publish plus otopcua.script.run_refused, and no longer a quarantine strike. Only a timeout (and an allocation breach) counts; a refusal neither adds to nor resets the timeout streak; a completed run (value or throw) resets it. On a 1 Hz trigger with a 2 s budget a single 4 s overrun is one timeout plus two refusals and must not quarantine. Tests updated across the guard, the alarm engine, the Calculation evaluator and the VT evaluator, plus Refusals_neither_count_toward_nor_reset_the_timeout_streak. - I3 ScriptedAlarmEngine: the quality bucket is tracked only after the predicate completes (or in the inputs-not-ready path), so the tracked bucket is the last one annotated. A refused or quarantined predicate used to re-track the input bucket Good before the attempt and then flip it Bad, emitting QualityChanged on every push. The run-guard tests now count emissions (exactly 1 across N pushes, refused and quarantined cases), with a healthy sibling's dispatch count as the positive control that every push was processed. Both were red before the fix. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
88 lines
4.2 KiB
C#
88 lines
4.2 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Serilog;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
|
|
|
|
/// <summary>
|
|
/// <see cref="CalculationEvaluator"/> runs every compiled script through a <see cref="ScriptRunGuard"/>
|
|
/// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every
|
|
/// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now
|
|
/// every later trigger is refused ("previous evaluation still running" — Bad, but not a quarantine
|
|
/// strike; only consecutive timeouts quarantine), and a healthy sibling on the same evaluator keeps computing.
|
|
/// </summary>
|
|
public sealed class CalculationEvaluatorRunGuardTests
|
|
{
|
|
private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
|
|
|
|
/// <summary>Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself.</summary>
|
|
private const string SpinScript =
|
|
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
|
|
|
|
private static readonly Dictionary<string, object?> NoDeps = new();
|
|
|
|
[Fact]
|
|
public async Task Triggers_during_the_previous_orphan_are_refused_not_quarantined_and_a_sibling_keeps_computing()
|
|
{
|
|
using var sut = new CalculationEvaluator(
|
|
NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
|
|
var ct = TestContext.Current.CancellationToken;
|
|
|
|
var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
|
|
first.Success.ShouldBeFalse();
|
|
first.Reason.ShouldNotBeNull();
|
|
first.Reason.ShouldContain("timed out");
|
|
|
|
// Every later trigger while the orphan runs is a Bad "previous evaluation still running" — a
|
|
// refusal is not a strike, so the tag is NOT quarantined by them (fix round 1, I1).
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
var refused = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct);
|
|
refused.Success.ShouldBeFalse();
|
|
refused.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
|
|
}
|
|
sut.QuarantinedScripts.ShouldBeEmpty();
|
|
sut.RunGuard.RefusedWhileRunningCount.ShouldBe(3);
|
|
|
|
// Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger.
|
|
sut.RunGuard.StartedRunCount.ShouldBe(1);
|
|
sut.RunGuard.InFlightCount.ShouldBe(1);
|
|
|
|
// Positive control: a sibling tag on the same evaluator still computes.
|
|
var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct);
|
|
sibling.Success.ShouldBeTrue(sibling.Reason);
|
|
sibling.Value.ShouldBe(42);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does()
|
|
{
|
|
using var sut = new CalculationEvaluator(
|
|
NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
|
|
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
|
|
var ct = TestContext.Current.CancellationToken;
|
|
|
|
var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
|
|
r.Reason.ShouldNotBeNull();
|
|
r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
|
|
|
|
sut.ClearCompiledScripts();
|
|
|
|
sut.QuarantinedScripts.ShouldBeEmpty();
|
|
// The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag.
|
|
var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct);
|
|
fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
|
|
fixedRun.Value.ShouldBe(7);
|
|
}
|
|
|
|
private static Dictionary<string, Core.Abstractions.DataValueSnapshot> Snap(Dictionary<string, object?> deps)
|
|
{
|
|
var now = DateTime.UtcNow;
|
|
var d = new Dictionary<string, Core.Abstractions.DataValueSnapshot>(StringComparer.Ordinal);
|
|
foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now);
|
|
return d;
|
|
}
|
|
}
|