using Serilog; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; /// /// runs every predicate through a keyed by /// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation /// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is /// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a /// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine. /// [Trait("Category", "Unit")] public sealed class ScriptedAlarmEngineRunGuardTests { private const uint StatusBadInternalError = 0x80020000u; private const string SpinPredicate = "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;"; private static ScriptedAlarmDefinition Alarm(string id, string predicate) => new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null) { var logger = new LoggerConfiguration().CreateLogger(); // 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and // must finish inside the budget on a loaded CI box, or the positive control itself times out. The // spin predicate outlives this by 12x, so the runaway alarm still cannot finish first. return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting); } [Fact] public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating() { var up = new FakeUpstream(); up.Set("Temp", 50); using var eng = Build(up); var events = new List(); eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; // The load itself evaluates once (startup recovery) — that is the first, abandoned run. await eng.LoadAsync( [Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")], TestContext.Current.CancellationToken); // Two more pushes: refused (orphan still running), then quarantined. up.Push("Temp", 60); await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1); up.Push("Temp", 70); await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _)); eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); eng.RunGuard.InFlightCount.ShouldBe(1); // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving. ScriptedAlarmEvent[] snapshot; lock (events) snapshot = events.ToArray(); var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray(); quality.ShouldNotBeEmpty(); quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented"); // Positive control: the healthy sibling still transitions on the same engine. up.Push("Temp", 150); await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active); } [Fact] public async Task A_reload_lifts_the_quarantine() { var up = new FakeUpstream(); up.Set("Temp", 50); using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")], TestContext.Current.CancellationToken); eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue(); // The operator fixed the predicate and redeployed. await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], TestContext.Current.CancellationToken); eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse(); up.Push("Temp", 150); await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active); } private static async Task WaitForAsync(Func cond, int timeoutMs = 10_000) { var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); while (DateTime.UtcNow < deadline) { if (cond()) return; await Task.Delay(25); } throw new TimeoutException("Condition did not become true in time"); } }