using System.Collections.Concurrent; using System.Diagnostics; using Serilog; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// A minimal : set or push a value; push notifies subscribers synchronously. internal sealed class TestUpstream : ITagUpstreamSource { private readonly ConcurrentDictionary _values = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _subs = new(StringComparer.Ordinal); public void Set(string path, object? value, uint status = 0u) => _values[path] = new DataValueSnapshot(value, status, DateTime.UtcNow, DateTime.UtcNow); public void Push(string path, object? value, uint status = 0u) { Set(path, value, status); if (!_subs.TryGetValue(path, out var list)) return; Action[] observers; lock (list) observers = [.. list]; foreach (var observer in observers) observer(path, _values[path]); } public DataValueSnapshot ReadTag(string path) => _values.TryGetValue(path, out var v) ? v : new DataValueSnapshot(null, 0x80320000u, null, DateTime.UtcNow); public IDisposable SubscribeTag(string path, Action observer) { var list = _subs.GetOrAdd(path, _ => []); lock (list) list.Add(observer); return new Unsubscribe(() => { lock (list) list.Remove(observer); }); } private sealed class Unsubscribe(Action dispose) : IDisposable { public void Dispose() => dispose(); } } /// /// lmxopcua#780 — the scripted-alarm seam over the REAL worker (design §9, alarm column): a runaway predicate is killed /// at T + G and quarantined, its condition frozen with a BadInternalError annotation and never transitioned; a /// sibling alarm on the same shard keeps evaluating; and a worker that dies under an in-flight predicate freezes its /// condition BadResourceUnavailable — it never clears it — and the alarm is re-evaluated by itself on return. /// [Collection(ScriptWorkerCollection.Name)] [Trait("Category", "ScriptWorker")] public sealed class AlarmSeamTests { private const uint StatusBadInternalError = 0x80020000u; private const uint StatusBadResourceUnavailable = 0x80040000u; private static CancellationToken Ct => TestContext.Current.CancellationToken; private static ScriptedAlarmDefinition Alarm(string id, string predicate) => new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner runner, TimeSpan budget) { var logger = new LoggerConfiguration().CreateLogger(); return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, scriptTimeout: budget, scriptRunner: runner); } [Fact] public async Task A_runaway_predicate_is_killed_and_quarantined_its_alarm_frozen_Bad_and_a_sibling_keeps_evaluating() { var budget = TimeSpan.FromMilliseconds(300); var log = new CapturingSink(); await using var runner = Scripts.Runner(Scripts.Options(hardGrace: TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2), log.CreateLogger()); var up = new TestUpstream(); up.Set("x", 0); using var engine = Engine(up, runner, budget); var events = new ConcurrentQueue(); engine.OnEvent += (_, e) => events.Enqueue(e); await engine.LoadAsync( [ Alarm("runaway", "if ((int)ctx.GetTag(\"x\").Value > 5) { while (true) { } } return false;"), Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;"), ], Ct); var first = runner.Shards[0].Current.ShouldNotBeNull(); up.Push("x", 10); await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Activated }), TimeSpan.FromSeconds(10), "the sibling's activation"); await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "runaway", Emission: EmissionKind.QualityChanged }), TimeSpan.FromSeconds(10), "the runaway's annotation"); events.Single(e => e.AlarmId == "runaway").WorstInputStatusCode.ShouldBe(StatusBadInternalError); (await first.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct)).CulpritKey .ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.AlarmPredicate, "runaway")); // The second kill quarantines it; the sibling (same shard, worker restarted twice) keeps evaluating. Re-pushed: // Exited completes before the dead worker's evaluations are released, so the first push after it can find the // runaway's slot still held and be refused (not a strike, exactly as in process) — the next push runs it. var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60); for (var x = 11; !engine.QuarantinedAlarmIds.Contains("runaway"); x++) { if (DateTime.UtcNow > deadline) throw new TimeoutException("no kill quarantine; events: " + string.Join(", ", events.Select(e => $"{e.AlarmId}:{e.Emission}:{e.WorstInputStatusCode:X8}")) + Environment.NewLine + string.Join(Environment.NewLine, log.Rendered)); up.Push("x", x); await Task.Delay(TimeSpan.FromSeconds(2), Ct); } engine.QuarantinedAlarmIds.ShouldBe(["runaway"]); up.Push("x", 0); await Scripts.Eventually(() => events.Any(e => e is { AlarmId: "sibling", Emission: EmissionKind.Cleared }), TimeSpan.FromSeconds(60), "the sibling's clear"); engine.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "a predicate that never returned never transitions"); events.Where(e => e.AlarmId == "runaway").ShouldAllBe(e => e.Emission == EmissionKind.QualityChanged); // A redeploy (reload) lifts the quarantine, as in process. await engine.LoadAsync([Alarm("runaway", "return (int)ctx.GetTag(\"x\").Value > 5;"), Alarm("sibling", "return (int)ctx.GetTag(\"x\").Value > 1;")], Ct); engine.QuarantinedAlarmIds.ShouldBeEmpty(); } [Fact] public async Task A_worker_that_dies_under_a_predicate_freezes_it_BadResourceUnavailable_never_clears_it_and_it_is_re_evaluated_on_return() { await using var runner = Scripts.Runner(Scripts.Options(probation: 1)); var up = new TestUpstream(); up.Set("ms", 1); using var engine = Engine(up, runner, TimeSpan.FromSeconds(5)); var events = new ConcurrentQueue(); engine.OnEvent += (_, e) => events.Enqueue(e); await engine.LoadAsync([Alarm("slow", "var ms = (int)ctx.GetTag(\"ms\").Value; var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms > 0;")], Ct); engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active); var worker = runner.Shards[0].Current.ShouldNotBeNull(); events.Clear(); up.Push("ms", 1500); await Task.Delay(300, Ct); // the predicate is spinning in the worker Process.GetProcessById(worker.Pid).Kill(); await Scripts.Eventually(() => events.Any(e => e.Emission == EmissionKind.QualityChanged), TimeSpan.FromSeconds(10), "the runner-failure annotation"); events.First().WorstInputStatusCode.ShouldBe(StatusBadResourceUnavailable); engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active, "a dead worker never clears an alarm"); // No further input change: the runner's recovery re-evaluates it (in the isolation worker) and the quality returns. await Scripts.Eventually(() => events.Any(e => e is { Emission: EmissionKind.QualityChanged, WorstInputStatusCode: 0u }), TimeSpan.FromSeconds(60), "the re-evaluation on return"); engine.GetState("slow")!.Active.ShouldBe(AlarmActiveState.Active); events.ShouldNotContain(e => e.Emission == EmissionKind.Cleared); } }