fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4)

- R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:27:49 -04:00
parent 6cbf763719
commit 4d3fc9c816
15 changed files with 921 additions and 63 deletions
@@ -0,0 +1,145 @@
+using Akka.Actor;
+using Akka.TestKit;
+using Serilog;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer;
+using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
+using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
+using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms;
+
+/// <summary>
+///     R02:H1 — a supervision restart of <see cref="ScriptedAlarmHostActor"/> used to dispose the
+///     Props-captured engine and re-create the actor around the disposed instance, so every later
+///     <c>ApplyScriptedAlarms</c> failed and the node's scripted alarms silently died. With the factory
+///     Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both
+///     shapes must load cleanly after a throw inside a <see cref="VirtualTagActor.DependencyValueBatch"/>
+///     handler. Also R02:H3 — an inbound alarm command's engine op is bounded.
+/// </summary>
+public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase
+{
+    private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
+
+    private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") =>
+        new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition",
+            Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script",
+            PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;",
+            DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true);
+
+    private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null)
+    {
+        var logger = new LoggerConfiguration().CreateLogger();
+        return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
+    }
+
+    private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair()
+    {
+        var up = new DependencyMuxTagUpstreamSource();
+        return (up, BuildEngine(up));
+    }
+
+    /// <summary>Apply, crash the handler (a null TagId makes <c>Push</c> throw → restart), apply again
+    /// and prove the second load registers interest and the alarm still activates.</summary>
+    private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux)
+    {
+        host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
+        mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
+
+        // The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler
+        // (the documented, deliberately unguarded path) → supervision restarts the actor.
+        host.Tell(new VirtualTagActor.DependencyValueBatch(new[]
+        {
+            new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow),
+        }));
+        mux.ExpectMsg<DependencyMuxActor.UnregisterInterest>(Timeout); // PostStop of the crashed incarnation
+
+        // A later apply must load cleanly on the restarted incarnation…
+        host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
+        var reg = mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
+        reg.TagRefs.ShouldContain("M.T");
+
+        // …and the alarm must still work end to end.
+        host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
+        var state = publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
+        state.AlarmNodeId.ShouldBe("alm-1");
+        state.State.Active.ShouldBeTrue();
+    }
+
+    [Fact]
+    public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards()
+    {
+        var publish = CreateTestProbe();
+        var mux = CreateTestProbe();
+        var built = 0;
+        var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); }));
+
+        RestartThenReload(host, publish, mux);
+
+        built.ShouldBe(2, "the restarted incarnation built its own engine");
+    }
+
+    [Fact]
+    public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches()
+    {
+        var publish = CreateTestProbe();
+        var mux = CreateTestProbe();
+        var (upstream, engine) = Pair();
+        var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine));
+
+        RestartThenReload(host, publish, mux);
+    }
+
+    [Fact]
+    public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on()
+    {
+        var publish = CreateTestProbe();
+        var mux = CreateTestProbe();
+        var store = new HangingSaveStore();
+        var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref,
+            () => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); },
+            commandTimeout: TimeSpan.FromMilliseconds(300)));
+
+        host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
+        mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
+        host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
+        publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeTrue();
+
+        store.HangNext = true;
+        host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null));
+
+        // The hung save would have suspended the mailbox indefinitely; the bound releases it, so the
+        // next dependency value is processed and the alarm clears.
+        host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow));
+        publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeFalse();
+        store.ObservedCancellation.ShouldBeTrue();
+    }
+
+    /// <summary>Store whose next SaveAsync blocks until its token is cancelled.</summary>
+    private sealed class HangingSaveStore : IAlarmStateStore
+    {
+        private readonly InMemoryAlarmStateStore _inner = new();
+        public volatile bool HangNext;
+        public volatile bool ObservedCancellation;
+
+        public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
+        public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
+        public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
+
+        public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
+        {
+            if (HangNext)
+            {
+                HangNext = false;
+                try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); }
+                catch (OperationCanceledException) { ObservedCancellation = true; throw; }
+            }
+            await _inner.SaveAsync(state, ct);
+        }
+    }
+}