From 89204ac0faa3c337daba4a1de52a8cde427a0574 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:13:02 -0400 Subject: [PATCH] fix(alarms): re-apply scripted alarms after a supervision restart of their host Review I-2. Under factory Props a supervision restart of ScriptedAlarmHostActor builds an empty engine, and nothing re-sent ApplyScriptedAlarms. Until the next deploy the host had no mux interest (inputs stopped), and its ownership filter silently dropped every operator ack and shelve. The host now tells its parent ScriptedAlarmHostRestarted from PostRestart. DriverHostActor keeps the last plans it applied and re-sends ApplyScriptedAlarms when the signal comes from the current host, in Steady, Applying and Stale. This mirrors how a driver child's ReprovisionRequested is answered. The OnAlarmsLoaded comment, which assumed a re-apply that never happened, now says where it comes from. DriverHostActorScriptedAlarmRestartTests deploys one scripted alarm through a real host, crashes the alarm host, does NOT re-apply by hand, and asserts that interest is re-registered, a dependency push activates the alarm, and an operator ack reaches the engine. It is red without the signal (times out waiting for the re-registration). Refs #731 #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Drivers/DriverHostActor.cs | 27 +++++ .../ScriptedAlarms/ScriptedAlarmHostActor.cs | 31 +++++- ...riverHostActorScriptedAlarmRestartTests.cs | 103 ++++++++++++++++++ 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorScriptedAlarmRestartTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index b751214f..fd29c0b8 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -278,6 +278,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// from . private IActorRef? _scriptedAlarmHost; + /// The scripted-alarm plans last sent to , re-sent when it reports a + /// supervision restart (). Null until the + /// first apply. + private IReadOnlyList? _lastScriptedAlarmPlans; + private RevisionHash? _currentRevision; /// @@ -1528,6 +1533,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleGetDiagnostics); Receive(ForwardBatchToMux); Receive(OnRetiredChildTerminated); + Receive(_ => HandleScriptedAlarmHostRestarted()); Receive(ForwardNativeAlarm); Receive(OnDriverConnectivityChanged); Receive(HandleCommsRecoveryAudit); @@ -1591,6 +1597,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(HandleGetDiagnostics); Receive(ForwardBatchToMux); Receive(OnRetiredChildTerminated); + Receive(_ => HandleScriptedAlarmHostRestarted()); Receive(ForwardNativeAlarm); Receive(OnDriverConnectivityChanged); Receive(HandleCommsRecoveryAudit); @@ -2541,6 +2548,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS _log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)", _localNode, m.Items.Count, m.DriverInstanceId)); Receive(OnRetiredChildTerminated); + Receive(_ => HandleScriptedAlarmHostRestarted()); Receive(m => _log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale", _localNode, m.Args.ConditionId, m.DriverInstanceId)); @@ -3692,6 +3700,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // (re)loads its engine + re-registers mux interest for the union of dependency refs. Covers // both the fresh-apply and bootstrap-restore paths (both call this method); the Stale-recovery // path deliberately does not, matching driver + VirtualTag recovery. + _lastScriptedAlarmPlans = composition.EquipmentScriptedAlarms; _scriptedAlarmHost?.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(composition.EquipmentScriptedAlarms)); if (composition.EquipmentScriptedAlarms.Count > 0) { @@ -3906,6 +3915,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Context.Watch(child); } + /// + /// The scripted-alarm host was restarted by supervision and has an empty engine: re-send the last plans so + /// its inputs and operator commands work again without waiting for the next deploy. Only the CURRENT host's + /// signal is honoured. + /// + private void HandleScriptedAlarmHostRestarted() + { + if (_scriptedAlarmHost is null || !Sender.Equals(_scriptedAlarmHost)) return; + if (_lastScriptedAlarmPlans is null) + { + _log.Debug("DriverHost {Node}: ScriptedAlarm host restarted before any apply; nothing to re-send", _localNode); + return; + } + _log.Warning("DriverHost {Node}: ScriptedAlarm host was restarted by supervision; re-applying {Count} scripted alarm(s)", + _localNode, _lastScriptedAlarmPlans.Count); + _scriptedAlarmHost.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(_lastScriptedAlarmPlans)); + } + /// A retired child has terminated; nothing more can arrive from it. /// The termination notice. private void OnRetiredChildTerminated(Terminated msg) => _retiredChildren.Remove(msg.ActorRef); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index 210a84ca..60f05729 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -86,6 +86,21 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// The desired Equipment-namespace scripted-alarm plans. public sealed record ApplyScriptedAlarms(IReadOnlyList Plans); + /// + /// Told to the parent from . A supervision restart is otherwise invisible to the + /// parent, and the new incarnation has an EMPTY engine (factory Props) and no mux interest, so until the + /// next its inputs stop and every operator command is dropped as not + /// owned. The parent (DriverHostActor) answers by re-sending the last plans it applied, mirroring + /// how a driver child's ReprovisionRequested is answered. + /// + public sealed class ScriptedAlarmHostRestarted + { + /// The single instance. + public static readonly ScriptedAlarmHostRestarted Instance = new(); + + private ScriptedAlarmHostRestarted() { } + } + /// Marshals an engine emission off the engine's worker thread onto the actor thread. /// Carries the the engine raised on OnEvent. private sealed record EngineEmission(ScriptedAlarmEvent Event); @@ -384,9 +399,10 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor // subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the // engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor), // so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed. - // ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a - // supervision restart builds one around an EMPTY upstream) converges immediately instead of - // waiting for every input to change again. + // ReplayRetained: the mux replays the last value it saw for each ref, so a freshly loaded engine + // converges immediately instead of waiting for every input to change again. That includes the engine + // a supervision restart builds around an EMPTY upstream, but only once the parent has re-sent + // ApplyScriptedAlarms in answer to ScriptedAlarmHostRestarted (PostRestart); nothing reloads it before. _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true)); // Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references // any more will never be pushed again, so its last value is dead weight for the process lifetime. @@ -791,6 +807,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor base.PreRestart(reason, message); } + /// + /// Tells the parent so it re-sends the last + /// : the new incarnation has nothing loaded and no mux interest. + protected override void PostRestart(Exception reason) + { + base.PostRestart(reason); + Context.Parent.Tell(ScriptedAlarmHostRestarted.Instance); + } + /// protected override void PostStop() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorScriptedAlarmRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorScriptedAlarmRestartTests.cs new file mode 100644 index 00000000..2ff682f2 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorScriptedAlarmRestartTests.cs @@ -0,0 +1,103 @@ +using System.Text.Json; +using Akka.Actor; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Engines; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; + +/// +/// Review I-2: under the factory Props a supervision restart of the scripted-alarm host builds a fresh, EMPTY +/// engine. Nothing re-sent ApplyScriptedAlarms, so until the next deploy the host had no mux interest +/// (inputs stopped) and its ownership filter dropped every operator ack/shelve as "not owned". The host now +/// tells its parent from PostRestart, +/// and re-sends the last plans it applied. This test never re-applies by hand. +/// +public sealed class DriverHostActorScriptedAlarmRestartTests : RuntimeActorTestBase +{ + private static readonly NodeId TestNode = NodeId.Parse("driver-alarmhost-restart-test"); + private static readonly RevisionHash RevA = RevisionHash.Parse(new string('d', 64)); + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + private static DeploymentId SeedAlarmDeployment(Microsoft.EntityFrameworkCore.IDbContextFactory db) + { + var artifact = JsonSerializer.SerializeToUtf8Bytes(new + { + ScriptedAlarms = new[] + { + new + { + ScriptedAlarmId = "alm-1", EquipmentId = "eq-1", Name = "HighTemp", AlarmType = "AlarmCondition", + Severity = 800, MessageTemplate = "hot", PredicateScriptId = "s-1", + HistorizeToAveva = false, Retain = true, Enabled = true, + }, + }, + Scripts = new[] { new { ScriptId = "s-1", SourceCode = "return (int)ctx.GetTag(\"M.T\").Value > 90;" } }, + }); + var id = DeploymentId.NewId(); + using var ctx = db.CreateDbContext(); + ctx.Deployments.Add(new Deployment + { + DeploymentId = id.Value, RevisionHash = RevA.Value, Status = DeploymentStatus.Sealed, + CreatedBy = "test", SealedAtUtc = DateTime.UtcNow, ArtifactBlob = artifact, + }); + ctx.SaveChanges(); + return id; + } + + [Fact] + public void After_a_supervision_restart_the_alarm_host_is_re_applied_and_inputs_and_acks_work_again() + { + var db = NewInMemoryDbFactory(); + var deploymentId = SeedAlarmDeployment(db); + var publish = this.CreateCleanOutcomePublishProbe(); + var mux = CreateTestProbe(); + var coordinator = CreateTestProbe(); + var host = Sys.ActorOf(DriverHostActor.Props( + db, TestNode, coordinator.Ref, + localRoles: new HashSet { "driver" }, + dependencyMux: mux.Ref, + opcUaPublishActor: publish.Ref, + virtualTagEvaluator: NullVirtualTagEvaluator.Instance, + virtualTagHostOverride: CreateTestProbe().Ref, + scriptRootLogger: new ScriptRootLogger(new LoggerConfiguration().CreateLogger()), + alarmStateStore: new InMemoryAlarmStateStore())); + V3ArtifactSeeder.Dispatch(host, coordinator, deploymentId, RevA); + + var first = mux.FishForMessage(r => r.TagRefs.Contains("M.T"), Timeout); + var alarmHost = first.Subscriber; + + // The crash: a batch item with a null TagId makes the upstream Push throw (the documented unguarded + // path), so supervision restarts the alarm host. NO manual re-apply follows. + alarmHost.Tell(new VirtualTagActor.DependencyValueBatch(new[] + { + new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow), + })); + mux.FishForMessage(u => u.Subscriber.Equals(alarmHost), Timeout); + + // The parent re-applies on the restart signal: interest is registered again by the new incarnation. + mux.FishForMessage(r => r.TagRefs.Contains("M.T"), Timeout); + + // Inputs work: a dependency push activates the alarm. + alarmHost.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + publish.FishForMessage( + m => m.AlarmNodeId == "alm-1" && m.State.Active && !m.State.Acknowledged, Timeout); + + // Acks work: the restarted engine owns alm-1 again, so the command is not filtered out as unowned. + alarmHost.Tell(new AlarmCommand(AlarmId: "alm-1", Operation: "Acknowledge", User: "alice", Comment: null, UnshelveAtUtc: null)); + publish.FishForMessage( + m => m.AlarmNodeId == "alm-1" && m.State.Acknowledged, Timeout) + .State.Acknowledged.ShouldBeTrue(); + } +}