dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(alarms): re-apply scripted alarms after a supervision restart of their host

Browse Source 
Review I-2. Under factory Props a supervision restart of
ScriptedAlarmHostActor builds an empty engine, and nothing re-sent
ApplyScriptedAlarms. Until the next deploy the host had no mux interest
(inputs stopped), and its ownership filter silently dropped every
operator ack and shelve.

The host now tells its parent ScriptedAlarmHostRestarted from
PostRestart. DriverHostActor keeps the last plans it applied and
re-sends ApplyScriptedAlarms when the signal comes from the current
host, in Steady, Applying and Stale. This mirrors how a driver child's
ReprovisionRequested is answered. The OnAlarmsLoaded comment, which
assumed a re-apply that never happened, now says where it comes from.

DriverHostActorScriptedAlarmRestartTests deploys one scripted alarm
through a real host, crashes the alarm host, does NOT re-apply by hand,
and asserts that interest is re-registered, a dependency push activates
the alarm, and an operator ack reaches the engine. It is red without
the signal (times out waiting for the re-registration).

Refs #731 #734
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 15:13:02 -04:00
parent acdf6d83a3
commit 89204ac0fa
3 changed files with 158 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+27
View File
@@ -278,6 +278,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// from <see cref="PushDesiredSubscriptions"/>.</summary>
private IActorRef? _scriptedAlarmHost;
/// <summary>The scripted-alarm plans last sent to <see cref="_scriptedAlarmHost"/>, re-sent when it reports a
/// supervision restart (<see cref="ScriptedAlarmHostActor.ScriptedAlarmHostRestarted"/>). Null until the
/// first apply.</summary>
private IReadOnlyList<EquipmentScriptedAlarmPlan>? _lastScriptedAlarmPlans;
private RevisionHash? _currentRevision;
/// <summary>
@@ -1528,6 +1533,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<Terminated>(OnRetiredChildTerminated);
Receive<ScriptedAlarmHostActor.ScriptedAlarmHostRestarted>(_ => HandleScriptedAlarmHostRestarted());
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
@@ -1591,6 +1597,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<Terminated>(OnRetiredChildTerminated);
Receive<ScriptedAlarmHostActor.ScriptedAlarmHostRestarted>(_ => HandleScriptedAlarmHostRestarted());
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
@@ -2541,6 +2548,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
_log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)",
_localNode, m.Items.Count, m.DriverInstanceId));
Receive<Terminated>(OnRetiredChildTerminated);
Receive<ScriptedAlarmHostActor.ScriptedAlarmHostRestarted>(_ => HandleScriptedAlarmHostRestarted());
Receive<DriverInstanceActor.AttributeAlarmPublished>(m =>
_log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale",
_localNode, m.Args.ConditionId, m.DriverInstanceId));
@@ -3692,6 +3700,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// (re)loads its engine + re-registers mux interest for the union of dependency refs. Covers
// both the fresh-apply and bootstrap-restore paths (both call this method); the Stale-recovery
// path deliberately does not, matching driver + VirtualTag recovery.
_lastScriptedAlarmPlans = composition.EquipmentScriptedAlarms;
_scriptedAlarmHost?.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(composition.EquipmentScriptedAlarms));
if (composition.EquipmentScriptedAlarms.Count > 0)
{
@@ -3906,6 +3915,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Context.Watch(child);
}
/// <summary>
/// The scripted-alarm host was restarted by supervision and has an empty engine: re-send the last plans so
/// its inputs and operator commands work again without waiting for the next deploy. Only the CURRENT host's
/// signal is honoured.
/// </summary>
private void HandleScriptedAlarmHostRestarted()
{
if (_scriptedAlarmHost is null || !Sender.Equals(_scriptedAlarmHost)) return;
if (_lastScriptedAlarmPlans is null)
{
_log.Debug("DriverHost {Node}: ScriptedAlarm host restarted before any apply; nothing to re-send", _localNode);
return;
}
_log.Warning("DriverHost {Node}: ScriptedAlarm host was restarted by supervision; re-applying {Count} scripted alarm(s)",
_localNode, _lastScriptedAlarmPlans.Count);
_scriptedAlarmHost.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(_lastScriptedAlarmPlans));
}
/// <summary>A retired child has terminated; nothing more can arrive from it.</summary>
/// <param name="msg">The termination notice.</param>
private void OnRetiredChildTerminated(Terminated msg) => _retiredChildren.Remove(msg.ActorRef);
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs
+28 -3
View File
@@ -86,6 +86,21 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
/// <param name="Plans">The desired Equipment-namespace scripted-alarm plans.</param>
public sealed record ApplyScriptedAlarms(IReadOnlyList<EquipmentScriptedAlarmPlan> Plans);
/// <summary>
/// Told to the parent from <see cref="PostRestart"/>. A supervision restart is otherwise invisible to the
/// parent, and the new incarnation has an EMPTY engine (factory Props) and no mux interest, so until the
/// next <see cref="ApplyScriptedAlarms"/> its inputs stop and every operator command is dropped as not
/// owned. The parent (<c>DriverHostActor</c>) answers by re-sending the last plans it applied, mirroring
/// how a driver child's <c>ReprovisionRequested</c> is answered.
/// </summary>
public sealed class ScriptedAlarmHostRestarted
{
/// <summary>The single instance.</summary>
public static readonly ScriptedAlarmHostRestarted Instance = new();
private ScriptedAlarmHostRestarted() { }
}
/// <summary>Marshals an engine emission off the engine's worker thread onto the actor thread.
/// Carries the <see cref="ScriptedAlarmEvent"/> the engine raised on <c>OnEvent</c>.</summary>
private sealed record EngineEmission(ScriptedAlarmEvent Event);
@@ -384,9 +399,10 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
// subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the
// engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor),
// so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed.
// ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a
// supervision restart builds one around an EMPTY upstream) converges immediately instead of
// waiting for every input to change again.
// ReplayRetained: the mux replays the last value it saw for each ref, so a freshly loaded engine
// converges immediately instead of waiting for every input to change again. That includes the engine
// a supervision restart builds around an EMPTY upstream, but only once the parent has re-sent
// ApplyScriptedAlarms in answer to ScriptedAlarmHostRestarted (PostRestart); nothing reloads it before.
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true));
// Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references
// any more will never be pushed again, so its last value is dead weight for the process lifetime.
@@ -791,6 +807,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
base.PreRestart(reason, message);
}
/// <inheritdoc />
/// <remarks>Tells the parent <see cref="ScriptedAlarmHostRestarted"/> so it re-sends the last
/// <see cref="ApplyScriptedAlarms"/>: the new incarnation has nothing loaded and no mux interest.</remarks>
protected override void PostRestart(Exception reason)
{
base.PostRestart(reason);
Context.Parent.Tell(ScriptedAlarmHostRestarted.Instance);
}
/// <inheritdoc />
protected override void PostStop()
{
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorScriptedAlarmRestartTests.cs
+103
View File
@@ -0,0 +1,103 @@
using System.Text.Json;
using Akka.Actor;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review I-2: under the factory Props a supervision restart of the scripted-alarm host builds a fresh, EMPTY
/// engine. Nothing re-sent <c>ApplyScriptedAlarms</c>, so until the next deploy the host had no mux interest
/// (inputs stopped) and its ownership filter dropped every operator ack/shelve as "not owned". The host now
/// tells its parent <see cref="ScriptedAlarms.ScriptedAlarmHostActor.ScriptedAlarmHostRestarted"/> from <c>PostRestart</c>,
/// and <see cref="DriverHostActor"/> re-sends the last plans it applied. This test never re-applies by hand.
/// </summary>
public sealed class DriverHostActorScriptedAlarmRestartTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-alarmhost-restart-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('d', 64));
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
private static DeploymentId SeedAlarmDeployment(Microsoft.EntityFrameworkCore.IDbContextFactory<ZB.MOM.WW.OtOpcUa.Configuration.OtOpcUaConfigDbContext> db)
{
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
{
ScriptedAlarms = new[]
{
new
{
ScriptedAlarmId = "alm-1", EquipmentId = "eq-1", Name = "HighTemp", AlarmType = "AlarmCondition",
Severity = 800, MessageTemplate = "hot", PredicateScriptId = "s-1",
HistorizeToAveva = false, Retain = true, Enabled = true,
},
},
Scripts = new[] { new { ScriptId = "s-1", SourceCode = "return (int)ctx.GetTag(\"M.T\").Value > 90;" } },
});
var id = DeploymentId.NewId();
using var ctx = db.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id.Value, RevisionHash = RevA.Value, Status = DeploymentStatus.Sealed,
CreatedBy = "test", SealedAtUtc = DateTime.UtcNow, ArtifactBlob = artifact,
});
ctx.SaveChanges();
return id;
}
[Fact]
public void After_a_supervision_restart_the_alarm_host_is_re_applied_and_inputs_and_acks_work_again()
{
var db = NewInMemoryDbFactory();
var deploymentId = SeedAlarmDeployment(db);
var publish = this.CreateCleanOutcomePublishProbe();
var mux = CreateTestProbe();
var coordinator = CreateTestProbe();
var host = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
localRoles: new HashSet<string> { "driver" },
dependencyMux: mux.Ref,
opcUaPublishActor: publish.Ref,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: CreateTestProbe().Ref,
scriptRootLogger: new ScriptRootLogger(new LoggerConfiguration().CreateLogger()),
alarmStateStore: new InMemoryAlarmStateStore()));
V3ArtifactSeeder.Dispatch(host, coordinator, deploymentId, RevA);
var first = mux.FishForMessage<DependencyMuxActor.RegisterInterest>(r => r.TagRefs.Contains("M.T"), Timeout);
var alarmHost = first.Subscriber;
// The crash: a batch item with a null TagId makes the upstream Push throw (the documented unguarded
// path), so supervision restarts the alarm host. NO manual re-apply follows.
alarmHost.Tell(new VirtualTagActor.DependencyValueBatch(new[]
{
new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow),
}));
mux.FishForMessage<DependencyMuxActor.UnregisterInterest>(u => u.Subscriber.Equals(alarmHost), Timeout);
// The parent re-applies on the restart signal: interest is registered again by the new incarnation.
mux.FishForMessage<DependencyMuxActor.RegisterInterest>(r => r.TagRefs.Contains("M.T"), Timeout);
// Inputs work: a dependency push activates the alarm.
alarmHost.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(
m => m.AlarmNodeId == "alm-1" && m.State.Active && !m.State.Acknowledged, Timeout);
// Acks work: the restarted engine owns alm-1 again, so the command is not filtered out as unowned.
alarmHost.Tell(new AlarmCommand(AlarmId: "alm-1", Operation: "Acknowledge", User: "alice", Comment: null, UnshelveAtUtc: null));
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(
m => m.AlarmNodeId == "alm-1" && m.State.Acknowledged, Timeout)
.State.Acknowledged.ShouldBeTrue();
}
}