fix(alarms): re-apply scripted alarms after a supervision restart of their host

Review I-2. Under factory Props a supervision restart of
ScriptedAlarmHostActor builds an empty engine, and nothing re-sent
ApplyScriptedAlarms. Until the next deploy the host had no mux interest
(inputs stopped), and its ownership filter silently dropped every
operator ack and shelve.

The host now tells its parent ScriptedAlarmHostRestarted from
PostRestart. DriverHostActor keeps the last plans it applied and
re-sends ApplyScriptedAlarms when the signal comes from the current
host, in Steady, Applying and Stale. This mirrors how a driver child's
ReprovisionRequested is answered. The OnAlarmsLoaded comment, which
assumed a re-apply that never happened, now says where it comes from.

DriverHostActorScriptedAlarmRestartTests deploys one scripted alarm
through a real host, crashes the alarm host, does NOT re-apply by hand,
and asserts that interest is re-registered, a dependency push activates
the alarm, and an operator ack reaches the engine. It is red without
the signal (times out waiting for the re-registration).

Refs #731 #734
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 15:13:02 -04:00
parent acdf6d83a3
commit 89204ac0fa
3 changed files with 158 additions and 3 deletions
@@ -278,6 +278,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// from <see cref="PushDesiredSubscriptions"/>.</summary>
private IActorRef? _scriptedAlarmHost;
/// <summary>The scripted-alarm plans last sent to <see cref="_scriptedAlarmHost"/>, re-sent when it reports a
/// supervision restart (<see cref="ScriptedAlarmHostActor.ScriptedAlarmHostRestarted"/>). Null until the
/// first apply.</summary>
private IReadOnlyList<EquipmentScriptedAlarmPlan>? _lastScriptedAlarmPlans;
private RevisionHash? _currentRevision;
/// <summary>
@@ -1528,6 +1533,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<Terminated>(OnRetiredChildTerminated);
Receive<ScriptedAlarmHostActor.ScriptedAlarmHostRestarted>(_ => HandleScriptedAlarmHostRestarted());
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
@@ -1591,6 +1597,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(ForwardBatchToMux);
Receive<Terminated>(OnRetiredChildTerminated);
Receive<ScriptedAlarmHostActor.ScriptedAlarmHostRestarted>(_ => HandleScriptedAlarmHostRestarted());
Receive<DriverInstanceActor.AttributeAlarmPublished>(ForwardNativeAlarm);
Receive<DriverInstanceActor.ConnectivityChanged>(OnDriverConnectivityChanged);
Receive<CommsRecoveryAudit>(HandleCommsRecoveryAudit);
@@ -2541,6 +2548,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
_log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)",
_localNode, m.Items.Count, m.DriverInstanceId));
Receive<Terminated>(OnRetiredChildTerminated);
Receive<ScriptedAlarmHostActor.ScriptedAlarmHostRestarted>(_ => HandleScriptedAlarmHostRestarted());
Receive<DriverInstanceActor.AttributeAlarmPublished>(m =>
_log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale",
_localNode, m.Args.ConditionId, m.DriverInstanceId));
@@ -3692,6 +3700,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// (re)loads its engine + re-registers mux interest for the union of dependency refs. Covers
// both the fresh-apply and bootstrap-restore paths (both call this method); the Stale-recovery
// path deliberately does not, matching driver + VirtualTag recovery.
_lastScriptedAlarmPlans = composition.EquipmentScriptedAlarms;
_scriptedAlarmHost?.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(composition.EquipmentScriptedAlarms));
if (composition.EquipmentScriptedAlarms.Count > 0)
{
@@ -3906,6 +3915,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Context.Watch(child);
}
/// <summary>
/// The scripted-alarm host was restarted by supervision and has an empty engine: re-send the last plans so
/// its inputs and operator commands work again without waiting for the next deploy. Only the CURRENT host's
/// signal is honoured.
/// </summary>
private void HandleScriptedAlarmHostRestarted()
{
if (_scriptedAlarmHost is null || !Sender.Equals(_scriptedAlarmHost)) return;
if (_lastScriptedAlarmPlans is null)
{
_log.Debug("DriverHost {Node}: ScriptedAlarm host restarted before any apply; nothing to re-send", _localNode);
return;
}
_log.Warning("DriverHost {Node}: ScriptedAlarm host was restarted by supervision; re-applying {Count} scripted alarm(s)",
_localNode, _lastScriptedAlarmPlans.Count);
_scriptedAlarmHost.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(_lastScriptedAlarmPlans));
}
/// <summary>A retired child has terminated; nothing more can arrive from it.</summary>
/// <param name="msg">The termination notice.</param>
private void OnRetiredChildTerminated(Terminated msg) => _retiredChildren.Remove(msg.ActorRef);