fix(runtime): keep native-alarm feed across data re-subscribes; reconcile retries alarms (#662)

A data re-subscribe (every deploy's bulk SetDesiredSubscriptions push to a
kept driver) dropped the prior data subscription through the full
DetachSubscription, which also unhooked OnAlarmEvent and forgot the alarm
handle. On any deploy that did not respawn the driver, the alarm
subscription kept logging "established" while every event reached nobody
until a reconnect or respawn. The data path now uses DetachDataSubscription;
only a lost connection and PostStop detach both.

The #488 health-poll reconcile only looked at data, so an alarm subscribe
that failed on (re)connect gated the feed for the life of the connection.
ReconcileAlarmSubscription now re-attaches a missing handler and
re-subscribes a stale/missing alarm handle every tick.

OpcUaClient removes a half-built alarm subscription from its session when
SubscribeAlarmsAsync throws, so the new retries cannot accumulate.

Tests: actor-level data-resubscribe + reconcile-retry tests and an
end-to-end NativeAlarmFeedSurvivesDeployTests through DriverHostActor; all
three fail against the unfixed actor.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-09 07:30:25 -04:00
parent ea220363f3
commit 5067509dd4
7 changed files with 295 additions and 21 deletions
@@ -690,6 +690,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// </remarks>
private void ReconcileSubscription()
{
ReconcileAlarmSubscription();
if (_driver is not ISubscribable) return;
if (_desiredRefs.Count == 0 || _subscriptionHandle is not null) return;
@@ -699,6 +700,34 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Self.Tell(new Subscribe(_desiredRefs, _desiredInterval));
}
/// <summary>
/// #662 — the native-alarm half of <see cref="ReconcileSubscription"/>. The data reconcile above never
/// looked at alarms, so an alarm subscribe that failed on (re)connect — e.g. a session not yet ready
/// after an outage — left the feed gated for the life of the connection, with the driver Healthy and
/// its data streaming. Re-asserts both halves the feed needs: the <c>OnAlarmEvent</c> handler
/// (idempotent attach) and a subscription matching the desired set (<see cref="SubscribeDesiredAlarms"/>
/// is a no-op while the live handle already matches). Gated on <see cref="IAlarmSource"/> + a
/// non-empty desired set, so it costs nothing on a driver with no alarm tags; a persistently failing
/// subscribe retries once per tick, the same cadence as the data reconcile.
/// </summary>
private void ReconcileAlarmSubscription()
{
if (_driver is not IAlarmSource || _desiredAlarmRefs.Count == 0) return;
if (_alarmEventHandler is null)
{
_log.Info("DriverInstance {Id}: connected with desired alarm refs but no native-alarm handler — re-attaching (periodic reconcile)",
_driverInstanceId);
AttachAlarmSource();
}
if (AlarmSubscriptionIsStale())
{
_log.Info(
"DriverInstance {Id}: connected with {Count} desired alarm ref(s) but no matching native-alarm subscription — re-subscribing (periodic reconcile)",
_driverInstanceId, _desiredAlarmRefs.Count);
Self.Tell(new SubscribeAlarms());
}
}
private void Reconnecting()
{
// #477 — announce comms loss to the host so it annotates this driver's native alarm conditions Bad
@@ -1002,7 +1031,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
}
catch (Exception ex)
{
DetachSubscription();
DetachDataSubscription();
_log.Warning(ex, "DriverInstance {Id}: subscribe failed", _driverInstanceId);
replyTo.Tell(new SubscriptionFailed(ex.Message));
}
@@ -1012,7 +1041,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
{
if (_driver is not ISubscribable subscribable || _subscriptionHandle is null)
{
DetachSubscription();
DetachDataSubscription();
return;
}
try
@@ -1031,17 +1060,32 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
}
finally
{
DetachSubscription();
DetachDataSubscription();
}
}
/// <summary>Tear down the data-change (per-tag AND batch) + native-alarm event handlers + null the handle.
/// Called from the Unsubscribe path, on PostStop, and on Connected → Reconnecting transitions so a stale
/// handler doesn't push data-change / alarm events to an actor that has lost its driver connection.
/// <summary>Tear down the data-change (per-tag AND batch) + native-alarm event handlers + null both handles.
/// Called on PostStop and on Connected → Reconnecting transitions so a stale handler doesn't push
/// data-change / alarm events to an actor that has lost its driver connection.
/// <para>Not for the data Unsubscribe / re-subscribe path — that uses <see cref="DetachDataSubscription"/>
/// (#662).</para></summary>
private void DetachSubscription()
{
DetachDataSubscription();
DetachAlarmSource();
}
/// <summary>Tear down the data-change (per-tag AND batch) handlers + null the data handle, leaving the
/// native-alarm handler and handle alone.
/// <para>#662: the data Unsubscribe / re-subscribe path used to run the full <see cref="DetachSubscription"/>,
/// so every deploy that re-subscribed a Connected driver's DATA also unhooked <c>OnAlarmEvent</c> and
/// forgot the alarm handle — the alarm subscription still logged "established" but its events reached
/// nobody until a reconnect or respawn. The two subscriptions are independent at the driver; only a lost
/// connection ends both.</para>
/// <para>Both data-change fields are cleared unconditionally even though only one is ever attached — the
/// IDriver object outlives the actor when the host respawns a child around it, so a handler left on the
/// driver's event would keep Telling a dead actor.</para></summary>
private void DetachSubscription()
private void DetachDataSubscription()
{
if (_driver is ISubscribable subscribable && _dataChangeHandler is not null)
{
@@ -1054,7 +1098,6 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
}
_batchChangeHandler = null;
_subscriptionHandle = null;
DetachAlarmSource();
}
/// <summary>Subscribe the driver's native alarm event (if it is an <see cref="IAlarmSource"/>),
@@ -1229,7 +1272,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// <summary>Calls the driver's <see cref="IAlarmSource.SubscribeAlarmsAsync"/> (bounded) to register the
/// alarm subscription that un-gates its native feed, caching the returned handle. Re-checks the guard
/// (the desired set may have cleared, or another SubscribeAlarms may have already established a handle,
/// while this was queued). Failures are logged and retried on the next Connected entry — the feed simply
/// while this was queued). Failures are logged and retried on the next Connected entry or health-poll reconcile tick (#662) — the feed
/// stays gated until then.</summary>
private async Task HandleSubscribeAlarmsAsync(SubscribeAlarms _)
{
@@ -1276,7 +1319,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
}
catch (Exception ex)
{
_log.Warning(ex, "DriverInstance {Id}: native-alarm subscription failed — feed stays gated until reconnect",
_log.Warning(ex, "DriverInstance {Id}: native-alarm subscription failed — feed stays gated until the next reconcile tick retries it",
_driverInstanceId);
}
}