fix(runtime): the passive half of a LocalDb pair stands its alarm drain down for its mesh partner (review R6 MINOR-4)

The drain gate's 'is my queue-sharing peer Primary?' used only the host of
LocalDb:Replication:PeerAddress, which the passive half (ApiKey +
SyncListenPort) does not set - so as Secondary it kept draining beside its
Primary partner and every alarm event reached the historian twice.
DriverHostActor takes a sharesAlarmQueueWithLocal predicate used when no
peer host is known; WithOtOpcUaRuntimeActors builds it
(BuildAlarmQueuePeerPredicate) for a SharesRowsWithPeer node with no
PeerAddress as 'a member of my own cluster-{ClusterId} mesh other than
me'. A node sharing no rows, or with no mesh role, gets none and keeps
draining. AlarmHistorian.md / Redundancy.md / pair-replication runbook
updated.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 08:43:06 -04:00
parent e93d115cbe
commit 3be8a1217e
7 changed files with 203 additions and 11 deletions
@@ -472,6 +472,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// </summary>
private readonly string? _replicationPeerHost;
/// <summary>
/// Review R6 MINOR-4: identifies the alarm-queue-sharing peer when <see cref="_replicationPeerHost"/> is
/// unknown — the passive (listening) half of a pair, which replicates the same rows but dials nobody. Before
/// this the passive half had no peer to defer to, so as Secondary it kept draining alongside its Primary
/// partner and every alarm event reached the historian twice. Null ⇒ no peer known ⇒ never stand down.
/// </summary>
private readonly Func<CommonsNodeId, bool>? _sharesAlarmQueueWithLocal;
/// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
/// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
private bool _warnedSnapshotMissingLocalNode;
@@ -617,7 +625,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null) =>
TimeSpan? commsRecoveryAuditDelay = null,
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -629,7 +638,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay));
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -691,6 +701,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; a test shortens it for the same reason.</param>
/// <param name="sharesAlarmQueueWithLocal">Review R6 MINOR-4: tests whether a node named in a redundancy
/// snapshot holds this node's replicated alarm queue, for a node that knows no
/// <paramref name="replicationPeerHost"/> — the PASSIVE half of a LocalDb pair (ApiKey + SyncListenPort, no
/// PeerAddress), whose partner is its per-cluster mesh partner. Ignored when
/// <paramref name="replicationPeerHost"/> is set. Null ⇒ no such peer is known and this node never stands its
/// drain down.</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
@@ -720,8 +736,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null)
TimeSpan? commsRecoveryAuditDelay = null,
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null)
{
_sharesAlarmQueueWithLocal = sharesAlarmQueueWithLocal;
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
_reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
_commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
@@ -2022,6 +2040,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
}
}
/// <summary>Whether <paramref name="nodeId"/> holds this node's replicated alarm queue: the PeerAddress host when
/// this node dials, else the passive-half predicate (review R6 MINOR-4). This node itself never counts.</summary>
private bool SharesAlarmQueue(CommonsNodeId nodeId) =>
nodeId != _localNode
&& (_replicationPeerHost is not null
? HostOf(nodeId) == _replicationPeerHost
: _sharesAlarmQueueWithLocal?.Invoke(nodeId) == true);
/// <summary>Host part of a <c>host:port</c> node id.</summary>
private static string HostOf(NodeId nodeId)
{
@@ -2071,10 +2097,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// Defer ONLY to the node that holds a copy of our rows. The redundancy election is
// cluster-wide and the queue is pair-local, so "somebody is Primary" is not a licence to stop
// draining — that Primary is usually in another pair entirely, and cannot deliver our events.
var peerIsPrimary =
_replicationPeerHost is not null
&& msg.Nodes.Any(n => n.Role == RedundancyRole.Primary
&& HostOf(n.NodeId) == _replicationPeerHost);
//
// The dialling half knows its peer by PeerAddress host. The passive half (review R6 MINOR-4) knows it
// only as its mesh partner — the predicate — and must be gated too, or both halves drain the same rows.
var peerIsPrimary = msg.Nodes.Any(n => n.Role == RedundancyRole.Primary && SharesAlarmQueue(n.NodeId));
_redundancyRoleView?.Publish(
PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary));
@@ -279,6 +279,33 @@ public static class ServiceCollectionExtensions
return services;
}
/// <summary>
/// Review R6 MINOR-4 — "does this snapshot node hold my replicated alarm queue?" for a node that does not
/// dial its peer. Both halves of a LocalDb pair hold the same rows, but only the dialling half knows its
/// partner (<c>LocalDb:Replication:PeerAddress</c>); the passive half (<c>ApiKey</c> + <c>SyncListenPort</c>)
/// knew nobody, so as Secondary it never stood its drain down and both nodes delivered every alarm event.
/// Its partner is its per-cluster mesh partner, which a cluster-scoped role identifies.
/// </summary>
/// <param name="configuration">The host configuration (the LocalDb replication keys).</param>
/// <param name="roleInfo">This node's cluster role info.</param>
/// <param name="replicationPeerHost">The dialled peer's host, when this node dials one.</param>
/// <returns>
/// <c>null</c> when the node dials its peer (the host match covers it), when it shares its queue with no
/// peer (<see cref="LocalDbReplicationPolicy.SharesRowsWithPeer"/> false — it must never stand down), or
/// when it carries no cluster-scoped role (a legacy node cannot prove which node is its partner, and
/// standing down for the wrong Primary would strand its rows); otherwise "is a member of my own mesh other
/// than me".
/// </returns>
internal static Func<Commons.Types.NodeId, bool>? BuildAlarmQueuePeerPredicate(
IConfiguration? configuration, IClusterRoleInfo? roleInfo, string? replicationPeerHost)
{
if (replicationPeerHost is not null || configuration is null || roleInfo is null) return null;
if (!LocalDbReplicationPolicy.SharesRowsWithPeer(configuration)) return null;
if (roleInfo.ClusterRole is not { } meshRole) return null;
var local = roleInfo.LocalNode;
return nodeId => nodeId != local && roleInfo.MembersWithRole(meshRole).Contains(nodeId);
}
/// <summary>
/// Spawns the per-node driver-role actors on the host's <see cref="ActorSystem"/>:
/// <see cref="DriverHostActor"/> (one per node), <see cref="DbHealthProbeActor"/>
@@ -361,7 +388,8 @@ public static class ServiceCollectionExtensions
var redundancyRoleView = resolver.GetService<IRedundancyRoleView>();
// Host of this node's LocalDb replication partner: the ONLY node that holds a copy of this
// node's alarm queue, and so the only node it may stand down in favour of. Null when this
// node dials nobody, which correctly means "never stand down".
// node dials nobody — unpaired (never stand down), or the PASSIVE half of a pair, which is
// told its partner through BuildAlarmQueuePeerPredicate instead (review R6 MINOR-4).
var replicationPeerHost =
Uri.TryCreate(
resolver.GetService<IConfiguration>()?["LocalDb:Replication:PeerAddress"],
@@ -592,7 +620,11 @@ public static class ServiceCollectionExtensions
// Gitea #537: where this actor publishes the continuous-historization CAPTURE
// verdict, and how it decides which peer is worth standing down for.
historizationRoleView: resolver.GetService<IHistorizationRoleView>(),
sharesHistorizationScopeWithLocal: historizationScopePeer),
sharesHistorizationScopeWithLocal: historizationScopePeer,
// Review R6 MINOR-4: the passive half of a LocalDb pair has no PeerAddress, so it is told
// its queue-sharing peer as its mesh partner instead — otherwise it never stands down.
sharesAlarmQueueWithLocal: BuildAlarmQueuePeerPredicate(
resolver.GetService<IConfiguration>(), roleInfo, replicationPeerHost)),
DriverHostActorName);
registry.Register<DriverHostActorKey>(driverHost);