fix(runtime): the passive half of a LocalDb pair stands its alarm drain down for its mesh partner (review R6 MINOR-4)
The drain gate's 'is my queue-sharing peer Primary?' used only the host of
LocalDb:Replication:PeerAddress, which the passive half (ApiKey +
SyncListenPort) does not set - so as Secondary it kept draining beside its
Primary partner and every alarm event reached the historian twice.
DriverHostActor takes a sharesAlarmQueueWithLocal predicate used when no
peer host is known; WithOtOpcUaRuntimeActors builds it
(BuildAlarmQueuePeerPredicate) for a SharesRowsWithPeer node with no
PeerAddress as 'a member of my own cluster-{ClusterId} mesh other than
me'. A node sharing no rows, or with no mesh role, gets none and keeps
draining. AlarmHistorian.md / Redundancy.md / pair-replication runbook
updated.
Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
@@ -472,6 +472,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
private readonly string? _replicationPeerHost;
|
||||
|
||||
/// <summary>
|
||||
/// Review R6 MINOR-4: identifies the alarm-queue-sharing peer when <see cref="_replicationPeerHost"/> is
|
||||
/// unknown — the passive (listening) half of a pair, which replicates the same rows but dials nobody. Before
|
||||
/// this the passive half had no peer to defer to, so as Secondary it kept draining alongside its Primary
|
||||
/// partner and every alarm event reached the historian twice. Null ⇒ no peer known ⇒ never stand down.
|
||||
/// </summary>
|
||||
private readonly Func<CommonsNodeId, bool>? _sharesAlarmQueueWithLocal;
|
||||
|
||||
/// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
|
||||
/// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
|
||||
private bool _warnedSnapshotMissingLocalNode;
|
||||
@@ -617,7 +625,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
|
||||
TimeSpan? applyWedgeTimeout = null,
|
||||
TimeSpan? reprovisionBackoff = null,
|
||||
TimeSpan? commsRecoveryAuditDelay = null) =>
|
||||
TimeSpan? commsRecoveryAuditDelay = null,
|
||||
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -629,7 +638,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
|
||||
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay));
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
|
||||
sharesAlarmQueueWithLocal));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -691,6 +701,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <param name="commsRecoveryAuditDelay">Archreview 03/S15: how long after a driver reconnects the
|
||||
/// recovery audit waits before reporting refs still latched <c>BadNoCommunication</c>. Defaults to
|
||||
/// <see cref="DefaultCommsRecoveryAuditDelay"/>; a test shortens it for the same reason.</param>
|
||||
/// <param name="sharesAlarmQueueWithLocal">Review R6 MINOR-4: tests whether a node named in a redundancy
|
||||
/// snapshot holds this node's replicated alarm queue, for a node that knows no
|
||||
/// <paramref name="replicationPeerHost"/> — the PASSIVE half of a LocalDb pair (ApiKey + SyncListenPort, no
|
||||
/// PeerAddress), whose partner is its per-cluster mesh partner. Ignored when
|
||||
/// <paramref name="replicationPeerHost"/> is set. Null ⇒ no such peer is known and this node never stands its
|
||||
/// drain down.</param>
|
||||
public DriverHostActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
|
||||
CommonsNodeId localNode,
|
||||
@@ -720,8 +736,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
|
||||
TimeSpan? applyWedgeTimeout = null,
|
||||
TimeSpan? reprovisionBackoff = null,
|
||||
TimeSpan? commsRecoveryAuditDelay = null)
|
||||
TimeSpan? commsRecoveryAuditDelay = null,
|
||||
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null)
|
||||
{
|
||||
_sharesAlarmQueueWithLocal = sharesAlarmQueueWithLocal;
|
||||
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
|
||||
_reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
|
||||
_commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
|
||||
@@ -2022,6 +2040,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="nodeId"/> holds this node's replicated alarm queue: the PeerAddress host when
|
||||
/// this node dials, else the passive-half predicate (review R6 MINOR-4). This node itself never counts.</summary>
|
||||
private bool SharesAlarmQueue(CommonsNodeId nodeId) =>
|
||||
nodeId != _localNode
|
||||
&& (_replicationPeerHost is not null
|
||||
? HostOf(nodeId) == _replicationPeerHost
|
||||
: _sharesAlarmQueueWithLocal?.Invoke(nodeId) == true);
|
||||
|
||||
/// <summary>Host part of a <c>host:port</c> node id.</summary>
|
||||
private static string HostOf(NodeId nodeId)
|
||||
{
|
||||
@@ -2071,10 +2097,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// Defer ONLY to the node that holds a copy of our rows. The redundancy election is
|
||||
// cluster-wide and the queue is pair-local, so "somebody is Primary" is not a licence to stop
|
||||
// draining — that Primary is usually in another pair entirely, and cannot deliver our events.
|
||||
var peerIsPrimary =
|
||||
_replicationPeerHost is not null
|
||||
&& msg.Nodes.Any(n => n.Role == RedundancyRole.Primary
|
||||
&& HostOf(n.NodeId) == _replicationPeerHost);
|
||||
//
|
||||
// The dialling half knows its peer by PeerAddress host. The passive half (review R6 MINOR-4) knows it
|
||||
// only as its mesh partner — the predicate — and must be gated too, or both halves drain the same rows.
|
||||
var peerIsPrimary = msg.Nodes.Any(n => n.Role == RedundancyRole.Primary && SharesAlarmQueue(n.NodeId));
|
||||
|
||||
_redundancyRoleView?.Publish(
|
||||
PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary));
|
||||
|
||||
@@ -279,6 +279,33 @@ public static class ServiceCollectionExtensions
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review R6 MINOR-4 — "does this snapshot node hold my replicated alarm queue?" for a node that does not
|
||||
/// dial its peer. Both halves of a LocalDb pair hold the same rows, but only the dialling half knows its
|
||||
/// partner (<c>LocalDb:Replication:PeerAddress</c>); the passive half (<c>ApiKey</c> + <c>SyncListenPort</c>)
|
||||
/// knew nobody, so as Secondary it never stood its drain down and both nodes delivered every alarm event.
|
||||
/// Its partner is its per-cluster mesh partner, which a cluster-scoped role identifies.
|
||||
/// </summary>
|
||||
/// <param name="configuration">The host configuration (the LocalDb replication keys).</param>
|
||||
/// <param name="roleInfo">This node's cluster role info.</param>
|
||||
/// <param name="replicationPeerHost">The dialled peer's host, when this node dials one.</param>
|
||||
/// <returns>
|
||||
/// <c>null</c> when the node dials its peer (the host match covers it), when it shares its queue with no
|
||||
/// peer (<see cref="LocalDbReplicationPolicy.SharesRowsWithPeer"/> false — it must never stand down), or
|
||||
/// when it carries no cluster-scoped role (a legacy node cannot prove which node is its partner, and
|
||||
/// standing down for the wrong Primary would strand its rows); otherwise "is a member of my own mesh other
|
||||
/// than me".
|
||||
/// </returns>
|
||||
internal static Func<Commons.Types.NodeId, bool>? BuildAlarmQueuePeerPredicate(
|
||||
IConfiguration? configuration, IClusterRoleInfo? roleInfo, string? replicationPeerHost)
|
||||
{
|
||||
if (replicationPeerHost is not null || configuration is null || roleInfo is null) return null;
|
||||
if (!LocalDbReplicationPolicy.SharesRowsWithPeer(configuration)) return null;
|
||||
if (roleInfo.ClusterRole is not { } meshRole) return null;
|
||||
var local = roleInfo.LocalNode;
|
||||
return nodeId => nodeId != local && roleInfo.MembersWithRole(meshRole).Contains(nodeId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Spawns the per-node driver-role actors on the host's <see cref="ActorSystem"/>:
|
||||
/// <see cref="DriverHostActor"/> (one per node), <see cref="DbHealthProbeActor"/>
|
||||
@@ -361,7 +388,8 @@ public static class ServiceCollectionExtensions
|
||||
var redundancyRoleView = resolver.GetService<IRedundancyRoleView>();
|
||||
// Host of this node's LocalDb replication partner: the ONLY node that holds a copy of this
|
||||
// node's alarm queue, and so the only node it may stand down in favour of. Null when this
|
||||
// node dials nobody, which correctly means "never stand down".
|
||||
// node dials nobody — unpaired (never stand down), or the PASSIVE half of a pair, which is
|
||||
// told its partner through BuildAlarmQueuePeerPredicate instead (review R6 MINOR-4).
|
||||
var replicationPeerHost =
|
||||
Uri.TryCreate(
|
||||
resolver.GetService<IConfiguration>()?["LocalDb:Replication:PeerAddress"],
|
||||
@@ -592,7 +620,11 @@ public static class ServiceCollectionExtensions
|
||||
// Gitea #537: where this actor publishes the continuous-historization CAPTURE
|
||||
// verdict, and how it decides which peer is worth standing down for.
|
||||
historizationRoleView: resolver.GetService<IHistorizationRoleView>(),
|
||||
sharesHistorizationScopeWithLocal: historizationScopePeer),
|
||||
sharesHistorizationScopeWithLocal: historizationScopePeer,
|
||||
// Review R6 MINOR-4: the passive half of a LocalDb pair has no PeerAddress, so it is told
|
||||
// its queue-sharing peer as its mesh partner instead — otherwise it never stands down.
|
||||
sharesAlarmQueueWithLocal: BuildAlarmQueuePeerPredicate(
|
||||
resolver.GetService<IConfiguration>(), roleInfo, replicationPeerHost)),
|
||||
DriverHostActorName);
|
||||
registry.Register<DriverHostActorKey>(driverHost);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user