dohertj2/lmxopcua
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(r2-04): primary gates default-DENY on multi-driver clusters while role unknown (03/S4)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-13 10:59:43 -04:00
parent f2e899aaad
commit 72b1600245
2 changed files with 120 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
archreview/plans/R2-04-failure-visibility-plan.md.tasks.json
+1 -1
View File
@@ -11,7 +11,7 @@
{ "id": "T7", "subject": "Galaxy counters galaxy.writes.advise_failed + galaxy.writes.unconfirmed (empty-statuses Good metered)", "status": "completed", "blockedBy": ["T6"] },
{ "id": "T8", "subject": "Galaxy skip-gated live smoke: normal advised write still returns Good post-change", "status": "completed", "blockedBy": ["T6"] },
{ "id": "T9", "subject": "PrimaryGatePolicy pure class + truth-table tests + otopcua.redundancy.primary_gate_denied counter (03/S4)", "status": "completed", "blockedBy": [] },
{ "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "pending", "blockedBy": ["T9"] },
{ "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "completed", "blockedBy": ["T9"] },
{ "id": "T11", "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", "status": "pending", "blockedBy": ["T10"] },
{ "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "pending", "blockedBy": ["T9"] },
{ "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "pending", "blockedBy": ["T11"] },
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+119 -25
View File
@@ -192,14 +192,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// <summary>
/// Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
/// snapshot (null = unknown until the first snapshot arrives, or no local node match). The inbound
/// write gate in <see cref="HandleRouteNodeWrite"/> reuses this signal — the SAME one the
/// scripted-alarm emit gate uses (<c>ScriptedAlarmHostActor._localRole</c>): only the Primary
/// services writes, default-allow while unknown so single-node deploys + the boot window never
/// reject (a node is the sole Primary until told otherwise).
/// snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary
/// data-plane gates (<see cref="HandleRouteNodeWrite"/>, <see cref="HandleRouteNativeAlarmAck"/>,
/// <see cref="ForwardNativeAlarm"/>) resolve this through <see cref="PrimaryGatePolicy"/>: a KNOWN
/// role wins outright; an UNKNOWN role is resolved by cluster membership — a single-driver cluster
/// stays default-ALLOW (boot-window / single-node), a multi-driver cluster default-DENIES until a
/// snapshot proves this node is Primary (archreview 03/S4 — closes the dual-primary boot window).
/// </summary>
private RedundancyRole? _localRole;
/// <summary>Test seam (archreview 03/S4): overrides the count of Up <c>driver</c>-role cluster members
/// the Primary gate reads while the role is unknown. Null ⇒ read the live cluster state (0 on a
/// non-cluster ActorRefProvider). See <see cref="DriverMemberCount"/>.</summary>
private readonly Func<int>? _driverMemberCountProvider;
/// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
/// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
private bool _warnedSnapshotMissingLocalNode;
/// <summary>Cached cluster DistributedPubSub mediator, resolved once in <see cref="PreStart"/> (on the
/// actor thread) and reused for the Primary-gated native-alarm <c>alerts</c> fan-out in
/// <see cref="ForwardNativeAlarm"/> instead of re-resolving it per-publish. Mirrors
@@ -308,11 +318,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null) =>
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null) =>
Akka.Actor.Props.Create(() => new DriverHostActor(
dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory));
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -338,6 +349,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// <param name="invokerFactory">Phase 6.1 resilience-invoker factory used to attach an
/// <see cref="IDriverCapabilityInvoker"/> to each spawned driver; defaults to
/// <see cref="NullDriverCapabilityInvokerFactory"/> (pass-through) when null.</param>
/// <param name="driverMemberCountProvider">Test seam (archreview 03/S4): overrides the count of Up
/// <c>driver</c>-role cluster members the Primary gate reads while the role is unknown. When null the
/// default reads <c>Cluster.Get(Context.System).State.Members</c> (0 on a non-cluster ActorRefProvider).</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
CommonsNodeId localNode,
@@ -353,13 +367,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
IDriverCapabilityInvokerFactory? invokerFactory = null)
IDriverCapabilityInvokerFactory? invokerFactory = null,
Func<int>? driverMemberCountProvider = null)
{
_dbFactory = dbFactory;
_localNode = localNode;
_coordinatorOverride = coordinator;
_driverFactory = driverFactory ?? NullDriverFactory.Instance;
_invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
_driverMemberCountProvider = driverMemberCountProvider;
_localRoles = localRoles ?? new HashSet<string>(StringComparer.Ordinal);
_dependencyMux = dependencyMux;
_opcUaPublishActor = opcUaPublishActor;
@@ -964,17 +980,29 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
return;
}
// PRIMARY GATE (archreview 03/S4) — decided ONCE for this transition: the OPC UA condition write
// below stays UNGATED (a Secondary keeps its address space warm for failover), but only the Primary
// publishes the single cluster-wide alerts copy. Denied on a Secondary/Detached, OR on a multi-driver
// cluster while the role is unknown (default-deny closes the dual-primary boot window that would
// otherwise historize duplicate AVEVA alarm rows). Metered + Debug-logged once when denied.
var serviceAlertsAsPrimary = ShouldServiceAsPrimary();
if (!serviceAlertsAsPrimary)
{
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair<string, object?>("site", "alarm-emit"),
new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
_log.Debug("DriverHost {Node}: native-alarm alerts publish suppressed for ({Driver},{Ref}) — not primary",
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
}
foreach (var nodeId in nodeIds)
{
var snapshot = _nativeAlarmProjector.Project(nodeId, msg.Args);
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate(
nodeId, snapshot, msg.Args.SourceTimestampUtc));
// Warm-standby dedup: the OPC UA condition write above is UNGATED (the secondary keeps its
// address space warm), but only the Primary publishes the cluster-wide alerts transition.
// Default-emit until told we are Secondary/Detached so single-node deploys + the boot window
// never drop transitions — the SAME signal HandleRouteNodeWrite gates writes on.
if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) continue;
// Only the Primary publishes the cluster-wide alerts transition (see the gate above).
if (!serviceAlertsAsPrimary) continue;
var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m)
? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null);
@@ -1034,11 +1062,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// </summary>
private void HandleRouteNodeWrite(RouteNodeWrite msg)
{
// PRIMARY GATE FIRST — only the Primary services operator writes (same signal as the alarm-emit
// gate; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked).
if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
// PRIMARY GATE FIRST (archreview 03/S4) — only the Primary services operator writes. A KNOWN
// Secondary/Detached is denied "not primary" (unchanged); an UNKNOWN role denies ONLY on a
// multi-driver cluster ("not primary (role unknown)"), while a single-driver/boot-window cluster is
// still serviced. The rejection rides the established optimistic-write self-correction surface (the
// node reverts to its prior value with a Bad blip + a Part 8 audit event) — writes are NOT queued.
if (!ShouldServiceAsPrimary())
{
Sender.Tell(new NodeWriteResult(false, "not primary"));
var reason = _localRole is null ? "not primary (role unknown)" : "not primary";
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair<string, object?>("site", "write"),
new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
Sender.Tell(new NodeWriteResult(false, reason));
return;
}
@@ -1090,12 +1125,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// </summary>
private void HandleRouteNativeAlarmAck(RouteNativeAlarmAck msg)
{
// PRIMARY GATE FIRST — only the Primary services operator acks (same signal as the inbound-write +
// alarm-emit gates; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked).
if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
// PRIMARY GATE FIRST (archreview 03/S4) — only the Primary pushes the ack upstream. Same policy as
// the inbound-write gate. A KNOWN Secondary/Detached is dropped at Debug (unchanged); an UNKNOWN role
// on a multi-driver cluster is dropped at WARNING (an operator ack silently not reaching the upstream
// alarm system is a swallowed failure worth surfacing).
if (!ShouldServiceAsPrimary())
{
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
_localNode, msg.ConditionNodeId);
OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
new KeyValuePair<string, object?>("site", "ack"),
new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
if (_localRole is null)
{
_log.Warning("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary (role unknown on multi-driver cluster)",
_localNode, msg.ConditionNodeId);
}
else
{
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
_localNode, msg.ConditionNodeId);
}
return;
}
@@ -1119,9 +1167,43 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(target.FullName, msg.Comment, msg.OperatorUser));
}
/// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so
/// <see cref="HandleRouteNodeWrite"/> can gate inbound writes to the Primary. A snapshot that doesn't
/// mention this node leaves the cached role unchanged ⇒ default-allow. Mirrors
/// <summary>Whether this node should service a Primary-only data-plane operation right now — the single
/// gate decision (archreview 03/S4), delegating to <see cref="PrimaryGatePolicy"/> with the live driver
/// member count.</summary>
/// <returns><c>true</c> to service as Primary; <c>false</c> to deny.</returns>
private bool ShouldServiceAsPrimary() =>
PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount());
/// <summary>Count of Up cluster members carrying the <c>driver</c> role. Uses the injected test seam when
/// present; otherwise reads the live cluster state guarded by try/catch so a non-cluster
/// ActorRefProvider (legacy/test harnesses) yields 0 ⇒ the single-node default-allow posture.</summary>
/// <returns>The number of Up driver-role members, or 0 when the cluster extension is unavailable.</returns>
private int DriverMemberCount()
{
if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
try
{
return Akka.Cluster.Cluster.Get(Context.System).State.Members
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
}
catch (Exception)
{
return 0; // non-cluster ActorRefProvider (legacy harnesses) ⇒ single-node posture
}
}
/// <summary>The Primary-gate deny reason tag for the denial meter (<c>secondary|detached|role-unknown</c>).</summary>
private string PrimaryGateDenyReason() => _localRole switch
{
RedundancyRole.Secondary => "secondary",
RedundancyRole.Detached => "detached",
_ => "role-unknown",
};
/// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so the
/// Primary data-plane gates can resolve their decision. A snapshot that doesn't mention this node leaves
/// the cached role unchanged; when this node has a real driver peer (count &gt; 1) that omission is the
/// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
{
@@ -1129,6 +1211,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
if (local is not null)
{
_localRole = local.Role;
return;
}
// S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
// Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
// identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
{
_warnedSnapshotMissingLocalNode = true;
_log.Warning(
"DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
_localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
}
}