diff --git a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json
index 703f3baf..2c5d4a97 100644
--- a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json
+++ b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json
@@ -11,7 +11,7 @@
{ "id": "T7", "subject": "Galaxy counters galaxy.writes.advise_failed + galaxy.writes.unconfirmed (empty-statuses Good metered)", "status": "completed", "blockedBy": ["T6"] },
{ "id": "T8", "subject": "Galaxy skip-gated live smoke: normal advised write still returns Good post-change", "status": "completed", "blockedBy": ["T6"] },
{ "id": "T9", "subject": "PrimaryGatePolicy pure class + truth-table tests + otopcua.redundancy.primary_gate_denied counter (03/S4)", "status": "completed", "blockedBy": [] },
- { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "pending", "blockedBy": ["T9"] },
+ { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "completed", "blockedBy": ["T9"] },
{ "id": "T11", "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", "status": "pending", "blockedBy": ["T10"] },
{ "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "pending", "blockedBy": ["T9"] },
{ "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "pending", "blockedBy": ["T11"] },
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
index 42ffbe84..97e3eb68 100644
--- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
@@ -192,14 +192,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
///
/// Cached local from the latest
- /// snapshot (null = unknown until the first snapshot arrives, or no local node match). The inbound
- /// write gate in reuses this signal — the SAME one the
- /// scripted-alarm emit gate uses (ScriptedAlarmHostActor._localRole): only the Primary
- /// services writes, default-allow while unknown so single-node deploys + the boot window never
- /// reject (a node is the sole Primary until told otherwise).
+ /// snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary
+ /// data-plane gates (, ,
+ /// ) resolve this through : a KNOWN
+ /// role wins outright; an UNKNOWN role is resolved by cluster membership — a single-driver cluster
+ /// stays default-ALLOW (boot-window / single-node), a multi-driver cluster default-DENIES until a
+ /// snapshot proves this node is Primary (archreview 03/S4 — closes the dual-primary boot window).
///
private RedundancyRole? _localRole;
+ /// Test seam (archreview 03/S4): overrides the count of Up driver-role cluster members
+ /// the Primary gate reads while the role is unknown. Null ⇒ read the live cluster state (0 on a
+ /// non-cluster ActorRefProvider). See .
+ private readonly Func? _driverMemberCountProvider;
+
+ /// Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
+ /// identity mismatch (03/S5) would otherwise log on every snapshot.
+ private bool _warnedSnapshotMissingLocalNode;
+
/// Cached cluster DistributedPubSub mediator, resolved once in (on the
/// actor thread) and reused for the Primary-gated native-alarm alerts fan-out in
/// instead of re-resolving it per-publish. Mirrors
@@ -308,11 +318,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
- IDriverCapabilityInvokerFactory? invokerFactory = null) =>
+ IDriverCapabilityInvokerFactory? invokerFactory = null,
+ Func? driverMemberCountProvider = null) =>
Akka.Actor.Props.Create(() => new DriverHostActor(
dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
- loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory));
+ loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider));
/// Initializes a new DriverHostActor with the specified dependencies.
/// Database context factory for configuration database access.
@@ -338,6 +349,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// Phase 6.1 resilience-invoker factory used to attach an
/// to each spawned driver; defaults to
/// (pass-through) when null.
+ /// Test seam (archreview 03/S4): overrides the count of Up
+ /// driver-role cluster members the Primary gate reads while the role is unknown. When null the
+ /// default reads Cluster.Get(Context.System).State.Members (0 on a non-cluster ActorRefProvider).
public DriverHostActor(
IDbContextFactory dbFactory,
CommonsNodeId localNode,
@@ -353,13 +367,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
ILoggerFactory? loggerFactory = null,
ScriptRootLogger? scriptRootLogger = null,
IActorRef? scriptedAlarmHostOverride = null,
- IDriverCapabilityInvokerFactory? invokerFactory = null)
+ IDriverCapabilityInvokerFactory? invokerFactory = null,
+ Func? driverMemberCountProvider = null)
{
_dbFactory = dbFactory;
_localNode = localNode;
_coordinatorOverride = coordinator;
_driverFactory = driverFactory ?? NullDriverFactory.Instance;
_invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
+ _driverMemberCountProvider = driverMemberCountProvider;
_localRoles = localRoles ?? new HashSet(StringComparer.Ordinal);
_dependencyMux = dependencyMux;
_opcUaPublishActor = opcUaPublishActor;
@@ -964,17 +980,29 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
_localNode, msg.DriverInstanceId, msg.Args.ConditionId);
return;
}
+ // PRIMARY GATE (archreview 03/S4) — decided ONCE for this transition: the OPC UA condition write
+ // below stays UNGATED (a Secondary keeps its address space warm for failover), but only the Primary
+ // publishes the single cluster-wide alerts copy. Denied on a Secondary/Detached, OR on a multi-driver
+ // cluster while the role is unknown (default-deny closes the dual-primary boot window that would
+ // otherwise historize duplicate AVEVA alarm rows). Metered + Debug-logged once when denied.
+ var serviceAlertsAsPrimary = ShouldServiceAsPrimary();
+ if (!serviceAlertsAsPrimary)
+ {
+ OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+ new KeyValuePair("site", "alarm-emit"),
+ new KeyValuePair("reason", PrimaryGateDenyReason()));
+ _log.Debug("DriverHost {Node}: native-alarm alerts publish suppressed for ({Driver},{Ref}) — not primary",
+ _localNode, msg.DriverInstanceId, msg.Args.ConditionId);
+ }
+
foreach (var nodeId in nodeIds)
{
var snapshot = _nativeAlarmProjector.Project(nodeId, msg.Args);
_opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate(
nodeId, snapshot, msg.Args.SourceTimestampUtc));
- // Warm-standby dedup: the OPC UA condition write above is UNGATED (the secondary keeps its
- // address space warm), but only the Primary publishes the cluster-wide alerts transition.
- // Default-emit until told we are Secondary/Detached so single-node deploys + the boot window
- // never drop transitions — the SAME signal HandleRouteNodeWrite gates writes on.
- if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) continue;
+ // Only the Primary publishes the cluster-wide alerts transition (see the gate above).
+ if (!serviceAlertsAsPrimary) continue;
var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m)
? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null);
@@ -1034,11 +1062,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
///
private void HandleRouteNodeWrite(RouteNodeWrite msg)
{
- // PRIMARY GATE FIRST — only the Primary services operator writes (same signal as the alarm-emit
- // gate; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked).
- if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
+ // PRIMARY GATE FIRST (archreview 03/S4) — only the Primary services operator writes. A KNOWN
+ // Secondary/Detached is denied "not primary" (unchanged); an UNKNOWN role denies ONLY on a
+ // multi-driver cluster ("not primary (role unknown)"), while a single-driver/boot-window cluster is
+ // still serviced. The rejection rides the established optimistic-write self-correction surface (the
+ // node reverts to its prior value with a Bad blip + a Part 8 audit event) — writes are NOT queued.
+ if (!ShouldServiceAsPrimary())
{
- Sender.Tell(new NodeWriteResult(false, "not primary"));
+ var reason = _localRole is null ? "not primary (role unknown)" : "not primary";
+ OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+ new KeyValuePair("site", "write"),
+ new KeyValuePair("reason", PrimaryGateDenyReason()));
+ Sender.Tell(new NodeWriteResult(false, reason));
return;
}
@@ -1090,12 +1125,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
///
private void HandleRouteNativeAlarmAck(RouteNativeAlarmAck msg)
{
- // PRIMARY GATE FIRST — only the Primary services operator acks (same signal as the inbound-write +
- // alarm-emit gates; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked).
- if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
+ // PRIMARY GATE FIRST (archreview 03/S4) — only the Primary pushes the ack upstream. Same policy as
+ // the inbound-write gate. A KNOWN Secondary/Detached is dropped at Debug (unchanged); an UNKNOWN role
+ // on a multi-driver cluster is dropped at WARNING (an operator ack silently not reaching the upstream
+ // alarm system is a swallowed failure worth surfacing).
+ if (!ShouldServiceAsPrimary())
{
- _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
- _localNode, msg.ConditionNodeId);
+ OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+ new KeyValuePair("site", "ack"),
+ new KeyValuePair("reason", PrimaryGateDenyReason()));
+ if (_localRole is null)
+ {
+ _log.Warning("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary (role unknown on multi-driver cluster)",
+ _localNode, msg.ConditionNodeId);
+ }
+ else
+ {
+ _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
+ _localNode, msg.ConditionNodeId);
+ }
return;
}
@@ -1119,9 +1167,43 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(target.FullName, msg.Comment, msg.OperatorUser));
}
- /// Caches this node's from a cluster redundancy snapshot so
- /// can gate inbound writes to the Primary. A snapshot that doesn't
- /// mention this node leaves the cached role unchanged ⇒ default-allow. Mirrors
+ /// Whether this node should service a Primary-only data-plane operation right now — the single
+ /// gate decision (archreview 03/S4), delegating to with the live driver
+ /// member count.
+ /// true to service as Primary; false to deny.
+ private bool ShouldServiceAsPrimary() =>
+ PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount());
+
+ /// Count of Up cluster members carrying the driver role. Uses the injected test seam when
+ /// present; otherwise reads the live cluster state guarded by try/catch so a non-cluster
+ /// ActorRefProvider (legacy/test harnesses) yields 0 ⇒ the single-node default-allow posture.
+ /// The number of Up driver-role members, or 0 when the cluster extension is unavailable.
+ private int DriverMemberCount()
+ {
+ if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
+ try
+ {
+ return Akka.Cluster.Cluster.Get(Context.System).State.Members
+ .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
+ }
+ catch (Exception)
+ {
+ return 0; // non-cluster ActorRefProvider (legacy harnesses) ⇒ single-node posture
+ }
+ }
+
+ /// The Primary-gate deny reason tag for the denial meter (secondary|detached|role-unknown).
+ private string PrimaryGateDenyReason() => _localRole switch
+ {
+ RedundancyRole.Secondary => "secondary",
+ RedundancyRole.Detached => "detached",
+ _ => "role-unknown",
+ };
+
+ /// Caches this node's from a cluster redundancy snapshot so the
+ /// Primary data-plane gates can resolve their decision. A snapshot that doesn't mention this node leaves
+ /// the cached role unchanged; when this node has a real driver peer (count > 1) that omission is the
+ /// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
/// ScriptedAlarmHostActor.OnRedundancyStateChanged / OpcUaPublishActor.
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
{
@@ -1129,6 +1211,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
if (local is not null)
{
_localRole = local.Role;
+ return;
+ }
+
+ // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
+ // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
+ // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
+ if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
+ {
+ _warnedSnapshotMissingLocalNode = true;
+ _log.Warning(
+ "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
+ _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
}
}