diff --git a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json index 703f3baf..2c5d4a97 100644 --- a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json +++ b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json @@ -11,7 +11,7 @@ { "id": "T7", "subject": "Galaxy counters galaxy.writes.advise_failed + galaxy.writes.unconfirmed (empty-statuses Good metered)", "status": "completed", "blockedBy": ["T6"] }, { "id": "T8", "subject": "Galaxy skip-gated live smoke: normal advised write still returns Good post-change", "status": "completed", "blockedBy": ["T6"] }, { "id": "T9", "subject": "PrimaryGatePolicy pure class + truth-table tests + otopcua.redundancy.primary_gate_denied counter (03/S4)", "status": "completed", "blockedBy": [] }, - { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "pending", "blockedBy": ["T9"] }, + { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "completed", "blockedBy": ["T9"] }, { "id": "T11", "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", "status": "pending", "blockedBy": ["T10"] }, { "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "pending", "blockedBy": ["T9"] }, { "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "pending", "blockedBy": ["T11"] }, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 42ffbe84..97e3eb68 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -192,14 +192,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// /// Cached local from the latest - /// snapshot (null = unknown until the first snapshot arrives, or no local node match). The inbound - /// write gate in reuses this signal — the SAME one the - /// scripted-alarm emit gate uses (ScriptedAlarmHostActor._localRole): only the Primary - /// services writes, default-allow while unknown so single-node deploys + the boot window never - /// reject (a node is the sole Primary until told otherwise). + /// snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary + /// data-plane gates (, , + /// ) resolve this through : a KNOWN + /// role wins outright; an UNKNOWN role is resolved by cluster membership — a single-driver cluster + /// stays default-ALLOW (boot-window / single-node), a multi-driver cluster default-DENIES until a + /// snapshot proves this node is Primary (archreview 03/S4 — closes the dual-primary boot window). /// private RedundancyRole? _localRole; + /// Test seam (archreview 03/S4): overrides the count of Up driver-role cluster members + /// the Primary gate reads while the role is unknown. Null ⇒ read the live cluster state (0 on a + /// non-cluster ActorRefProvider). See . + private readonly Func? _driverMemberCountProvider; + + /// Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent + /// identity mismatch (03/S5) would otherwise log on every snapshot. + private bool _warnedSnapshotMissingLocalNode; + /// Cached cluster DistributedPubSub mediator, resolved once in (on the /// actor thread) and reused for the Primary-gated native-alarm alerts fan-out in /// instead of re-resolving it per-publish. Mirrors @@ -308,11 +318,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers ILoggerFactory? loggerFactory = null, ScriptRootLogger? scriptRootLogger = null, IActorRef? scriptedAlarmHostOverride = null, - IDriverCapabilityInvokerFactory? invokerFactory = null) => + IDriverCapabilityInvokerFactory? invokerFactory = null, + Func? driverMemberCountProvider = null) => Akka.Actor.Props.Create(() => new DriverHostActor( dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor, healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride, - loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory)); + loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider)); /// Initializes a new DriverHostActor with the specified dependencies. /// Database context factory for configuration database access. @@ -338,6 +349,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// Phase 6.1 resilience-invoker factory used to attach an /// to each spawned driver; defaults to /// (pass-through) when null. + /// Test seam (archreview 03/S4): overrides the count of Up + /// driver-role cluster members the Primary gate reads while the role is unknown. When null the + /// default reads Cluster.Get(Context.System).State.Members (0 on a non-cluster ActorRefProvider). public DriverHostActor( IDbContextFactory dbFactory, CommonsNodeId localNode, @@ -353,13 +367,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers ILoggerFactory? loggerFactory = null, ScriptRootLogger? scriptRootLogger = null, IActorRef? scriptedAlarmHostOverride = null, - IDriverCapabilityInvokerFactory? invokerFactory = null) + IDriverCapabilityInvokerFactory? invokerFactory = null, + Func? driverMemberCountProvider = null) { _dbFactory = dbFactory; _localNode = localNode; _coordinatorOverride = coordinator; _driverFactory = driverFactory ?? NullDriverFactory.Instance; _invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance; + _driverMemberCountProvider = driverMemberCountProvider; _localRoles = localRoles ?? new HashSet(StringComparer.Ordinal); _dependencyMux = dependencyMux; _opcUaPublishActor = opcUaPublishActor; @@ -964,17 +980,29 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers _localNode, msg.DriverInstanceId, msg.Args.ConditionId); return; } + // PRIMARY GATE (archreview 03/S4) — decided ONCE for this transition: the OPC UA condition write + // below stays UNGATED (a Secondary keeps its address space warm for failover), but only the Primary + // publishes the single cluster-wide alerts copy. Denied on a Secondary/Detached, OR on a multi-driver + // cluster while the role is unknown (default-deny closes the dual-primary boot window that would + // otherwise historize duplicate AVEVA alarm rows). Metered + Debug-logged once when denied. + var serviceAlertsAsPrimary = ShouldServiceAsPrimary(); + if (!serviceAlertsAsPrimary) + { + OtOpcUaTelemetry.PrimaryGateDenied.Add(1, + new KeyValuePair("site", "alarm-emit"), + new KeyValuePair("reason", PrimaryGateDenyReason())); + _log.Debug("DriverHost {Node}: native-alarm alerts publish suppressed for ({Driver},{Ref}) — not primary", + _localNode, msg.DriverInstanceId, msg.Args.ConditionId); + } + foreach (var nodeId in nodeIds) { var snapshot = _nativeAlarmProjector.Project(nodeId, msg.Args); _opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate( nodeId, snapshot, msg.Args.SourceTimestampUtc)); - // Warm-standby dedup: the OPC UA condition write above is UNGATED (the secondary keeps its - // address space warm), but only the Primary publishes the cluster-wide alerts transition. - // Default-emit until told we are Secondary/Detached so single-node deploys + the boot window - // never drop transitions — the SAME signal HandleRouteNodeWrite gates writes on. - if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) continue; + // Only the Primary publishes the cluster-wide alerts transition (see the gate above). + if (!serviceAlertsAsPrimary) continue; var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m) ? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null); @@ -1034,11 +1062,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// private void HandleRouteNodeWrite(RouteNodeWrite msg) { - // PRIMARY GATE FIRST — only the Primary services operator writes (same signal as the alarm-emit - // gate; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked). - if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) + // PRIMARY GATE FIRST (archreview 03/S4) — only the Primary services operator writes. A KNOWN + // Secondary/Detached is denied "not primary" (unchanged); an UNKNOWN role denies ONLY on a + // multi-driver cluster ("not primary (role unknown)"), while a single-driver/boot-window cluster is + // still serviced. The rejection rides the established optimistic-write self-correction surface (the + // node reverts to its prior value with a Bad blip + a Part 8 audit event) — writes are NOT queued. + if (!ShouldServiceAsPrimary()) { - Sender.Tell(new NodeWriteResult(false, "not primary")); + var reason = _localRole is null ? "not primary (role unknown)" : "not primary"; + OtOpcUaTelemetry.PrimaryGateDenied.Add(1, + new KeyValuePair("site", "write"), + new KeyValuePair("reason", PrimaryGateDenyReason())); + Sender.Tell(new NodeWriteResult(false, reason)); return; } @@ -1090,12 +1125,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers /// private void HandleRouteNativeAlarmAck(RouteNativeAlarmAck msg) { - // PRIMARY GATE FIRST — only the Primary services operator acks (same signal as the inbound-write + - // alarm-emit gates; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked). - if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) + // PRIMARY GATE FIRST (archreview 03/S4) — only the Primary pushes the ack upstream. Same policy as + // the inbound-write gate. A KNOWN Secondary/Detached is dropped at Debug (unchanged); an UNKNOWN role + // on a multi-driver cluster is dropped at WARNING (an operator ack silently not reaching the upstream + // alarm system is a swallowed failure worth surfacing). + if (!ShouldServiceAsPrimary()) { - _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary", - _localNode, msg.ConditionNodeId); + OtOpcUaTelemetry.PrimaryGateDenied.Add(1, + new KeyValuePair("site", "ack"), + new KeyValuePair("reason", PrimaryGateDenyReason())); + if (_localRole is null) + { + _log.Warning("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary (role unknown on multi-driver cluster)", + _localNode, msg.ConditionNodeId); + } + else + { + _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary", + _localNode, msg.ConditionNodeId); + } return; } @@ -1119,9 +1167,43 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(target.FullName, msg.Comment, msg.OperatorUser)); } - /// Caches this node's from a cluster redundancy snapshot so - /// can gate inbound writes to the Primary. A snapshot that doesn't - /// mention this node leaves the cached role unchanged ⇒ default-allow. Mirrors + /// Whether this node should service a Primary-only data-plane operation right now — the single + /// gate decision (archreview 03/S4), delegating to with the live driver + /// member count. + /// true to service as Primary; false to deny. + private bool ShouldServiceAsPrimary() => + PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount()); + + /// Count of Up cluster members carrying the driver role. Uses the injected test seam when + /// present; otherwise reads the live cluster state guarded by try/catch so a non-cluster + /// ActorRefProvider (legacy/test harnesses) yields 0 ⇒ the single-node default-allow posture. + /// The number of Up driver-role members, or 0 when the cluster extension is unavailable. + private int DriverMemberCount() + { + if (_driverMemberCountProvider is not null) return _driverMemberCountProvider(); + try + { + return Akka.Cluster.Cluster.Get(Context.System).State.Members + .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole)); + } + catch (Exception) + { + return 0; // non-cluster ActorRefProvider (legacy harnesses) ⇒ single-node posture + } + } + + /// The Primary-gate deny reason tag for the denial meter (secondary|detached|role-unknown). + private string PrimaryGateDenyReason() => _localRole switch + { + RedundancyRole.Secondary => "secondary", + RedundancyRole.Detached => "detached", + _ => "role-unknown", + }; + + /// Caches this node's from a cluster redundancy snapshot so the + /// Primary data-plane gates can resolve their decision. A snapshot that doesn't mention this node leaves + /// the cached role unchanged; when this node has a real driver peer (count > 1) that omission is the + /// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors /// ScriptedAlarmHostActor.OnRedundancyStateChanged / OpcUaPublishActor. private void OnRedundancyStateChanged(RedundancyStateChanged msg) { @@ -1129,6 +1211,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers if (local is not null) { _localRole = local.Role; + return; + } + + // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the + // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent + // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable. + if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1) + { + _warnedSnapshotMissingLocalNode = true; + _log.Warning( + "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster", + _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId))); } }