diff --git a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json index 20ea2898..3249880b 100644 --- a/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json +++ b/archreview/plans/R2-04-failure-visibility-plan.md.tasks.json @@ -13,7 +13,7 @@ { "id": "T9", "subject": "PrimaryGatePolicy pure class + truth-table tests + otopcua.redundancy.primary_gate_denied counter (03/S4)", "status": "completed", "blockedBy": [] }, { "id": "T10", "subject": "Wire DriverHostActor gates (:977/:1039/:1095) through PrimaryGatePolicy with member-count provider + S5 log-once", "status": "completed", "blockedBy": ["T9"] }, { "id": "T11", "subject": "DriverHostActorPrimaryGateTests: deny-on-unknown multi-node, allow single-node, ack/emit variants, meter tags", "status": "completed", "blockedBy": ["T10"] }, - { "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "pending", "blockedBy": ["T9"] }, + { "id": "T12", "subject": "ScriptedAlarmHostActor alerts-emit gate adopts PrimaryGatePolicy (consistency extension)", "status": "completed", "blockedBy": ["T9"] }, { "id": "T13", "subject": "In-process 2-node delivery-path test (TwoNodeClusterHarness): DPS-delivered snapshot drives the gate", "status": "pending", "blockedBy": ["T11"] }, { "id": "T14", "subject": "docs/Redundancy.md: gate policy table, boot-window client experience, new meters, Galaxy fail-closed semantics", "status": "pending", "blockedBy": [] }, { "id": "T15", "subject": "LIVE GATE: 2-node docker-dev rig verify (ServiceLevel 240/100, boot-window write rejected+reverted, single /alerts row)", "status": "pending", "blockedBy": ["T10", "T11", "T12", "T13"] } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index 7d93e51d..a8d790b8 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -3,11 +3,13 @@ using Akka.Cluster.Tools.PublishSubscribe; using Akka.Event; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; @@ -104,10 +106,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Cached local from the latest /// snapshot (null = unknown until the first snapshot arrives, or no wired). The - /// cluster-wide alerts publish in is gated on this: only the Primary - /// publishes (default-emit while unknown so single-node deploys + the boot window never drop transitions). + /// cluster-wide alerts publish in resolves this through + /// : only the Primary publishes; an unknown role default-emits on a + /// single-driver cluster but default-DENIES on a multi-driver one, so the dual-primary boot window can't + /// historize duplicate AVEVA alarm rows (archreview 03/S4). private RedundancyRole? _localRole; + /// Test seam (archreview 03/S4): overrides the count of Up driver-role cluster members the + /// alerts-emit gate reads while the role is unknown. Null ⇒ read the live cluster state. + private readonly Func? _driverMemberCountProvider; + /// Monotonic load generation, bumped on every . The continuation that /// pipes back an captures the generation it was started under; a stale /// completion (an earlier generation arriving after a newer apply) is discarded in @@ -136,8 +144,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor IActorRef? mux, DependencyMuxTagUpstreamSource upstream, ScriptedAlarmEngine engine, - NodeId? localNode = null) => - Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode)); + NodeId? localNode = null, + Func? driverMemberCountProvider = null) => + Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider)); /// Initializes a new instance of the class. /// The OPC UA publish actor emissions are bridged to. @@ -147,12 +156,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// The local cluster node id, used to read this node's /// from the redundancy-state topic so only the Primary publishes the cluster-wide alerts /// transition. Null leaves the role unknown ⇒ default-emit (single-node deploys + tests). + /// Test seam (archreview 03/S4): overrides the count of Up + /// driver-role cluster members the alerts-emit gate reads while the role is unknown. Null reads + /// the live cluster state (0 on a non-cluster ActorRefProvider ⇒ single-node default-emit). public ScriptedAlarmHostActor( IActorRef publishActor, IActorRef? mux, DependencyMuxTagUpstreamSource upstream, ScriptedAlarmEngine engine, - NodeId? localNode = null) + NodeId? localNode = null, + Func? driverMemberCountProvider = null) { ArgumentNullException.ThrowIfNull(publishActor); ArgumentNullException.ThrowIfNull(upstream); @@ -162,6 +175,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor _upstream = upstream; _engine = engine; _localNode = localNode; + _driverMemberCountProvider = driverMemberCountProvider; // OnEvent fires on the engine's worker thread. NEVER touch Context / actor state here — // marshal onto the actor thread via the thread-safe Self.Tell. Keep the handler in a field @@ -307,18 +321,45 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor // suppress only the sink write. This publish (and the live `/alerts` fan-out) is NOT gated on it. HistorizeToAveva: e.HistorizeToAveva); - // Warm-standby dedup: only the Primary (driver-role leader) publishes the cluster-wide - // transition. Default-emit until told we are Secondary/Detached so single-node deploys + the - // boot window never drop transitions. The OPC UA node write above + inbound command processing - // stay ungated (the secondary keeps its address space + engine state warm for failover). - if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) + // Warm-standby dedup: only the Primary publishes the cluster-wide transition (archreview 03/S4 — + // same PrimaryGatePolicy as DriverHostActor). A KNOWN Secondary/Detached is denied; an UNKNOWN role + // default-emits on a single-driver cluster but default-DENIES on a multi-driver one (so the + // dual-primary boot window can't historize duplicate AVEVA alarm rows). The OPC UA node write above + + // inbound command processing stay ungated (the secondary keeps its state warm for failover). + if (!PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount())) { + OtOpcUaTelemetry.PrimaryGateDenied.Add(1, + new KeyValuePair("site", "alarm-emit"), + new KeyValuePair("reason", _localRole switch + { + RedundancyRole.Secondary => "secondary", + RedundancyRole.Detached => "detached", + _ => "role-unknown", + })); return; } _mediator.Tell(new Publish(AlertsTopic, evt)); } + /// Count of Up cluster members carrying the driver role, for the alerts-emit gate. Uses + /// the injected test seam when present; otherwise reads the live cluster state guarded by try/catch so a + /// non-cluster ActorRefProvider yields 0 ⇒ the single-node default-emit posture (archreview 03/S4). + /// The number of Up driver-role members, or 0 when the cluster extension is unavailable. + private int DriverMemberCount() + { + if (_driverMemberCountProvider is not null) return _driverMemberCountProvider(); + try + { + return Akka.Cluster.Cluster.Get(Context.System).State.Members + .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole)); + } + catch (Exception) + { + return 0; + } + } + /// Caches this node's from a cluster redundancy snapshot so /// can gate the cluster-wide alerts publish to the Primary. A /// snapshot that doesn't mention (or no local node wired) leaves the cached diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorTests.cs index 94e7a78c..2bd182bd 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorTests.cs @@ -519,6 +519,58 @@ public sealed class ScriptedAlarmHostActorTests : RuntimeActorTestBase evt.TransitionKind.ShouldBe("Activated"); } + /// Boot-window multi-driver suppression (archreview 03/S4): role unknown + a real driver peer + /// (member count 2) ⇒ the alerts publish is DENIED (default-deny closes the dual-primary window that would + /// historize duplicate AVEVA rows), while the OPC UA node write stays ungated. + [Fact] + public void Unknown_role_multi_driver_suppresses_alerts_but_still_writes_opcua() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var alerts = CreateTestProbe(); + SubscribeToAlerts(alerts); + + var (host, _) = SpawnWithMemberCount(publish, mux, LocalNode, driverMemberCount: 2); + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan(severity: 800) })); + mux.ExpectMsg(Timeout); + + // No snapshot sent ⇒ role unknown; count 2 ⇒ default-DENY. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + + publish.FishForMessage(m => m.State.Active, Timeout); + alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); + } + + /// Boot-window single-driver emit (archreview 03/S4): role unknown + no driver peer (member + /// count 1) ⇒ the alerts publish proceeds (single-node posture preserved). + [Fact] + public void Unknown_role_single_driver_publishes_alerts() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var alerts = CreateTestProbe(); + SubscribeToAlerts(alerts); + + var (host, _) = SpawnWithMemberCount(publish, mux, LocalNode, driverMemberCount: 1); + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan(severity: 800) })); + mux.ExpectMsg(Timeout); + + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + + publish.FishForMessage(m => m.State.Active, Timeout); + alerts.ExpectMsg(Timeout).AlarmId.ShouldBe("alm-1"); + } + + private (IActorRef Host, DependencyMuxTagUpstreamSource Upstream) SpawnWithMemberCount( + TestProbe publish, TestProbe mux, NodeId localNode, int driverMemberCount) + { + var upstream = new DependencyMuxTagUpstreamSource(); + var engine = BuildEngine(upstream); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props( + publish.Ref, mux.Ref, upstream, engine, localNode, driverMemberCountProvider: () => driverMemberCount)); + return (host, upstream); + } + /// Inbound command ungated by role (T1): the alerts-publish gate must NOT affect inbound /// command processing. Under a Secondary role, an AlarmCommand("Acknowledge") for an owned, active /// alarm still drives the engine — observed via the resulting AlarmStateUpdate(Acknowledged=true)