diff --git a/docs/AlarmTracking.md b/docs/AlarmTracking.md index 445037d0..6045d563 100644 --- a/docs/AlarmTracking.md +++ b/docs/AlarmTracking.md @@ -357,6 +357,11 @@ is client-visible. > pair (role still unknown, a driver peer Up) the gate already denies, exactly > as the route does (review R1 m2 — previously the view sat on its single-node > "allow" seed until the first snapshot, reopening the flap for that window). +> It mirrors, it does not deny from start: until this node has **joined** and +> sees its peer Up, its member count is 0 and both the published verdict and the +> live route **allow** (seconds normally; up to the bootstrap guard's 25 s +> partner probe). The two agree there, so the flap cannot occur, but a native +> ack in that pre-join window is serviced (review R6 MINOR-2). > **Best-effort during reconnect.** If a native ack arrives while the owning > driver is `Connecting`/`Reconnecting`, it is dropped with a debug log and **not** diff --git a/docs/Redundancy.md b/docs/Redundancy.md index c6e73f8b..c8f88d59 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -526,7 +526,7 @@ Three data-plane surfaces are **Primary-gated** so a warm standby never touches Before archreview 03/S4 the unknown-role case defaulted to *allow* unconditionally, opening a dual-primary window: a freshly-booted secondary on a multi-node cluster serviced shared-device writes and emitted duplicate fleet-wide alerts for up to the ~10 s `RedundancyStateActor` heartbeat (and indefinitely if the snapshot's NodeId never matched this node). Membership is authoritative much earlier than DPS delivery, so an unknown role on a **multi-driver** cluster now default-**denies**. The driver member count is read live from `Cluster.State.Members` (Up members with the `driver` role); a non-cluster ActorRefProvider yields 0 ⇒ single-node posture. If a snapshot arrives that never mentions this node while a driver peer exists (the 03/S5 identity-mismatch shape), `DriverHostActor` logs a one-time Warning so the silent mismatch is diagnosable. -**What a client sees during the boot window (multi-node, role unknown):** an inbound write is **rejected** (not queued — OT actuation seconds late is a surprise). The reject rides the sanctioned optimistic-write self-correction surface: the node reverts to its prior value with a transient Bad-quality blip and a Part 8 `AuditWriteUpdateEvent`; the client's synchronous write call still returned Good (exactly as any failed device write behaves since the #5 write-outcome self-correction). The rejection reason distinguishes the boot window (`"not primary (role unknown)"`) from a steady-state Secondary (`"not primary"`). Acks are dropped (Warning-logged when role-unknown) — and since review R1 m2 a native OPC UA Acknowledge is refused up front (`BadInvalidState`) in the same window: `DriverHostActor` publishes its Primary verdict to `IDriverPrimaryRoleSource` (the native-ack gate, and the MQTT Sparkplug primary-host claim) once at start and again on every cluster membership change, not only on redundancy snapshots, so the published verdict denies as soon as a driver peer is Up rather than sitting on the single-node "allow" seed until the first snapshot. The alerts emit is skipped (the Primary peer publishes the single fleet copy). The window closes on the first delivered snapshot. +**What a client sees during the boot window (multi-node, role unknown):** an inbound write is **rejected** (not queued — OT actuation seconds late is a surprise). The reject rides the sanctioned optimistic-write self-correction surface: the node reverts to its prior value with a transient Bad-quality blip and a Part 8 `AuditWriteUpdateEvent`; the client's synchronous write call still returned Good (exactly as any failed device write behaves since the #5 write-outcome self-correction). The rejection reason distinguishes the boot window (`"not primary (role unknown)"`) from a steady-state Secondary (`"not primary"`). Acks are dropped (Warning-logged when role-unknown) — and since review R1 m2 a native OPC UA Acknowledge is refused up front (`BadInvalidState`) in the same window: `DriverHostActor` publishes its Primary verdict to `IDriverPrimaryRoleSource` (the native-ack gate, and the MQTT Sparkplug primary-host claim) once at start and again on every cluster membership change, not only on redundancy snapshots, so the published verdict denies as soon as a driver peer is Up rather than sitting on the single-node "allow" seed until the first snapshot. ⚠️ "As soon as a driver peer is Up" means **in this node's own cluster view**, i.e. after it has joined: the host is spawned before the join, when the view holds no members, and a member count of 0 takes the single-node *allow* posture — for the published verdict and the live gate alike. So a node cold-starting into a pair whose peer is already Primary allows until its own join completes (seconds normally; up to `Cluster:BootstrapGuard:PartnerProbeSeconds`, 25 s, under the bootstrap guard), and in that window the MQTT Sparkplug primary-host claim can be held on both nodes. Native ack is not exposed (the view and the route agree, so there is no acked → unacked revert). Denying until self is Up was not adopted: it would also deny writes/acks during a single-node cluster's own join (review R6 MINOR-2). The alerts emit is skipped (the Primary peer publishes the single fleet copy). The window closes on the first delivered snapshot. Under warm/hot redundancy both cluster nodes run `ScriptedAlarmHostActor` and evaluate scripted alarms, keeping each node's address space and engine state warm for instant failover. To avoid duplicate rows on `/alerts` and duplicate historian writes, only the Primary node publishes externally: diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index 5a064550..8804f04f 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -363,9 +363,14 @@ Hence: bind the claim to a connection, and rebuild the connection on role change | Role unknown, no driver peer (single node) | open | `ONLINE` | The verdict comes from `PrimaryGatePolicy.ShouldServiceAsPrimary` via `IDriverPrimaryRoleSource` -(`Core.Abstractions`), published by `DriverHostActor` on every redundancy snapshot and **sampled on a -1-second timer** rather than evented — a process-singleton raising events at per-instance drivers -outlives every driver it holds a reference to. +(`Core.Abstractions`), published by `DriverHostActor` at host start, on every cluster membership change and on +every redundancy snapshot, and **sampled on a 1-second timer** rather than evented — a process-singleton +raising events at per-instance drivers outlives every driver it holds a reference to. + +> ⚠️ **"Driver peer present" is this node's own cluster view.** Before the node has joined, it sees no +> members, which is the single-node row: a node cold-starting into a pair whose peer is already Primary +> can open the claim until its join completes (seconds; up to the bootstrap guard's 25 s partner probe), +> so both nodes may briefly hold it (review R6 MINOR-2). > **Accepted trade: a brief edge-visible `OFFLINE` during failover**, while the old Primary stands down > and the new one connects and republishes. That is a genuine state transition — for that window there diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index f9c3b292..5c72c36d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -802,6 +802,19 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS /// membership changes (a driver peer coming Up turns an unknown role from allow into deny before any /// snapshot names this node). A non-cluster ActorRefProvider (legacy/test harnesses) just keeps the /// initial publish. + /// + /// ⚠️ It MIRRORS the live gate; it does not deny from host start. The host is spawned at + /// ActorSystem start, before this node has joined, when its view of Cluster.State.Members is + /// empty — a member count of 0, which the single-node posture ALLOWS. So the verdict published here is + /// allow, and it flips to deny only once this node's own cluster state shows a second Up driver member: + /// normally seconds, up to Cluster:BootstrapGuard:PartnerProbeSeconds (25 s) under the + /// bootstrap guard. In that pre-join window the Sparkplug primary-host claim can be held on both nodes of + /// a pair. Native ack is NOT exposed by it: the live route gate reads the same count and also allows, so + /// the view and the route still agree and the acked → unacked revert cannot occur (review R6 MINOR-2). + /// Denying until self is Up was considered and NOT done: it would change the live data-plane gate (the + /// two must stay one decision) and so delay writes and acks on a genuinely single-node cluster during its + /// own join, and this actor has no reliable way to know that a partner is configured. + /// /// private void StartPrimaryVerdictPublishing() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorPrimaryRoleSourceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorPrimaryRoleSourceTests.cs index b846dea9..6043f7dd 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorPrimaryRoleSourceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorPrimaryRoleSourceTests.cs @@ -66,6 +66,31 @@ public sealed class DriverHostActorPrimaryRoleSourceTests : RuntimeActorTestBase AwaitAssert(() => source.Published.ShouldBe([true, false]), duration: Timeout); } + /// + /// Review R6 MINOR-3 — the same flip driven by a real cluster MEMBER EVENT rather than a direct + /// tell, so it fails if the host's + /// Cluster.Subscribe(… IMemberEvent) or its IMemberEvent handler is deleted. The event is published + /// on the ActorSystem EventStream, which is exactly how the cluster's domain-event publisher delivers it to a + /// Cluster.Subscribe subscriber — and to nothing else. Re-published inside the wait because the + /// subscription is registered asynchronously by the publisher actor after PreStart. + /// + [Fact] + public void A_cluster_member_event_flips_the_published_verdict_without_a_direct_republish() + { + var source = new RecordingPrimaryRoleSource(); + var members = 1; + SpawnHost(source, () => Volatile.Read(ref members)); + AwaitAssert(() => source.Published.ShouldBe([true]), duration: Timeout); + + Volatile.Write(ref members, 2); + var memberUp = new Akka.Cluster.ClusterEvent.MemberUp(Akka.Cluster.Cluster.Get(Sys).SelfMember); + AwaitAssert(() => + { + Sys.EventStream.Publish(memberUp); + source.Published.ShouldBe([true, false]); + }, duration: Timeout, interval: TimeSpan.FromMilliseconds(100)); + } + private IActorRef SpawnHost(IDriverPrimaryRoleSource source, Func driverMemberCount) => Sys.ActorOf(DriverHostActor.Props( NewInMemoryDbFactory(),