fix(redundancy): elect the Primary by cluster age, not by address

RedundancyStateActor derived the driver Primary from ClusterState.RoleLeader("driver").
Akka offers two different notions of a "first" member and they are not the same
one: role leader is the lowest-ADDRESSED Up member (host, then port), while
ClusterSingletonManager places singletons on the OLDEST — lowest up-number.

They agree on a freshly-formed cluster, which is why every existing test passed.
They diverge after any restart: the restarted node re-joins as the youngest while
keeping its address, so if it holds the lower address it becomes role leader while
the singletons stay put. The snapshot would then name a Primary that is not
hosting the work, and every Primary-gated surface follows it — inbound device
writes, native-alarm acks, the fleet-wide alerts emit, and the alarm-history
drain would all enable on the wrong node while the node actually running the
singletons stayed gated off.

BuildSnapshot now selects the oldest Up member carrying the driver role, matching
singleton placement. Leaving members are excluded: a node handing its singletons
over must not be named Primary.

NodeRedundancyState.IsRoleLeaderForDriver is renamed IsDriverPrimary, and
NodeHealthInputs.IsDriverRoleLeader likewise. Keeping the old names would have
left the wire contract asserting a derivation the code no longer uses — the same
drift that made this defect invisible.

Proven by a real two-node cluster rather than a mock. RedundancyPrimaryElectionTests
binds the first-joining node to the HIGHER port, so oldest and lowest-address name
different nodes, and includes a fixture assertion that the divergence actually
occurred — without it the real assertion could pass for the wrong reason. Positive
control: restoring the RoleLeader derivation turns exactly the two election tests
red while the fixture check stays green.

Runtime.Tests 440 passed, ControlPlane.Tests 82, Cluster.Tests 36.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Redundancy/ServiceLevelCalculator.cs

@@ -7,7 +7,7 @@ public readonly record struct NodeHealthInputs(
     bool DbReachable,
     bool OpcUaProbeOk,
     bool Stale,
-    bool IsDriverRoleLeader);
+    bool IsDriverPrimary);
 
 /// <summary>
 /// Pure ServiceLevel computation per design §6. Output range 0–255, where higher = "more
@@ -18,7 +18,9 @@ public readonly record struct NodeHealthInputs(
 /// - Member not Up/Joining: 0 (cluster cannot trust this node).
 /// - DB reachable + OPC UA probe ok + not stale: 240 (full service).
 /// - Stale config (DB reachable or not, OPC UA probe state ignored): 100 or 200 depending on DB.
-/// - +10 bonus when this node holds the role-leader lease for the "driver" role.
+/// - +10 bonus when this node is the driver Primary (the oldest Up member with the "driver" role —
+///   where the cluster singletons live). Previously this keyed off Akka's role *leader*, the
+///   lowest-addressed member, which names a different node once any node has restarted.
 /// </summary>
 public static class ServiceLevelCalculator
 {
@@ -38,6 +40,6 @@ public static class ServiceLevelCalculator
             _ => 0,
         };
 
-        return (byte)Math.Clamp(basis + (h.IsDriverRoleLeader ? 10 : 0), 0, 255);
+        return (byte)Math.Clamp(basis + (h.IsDriverPrimary ? 10 : 0), 0, 255);
     }
 }

src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Redundancy/NodeRedundancyState.cs

@@ -8,9 +8,20 @@ public enum RedundancyRole { Primary, Secondary, Detached }
 /// Snapshot of a single node's redundancy state. Aggregated by <c>RedundancyStateActor</c>
 /// to compute fleet-wide ServiceLevel.
 /// </summary>
+/// <param name="NodeId">Canonical <c>host:port</c> id of the node this entry describes.</param>
+/// <param name="Role">The node's redundancy role in the current snapshot.</param>
+/// <param name="IsClusterLeader">Whether the node is the Akka cluster leader.</param>
+/// <param name="IsDriverPrimary">
+/// Whether the node is the driver Primary — the <b>oldest</b> Up member carrying the <c>driver</c>
+/// role, which is where <c>ClusterSingletonManager</c> places singletons. Renamed from
+/// <c>IsRoleLeaderForDriver</c>: it was derived from <c>ClusterState.RoleLeader("driver")</c>, the
+/// lowest-<i>addressed</i> member, which diverges from the oldest after any node restart. The name
+/// now describes what the value means rather than how it used to be computed.
+/// </param>
+/// <param name="AsOfUtc">When the snapshot was computed.</param>
 public sealed record NodeRedundancyState(
     NodeId NodeId,
     RedundancyRole Role,
     bool IsClusterLeader,
-    bool IsRoleLeaderForDriver,
+    bool IsDriverPrimary,
     DateTime AsOfUtc);

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs

@@ -111,9 +111,49 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
         _log.Debug("Published RedundancyStateChanged with {Count} nodes", snapshot.Count);
     }
 
+    /// <summary>The cluster role that marks a node as carrying the driver runtime.</summary>
+    public const string DriverRole = "driver";
+
+    /// <summary>
+    /// Selects the driver Primary: the <b>oldest</b> Up member carrying the <see cref="DriverRole"/>.
+    /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         This deliberately does <b>not</b> use <c>ClusterState.RoleLeader(role)</c>, which it
+    ///         previously did. Role leader is the <i>lowest-addressed</i> Up member with the role —
+    ///         address order (host, then port), with no relationship to time. Oldest is the member
+    ///         with the lowest up-number, and it is what <c>ClusterSingletonManager</c> uses to place
+    ///         singletons.
+    ///     </para>
+    ///     <para>
+    ///         On a freshly-formed cluster the two agree, which is why the discrepancy was invisible.
+    ///         They diverge after any restart: the restarted node re-joins as the youngest but keeps
+    ///         its address, so if it holds the lower address it becomes role leader while the
+    ///         singletons — and all the work they own — stay on the other node. Electing it Primary
+    ///         would enable the Primary-gated data plane (inbound writes, alarm acks, the fleet-wide
+    ///         alerts emit, the alarm-history drain) on the node that is not hosting the work.
+    ///     </para>
+    ///     <para>
+    ///         Only <see cref="MemberStatus.Up"/> members are eligible, matching singleton placement:
+    ///         a <c>Leaving</c> node is handing its singletons over and must not be named Primary.
+    ///     </para>
+    /// </remarks>
+    /// <param name="members">The current cluster members, in any order.</param>
+    /// <returns>The oldest Up driver member's address, or <c>null</c> when there is none.</returns>
+    public static Address? SelectDriverPrimary(IEnumerable<Member> members)
+    {
+        ArgumentNullException.ThrowIfNull(members);
+
+        return members
+            .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole))
+            .OrderBy(m => m, Member.AgeOrdering)
+            .FirstOrDefault()
+            ?.Address;
+    }
+
     private IReadOnlyList<NodeRedundancyState> BuildSnapshot()
     {
-        var driverLeader = _cluster.State.RoleLeader("driver");
+        var driverPrimary = SelectDriverPrimary(_cluster.State.Members);
         var clusterLeader = _cluster.State.Leader;
         var now = DateTime.UtcNow;
 
@@ -123,15 +163,16 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
             var host = member.Address.Host;
             if (string.IsNullOrWhiteSpace(host)) continue;
 
-            var role = member.Roles.Contains("driver")
-                ? (driverLeader == member.Address ? CommonsRedundancyRole.Primary : CommonsRedundancyRole.Secondary)
+            var isPrimary = driverPrimary is not null && driverPrimary == member.Address;
+            var role = member.Roles.Contains(DriverRole)
+                ? (isPrimary ? CommonsRedundancyRole.Primary : CommonsRedundancyRole.Secondary)
                 : CommonsRedundancyRole.Detached;
 
             list.Add(new NodeRedundancyState(
                 ToNodeId(member.Address),
                 role,
                 IsClusterLeader: clusterLeader == member.Address,
-                IsRoleLeaderForDriver: driverLeader == member.Address,
+                IsDriverPrimary: isPrimary,
                 AsOfUtc: now));
         }
         return list;

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs

@@ -652,7 +652,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             Stale: !_lastDbHealth.Reachable
                    || (now - _lastDbHealth.AsOfUtc) > _staleWindow
                    || (now - entry.AsOfUtc) > _staleWindow,
-            IsDriverRoleLeader: entry.IsRoleLeaderForDriver);
+            IsDriverPrimary: entry.IsDriverPrimary);
 
         Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
     }
@@ -661,7 +661,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
     /// secondary → 100, _ → 0). Preserved as the back-compat / bootstrap seam.</summary>
     private static byte LegacyRoleOnly(NodeRedundancyState entry) => entry.Role switch
     {
-        RedundancyRole.Primary when entry.IsRoleLeaderForDriver => 240,
+        RedundancyRole.Primary when entry.IsDriverPrimary => 240,
         RedundancyRole.Primary => 200,
         RedundancyRole.Secondary => 100,
         _ => 0,