feat(mesh): re-home the redundancy singleton to a per-cluster driver singleton

Phase 6 splits the single fleet-wide Akka mesh into one 2-node mesh per
application Cluster. RedundancyStateActor was an admin-scoped cluster singleton
that elected ONE fleet-wide driver Primary — but after the split a driver-only
site pair has no admin node in its mesh to host it, so its Primary would never
be elected.

Re-home it: remove the redundancy WithSingleton block from the admin
WithOtOpcUaControlPlaneSingletons set, and add WithOtOpcUaClusterRedundancySingleton,
a singleton spawned from Program.cs's hasDriver branch on EVERY driver node (the
fused central included). It scopes to the node's own cluster-{ClusterId} role when
present, and falls back to the fixed `driver` role otherwise. Each mesh then has
exactly one, electing its own pair-local Primary and publishing redundancy-state
on its own mesh's DistributedPubSub. The election logic (oldest Up driver member)
and the DPS publish are unchanged — they become pair-local after the split.

The driver-role fallback (Decision 2) deliberately does NOT throw when a node has
no cluster role: that is the pre-Phase-6 fleet-wide behavior on a legacy single
mesh, so legacy/harness (TwoNodeClusterHarness: admin,driver) and not-yet-migrated
deployments keep booting unchanged. On a genuinely split 2-node mesh the `driver`
role is already pair-local (the mesh IS the pair), so the fallback is correct in
both worlds. Cluster-scope, when present, additionally survives an accidental
two-mesh merge by keeping one singleton per cluster.

The role scope + driver-role fallback are pinned by unit tests against the
extracted BuildClusterRedundancySingletonOptions helper (the BuildDowningHocon
pattern); admin-removal + driver-registration are pinned behaviorally against a
booted node's ActorRegistry.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs

@@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
+using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
@@ -130,10 +131,12 @@ public static class ServiceCollectionExtensions
             (system, registry, resolver) => FleetStatusBroadcaster.Props(),
             singletonOptions);
 
-        builder.WithSingleton<RedundancyStateActorKey>(
-            RedundancyStateSingletonName,
-            (system, registry, resolver) => RedundancyStateActor.Props(),
-            singletonOptions);
+        // Per-cluster mesh Phase 6: the redundancy-state singleton is NO LONGER registered here.
+        // It was an admin-scoped, fleet-wide singleton — one Primary elected across the whole mesh.
+        // After the mesh split a driver-only site pair has no admin node to host it, so it moved to
+        // WithOtOpcUaClusterRedundancySingleton, a cluster-{ClusterId}-scoped singleton spawned on
+        // EVERY driver node (the fused central included). One per mesh, electing its own pair-local
+        // Primary and publishing on its own mesh's DistributedPubSub.
 
         // Per-cluster mesh Phase 1: ClusterNode.AkkaPort duplicates the node's own Cluster:Port and
         // nothing makes them agree. Runs here rather than on each driver node because Phase 4 removes
@@ -157,6 +160,61 @@ public static class ServiceCollectionExtensions
 
         return builder;
     }
+
+    /// <summary>
+    /// Registers <see cref="RedundancyStateActor"/> as a cluster singleton scoped to the local node's
+    /// OWN <c>cluster-{ClusterId}</c> role (or the fixed <c>driver</c> role when the node carries no
+    /// cluster role), and spawns it on EVERY driver node (per-cluster mesh, Phase 6). Wire from the
+    /// driver branch of the fused Host's Program.cs — the fused central node is <c>hasDriver</c> too,
+    /// so it hosts exactly one redundancy singleton scoped to its own cluster role (e.g.
+    /// <c>cluster-MAIN</c>), while the admin singleton set it also registers no longer carries
+    /// redundancy.
+    /// </summary>
+    /// <param name="builder">The Akka configuration builder.</param>
+    /// <param name="roleInfo">The local node's cluster-role view (Task 1's <c>ClusterRoleInfo</c>).</param>
+    /// <returns>The builder for fluent chaining.</returns>
+    /// <remarks>
+    /// After the mesh split, central's mesh sees only its own pair — a driver-only site pair therefore
+    /// has NO admin node to elect its Primary. Scoping this singleton to the node's cluster role and
+    /// spawning it on every driver node gives each mesh exactly one, electing its own pair-local
+    /// Primary and publishing <c>redundancy-state</c> on its own mesh's DistributedPubSub.
+    /// </remarks>
+    public static AkkaConfigurationBuilder WithOtOpcUaClusterRedundancySingleton(
+        this AkkaConfigurationBuilder builder, IClusterRoleInfo roleInfo)
+    {
+        var singletonOptions = BuildClusterRedundancySingletonOptions(roleInfo);
+
+        builder.WithSingleton<RedundancyStateActorKey>(
+            RedundancyStateSingletonName,
+            (system, registry, resolver) => RedundancyStateActor.Props(),
+            singletonOptions);
+
+        return builder;
+    }
+
+    /// <summary>
+    /// Builds the <see cref="ClusterSingletonOptions"/> that scope the redundancy-state singleton to
+    /// the local node's <c>cluster-{ClusterId}</c> role, falling back to the fixed <c>driver</c> role
+    /// when the node carries no cluster role. Extracted (like
+    /// <c>SplitBrainResolverActivationTests</c>'s <c>BuildDowningHocon</c>) so the exact role scope is
+    /// unit-assertable without booting a cluster.
+    /// </summary>
+    /// <remarks>
+    /// Cluster-scope is the robust choice — it survives an accidental two-mesh merge by keeping one
+    /// singleton per cluster. The <c>driver</c>-role fallback deliberately does NOT throw when a node
+    /// has no cluster role, because (a) that is the pre-Phase-6 fleet-wide behavior on a legacy single
+    /// mesh, keeping legacy/harness nodes and any not-yet-migrated deployment booting unchanged, and
+    /// (b) on a genuinely split 2-node mesh the <c>driver</c> role is already pair-local — the mesh IS
+    /// the pair. So the fallback is correct in both worlds and costs no availability.
+    /// </remarks>
+    /// <param name="roleInfo">The local node's cluster-role view.</param>
+    /// <returns>Singleton options scoped to the node's cluster role, or the <c>driver</c> role.</returns>
+    public static ClusterSingletonOptions BuildClusterRedundancySingletonOptions(IClusterRoleInfo roleInfo)
+    {
+        ArgumentNullException.ThrowIfNull(roleInfo);
+
+        return new ClusterSingletonOptions { Role = roleInfo.ClusterRole ?? RoleParser.Driver };
+    }
 }
 
 /// <summary>Marker key types used by <c>Akka.Hosting</c> to resolve singletons from the registry.</summary>

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -12,6 +12,7 @@ using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
 using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
 using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis;
 using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Configuration.Services;
 using ZB.MOM.WW.OtOpcUa.ControlPlane;
@@ -371,7 +372,17 @@ builder.Services.AddAkka("otopcua", (ab, sp) =>
         ab.WithOtOpcUaSignalRBridges();
     }
     if (hasDriver)
+    {
         ab.WithOtOpcUaRuntimeActors();
+        // Per-cluster mesh Phase 6: re-home the redundancy-state singleton off the admin node onto
+        // EVERY driver node, scoped to the node's own cluster-{ClusterId} role, so each mesh elects
+        // its own pair-local Primary (a driver-only site pair has no admin node to do it). Runs on
+        // the fused central node too (it is hasDriver), where it scopes to cluster-MAIN — exactly one
+        // redundancy singleton per node, since the admin branch above no longer registers one.
+        // IClusterRoleInfo is registered by AddOtOpcUaCluster (resolved lazily here); it throws at
+        // host start if this driver node carries no cluster-{ClusterId} role.
+        ab.WithOtOpcUaClusterRedundancySingleton(sp.GetRequiredService<IClusterRoleInfo>());
+    }
 });
 
 // Down-if-alone recovery watchdog (#459). Registered AFTER AddAkka so it starts after Akka's own

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/RedundancyStateSingletonRehomeTests.cs

@@ -0,0 +1,152 @@
+using Akka.Actor;
+using Akka.Cluster.Hosting;
+using Akka.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+/// Guards Phase 6's re-homing of <c>RedundancyStateActor</c> off the fleet-wide admin singleton onto
+/// a per-node <c>cluster-{ClusterId}</c> singleton (see
+/// <see cref="ServiceCollectionExtensions.WithOtOpcUaClusterRedundancySingleton"/>). After the mesh
+/// split a driver-only site pair has NO admin node to elect its Primary, so the singleton must be
+/// scoped to the node's own cluster role and spawned on every driver node — one per mesh.
+///
+/// <para>The role scope + the missing-role guard are asserted against the extracted
+/// <see cref="ServiceCollectionExtensions.BuildClusterRedundancySingletonOptions"/> helper directly —
+/// the same pattern <c>SplitBrainResolverActivationTests</c> uses for <c>BuildDowningHocon</c> /
+/// <c>BuildClusterOptions</c>: it pins the exact <see cref="ClusterSingletonOptions.Role"/> without
+/// booting a cluster, which no builder-introspection API can otherwise reach. The admin-removal claim
+/// is behavioral (a booted admin node's <see cref="ActorRegistry"/> must no longer carry the
+/// redundancy key) so it needs a real host.</para>
+/// </summary>
+public sealed class RedundancyStateSingletonRehomeTests
+{
+    /// <summary>The singleton options scope to the node's OWN cluster role, not the fixed admin role.</summary>
+    [Fact]
+    public void Options_scope_the_singleton_to_the_nodes_cluster_role()
+    {
+        var options = ServiceCollectionExtensions.BuildClusterRedundancySingletonOptions(
+            new FakeClusterRoleInfo(clusterRole: "cluster-SITE-A", clusterId: "SITE-A"));
+
+        options.Role.ShouldBe("cluster-SITE-A");
+    }
+
+    /// <summary>
+    /// A node with no <c>cluster-{ClusterId}</c> role (a legacy single-mesh node, or the
+    /// <c>TwoNodeClusterHarness</c> with roles <c>admin,driver</c>) falls back to the fixed
+    /// <c>driver</c> role rather than throwing — so legacy/not-yet-migrated deployments keep booting.
+    /// On a split 2-node mesh the <c>driver</c> role is already pair-local (the mesh IS the pair).
+    /// </summary>
+    [Fact]
+    public void Missing_cluster_role_falls_back_to_the_driver_role()
+    {
+        var options = ServiceCollectionExtensions.BuildClusterRedundancySingletonOptions(
+            new FakeClusterRoleInfo(clusterRole: null, clusterId: null));
+
+        options.Role.ShouldBe(RoleParser.Driver);
+    }
+
+    /// <summary>
+    /// The admin singleton set no longer registers <c>redundancy-state</c>: a booted admin node's
+    /// registry carries the OTHER admin singletons (proxy) but NOT the redundancy key.
+    /// </summary>
+    [Fact]
+    public async Task Admin_singletons_no_longer_register_the_redundancy_singleton()
+    {
+        var registry = await BootAndGetRegistryAsync(
+            roles: new[] { "admin" },
+            configure: ab => ab.WithOtOpcUaControlPlaneSingletons());
+
+        registry.TryGet<RedundancyStateActorKey>(out _).ShouldBeFalse(
+            "the redundancy singleton was re-homed off the admin node onto every driver node (Phase 6)");
+        registry.TryGet<FleetStatusBroadcasterKey>(out _).ShouldBeTrue(
+            "the remaining admin singletons must still register");
+    }
+
+    /// <summary>
+    /// The re-home target: a driver node carrying a <c>cluster-{ClusterId}</c> role registers the
+    /// redundancy singleton (proxy) via the new extension.
+    /// </summary>
+    [Fact]
+    public async Task Cluster_redundancy_extension_registers_the_singleton_on_a_driver_node()
+    {
+        var registry = await BootAndGetRegistryAsync(
+            roles: new[] { "driver", "cluster-SITE-A" },
+            configure: ab => ab.WithOtOpcUaClusterRedundancySingleton(
+                new FakeClusterRoleInfo(clusterRole: "cluster-SITE-A", clusterId: "SITE-A")));
+
+        registry.TryGet<RedundancyStateActorKey>(out _).ShouldBeTrue(
+            "every driver node hosts its own mesh's redundancy singleton (Phase 6)");
+    }
+
+    /// <summary>
+    /// Boots a real host through the production cluster bootstrap (Port=0, no seeds, so it never forms
+    /// a cluster and no singleton props factory runs — only the proxies register), applies the
+    /// supplied singleton registration, and returns the <see cref="ActorRegistry"/> after teardown.
+    /// Start/stop live here rather than the test bodies to keep the CancellationToken-less calls out
+    /// of the way (mirrors <c>ClusterRoleInfoTests.ResolveAsync</c>).
+    /// </summary>
+    private static async Task<ActorRegistry> BootAndGetRegistryAsync(
+        string[] roles, Action<AkkaConfigurationBuilder> configure)
+    {
+        var options = new AkkaClusterOptions
+        {
+            Port = 0,
+            Hostname = "127.0.0.1",
+            PublicHostname = "127.0.0.1",
+            SeedNodes = Array.Empty<string>(),
+            Roles = roles,
+        };
+
+        var builder = Host.CreateDefaultBuilder();
+        builder.ConfigureServices(services =>
+        {
+            services.AddSingleton<IOptions<AkkaClusterOptions>>(Options.Create(options));
+            // The admin WithActors block reads IOptions<MeshTransportOptions>.Value at spawn time.
+            services.AddSingleton<IOptions<MeshTransportOptions>>(Options.Create(new MeshTransportOptions()));
+            services.AddAkka("otopcua-rehome-test", (ab, sp) =>
+            {
+                ab.WithOtOpcUaClusterBootstrap(sp);
+                configure(ab);
+            });
+        });
+
+        using var host = builder.Build();
+        await host.StartAsync();
+        try
+        {
+            return host.Services.GetRequiredService<ActorRegistry>();
+        }
+        finally
+        {
+            await host.StopAsync();
+        }
+    }
+
+    /// <summary>Minimal <see cref="IClusterRoleInfo"/> stub exposing only the cluster-role identity.</summary>
+    private sealed class FakeClusterRoleInfo : IClusterRoleInfo
+    {
+        public FakeClusterRoleInfo(string? clusterRole, string? clusterId)
+        {
+            ClusterRole = clusterRole;
+            ClusterId = clusterId;
+        }
+
+        public string? ClusterRole { get; }
+        public string? ClusterId { get; }
+        public NodeId LocalNode => NodeId.Parse("127.0.0.1:0");
+        public IReadOnlySet<string> LocalRoles => new HashSet<string>();
+        public bool HasRole(string role) => false;
+        public IReadOnlyList<NodeId> MembersWithRole(string role) => Array.Empty<NodeId>();
+        public NodeId? RoleLeader(string role) => null;
+        public event EventHandler<RoleLeaderChangedEventArgs>? RoleLeaderChanged { add { } remove { } }
+    }
+}