diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index 86e1561b..149e03dd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; @@ -130,10 +131,12 @@ public static class ServiceCollectionExtensions (system, registry, resolver) => FleetStatusBroadcaster.Props(), singletonOptions); - builder.WithSingleton( - RedundancyStateSingletonName, - (system, registry, resolver) => RedundancyStateActor.Props(), - singletonOptions); + // Per-cluster mesh Phase 6: the redundancy-state singleton is NO LONGER registered here. + // It was an admin-scoped, fleet-wide singleton — one Primary elected across the whole mesh. + // After the mesh split a driver-only site pair has no admin node to host it, so it moved to + // WithOtOpcUaClusterRedundancySingleton, a cluster-{ClusterId}-scoped singleton spawned on + // EVERY driver node (the fused central included). One per mesh, electing its own pair-local + // Primary and publishing on its own mesh's DistributedPubSub. // Per-cluster mesh Phase 1: ClusterNode.AkkaPort duplicates the node's own Cluster:Port and // nothing makes them agree. Runs here rather than on each driver node because Phase 4 removes @@ -157,6 +160,61 @@ public static class ServiceCollectionExtensions return builder; } + + /// + /// Registers as a cluster singleton scoped to the local node's + /// OWN cluster-{ClusterId} role (or the fixed driver role when the node carries no + /// cluster role), and spawns it on EVERY driver node (per-cluster mesh, Phase 6). Wire from the + /// driver branch of the fused Host's Program.cs — the fused central node is hasDriver too, + /// so it hosts exactly one redundancy singleton scoped to its own cluster role (e.g. + /// cluster-MAIN), while the admin singleton set it also registers no longer carries + /// redundancy. + /// + /// The Akka configuration builder. + /// The local node's cluster-role view (Task 1's ClusterRoleInfo). + /// The builder for fluent chaining. + /// + /// After the mesh split, central's mesh sees only its own pair — a driver-only site pair therefore + /// has NO admin node to elect its Primary. Scoping this singleton to the node's cluster role and + /// spawning it on every driver node gives each mesh exactly one, electing its own pair-local + /// Primary and publishing redundancy-state on its own mesh's DistributedPubSub. + /// + public static AkkaConfigurationBuilder WithOtOpcUaClusterRedundancySingleton( + this AkkaConfigurationBuilder builder, IClusterRoleInfo roleInfo) + { + var singletonOptions = BuildClusterRedundancySingletonOptions(roleInfo); + + builder.WithSingleton( + RedundancyStateSingletonName, + (system, registry, resolver) => RedundancyStateActor.Props(), + singletonOptions); + + return builder; + } + + /// + /// Builds the that scope the redundancy-state singleton to + /// the local node's cluster-{ClusterId} role, falling back to the fixed driver role + /// when the node carries no cluster role. Extracted (like + /// SplitBrainResolverActivationTests's BuildDowningHocon) so the exact role scope is + /// unit-assertable without booting a cluster. + /// + /// + /// Cluster-scope is the robust choice — it survives an accidental two-mesh merge by keeping one + /// singleton per cluster. The driver-role fallback deliberately does NOT throw when a node + /// has no cluster role, because (a) that is the pre-Phase-6 fleet-wide behavior on a legacy single + /// mesh, keeping legacy/harness nodes and any not-yet-migrated deployment booting unchanged, and + /// (b) on a genuinely split 2-node mesh the driver role is already pair-local — the mesh IS + /// the pair. So the fallback is correct in both worlds and costs no availability. + /// + /// The local node's cluster-role view. + /// Singleton options scoped to the node's cluster role, or the driver role. + public static ClusterSingletonOptions BuildClusterRedundancySingletonOptions(IClusterRoleInfo roleInfo) + { + ArgumentNullException.ThrowIfNull(roleInfo); + + return new ClusterSingletonOptions { Role = roleInfo.ClusterRole ?? RoleParser.Driver }; + } } /// Marker key types used by Akka.Hosting to resolve singletons from the registry. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 8853dca4..b6bf8b31 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -12,6 +12,7 @@ using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc; using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs; using ZB.MOM.WW.OtOpcUa.AdminUI.ScriptAnalysis; using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Services; using ZB.MOM.WW.OtOpcUa.ControlPlane; @@ -371,7 +372,17 @@ builder.Services.AddAkka("otopcua", (ab, sp) => ab.WithOtOpcUaSignalRBridges(); } if (hasDriver) + { ab.WithOtOpcUaRuntimeActors(); + // Per-cluster mesh Phase 6: re-home the redundancy-state singleton off the admin node onto + // EVERY driver node, scoped to the node's own cluster-{ClusterId} role, so each mesh elects + // its own pair-local Primary (a driver-only site pair has no admin node to do it). Runs on + // the fused central node too (it is hasDriver), where it scopes to cluster-MAIN — exactly one + // redundancy singleton per node, since the admin branch above no longer registers one. + // IClusterRoleInfo is registered by AddOtOpcUaCluster (resolved lazily here); it throws at + // host start if this driver node carries no cluster-{ClusterId} role. + ab.WithOtOpcUaClusterRedundancySingleton(sp.GetRequiredService()); + } }); // Down-if-alone recovery watchdog (#459). Registered AFTER AddAkka so it starts after Akka's own diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/RedundancyStateSingletonRehomeTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/RedundancyStateSingletonRehomeTests.cs new file mode 100644 index 00000000..9f447776 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/RedundancyStateSingletonRehomeTests.cs @@ -0,0 +1,152 @@ +using Akka.Actor; +using Akka.Cluster.Hosting; +using Akka.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; +using ZB.MOM.WW.OtOpcUa.Commons.Types; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Guards Phase 6's re-homing of RedundancyStateActor off the fleet-wide admin singleton onto +/// a per-node cluster-{ClusterId} singleton (see +/// ). After the mesh +/// split a driver-only site pair has NO admin node to elect its Primary, so the singleton must be +/// scoped to the node's own cluster role and spawned on every driver node — one per mesh. +/// +/// The role scope + the missing-role guard are asserted against the extracted +/// helper directly — +/// the same pattern SplitBrainResolverActivationTests uses for BuildDowningHocon / +/// BuildClusterOptions: it pins the exact without +/// booting a cluster, which no builder-introspection API can otherwise reach. The admin-removal claim +/// is behavioral (a booted admin node's must no longer carry the +/// redundancy key) so it needs a real host. +/// +public sealed class RedundancyStateSingletonRehomeTests +{ + /// The singleton options scope to the node's OWN cluster role, not the fixed admin role. + [Fact] + public void Options_scope_the_singleton_to_the_nodes_cluster_role() + { + var options = ServiceCollectionExtensions.BuildClusterRedundancySingletonOptions( + new FakeClusterRoleInfo(clusterRole: "cluster-SITE-A", clusterId: "SITE-A")); + + options.Role.ShouldBe("cluster-SITE-A"); + } + + /// + /// A node with no cluster-{ClusterId} role (a legacy single-mesh node, or the + /// TwoNodeClusterHarness with roles admin,driver) falls back to the fixed + /// driver role rather than throwing — so legacy/not-yet-migrated deployments keep booting. + /// On a split 2-node mesh the driver role is already pair-local (the mesh IS the pair). + /// + [Fact] + public void Missing_cluster_role_falls_back_to_the_driver_role() + { + var options = ServiceCollectionExtensions.BuildClusterRedundancySingletonOptions( + new FakeClusterRoleInfo(clusterRole: null, clusterId: null)); + + options.Role.ShouldBe(RoleParser.Driver); + } + + /// + /// The admin singleton set no longer registers redundancy-state: a booted admin node's + /// registry carries the OTHER admin singletons (proxy) but NOT the redundancy key. + /// + [Fact] + public async Task Admin_singletons_no_longer_register_the_redundancy_singleton() + { + var registry = await BootAndGetRegistryAsync( + roles: new[] { "admin" }, + configure: ab => ab.WithOtOpcUaControlPlaneSingletons()); + + registry.TryGet(out _).ShouldBeFalse( + "the redundancy singleton was re-homed off the admin node onto every driver node (Phase 6)"); + registry.TryGet(out _).ShouldBeTrue( + "the remaining admin singletons must still register"); + } + + /// + /// The re-home target: a driver node carrying a cluster-{ClusterId} role registers the + /// redundancy singleton (proxy) via the new extension. + /// + [Fact] + public async Task Cluster_redundancy_extension_registers_the_singleton_on_a_driver_node() + { + var registry = await BootAndGetRegistryAsync( + roles: new[] { "driver", "cluster-SITE-A" }, + configure: ab => ab.WithOtOpcUaClusterRedundancySingleton( + new FakeClusterRoleInfo(clusterRole: "cluster-SITE-A", clusterId: "SITE-A"))); + + registry.TryGet(out _).ShouldBeTrue( + "every driver node hosts its own mesh's redundancy singleton (Phase 6)"); + } + + /// + /// Boots a real host through the production cluster bootstrap (Port=0, no seeds, so it never forms + /// a cluster and no singleton props factory runs — only the proxies register), applies the + /// supplied singleton registration, and returns the after teardown. + /// Start/stop live here rather than the test bodies to keep the CancellationToken-less calls out + /// of the way (mirrors ClusterRoleInfoTests.ResolveAsync). + /// + private static async Task BootAndGetRegistryAsync( + string[] roles, Action configure) + { + var options = new AkkaClusterOptions + { + Port = 0, + Hostname = "127.0.0.1", + PublicHostname = "127.0.0.1", + SeedNodes = Array.Empty(), + Roles = roles, + }; + + var builder = Host.CreateDefaultBuilder(); + builder.ConfigureServices(services => + { + services.AddSingleton>(Options.Create(options)); + // The admin WithActors block reads IOptions.Value at spawn time. + services.AddSingleton>(Options.Create(new MeshTransportOptions())); + services.AddAkka("otopcua-rehome-test", (ab, sp) => + { + ab.WithOtOpcUaClusterBootstrap(sp); + configure(ab); + }); + }); + + using var host = builder.Build(); + await host.StartAsync(); + try + { + return host.Services.GetRequiredService(); + } + finally + { + await host.StopAsync(); + } + } + + /// Minimal stub exposing only the cluster-role identity. + private sealed class FakeClusterRoleInfo : IClusterRoleInfo + { + public FakeClusterRoleInfo(string? clusterRole, string? clusterId) + { + ClusterRole = clusterRole; + ClusterId = clusterId; + } + + public string? ClusterRole { get; } + public string? ClusterId { get; } + public NodeId LocalNode => NodeId.Parse("127.0.0.1:0"); + public IReadOnlySet LocalRoles => new HashSet(); + public bool HasRole(string role) => false; + public IReadOnlyList MembersWithRole(string role) => Array.Empty(); + public NodeId? RoleLeader(string role) => null; + public event EventHandler? RoleLeaderChanged { add { } remove { } } + } +}