using Akka.Actor; using Akka.Cluster.Hosting; using Akka.Cluster.Tools.Client; using Akka.Hosting; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane; public static class ServiceCollectionExtensions { public const string AdminRole = "admin"; public const string ConfigPublishSingletonName = "config-publish"; public const string AdminOperationsSingletonName = "admin-operations"; public const string AuditWriterSingletonName = "audit-writer"; public const string FleetStatusSingletonName = "fleet-status"; public const string RedundancyStateSingletonName = "redundancy-state"; public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler"; /// /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — five with /// registry proxies, plus the address reconciler, which is proxy-less because nothing addresses it. /// Must be called against the same builder used by AkkaHostedService so the singletons /// share the host's ActorSystem. /// /// The Akka configuration builder. /// The builder for fluent chaining. /// /// Wire from the fused Host's Program.cs: /// /// builder.Services.AddAkka("otopcua", (ab, sp) => /// { /// ab.WithRemoting(/* ... */).WithClustering(/* ... */); /// ab.WithOtOpcUaControlPlaneSingletons(); /// }); /// /// public static AkkaConfigurationBuilder WithOtOpcUaControlPlaneSingletons(this AkkaConfigurationBuilder builder) { var singletonOptions = new ClusterSingletonOptions { Role = AdminRole }; var proxyOptions = new ClusterSingletonOptions { Role = AdminRole }; // Per-cluster mesh Phase 2: the central end of the central↔node command boundary. // // A plain WithActors, NOT a singleton — and that is the load-bearing part. A driver node's // ClusterClient rotates across contact points and must find a live comm actor at whichever // central node answers. As a singleton it would exist on one central node only, and the // rotation would fail silently against the other: acks would land sometimes, depending on // which contact the client happened to settle on. builder.WithActors((system, registry, resolver) => { var dbFactory = resolver.GetService>(); var options = resolver.GetService>().Value; var actor = system.ActorOf( CentralCommunicationActor.Props( dbFactory, options, new DefaultMeshClusterClientFactory(), // Resolved lazily, per message, so this actor never depends on the order in // which Akka.Hosting materialises the singleton proxy relative to this block. () => registry.TryGet(out var coordinator) ? coordinator : null), MeshPaths.CentralCommunicationName); // Registered unconditionally, in both transport modes. Under Dps nothing dials it, and // an idle registration costs nothing — whereas registering only under ClusterClient // would mean flipping the flag on a running fleet requires a central restart before any // node can ack, turning a config change back into a deployment. ClusterClientReceptionist.Get(system).RegisterService(actor); registry.Register(actor); }); builder.WithSingleton( ConfigPublishSingletonName, (system, registry, resolver) => { var dbFactory = resolver.GetService>(); // Registered above, so it is already in the registry — the same // ordering-by-registration the AdminOperations singleton relies on to resolve the // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks // inside a props factory and races the spawn it is waiting for. var meshRouter = registry.Get(); return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter); }, singletonOptions); builder.WithSingleton( AdminOperationsSingletonName, (system, registry, resolver) => { var dbFactory = resolver.GetService>(); var coordinator = registry.Get(); var probes = resolver.GetService>() ?? Enumerable.Empty(); // R2-11 (05/CONV-2): deploy-gate TagConfig strictness mode from appsettings // (Deployment:TagConfigValidationMode = Warn default | Error opt-in). var mode = Enum.TryParse( resolver.GetService()?["Deployment:TagConfigValidationMode"], ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn; var meshRouter = registry.Get(); return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter); }, singletonOptions); builder.WithSingleton( AuditWriterSingletonName, (system, registry, resolver) => { var dbFactory = resolver.GetService>(); return AuditWriterActor.Props(dbFactory); }, singletonOptions); builder.WithSingleton( FleetStatusSingletonName, (system, registry, resolver) => FleetStatusBroadcaster.Props(), singletonOptions); // Per-cluster mesh Phase 6: the redundancy-state singleton is NO LONGER registered here. // It was an admin-scoped, fleet-wide singleton — one Primary elected across the whole mesh. // After the mesh split a driver-only site pair has no admin node to host it, so it moved to // WithOtOpcUaClusterRedundancySingleton, a cluster-{ClusterId}-scoped singleton spawned on // EVERY driver node (the fused central included). One per mesh, electing its own pair-local // Primary and publishing on its own mesh's DistributedPubSub. // Per-cluster mesh Phase 1: ClusterNode.AkkaPort duplicates the node's own Cluster:Port and // nothing makes them agree. Runs here rather than on each driver node because Phase 4 removes // the driver nodes' ConfigDb connection entirely. // // createProxyToo: false — unlike the five singletons above, nothing ever resolves // ClusterNodeAddressReconcilerKey from the registry. This actor only reads cluster state and // logs; no one sends it messages. A proxy would be a second actor per node whose only job is // to hunt for a singleton nobody addresses — visible as the "ClusterSingletonProxy failed to // find an associated singleton after 30 seconds" startup warning, and as extra cluster // chatter in every two-node test harness the integration suite builds. builder.WithSingleton( ClusterNodeAddressReconcilerSingletonName, (system, registry, resolver) => { var dbFactory = resolver.GetService>(); // Per-cluster mesh Phase 6: scope BOTH halves of the reconcile to the admin node's OWN // cluster — rows by ClusterId, observed membership by cluster role. After the mesh split // central sees only its own pair via gossip, so an unscoped sweep would false-report // every other cluster's rows (EnabledRowNotInCluster). Scoping only the rows would swap // that for a WORSE Error-level RunningNodeHasNoRow in the mixed-mesh window (rows already // filtered, but foreign members still visible via gossip). A legacy node with no cluster // role has null ClusterId/ClusterRole ⇒ full-fleet reconcile, unchanged. var roleInfo = resolver.GetService(); return ClusterNodeAddressReconcilerActor.Props( dbFactory, roleInfo.ClusterId, roleInfo.ClusterRole); }, singletonOptions, createProxyToo: false); return builder; } /// /// Registers as a cluster singleton scoped to the local node's /// OWN cluster-{ClusterId} role (or the fixed driver role when the node carries no /// cluster role), and spawns it on EVERY driver node (per-cluster mesh, Phase 6). Wire from the /// driver branch of the fused Host's Program.cs — the fused central node is hasDriver too, /// so it hosts exactly one redundancy singleton scoped to its own cluster role (e.g. /// cluster-MAIN), while the admin singleton set it also registers no longer carries /// redundancy. /// /// The Akka configuration builder. /// The local node's cluster-role view (Task 1's ClusterRoleInfo). /// The builder for fluent chaining. /// /// After the mesh split, central's mesh sees only its own pair — a driver-only site pair therefore /// has NO admin node to elect its Primary. Scoping this singleton to the node's cluster role and /// spawning it on every driver node gives each mesh exactly one, electing its own pair-local /// Primary and publishing redundancy-state on its own mesh's DistributedPubSub. /// public static AkkaConfigurationBuilder WithOtOpcUaClusterRedundancySingleton( this AkkaConfigurationBuilder builder, AkkaClusterOptions clusterOptions) { var singletonOptions = BuildClusterRedundancySingletonOptions(clusterOptions); builder.WithSingleton( RedundancyStateSingletonName, (system, registry, resolver) => RedundancyStateActor.Props(), singletonOptions); return builder; } /// /// Builds the that scope the redundancy-state singleton to /// the local node's cluster-{ClusterId} role, falling back to the fixed driver role /// when the node carries no cluster role. Extracted (like /// SplitBrainResolverActivationTests's BuildDowningHocon) so the exact role scope is /// unit-assertable without booting a cluster. /// /// /// Cluster-scope is the robust choice — it survives an accidental two-mesh merge by keeping one /// singleton per cluster. The driver-role fallback deliberately does NOT throw when a node /// has no cluster role, because (a) that is the pre-Phase-6 fleet-wide behavior on a legacy single /// mesh, keeping legacy/harness nodes and any not-yet-migrated deployment booting unchanged, and /// (b) on a genuinely split 2-node mesh the driver role is already pair-local — the mesh IS /// the pair. So the fallback is correct in both worlds and costs no availability. /// /// The node's own cluster configuration (roles), read without the /// ActorSystem so this is safe to call inside the Akka configurator lambda. /// Singleton options scoped to the node's cluster role, or the driver role. public static ClusterSingletonOptions BuildClusterRedundancySingletonOptions(AkkaClusterOptions clusterOptions) { ArgumentNullException.ThrowIfNull(clusterOptions); // Derive the cluster-{ClusterId} scope from the node's OWN configured roles — NOT from // IClusterRoleInfo. IClusterRoleInfo's implementation depends on the ActorSystem (it is a live // Cluster.State view), and this runs inside the AddAkka configurator lambda WHILE the // ActorSystem is being built; resolving IClusterRoleInfo there recurses back into ActorSystem // construction and stack-overflows the host at boot (the Phase 6 live gate caught exactly this // on the fused central node). AkkaClusterOptions is pure configuration, available before the // cluster forms — the same source ClusterRoleInfo.ClusterRole itself derives from — so the // scope is identical. First cluster role wins, in configuration order (matching ClusterRoleInfo). var clusterRole = clusterOptions.Roles.FirstOrDefault(RoleParser.IsClusterRole); return new ClusterSingletonOptions { Role = clusterRole ?? RoleParser.Driver }; } } /// Marker key types used by Akka.Hosting to resolve singletons from the registry. public sealed class ConfigPublishCoordinatorKey { } public sealed class AdminOperationsActorKey { } public sealed class AuditWriterActorKey { } public sealed class FleetStatusBroadcasterKey { } public sealed class RedundancyStateActorKey { } public sealed class ClusterNodeAddressReconcilerKey { } public sealed class CentralCommunicationActorKey { }