using Akka.Actor;
using Akka.Cluster.Hosting;
using Akka.Cluster.Tools.Client;
using Akka.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane;
public static class ServiceCollectionExtensions
{
public const string AdminRole = "admin";
public const string ConfigPublishSingletonName = "config-publish";
public const string AdminOperationsSingletonName = "admin-operations";
public const string AuditWriterSingletonName = "audit-writer";
public const string FleetStatusSingletonName = "fleet-status";
public const string RedundancyStateSingletonName = "redundancy-state";
public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler";
///
/// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — five with
/// registry proxies, plus the address reconciler, which is proxy-less because nothing addresses it.
/// Must be called against the same builder used by AkkaHostedService so the singletons
/// share the host's ActorSystem.
///
/// The Akka configuration builder.
/// The builder for fluent chaining.
///
/// Wire from the fused Host's Program.cs:
///
/// builder.Services.AddAkka("otopcua", (ab, sp) =>
/// {
/// ab.WithRemoting(/* ... */).WithClustering(/* ... */);
/// ab.WithOtOpcUaControlPlaneSingletons();
/// });
///
///
public static AkkaConfigurationBuilder WithOtOpcUaControlPlaneSingletons(this AkkaConfigurationBuilder builder)
{
var singletonOptions = new ClusterSingletonOptions { Role = AdminRole };
var proxyOptions = new ClusterSingletonOptions { Role = AdminRole };
// Per-cluster mesh Phase 2: the central end of the central↔node command boundary.
//
// A plain WithActors, NOT a singleton — and that is the load-bearing part. A driver node's
// ClusterClient rotates across contact points and must find a live comm actor at whichever
// central node answers. As a singleton it would exist on one central node only, and the
// rotation would fail silently against the other: acks would land sometimes, depending on
// which contact the client happened to settle on.
builder.WithActors((system, registry, resolver) =>
{
var dbFactory = resolver.GetService>();
var options = resolver.GetService>().Value;
var actor = system.ActorOf(
CentralCommunicationActor.Props(
dbFactory,
options,
new DefaultMeshClusterClientFactory(),
// Resolved lazily, per message, so this actor never depends on the order in
// which Akka.Hosting materialises the singleton proxy relative to this block.
() => registry.TryGet(out var coordinator)
? coordinator
: null),
MeshPaths.CentralCommunicationName);
// Registered unconditionally, in both transport modes. Under Dps nothing dials it, and
// an idle registration costs nothing — whereas registering only under ClusterClient
// would mean flipping the flag on a running fleet requires a central restart before any
// node can ack, turning a config change back into a deployment.
ClusterClientReceptionist.Get(system).RegisterService(actor);
registry.Register(actor);
});
builder.WithSingleton(
ConfigPublishSingletonName,
(system, registry, resolver) =>
{
var dbFactory = resolver.GetService>();
// Registered above, so it is already in the registry — the same
// ordering-by-registration the AdminOperations singleton relies on to resolve the
// coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks
// inside a props factory and races the spawn it is waiting for.
var meshRouter = registry.Get();
return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter);
},
singletonOptions);
builder.WithSingleton(
AdminOperationsSingletonName,
(system, registry, resolver) =>
{
var dbFactory = resolver.GetService>();
var coordinator = registry.Get();
var probes = resolver.GetService>() ?? Enumerable.Empty();
// R2-11 (05/CONV-2): deploy-gate TagConfig strictness mode from appsettings
// (Deployment:TagConfigValidationMode = Warn default | Error opt-in).
var mode = Enum.TryParse(
resolver.GetService()?["Deployment:TagConfigValidationMode"],
ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn;
var meshRouter = registry.Get();
return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter);
},
singletonOptions);
builder.WithSingleton(
AuditWriterSingletonName,
(system, registry, resolver) =>
{
var dbFactory = resolver.GetService>();
return AuditWriterActor.Props(dbFactory);
},
singletonOptions);
builder.WithSingleton(
FleetStatusSingletonName,
(system, registry, resolver) => FleetStatusBroadcaster.Props(),
singletonOptions);
// Per-cluster mesh Phase 6: the redundancy-state singleton is NO LONGER registered here.
// It was an admin-scoped, fleet-wide singleton — one Primary elected across the whole mesh.
// After the mesh split a driver-only site pair has no admin node to host it, so it moved to
// WithOtOpcUaClusterRedundancySingleton, a cluster-{ClusterId}-scoped singleton spawned on
// EVERY driver node (the fused central included). One per mesh, electing its own pair-local
// Primary and publishing on its own mesh's DistributedPubSub.
// Per-cluster mesh Phase 1: ClusterNode.AkkaPort duplicates the node's own Cluster:Port and
// nothing makes them agree. Runs here rather than on each driver node because Phase 4 removes
// the driver nodes' ConfigDb connection entirely.
//
// createProxyToo: false — unlike the five singletons above, nothing ever resolves
// ClusterNodeAddressReconcilerKey from the registry. This actor only reads cluster state and
// logs; no one sends it messages. A proxy would be a second actor per node whose only job is
// to hunt for a singleton nobody addresses — visible as the "ClusterSingletonProxy failed to
// find an associated singleton after 30 seconds" startup warning, and as extra cluster
// chatter in every two-node test harness the integration suite builds.
builder.WithSingleton(
ClusterNodeAddressReconcilerSingletonName,
(system, registry, resolver) =>
{
var dbFactory = resolver.GetService>();
// Per-cluster mesh Phase 6: scope BOTH halves of the reconcile to the admin node's OWN
// cluster — rows by ClusterId, observed membership by cluster role. After the mesh split
// central sees only its own pair via gossip, so an unscoped sweep would false-report
// every other cluster's rows (EnabledRowNotInCluster). Scoping only the rows would swap
// that for a WORSE Error-level RunningNodeHasNoRow in the mixed-mesh window (rows already
// filtered, but foreign members still visible via gossip). A legacy node with no cluster
// role has null ClusterId/ClusterRole ⇒ full-fleet reconcile, unchanged.
var roleInfo = resolver.GetService();
return ClusterNodeAddressReconcilerActor.Props(
dbFactory, roleInfo.ClusterId, roleInfo.ClusterRole);
},
singletonOptions,
createProxyToo: false);
return builder;
}
///
/// Registers as a cluster singleton scoped to the local node's
/// OWN cluster-{ClusterId} role (or the fixed driver role when the node carries no
/// cluster role), and spawns it on EVERY driver node (per-cluster mesh, Phase 6). Wire from the
/// driver branch of the fused Host's Program.cs — the fused central node is hasDriver too,
/// so it hosts exactly one redundancy singleton scoped to its own cluster role (e.g.
/// cluster-MAIN), while the admin singleton set it also registers no longer carries
/// redundancy.
///
/// The Akka configuration builder.
/// The local node's cluster-role view (Task 1's ClusterRoleInfo).
/// The builder for fluent chaining.
///
/// After the mesh split, central's mesh sees only its own pair — a driver-only site pair therefore
/// has NO admin node to elect its Primary. Scoping this singleton to the node's cluster role and
/// spawning it on every driver node gives each mesh exactly one, electing its own pair-local
/// Primary and publishing redundancy-state on its own mesh's DistributedPubSub.
///
public static AkkaConfigurationBuilder WithOtOpcUaClusterRedundancySingleton(
this AkkaConfigurationBuilder builder, AkkaClusterOptions clusterOptions)
{
var singletonOptions = BuildClusterRedundancySingletonOptions(clusterOptions);
builder.WithSingleton(
RedundancyStateSingletonName,
(system, registry, resolver) => RedundancyStateActor.Props(),
singletonOptions);
return builder;
}
///
/// Builds the that scope the redundancy-state singleton to
/// the local node's cluster-{ClusterId} role, falling back to the fixed driver role
/// when the node carries no cluster role. Extracted (like
/// SplitBrainResolverActivationTests's BuildDowningHocon) so the exact role scope is
/// unit-assertable without booting a cluster.
///
///
/// Cluster-scope is the robust choice — it survives an accidental two-mesh merge by keeping one
/// singleton per cluster. The driver-role fallback deliberately does NOT throw when a node
/// has no cluster role, because (a) that is the pre-Phase-6 fleet-wide behavior on a legacy single
/// mesh, keeping legacy/harness nodes and any not-yet-migrated deployment booting unchanged, and
/// (b) on a genuinely split 2-node mesh the driver role is already pair-local — the mesh IS
/// the pair. So the fallback is correct in both worlds and costs no availability.
///
/// The node's own cluster configuration (roles), read without the
/// ActorSystem so this is safe to call inside the Akka configurator lambda.
/// Singleton options scoped to the node's cluster role, or the driver role.
public static ClusterSingletonOptions BuildClusterRedundancySingletonOptions(AkkaClusterOptions clusterOptions)
{
ArgumentNullException.ThrowIfNull(clusterOptions);
// Derive the cluster-{ClusterId} scope from the node's OWN configured roles — NOT from
// IClusterRoleInfo. IClusterRoleInfo's implementation depends on the ActorSystem (it is a live
// Cluster.State view), and this runs inside the AddAkka configurator lambda WHILE the
// ActorSystem is being built; resolving IClusterRoleInfo there recurses back into ActorSystem
// construction and stack-overflows the host at boot (the Phase 6 live gate caught exactly this
// on the fused central node). AkkaClusterOptions is pure configuration, available before the
// cluster forms — the same source ClusterRoleInfo.ClusterRole itself derives from — so the
// scope is identical. First cluster role wins, in configuration order (matching ClusterRoleInfo).
var clusterRole = clusterOptions.Roles.FirstOrDefault(RoleParser.IsClusterRole);
return new ClusterSingletonOptions { Role = clusterRole ?? RoleParser.Driver };
}
}
/// Marker key types used by Akka.Hosting to resolve singletons from the registry.
public sealed class ConfigPublishCoordinatorKey { }
public sealed class AdminOperationsActorKey { }
public sealed class AuditWriterActorKey { }
public sealed class FleetStatusBroadcasterKey { }
public sealed class RedundancyStateActorKey { }
public sealed class ClusterNodeAddressReconcilerKey { }
public sealed class CentralCommunicationActorKey { }