using Akka.Cluster.Hosting;
using Akka.Cluster.Hosting.SBR;
using Akka.Event;
using Akka.Hosting;
using Akka.Logger.Serilog;
using Akka.Remote.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Configuration;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
namespace ZB.MOM.WW.OtOpcUa.Cluster;
public static class ServiceCollectionExtensions
{
///
/// Binds and registers . The
/// actual ActorSystem + cluster bootstrap is layered on inside the host's AddAkka(...)
/// configurator via — keeping the entire Akka graph
/// under Akka.Hosting's management so cluster singletons land on the same ActorSystem.
///
///
/// The binding is validated at startup (ValidateOnStart) by
/// , so a seed list that cannot bootstrap this node —
/// self listed behind its partner — fails the host loudly instead of leaving it running but
/// permanently outside the cluster.
///
/// The service collection to configure.
/// The application configuration containing cluster options.
/// The same service collection, for chaining.
public static IServiceCollection AddOtOpcUaCluster(this IServiceCollection services, IConfiguration configuration)
{
services.AddValidatedOptions(
configuration, AkkaClusterOptions.SectionName);
// Structural identity: EffectiveRoles is resolved ONCE and registered as a
// singleton, off the SAME IConfiguration this call received — not off a DI-resolved
// IConfiguration, which a plain ServiceCollection (as opposed to a WebApplicationBuilder) never
// registers. Every consumer (Program.cs's hasAdmin/hasDriver, the four role-scoped validators,
// AkkaClusterOptions.Roles below) therefore reads the exact same instance, so there is no second
// derivation left to disagree with — the boot-guard OVERALL asked for is this identity, not a
// runtime assert (see EffectiveRoles + AkkaClusterOptionsValidator's post-overlay guard).
// TryAddSingleton keeps this usable from a harness that registers its own EffectiveRoles first.
services.TryAddSingleton(_ => EffectiveRoles.Resolve(configuration));
// Review finding (2026-08-20): AkkaClusterOptionsValidator's post-overlay guard needs to
// re-read Cluster:Roles LIVE, off the current (possibly overlay-mutated) configuration — not
// off the pre-overlay EffectiveRoles snapshot above, and not off the bound AkkaClusterOptions
// (the PostConfigure two lines below makes that equal to the snapshot BY CONSTRUCTION, so
// comparing the guard against it could never observe a divergence). A bare ServiceCollection
// (unlike WebApplicationBuilder, which auto-registers IConfiguration) never registers this
// otherwise; TryAddSingleton is a no-op on a real host, where it is already registered.
services.TryAddSingleton(configuration);
// Apply the OTOPCUA_ROLES fallback the Roles property has
// always DOCUMENTED, in the one place every consumer reads through. Before this, a node
// configured only through the env var — the knob that drives all of Program.cs's role wiring —
// bound an EMPTY Roles array and so joined the cluster role-less: invisible to
// RoleLeader/MembersWithRole, ineligible for every role-scoped singleton, and un-deployable
// ("no driver-role nodes are Up") with nothing in its own logs to say so. Doing it here rather
// than at each call site means BuildClusterOptions (Akka's own member roles), ClusterRoleInfo,
// the ControlPlane singleton scope and ConfigDbApiKeyStore's ClusterId all see the same answer.
//
// PostConfigure, not a bind trick: it runs after the section binds and before ValidateOnStart,
// so the validators below see the effective list. Now consumes the registered EffectiveRoles
// instance rather than re-deriving from options.Roles + the environment — options.Roles becomes
// BY CONSTRUCTION the same list Program.cs's hasAdmin/hasDriver are computed from.
services.AddOptions().PostConfigure(
(options, roles) => options.Roles = [.. roles.Gossiped]);
// Per-cluster mesh Phase 2: which transport carries central→node commands. Validated at
// startup for the same reason the cluster options are — a contact point that cannot resolve
// produces silence, not an error, and silence is indistinguishable from "nothing to do".
services.AddValidatedOptions(
configuration, MeshTransportOptions.SectionName);
// Per-cluster mesh Phase 3: where a driver reads its config (Direct SQL, or fetch-and-cache
// over gRPC). Validated at startup for the same reason — a FetchAndCache node with no endpoint
// or no key produces silence, not an error. ConfigServe (the central serve surface) needs no
// cross-field validation: a 0 port is a legitimate "disabled", so a plain Configure suffices.
services.AddValidatedOptions(
configuration, ConfigSourceOptions.SectionName);
services.Configure(configuration.GetSection(ConfigServeOptions.SectionName));
// Per-cluster mesh Phase 5: which transport carries a node's live-telemetry stream (node
// serve side + central dial side). Validated at startup for the same reason ConfigSource is —
// a Grpc-mode node with no listen port or no key produces silence, not an error.
services.AddValidatedOptions(
configuration, TelemetryOptions.SectionName);
services.AddValidatedOptions(
configuration, TelemetryDialOptions.SectionName);
// Per-cluster mesh Phase 6: a node carrying a cluster-{ClusterId} role (the split topology)
// must use the mesh-crossing transports — the DPS dark-switch branches cannot cross a split
// mesh and would deliver nothing silently. Hung on MeshTransportOptions (which already has
// ValidateOnStart from its own registration above) so it fires at boot alongside the sibling
// validators; TryAddEnumerable inside AddValidatedOptions keeps the second validator additive.
services.AddValidatedOptions(
configuration, MeshTransportOptions.SectionName);
services.AddSingleton();
return services;
}
///
/// Configures the Akka.Hosting builder with the embedded OtOpcUa HOCON (split-brain resolver,
/// pinned dispatcher, failure detector tuning) + remote endpoint + cluster bootstrap derived
/// from .
///
/// Wire from Program.cs:
///
/// services.AddAkka("otopcua", (ab, sp) =>
/// {
/// ab.WithOtOpcUaClusterBootstrap(sp);
/// if (hasAdmin) ab.WithOtOpcUaControlPlaneSingletons();
/// if (hasDriver) ab.WithOtOpcUaRuntimeActors();
/// });
///
///
/// The Akka configuration builder to configure.
/// The service provider for resolving cluster options.
/// The same builder, for chaining.
public static AkkaConfigurationBuilder WithOtOpcUaClusterBootstrap(
this AkkaConfigurationBuilder builder,
IServiceProvider serviceProvider)
{
var options = serviceProvider.GetRequiredService>().Value;
builder.AddHocon(HoconLoader.LoadBaseConfig(), HoconAddMode.Append);
// Route Akka's internal ILoggingAdapter (DriverHostActor, DriverInstanceActor, cluster
// events, …) into Serilog so those logs reach the same sinks as the MEL/Serilog application
// logs. Akka.Hosting owns logger setup, so HOCON `akka.loggers` alone is not honored — the
// logger must be registered through ConfigureLoggers. Without this the actor graph logs only
// to the default StandardOutLogger (discarded under the Windows service host), which is why
// the driver-role actors were invisible during the 2026-06 data-plane investigation.
builder.ConfigureLoggers(setup =>
{
setup.LogLevel = LogLevel.DebugLevel;
setup.ClearLoggers();
setup.AddLogger();
});
builder.WithRemoting(new RemoteOptions
{
HostName = options.Hostname,
Port = options.Port,
PublicHostName = options.PublicHostname,
});
builder.WithClustering(BuildClusterOptions(options));
// Must come AFTER WithClustering, which always emits a downing-provider-class of its own:
// Akka.Cluster.Hosting applies SplitBrainResolverOption.Default whenever the typed
// ClusterOptions.SplitBrainResolver is null, so there is always a competing value to beat.
// Prepend is the highest-precedence mode, which makes this block win irrespective of how
// many fragments are added or in what order.
//
// Do not take this comment's word for it. Which fragment actually wins is not obvious from
// the mode names — measured, Append also wins here, purely because it happens to be added
// last. SplitBrainResolverActivationTests reads the provider back off a *running*
// ActorSystem for exactly that reason: the effective value is the only one worth asserting.
builder.AddHocon(BuildDowningHocon(options), HoconAddMode.Prepend);
// NOTE (2026-07-22): no bootstrap watchdog is armed here any more. The cold-start-alone gap
// it existed for — Akka lets only seed-nodes[0] form a NEW cluster — is now closed by
// self-first seed ordering (AkkaClusterOptions.SeedNodes), enforced at boot by
// AkkaClusterOptionsValidator. The retired ClusterBootstrapFallback timer sat OUTSIDE Akka's
// join handshake, so it could not distinguish "no seed answered" from "a seed answered and
// the join is in flight", and Cluster.Join(SelfAddress) is not ignored mid-handshake — it
// wins. See docs/Redundancy.md → "Bootstrap: self-first seed ordering".
return builder;
}
///
/// How long a member must stay unreachable before it is downed. Shared by both strategies
/// (auto-down-unreachable-after and the SBR resolver's stable-after) so the two
/// have the same failover latency. Must stay above
/// akka.cluster.failure-detector.acceptable-heartbeat-pause or a merely-slow node gets
/// downed; both constraints are pinned by tests.
///
public static readonly TimeSpan DowningStableAfter = TimeSpan.FromSeconds(15);
///
/// Builds the HOCON that selects the downing provider, per
/// .
///
/// The bound cluster options carrying the strategy.
///
/// For auto-down, a block installing Akka's AutoDowning provider with a downing
/// window of . For keep-oldest, an empty string — the
/// typed from already installs
/// the SBR provider, and the resolver's own settings live in Resources/akka.conf.
///
/// The strategy is not a recognised value.
public static string BuildDowningHocon(AkkaClusterOptions options)
{
ArgumentNullException.ThrowIfNull(options);
if (IsAutoDown(options))
{
// auto-down-unreachable-after is load-bearing: it defaults to `off`, under which
// AutoDowning is installed but never downs anything — failover silently disabled.
return $$"""
akka.cluster {
downing-provider-class = "Akka.Cluster.AutoDowning, Akka.Cluster"
auto-down-unreachable-after = {{(int)DowningStableAfter.TotalMilliseconds}}ms
}
""";
}
if (IsKeepOldest(options))
{
return string.Empty;
}
throw new ArgumentOutOfRangeException(
nameof(options),
options.SplitBrainResolverStrategy,
$"Unknown {AkkaClusterOptions.SectionName}:{nameof(AkkaClusterOptions.SplitBrainResolverStrategy)}. "
+ "Expected 'auto-down' (default; survives a crash of either node) or 'keep-oldest' "
+ "(partition-safe, but a two-node pair cannot survive a crash of the oldest node).");
}
private static bool IsAutoDown(AkkaClusterOptions options) =>
string.Equals(options.SplitBrainResolverStrategy, "auto-down", StringComparison.OrdinalIgnoreCase);
private static bool IsKeepOldest(AkkaClusterOptions options) =>
string.Equals(options.SplitBrainResolverStrategy, "keep-oldest", StringComparison.OrdinalIgnoreCase);
///
/// Builds the for the fused-host cluster.
///
///
///
/// Activation note (arch-review 03/S1, corrected twice). Akka.Cluster.Hosting's
/// WithClustering always installs a downing provider: when
/// is null it applies
/// SplitBrainResolverOption.Default, which registers
/// Akka.Cluster.SBR.SplitBrainResolverProvider and reads the
/// split-brain-resolver block in Resources/akka.conf. The cluster is
/// therefore never NoDowning. What this method chooses is only which
/// provider — and under the default auto-down strategy the choice is not made
/// here at all but by the Prepended HOCON in
/// , which outranks whatever
/// WithClustering emits.
///
///
/// Why keep-oldest is no longer the default (2026-07-21). The previous
/// documentation here asserted that with
/// DownIfAlone=true was "the correct strategy for a 2-node warm-redundancy pair"
/// because down-if-alone would down a node that lost its peer. That is the
/// opposite of what Akka.NET does. In KeepOldest.OldestDecision the
/// down-if-alone branch requires the surviving side to hold >= 2 members, so
/// in a 1-vs-1 split the survivor falls through to DownReachable and downs
/// itself; with run-coordinated-shutdown-when-down = on it then exits. A
/// two-node pair running keep-oldest converts a crash of the oldest node into a total
/// outage — precisely the failure redundancy exists to absorb. Live-proven on the sister
/// project's rig; see docs/Redundancy.md and
/// for the trade.
///
///
/// The bound cluster options carrying seed nodes, roles and the strategy.
///
/// The cluster options with seed nodes and roles.
/// is set only under keep-oldest, where the SBR provider is the one wanted; under
/// auto-down it is left null because the Prepended HOCON replaces the provider anyway,
/// and naming a resolver that is not in force would be misleading.
///
public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options)
{
ArgumentNullException.ThrowIfNull(options);
return new ClusterOptions
{
// When the bootstrap guard is on, DO NOT hand Akka the seed list: Akka.Cluster.Hosting
// auto-joins from ClusterOptions.SeedNodes the instant the ActorSystem starts, which is
// exactly the self-first race the guard exists to arbitrate. Left empty, the node starts
// unjoined and ClusterBootstrapCoordinator picks the order (founder self-first / joiner
// peer-first, after a reachability probe) and issues the single Cluster.JoinSeedNodes.
// The config seed list is still read — by the coordinator, off AkkaClusterOptions.SeedNodes.
SeedNodes = options.BootstrapGuard.Enabled ? Array.Empty() : options.SeedNodes,
Roles = options.Roles,
SplitBrainResolver = IsKeepOldest(options) ? new KeepOldestOption { DownIfAlone = true } : null,
};
}
}