using Akka.Cluster.Hosting; using Akka.Cluster.Hosting.SBR; using Akka.Event; using Akka.Hosting; using Akka.Logger.Serilog; using Akka.Remote.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using ZB.MOM.WW.Configuration; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; namespace ZB.MOM.WW.OtOpcUa.Cluster; public static class ServiceCollectionExtensions { /// /// Binds and registers . The /// actual ActorSystem + cluster bootstrap is layered on inside the host's AddAkka(...) /// configurator via — keeping the entire Akka graph /// under Akka.Hosting's management so cluster singletons land on the same ActorSystem. /// /// /// The binding is validated at startup (ValidateOnStart) by /// , so a seed list that cannot bootstrap this node — /// self listed behind its partner — fails the host loudly instead of leaving it running but /// permanently outside the cluster. /// /// The service collection to configure. /// The application configuration containing cluster options. /// The same service collection, for chaining. public static IServiceCollection AddOtOpcUaCluster(this IServiceCollection services, IConfiguration configuration) { services.AddValidatedOptions( configuration, AkkaClusterOptions.SectionName); // Per-cluster mesh Phase 2: which transport carries central→node commands. Validated at // startup for the same reason the cluster options are — a contact point that cannot resolve // produces silence, not an error, and silence is indistinguishable from "nothing to do". services.AddValidatedOptions( configuration, MeshTransportOptions.SectionName); // Per-cluster mesh Phase 3: where a driver reads its config (Direct SQL, or fetch-and-cache // over gRPC). Validated at startup for the same reason — a FetchAndCache node with no endpoint // or no key produces silence, not an error. ConfigServe (the central serve surface) needs no // cross-field validation: a 0 port is a legitimate "disabled", so a plain Configure suffices. services.AddValidatedOptions( configuration, ConfigSourceOptions.SectionName); services.Configure(configuration.GetSection(ConfigServeOptions.SectionName)); // Per-cluster mesh Phase 5: which transport carries a node's live-telemetry stream (node // serve side + central dial side). Validated at startup for the same reason ConfigSource is — // a Grpc-mode node with no listen port or no key produces silence, not an error. services.AddValidatedOptions( configuration, TelemetryOptions.SectionName); services.AddValidatedOptions( configuration, TelemetryDialOptions.SectionName); services.AddSingleton(); return services; } /// /// Configures the Akka.Hosting builder with the embedded OtOpcUa HOCON (split-brain resolver, /// pinned dispatcher, failure detector tuning) + remote endpoint + cluster bootstrap derived /// from . /// /// Wire from Program.cs: /// /// services.AddAkka("otopcua", (ab, sp) => /// { /// ab.WithOtOpcUaClusterBootstrap(sp); /// if (hasAdmin) ab.WithOtOpcUaControlPlaneSingletons(); /// if (hasDriver) ab.WithOtOpcUaRuntimeActors(); /// }); /// /// /// The Akka configuration builder to configure. /// The service provider for resolving cluster options. /// The same builder, for chaining. public static AkkaConfigurationBuilder WithOtOpcUaClusterBootstrap( this AkkaConfigurationBuilder builder, IServiceProvider serviceProvider) { var options = serviceProvider.GetRequiredService>().Value; builder.AddHocon(HoconLoader.LoadBaseConfig(), HoconAddMode.Append); // Route Akka's internal ILoggingAdapter (DriverHostActor, DriverInstanceActor, cluster // events, …) into Serilog so those logs reach the same sinks as the MEL/Serilog application // logs. Akka.Hosting owns logger setup, so HOCON `akka.loggers` alone is not honored — the // logger must be registered through ConfigureLoggers. Without this the actor graph logs only // to the default StandardOutLogger (discarded under the Windows service host), which is why // the driver-role actors were invisible during the 2026-06 data-plane investigation. builder.ConfigureLoggers(setup => { setup.LogLevel = LogLevel.DebugLevel; setup.ClearLoggers(); setup.AddLogger(); }); builder.WithRemoting(new RemoteOptions { HostName = options.Hostname, Port = options.Port, PublicHostName = options.PublicHostname, }); builder.WithClustering(BuildClusterOptions(options)); // Must come AFTER WithClustering, which always emits a downing-provider-class of its own: // Akka.Cluster.Hosting applies SplitBrainResolverOption.Default whenever the typed // ClusterOptions.SplitBrainResolver is null, so there is always a competing value to beat. // Prepend is the highest-precedence mode, which makes this block win irrespective of how // many fragments are added or in what order. // // Do not take this comment's word for it. Which fragment actually wins is not obvious from // the mode names — measured, Append also wins here, purely because it happens to be added // last. SplitBrainResolverActivationTests reads the provider back off a *running* // ActorSystem for exactly that reason: the effective value is the only one worth asserting. builder.AddHocon(BuildDowningHocon(options), HoconAddMode.Prepend); // NOTE (2026-07-22): no bootstrap watchdog is armed here any more. The cold-start-alone gap // it existed for — Akka lets only seed-nodes[0] form a NEW cluster — is now closed by // self-first seed ordering (AkkaClusterOptions.SeedNodes), enforced at boot by // AkkaClusterOptionsValidator. The retired ClusterBootstrapFallback timer sat OUTSIDE Akka's // join handshake, so it could not distinguish "no seed answered" from "a seed answered and // the join is in flight", and Cluster.Join(SelfAddress) is not ignored mid-handshake — it // wins. See docs/Redundancy.md → "Bootstrap: self-first seed ordering". return builder; } /// /// How long a member must stay unreachable before it is downed. Shared by both strategies /// (auto-down-unreachable-after and the SBR resolver's stable-after) so the two /// have the same failover latency. Must stay above /// akka.cluster.failure-detector.acceptable-heartbeat-pause or a merely-slow node gets /// downed; both constraints are pinned by tests. /// public static readonly TimeSpan DowningStableAfter = TimeSpan.FromSeconds(15); /// /// Builds the HOCON that selects the downing provider, per /// . /// /// The bound cluster options carrying the strategy. /// /// For auto-down, a block installing Akka's AutoDowning provider with a downing /// window of . For keep-oldest, an empty string — the /// typed from already installs /// the SBR provider, and the resolver's own settings live in Resources/akka.conf. /// /// The strategy is not a recognised value. public static string BuildDowningHocon(AkkaClusterOptions options) { ArgumentNullException.ThrowIfNull(options); if (IsAutoDown(options)) { // auto-down-unreachable-after is load-bearing: it defaults to `off`, under which // AutoDowning is installed but never downs anything — failover silently disabled. return $$""" akka.cluster { downing-provider-class = "Akka.Cluster.AutoDowning, Akka.Cluster" auto-down-unreachable-after = {{(int)DowningStableAfter.TotalMilliseconds}}ms } """; } if (IsKeepOldest(options)) { return string.Empty; } throw new ArgumentOutOfRangeException( nameof(options), options.SplitBrainResolverStrategy, $"Unknown {AkkaClusterOptions.SectionName}:{nameof(AkkaClusterOptions.SplitBrainResolverStrategy)}. " + "Expected 'auto-down' (default; survives a crash of either node) or 'keep-oldest' " + "(partition-safe, but a two-node pair cannot survive a crash of the oldest node)."); } private static bool IsAutoDown(AkkaClusterOptions options) => string.Equals(options.SplitBrainResolverStrategy, "auto-down", StringComparison.OrdinalIgnoreCase); private static bool IsKeepOldest(AkkaClusterOptions options) => string.Equals(options.SplitBrainResolverStrategy, "keep-oldest", StringComparison.OrdinalIgnoreCase); /// /// Builds the for the fused-host cluster. /// /// /// /// Activation note (arch-review 03/S1, corrected twice). Akka.Cluster.Hosting's /// WithClustering always installs a downing provider: when /// is null it applies /// SplitBrainResolverOption.Default, which registers /// Akka.Cluster.SBR.SplitBrainResolverProvider and reads the /// split-brain-resolver block in Resources/akka.conf. The cluster is /// therefore never NoDowning. What this method chooses is only which /// provider — and under the default auto-down strategy the choice is not made /// here at all but by the Prepended HOCON in /// , which outranks whatever /// WithClustering emits. /// /// /// Why keep-oldest is no longer the default (2026-07-21). The previous /// documentation here asserted that with /// DownIfAlone=true was "the correct strategy for a 2-node warm-redundancy pair" /// because down-if-alone would down a node that lost its peer. That is the /// opposite of what Akka.NET does. In KeepOldest.OldestDecision the /// down-if-alone branch requires the surviving side to hold >= 2 members, so /// in a 1-vs-1 split the survivor falls through to DownReachable and downs /// itself; with run-coordinated-shutdown-when-down = on it then exits. A /// two-node pair running keep-oldest converts a crash of the oldest node into a total /// outage — precisely the failure redundancy exists to absorb. Live-proven on the sister /// project's rig; see docs/Redundancy.md and /// for the trade. /// /// /// The bound cluster options carrying seed nodes, roles and the strategy. /// /// The cluster options with seed nodes and roles. /// is set only under keep-oldest, where the SBR provider is the one wanted; under /// auto-down it is left null because the Prepended HOCON replaces the provider anyway, /// and naming a resolver that is not in force would be misleading. /// public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options) { ArgumentNullException.ThrowIfNull(options); return new ClusterOptions { SeedNodes = options.SeedNodes, Roles = options.Roles, SplitBrainResolver = IsKeepOldest(options) ? new KeepOldestOption { DownIfAlone = true } : null, }; } }