using Akka.Cluster.Hosting;
using Akka.Cluster.Hosting.SBR;
using Akka.Event;
using Akka.Hosting;
using Akka.Logger.Serilog;
using Akka.Remote.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Configuration;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
namespace ZB.MOM.WW.OtOpcUa.Cluster;
public static class ServiceCollectionExtensions
{
///
/// Binds and registers . The
/// actual ActorSystem + cluster bootstrap is layered on inside the host's AddAkka(...)
/// configurator via — keeping the entire Akka graph
/// under Akka.Hosting's management so cluster singletons land on the same ActorSystem.
///
///
/// The binding is validated at startup (ValidateOnStart) by
/// , so a seed list that cannot bootstrap this node —
/// self listed behind its partner — fails the host loudly instead of leaving it running but
/// permanently outside the cluster.
///
/// The service collection to configure.
/// The application configuration containing cluster options.
/// The same service collection, for chaining.
public static IServiceCollection AddOtOpcUaCluster(this IServiceCollection services, IConfiguration configuration)
{
services.AddValidatedOptions(
configuration, AkkaClusterOptions.SectionName);
// Per-cluster mesh Phase 2: which transport carries central→node commands. Validated at
// startup for the same reason the cluster options are — a contact point that cannot resolve
// produces silence, not an error, and silence is indistinguishable from "nothing to do".
services.AddValidatedOptions(
configuration, MeshTransportOptions.SectionName);
// Per-cluster mesh Phase 3: where a driver reads its config (Direct SQL, or fetch-and-cache
// over gRPC). Validated at startup for the same reason — a FetchAndCache node with no endpoint
// or no key produces silence, not an error. ConfigServe (the central serve surface) needs no
// cross-field validation: a 0 port is a legitimate "disabled", so a plain Configure suffices.
services.AddValidatedOptions(
configuration, ConfigSourceOptions.SectionName);
services.Configure(configuration.GetSection(ConfigServeOptions.SectionName));
// Per-cluster mesh Phase 5: which transport carries a node's live-telemetry stream (node
// serve side + central dial side). Validated at startup for the same reason ConfigSource is —
// a Grpc-mode node with no listen port or no key produces silence, not an error.
services.AddValidatedOptions(
configuration, TelemetryOptions.SectionName);
services.AddValidatedOptions(
configuration, TelemetryDialOptions.SectionName);
services.AddSingleton();
return services;
}
///
/// Configures the Akka.Hosting builder with the embedded OtOpcUa HOCON (split-brain resolver,
/// pinned dispatcher, failure detector tuning) + remote endpoint + cluster bootstrap derived
/// from .
///
/// Wire from Program.cs:
///
/// services.AddAkka("otopcua", (ab, sp) =>
/// {
/// ab.WithOtOpcUaClusterBootstrap(sp);
/// if (hasAdmin) ab.WithOtOpcUaControlPlaneSingletons();
/// if (hasDriver) ab.WithOtOpcUaRuntimeActors();
/// });
///
///
/// The Akka configuration builder to configure.
/// The service provider for resolving cluster options.
/// The same builder, for chaining.
public static AkkaConfigurationBuilder WithOtOpcUaClusterBootstrap(
this AkkaConfigurationBuilder builder,
IServiceProvider serviceProvider)
{
var options = serviceProvider.GetRequiredService>().Value;
builder.AddHocon(HoconLoader.LoadBaseConfig(), HoconAddMode.Append);
// Route Akka's internal ILoggingAdapter (DriverHostActor, DriverInstanceActor, cluster
// events, …) into Serilog so those logs reach the same sinks as the MEL/Serilog application
// logs. Akka.Hosting owns logger setup, so HOCON `akka.loggers` alone is not honored — the
// logger must be registered through ConfigureLoggers. Without this the actor graph logs only
// to the default StandardOutLogger (discarded under the Windows service host), which is why
// the driver-role actors were invisible during the 2026-06 data-plane investigation.
builder.ConfigureLoggers(setup =>
{
setup.LogLevel = LogLevel.DebugLevel;
setup.ClearLoggers();
setup.AddLogger();
});
builder.WithRemoting(new RemoteOptions
{
HostName = options.Hostname,
Port = options.Port,
PublicHostName = options.PublicHostname,
});
builder.WithClustering(BuildClusterOptions(options));
// Must come AFTER WithClustering, which always emits a downing-provider-class of its own:
// Akka.Cluster.Hosting applies SplitBrainResolverOption.Default whenever the typed
// ClusterOptions.SplitBrainResolver is null, so there is always a competing value to beat.
// Prepend is the highest-precedence mode, which makes this block win irrespective of how
// many fragments are added or in what order.
//
// Do not take this comment's word for it. Which fragment actually wins is not obvious from
// the mode names — measured, Append also wins here, purely because it happens to be added
// last. SplitBrainResolverActivationTests reads the provider back off a *running*
// ActorSystem for exactly that reason: the effective value is the only one worth asserting.
builder.AddHocon(BuildDowningHocon(options), HoconAddMode.Prepend);
// NOTE (2026-07-22): no bootstrap watchdog is armed here any more. The cold-start-alone gap
// it existed for — Akka lets only seed-nodes[0] form a NEW cluster — is now closed by
// self-first seed ordering (AkkaClusterOptions.SeedNodes), enforced at boot by
// AkkaClusterOptionsValidator. The retired ClusterBootstrapFallback timer sat OUTSIDE Akka's
// join handshake, so it could not distinguish "no seed answered" from "a seed answered and
// the join is in flight", and Cluster.Join(SelfAddress) is not ignored mid-handshake — it
// wins. See docs/Redundancy.md → "Bootstrap: self-first seed ordering".
return builder;
}
///
/// How long a member must stay unreachable before it is downed. Shared by both strategies
/// (auto-down-unreachable-after and the SBR resolver's stable-after) so the two
/// have the same failover latency. Must stay above
/// akka.cluster.failure-detector.acceptable-heartbeat-pause or a merely-slow node gets
/// downed; both constraints are pinned by tests.
///
public static readonly TimeSpan DowningStableAfter = TimeSpan.FromSeconds(15);
///
/// Builds the HOCON that selects the downing provider, per
/// .
///
/// The bound cluster options carrying the strategy.
///
/// For auto-down, a block installing Akka's AutoDowning provider with a downing
/// window of . For keep-oldest, an empty string — the
/// typed from already installs
/// the SBR provider, and the resolver's own settings live in Resources/akka.conf.
///
/// The strategy is not a recognised value.
public static string BuildDowningHocon(AkkaClusterOptions options)
{
ArgumentNullException.ThrowIfNull(options);
if (IsAutoDown(options))
{
// auto-down-unreachable-after is load-bearing: it defaults to `off`, under which
// AutoDowning is installed but never downs anything — failover silently disabled.
return $$"""
akka.cluster {
downing-provider-class = "Akka.Cluster.AutoDowning, Akka.Cluster"
auto-down-unreachable-after = {{(int)DowningStableAfter.TotalMilliseconds}}ms
}
""";
}
if (IsKeepOldest(options))
{
return string.Empty;
}
throw new ArgumentOutOfRangeException(
nameof(options),
options.SplitBrainResolverStrategy,
$"Unknown {AkkaClusterOptions.SectionName}:{nameof(AkkaClusterOptions.SplitBrainResolverStrategy)}. "
+ "Expected 'auto-down' (default; survives a crash of either node) or 'keep-oldest' "
+ "(partition-safe, but a two-node pair cannot survive a crash of the oldest node).");
}
private static bool IsAutoDown(AkkaClusterOptions options) =>
string.Equals(options.SplitBrainResolverStrategy, "auto-down", StringComparison.OrdinalIgnoreCase);
private static bool IsKeepOldest(AkkaClusterOptions options) =>
string.Equals(options.SplitBrainResolverStrategy, "keep-oldest", StringComparison.OrdinalIgnoreCase);
///
/// Builds the for the fused-host cluster.
///
///
///
/// Activation note (arch-review 03/S1, corrected twice). Akka.Cluster.Hosting's
/// WithClustering always installs a downing provider: when
/// is null it applies
/// SplitBrainResolverOption.Default, which registers
/// Akka.Cluster.SBR.SplitBrainResolverProvider and reads the
/// split-brain-resolver block in Resources/akka.conf. The cluster is
/// therefore never NoDowning. What this method chooses is only which
/// provider — and under the default auto-down strategy the choice is not made
/// here at all but by the Prepended HOCON in
/// , which outranks whatever
/// WithClustering emits.
///
///
/// Why keep-oldest is no longer the default (2026-07-21). The previous
/// documentation here asserted that with
/// DownIfAlone=true was "the correct strategy for a 2-node warm-redundancy pair"
/// because down-if-alone would down a node that lost its peer. That is the
/// opposite of what Akka.NET does. In KeepOldest.OldestDecision the
/// down-if-alone branch requires the surviving side to hold >= 2 members, so
/// in a 1-vs-1 split the survivor falls through to DownReachable and downs
/// itself; with run-coordinated-shutdown-when-down = on it then exits. A
/// two-node pair running keep-oldest converts a crash of the oldest node into a total
/// outage — precisely the failure redundancy exists to absorb. Live-proven on the sister
/// project's rig; see docs/Redundancy.md and
/// for the trade.
///
///
/// The bound cluster options carrying seed nodes, roles and the strategy.
///
/// The cluster options with seed nodes and roles.
/// is set only under keep-oldest, where the SBR provider is the one wanted; under
/// auto-down it is left null because the Prepended HOCON replaces the provider anyway,
/// and naming a resolver that is not in force would be misleading.
///
public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options)
{
ArgumentNullException.ThrowIfNull(options);
return new ClusterOptions
{
SeedNodes = options.SeedNodes,
Roles = options.Roles,
SplitBrainResolver = IsKeepOldest(options) ? new KeepOldestOption { DownIfAlone = true } : null,
};
}
}