Files
lmxopcua/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs
T
Joseph Doherty a81dea102c fix(cluster): activate split-brain resolver (arch-review 03/S1)
The split-brain-resolver HOCON block in akka.conf was inert — nothing
registered a downing provider, so the cluster ran Akka's default NoDowning:
a hard-crashed node was never downed, cluster singletons + the driver
role-leader never failed over, and a partition left both redundancy sides
at ServiceLevel 240 indefinitely.

Activate the resolver via the typed ClusterOptions.SplitBrainResolver
(KeepOldestOption { DownIfAlone = true }) in a new testable
BuildClusterOptions helper. keep-oldest + down-if-alone is correct for a
2-node warm-redundancy pair. HOCON keeps stable-after (typed option can't
express it) + a cross-reference comment. Deterministic unit guard added;
end-to-end verified the resolved ActorSystem config carries the provider
and stable-after survives the HOCON/Hosting merge. Redundancy.md corrected.

Hard-kill failover integration test tracked as a follow-up (effort-M harness).
2026-07-08 16:14:26 -04:00

117 lines
5.6 KiB
C#

using Akka.Cluster.Hosting;
using Akka.Cluster.Hosting.SBR;
using Akka.Event;
using Akka.Hosting;
using Akka.Logger.Serilog;
using Akka.Remote.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
namespace ZB.MOM.WW.OtOpcUa.Cluster;
public static class ServiceCollectionExtensions
{
/// <summary>
/// Binds <see cref="AkkaClusterOptions"/> and registers <see cref="IClusterRoleInfo"/>. The
/// actual ActorSystem + cluster bootstrap is layered on inside the host's <c>AddAkka(...)</c>
/// configurator via <see cref="WithOtOpcUaClusterBootstrap"/> — keeping the entire Akka graph
/// under Akka.Hosting's management so cluster singletons land on the same ActorSystem.
/// </summary>
/// <param name="services">The service collection to configure.</param>
/// <param name="configuration">The application configuration containing cluster options.</param>
/// <returns>The same service collection, for chaining.</returns>
public static IServiceCollection AddOtOpcUaCluster(this IServiceCollection services, IConfiguration configuration)
{
services.AddOptions<AkkaClusterOptions>()
.Bind(configuration.GetSection(AkkaClusterOptions.SectionName));
services.AddSingleton<IClusterRoleInfo, ClusterRoleInfo>();
return services;
}
/// <summary>
/// Configures the Akka.Hosting builder with the embedded OtOpcUa HOCON (split-brain resolver,
/// pinned dispatcher, failure detector tuning) + remote endpoint + cluster bootstrap derived
/// from <see cref="AkkaClusterOptions"/>.
///
/// Wire from Program.cs:
/// <code>
/// services.AddAkka("otopcua", (ab, sp) =>
/// {
/// ab.WithOtOpcUaClusterBootstrap(sp);
/// if (hasAdmin) ab.WithOtOpcUaControlPlaneSingletons();
/// if (hasDriver) ab.WithOtOpcUaRuntimeActors();
/// });
/// </code>
/// </summary>
/// <param name="builder">The Akka configuration builder to configure.</param>
/// <param name="serviceProvider">The service provider for resolving cluster options.</param>
/// <returns>The same builder, for chaining.</returns>
public static AkkaConfigurationBuilder WithOtOpcUaClusterBootstrap(
this AkkaConfigurationBuilder builder,
IServiceProvider serviceProvider)
{
var options = serviceProvider.GetRequiredService<IOptions<AkkaClusterOptions>>().Value;
builder.AddHocon(HoconLoader.LoadBaseConfig(), HoconAddMode.Append);
// Route Akka's internal ILoggingAdapter (DriverHostActor, DriverInstanceActor, cluster
// events, …) into Serilog so those logs reach the same sinks as the MEL/Serilog application
// logs. Akka.Hosting owns logger setup, so HOCON `akka.loggers` alone is not honored — the
// logger must be registered through ConfigureLoggers. Without this the actor graph logs only
// to the default StandardOutLogger (discarded under the Windows service host), which is why
// the driver-role actors were invisible during the 2026-06 data-plane investigation.
builder.ConfigureLoggers(setup =>
{
setup.LogLevel = LogLevel.DebugLevel;
setup.ClearLoggers();
setup.AddLogger<SerilogLogger>();
});
builder.WithRemoting(new RemoteOptions
{
HostName = options.Hostname,
Port = options.Port,
PublicHostName = options.PublicHostname,
});
builder.WithClustering(BuildClusterOptions(options));
return builder;
}
/// <summary>
/// Builds the <see cref="ClusterOptions"/> for the fused-host cluster, including the
/// <b>activating</b> split-brain-resolver registration.
///
/// The <c>split-brain-resolver</c> HOCON block in <c>Resources/akka.conf</c> is inert on its own —
/// Akka.NET only runs the resolver when a downing provider is registered, otherwise the cluster
/// falls back to <b>NoDowning</b> and a hard-crashed node is never downed (singletons and the
/// <c>driver</c> role-leader never fail over, and a partition leaves both redundancy sides at
/// ServiceLevel 240 forever). Setting <see cref="ClusterOptions.SplitBrainResolver"/> is what
/// activates <c>Akka.Cluster.SBR.SplitBrainResolverProvider</c> under the hood — this is the piece
/// that was missing (arch-review 03/S1).
///
/// <see cref="KeepOldestOption"/> with <c>DownIfAlone=true</c> mirrors the HOCON intent and is the
/// correct strategy for a 2-node warm-redundancy pair: on an even split the oldest member (typically
/// the long-running primary) survives, and <c>down-if-alone</c> downs a singleton that loses its
/// peer. <c>keep-majority</c>/<c>static-quorum</c> are wrong for two nodes. The strategy + down-if-alone
/// here MUST stay consistent with the HOCON block; <c>stable-after</c> lives only in HOCON because the
/// typed option cannot express it (it must stay ≥ <c>failure-detector.acceptable-heartbeat-pause</c>).
/// </summary>
/// <param name="options">The bound cluster options carrying seed nodes and roles.</param>
/// <returns>The cluster options with seed nodes, roles, and the activated split-brain resolver.</returns>
public static ClusterOptions BuildClusterOptions(AkkaClusterOptions options)
{
return new ClusterOptions
{
SeedNodes = options.SeedNodes,
Roles = options.Roles,
SplitBrainResolver = new KeepOldestOption { DownIfAlone = true },
};
}
}