feat(cluster): simultaneous-cold-start split-brain guard (opt-in)
Closes the residual Phase-7 gap: when BOTH nodes of a 2-node pair cold-start at the same instant, each self-first seed runs FirstSeedNodeProcess, times out waiting for the other, and forms its own 1-node cluster — two Primaries in one pair (the Phase 6 live gate reproduced this reliably on docker). The prior mitigation was operational (staggered start / compose depends_on), which does not exist on production hardware. The guard (dark switch Cluster:BootstrapGuard:Enabled, default OFF) prevents the split without giving up cold-start-alone: - The lower-address node is the preferred founder: self-first, forms immediately. - The higher node probes its partner's Akka port (TCP connect) up to PartnerProbeSeconds: reachable => peer-first (join the founder, never race it); unreachable => self-first (partner is dead, form alone). The order is decided BEFORE the single JoinSeedNodes, from an explicit reachability signal — never re-formed mid-handshake (the retired SelfFormAfter failure mode). When on, Akka gets no config seeds (BuildClusterOptions) and ClusterBootstrapCoordinator drives the join. Review-driven hardening: case-insensitive tie-break (a hostname-casing mismatch would reopen the split); fail-fast validation of the timing knobs; the residual "founder dies in the probe->join window" hang is documented and made operator-visible (warning + a restart recovers). Real-ActorSystem coordinator tests cover the load-bearing higher-node cold-start-alone case; 147/147 Cluster tests pass. Live-gated on docker-dev (site-a = enablement demo, serialization removed, guard on; site-b keeps depends_on-serialization + guard off as the A/B control): simultaneous start -> site-a-1 founds, site-a-2 probes-reachable-joins -> 250/240, NO split; higher-node-alone -> probes dead founder, self-forms -> 250; founder rejoin -> 240. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
@@ -79,4 +79,46 @@ public sealed class AkkaClusterOptions
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public string SplitBrainResolverStrategy { get; set; } = "auto-down";
|
||||
|
||||
/// <summary>
|
||||
/// The simultaneous-cold-start split-brain guard (<c>Cluster:BootstrapGuard</c>). Default OFF — a
|
||||
/// dark switch, so existing deployments and tests keep Akka's config-driven self-first auto-join
|
||||
/// unchanged. See <see cref="ClusterBootstrapGuard"/> for the decision logic and
|
||||
/// <c>ClusterBootstrapCoordinator</c> for the runtime.
|
||||
/// </summary>
|
||||
public ClusterBootstrapGuardOptions BootstrapGuard { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the simultaneous-cold-start split-brain guard. When
|
||||
/// <see cref="Enabled"/>, the node does NOT auto-join from its config seeds; a coordinator picks the
|
||||
/// join order (founder self-first / joiner peer-first) after a reachability probe. See
|
||||
/// <see cref="ClusterBootstrapGuard"/>.
|
||||
/// </summary>
|
||||
public sealed class ClusterBootstrapGuardOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets whether the guard is active. Default <see langword="false"/> — Akka auto-joins from
|
||||
/// the config seed list exactly as before. Only meaningful on a node that is one of its own two
|
||||
/// pair seeds; inert everywhere else.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets how long the higher-address node probes its partner's Akka endpoint before
|
||||
/// concluding the partner is dead and forming alone. Must comfortably exceed the partner's
|
||||
/// worst-case process-start-to-Akka-bind time so a slow-but-alive partner is never mistaken for a
|
||||
/// dead one (which would re-open the split). Default 25 s.
|
||||
/// </summary>
|
||||
public int PartnerProbeSeconds { get; set; } = 25;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the interval between partner reachability probes, in milliseconds. Default 500 ms.
|
||||
/// </summary>
|
||||
public int PartnerProbeIntervalMs { get; set; } = 500;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the per-probe TCP connect timeout, in milliseconds. Default 1000 ms.
|
||||
/// </summary>
|
||||
public int ProbeConnectTimeoutMs { get; set; } = 1000;
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase<AkkaClust
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
ValidateBootstrapGuard(builder, options.BootstrapGuard);
|
||||
|
||||
var seeds = options.SeedNodes ?? Array.Empty<string>();
|
||||
if (seeds.Length == 0)
|
||||
{
|
||||
@@ -68,6 +70,25 @@ public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase<AkkaClust
|
||||
+ "the entries — see docs/Redundancy.md → 'Bootstrap: self-first seed ordering'.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When the bootstrap guard is enabled, its timing knobs must be positive. A zero/negative
|
||||
/// <c>PartnerProbeSeconds</c> in particular silently degrades the guard to "never wait, always
|
||||
/// conclude the partner is dead" — it would form alone immediately and re-open the very split it
|
||||
/// exists to close. Fail-fast at boot rather than producing silence (the same rationale every
|
||||
/// sibling options validator in this project cites).
|
||||
/// </summary>
|
||||
private static void ValidateBootstrapGuard(ValidationBuilder builder, ClusterBootstrapGuardOptions guard)
|
||||
{
|
||||
if (guard is null || !guard.Enabled) return;
|
||||
|
||||
if (guard.PartnerProbeSeconds <= 0)
|
||||
builder.Add($"Cluster:BootstrapGuard:PartnerProbeSeconds must be > 0 when the guard is enabled (was {guard.PartnerProbeSeconds}).");
|
||||
if (guard.PartnerProbeIntervalMs <= 0)
|
||||
builder.Add($"Cluster:BootstrapGuard:PartnerProbeIntervalMs must be > 0 when the guard is enabled (was {guard.PartnerProbeIntervalMs}).");
|
||||
if (guard.ProbeConnectTimeoutMs <= 0)
|
||||
builder.Add($"Cluster:BootstrapGuard:ProbeConnectTimeoutMs must be > 0 when the guard is enabled (was {guard.ProbeConnectTimeoutMs}).");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="seed"/> addresses this node itself — host AND port.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net.Sockets;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Runs the simultaneous-cold-start split-brain guard: when <c>Cluster:BootstrapGuard:Enabled</c>, the
|
||||
/// node starts with NO config seed nodes (see <c>BuildClusterOptions</c>), and this coordinator picks
|
||||
/// the join order and issues the single <see cref="Akka.Cluster.Cluster.JoinSeedNodes"/> once the
|
||||
/// ActorSystem is up. See <see cref="ClusterBootstrapGuard"/> for the decision logic and the
|
||||
/// split-brain rationale.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The founder (lower canonical address) joins its self-first order immediately. The higher
|
||||
/// node probes its partner's Akka endpoint (a plain TCP connect — the partner has bound its
|
||||
/// port well before it forms) up to <see cref="ClusterBootstrapGuardOptions.PartnerProbeSeconds"/>:
|
||||
/// reachable ⇒ peer-first (join the founder); unreachable ⇒ self-first (partner is dead, form
|
||||
/// alone). The join runs on a background task so it never blocks host startup, and it is issued
|
||||
/// exactly once — the coordinator never re-forms a node mid-handshake, the failure mode of the
|
||||
/// retired <c>SelfFormAfter</c> watchdog.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ClusterBootstrapCoordinator : IHostedService
|
||||
{
|
||||
private readonly Func<ActorSystem> _system;
|
||||
private readonly AkkaClusterOptions _options;
|
||||
private readonly ILogger<ClusterBootstrapCoordinator> _log;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task? _joinTask;
|
||||
|
||||
/// <summary>Creates the coordinator.</summary>
|
||||
/// <param name="system">Lazy accessor for the node's ActorSystem (resolved after Akka's hosted service starts it).</param>
|
||||
/// <param name="options">The bound cluster options (seed list + guard settings).</param>
|
||||
/// <param name="log">Logger for the bootstrap decision.</param>
|
||||
public ClusterBootstrapCoordinator(
|
||||
Func<ActorSystem> system, IOptions<AkkaClusterOptions> options, ILogger<ClusterBootstrapCoordinator> log)
|
||||
{
|
||||
_system = system ?? throw new ArgumentNullException(nameof(system));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_log = log ?? throw new ArgumentNullException(nameof(log));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_options.BootstrapGuard.Enabled)
|
||||
return Task.CompletedTask; // dark switch off — Akka auto-joined from config seeds already.
|
||||
|
||||
// Fire-and-forget so a higher node with a dead partner (which waits the full probe window)
|
||||
// never blocks host startup. Exceptions are logged; a failed join leaves the node unjoined,
|
||||
// which is visible and recoverable, never a silent split.
|
||||
_joinTask = Task.Run(() => RunAsync(_cts.Token), _cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _cts.CancelAsync().ConfigureAwait(false);
|
||||
if (_joinTask is not null)
|
||||
{
|
||||
try { await _joinTask.ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { /* expected on shutdown */ }
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var seeds = _options.SeedNodes ?? Array.Empty<string>();
|
||||
var role = ClusterBootstrapGuard.Analyze(seeds, _options.PublicHostname, _options.Port);
|
||||
|
||||
string[] order;
|
||||
var committedPeerFirst = false;
|
||||
if (!role.Applies)
|
||||
{
|
||||
// Not a pair seed (single-seed legacy site node, self absent, malformed, or 3+ seeds):
|
||||
// join the configured seeds unchanged — the guard arbitrates only the 2-node pair race.
|
||||
order = seeds;
|
||||
_log.LogInformation(
|
||||
"Bootstrap guard: not a pair seed ({SeedCount} seed(s)); joining configured seeds unchanged.",
|
||||
seeds.Length);
|
||||
}
|
||||
else if (role.IsFounder)
|
||||
{
|
||||
order = role.SelfFirstOrder;
|
||||
_log.LogInformation(
|
||||
"Bootstrap guard: this node is the preferred founder (lower address); joining self-first, forming immediately if no peer answers.");
|
||||
}
|
||||
else
|
||||
{
|
||||
var reachable = await ProbePartnerAsync(role.PartnerHost!, role.PartnerPort, ct).ConfigureAwait(false);
|
||||
order = ClusterBootstrapGuard.HigherNodeOrder(role, reachable);
|
||||
committedPeerFirst = reachable;
|
||||
_log.LogInformation(
|
||||
"Bootstrap guard: this node is the higher address; partner {PartnerHost}:{PartnerPort} was {Reachability} within the probe window; joining {Order}.",
|
||||
role.PartnerHost, role.PartnerPort, reachable ? "REACHABLE" : "UNREACHABLE",
|
||||
reachable ? "peer-first (join the founder)" : "self-first (partner down, forming alone)");
|
||||
}
|
||||
|
||||
if (order.Length == 0)
|
||||
{
|
||||
_log.LogWarning("Bootstrap guard: no seed nodes to join; node stays unjoined until a seed is configured.");
|
||||
return;
|
||||
}
|
||||
|
||||
var cluster = Akka.Cluster.Cluster.Get(_system());
|
||||
var addresses = order.Select(Address.Parse).ToArray();
|
||||
cluster.JoinSeedNodes(addresses);
|
||||
|
||||
// Peer-first is a one-way commitment: JoinSeedNodeProcess never self-forms. If the founder
|
||||
// died in the probe→join window, this node hangs unjoined forever. We do NOT re-form here
|
||||
// (the retired SelfFormAfter mid-handshake failure mode) — we make the hang operator-visible
|
||||
// so a restart, which re-runs the guard against the now-dead founder and self-forms, recovers
|
||||
// it. Nothing is done on the founder / self-first paths, which always self-form on their own.
|
||||
if (committedPeerFirst)
|
||||
await WarnIfNotUpAsync(cluster, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Host shutting down before the join completed — nothing to do.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogError(ex, "Bootstrap guard: failed to issue JoinSeedNodes; node remains unjoined.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// After committing peer-first, waits a bounded time for this node to reach <c>Up</c> and logs a
|
||||
/// clear warning if it does not — the founder must have died in the probe→join window, leaving this
|
||||
/// node hung in <c>JoinSeedNodeProcess</c>. Read-only: it never re-forms or re-joins (that is the
|
||||
/// retired mid-handshake failure mode); it only surfaces the condition so an operator restarts the node.
|
||||
/// </summary>
|
||||
private async Task WarnIfNotUpAsync(Akka.Cluster.Cluster cluster, CancellationToken ct)
|
||||
{
|
||||
// Give the join a generous grace: the founder's own self-form (seed-node-timeout) plus this
|
||||
// node's join round-trip. Two probe windows is comfortably beyond both.
|
||||
var grace = TimeSpan.FromSeconds(Math.Max(10, _options.BootstrapGuard.PartnerProbeSeconds * 2));
|
||||
var sw = Stopwatch.StartNew();
|
||||
while (!ct.IsCancellationRequested && sw.Elapsed < grace)
|
||||
{
|
||||
if (cluster.SelfMember.Status == MemberStatus.Up) return; // joined — all good
|
||||
try { await Task.Delay(TimeSpan.FromSeconds(1), ct).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
|
||||
if (!ct.IsCancellationRequested && cluster.SelfMember.Status != MemberStatus.Up)
|
||||
_log.LogWarning(
|
||||
"Bootstrap guard: committed peer-first but this node is still {Status} after {Grace}s — its founder likely died in the probe→join window. This node will NOT self-form on its own (peer-first never does). RESTART it to recover: the guard will re-run, find the founder down, and form alone.",
|
||||
cluster.SelfMember.Status, (int)grace.TotalSeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Polls the partner's Akka endpoint with a plain TCP connect until it is reachable or the probe
|
||||
/// window elapses. Reachable-at-TCP is a sound proxy for "the partner is coming up": a node binds
|
||||
/// its Akka port near the start of startup, well before it forms or joins a cluster.
|
||||
/// </summary>
|
||||
private async Task<bool> ProbePartnerAsync(string host, int port, CancellationToken ct)
|
||||
{
|
||||
var window = TimeSpan.FromSeconds(Math.Max(0, _options.BootstrapGuard.PartnerProbeSeconds));
|
||||
var interval = TimeSpan.FromMilliseconds(Math.Max(50, _options.BootstrapGuard.PartnerProbeIntervalMs));
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
if (await TryConnectAsync(host, port, ct).ConfigureAwait(false)) return true;
|
||||
if (sw.Elapsed >= window) return false;
|
||||
try { await Task.Delay(interval, ct).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { return false; }
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<bool> TryConnectAsync(string host, int port, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var client = new TcpClient();
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
cts.CancelAfter(Math.Max(100, _options.BootstrapGuard.ProbeConnectTimeoutMs));
|
||||
await client.ConnectAsync(host, port, cts.Token).ConfigureAwait(false);
|
||||
return client.Connected;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw; // host shutdown — propagate
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false; // connect refused / timed out / DNS not resolvable yet — partner not up
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Pure decision logic for the simultaneous-cold-start split-brain guard.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Both nodes of a 2-node pair are self-first seeds so <b>either</b> can cold-start alone when
|
||||
/// its partner is dead (see <see cref="AkkaClusterOptions.SeedNodes"/>). The cost is that when
|
||||
/// BOTH cold-start at the same instant, each runs Akka's <c>FirstSeedNodeProcess</c>, times out
|
||||
/// waiting for the other, and forms its OWN single-node cluster — a split brain (two Primaries
|
||||
/// in one pair). Two independent clusters do not auto-merge, so the split persists until an
|
||||
/// operator intervenes.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This guard breaks the symmetry deterministically without giving up cold-start-alone. The
|
||||
/// node with the lexicographically <b>lower</b> canonical address is the <i>preferred
|
||||
/// founder</i>: it always uses self-first order and forms immediately. The <b>higher</b> node
|
||||
/// first probes whether its partner's Akka endpoint is reachable (see
|
||||
/// <see cref="ClusterBootstrapCoordinator"/>): reachable ⇒ peer-first order so it JOINS the
|
||||
/// founder rather than racing it; unreachable after the probe window ⇒ the partner is genuinely
|
||||
/// down, so it falls back to self-first and forms alone. Exactly one node founds when both are
|
||||
/// present; the lower node always founds when it starts alone.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Residual trade-off (accepted).</b> Once the higher node observes the partner reachable it
|
||||
/// commits to peer-first — Akka's <c>JoinSeedNodeProcess</c> retries <c>InitJoin</c> forever and
|
||||
/// never self-forms. If the founder dies in the small window between the probe succeeding and
|
||||
/// the join completing, the higher node hangs unjoined until it is restarted (a restart re-runs
|
||||
/// the guard, finds the founder unreachable, and self-forms). We deliberately do NOT re-decide
|
||||
/// mid-handshake — that is exactly the retired <c>SelfFormAfter</c> failure mode. Instead
|
||||
/// <see cref="ClusterBootstrapCoordinator"/> makes the hang operator-visible with a warning if
|
||||
/// the node has not come Up within a bounded time after committing peer-first.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This decides the join order BEFORE issuing a single <c>JoinSeedNodes</c>, from an explicit
|
||||
/// reachability signal — unlike the retired <c>SelfFormAfter</c> watchdog, which fired a
|
||||
/// <c>Join(self)</c> mid-handshake on a bare timeout and could not tell "no seed answered" from
|
||||
/// "a join is in flight", islanding a node a failover had just bounced.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class ClusterBootstrapGuard
|
||||
{
|
||||
private static readonly Regex SeedPattern =
|
||||
new(@"^akka(?:\.[a-z0-9]+)?://[^@/]+@(?<host>[^:/]+):(?<port>\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>The analyzed bootstrap role of this node within its seed list.</summary>
|
||||
/// <param name="Applies">True only when the guard engages: exactly two seeds, one of them this node.</param>
|
||||
/// <param name="IsFounder">True when this node is the preferred founder (lower address) — form immediately, no probe.</param>
|
||||
/// <param name="PartnerHost">The partner's advertised host (to probe when this node is the higher one); null when not applicable.</param>
|
||||
/// <param name="PartnerPort">The partner's Akka port.</param>
|
||||
/// <param name="SelfFirstOrder"><c>[self, partner]</c> — self-first; forms a new cluster when no peer answers.</param>
|
||||
/// <param name="PeerFirstOrder"><c>[partner, self]</c> — peer-first; joins the partner's cluster, never self-forms while it is up.</param>
|
||||
public sealed record BootstrapRole(
|
||||
bool Applies,
|
||||
bool IsFounder,
|
||||
string? PartnerHost,
|
||||
int PartnerPort,
|
||||
string[] SelfFirstOrder,
|
||||
string[] PeerFirstOrder);
|
||||
|
||||
/// <summary>
|
||||
/// Analyzes the configured seed list to decide this node's bootstrap role. The guard engages only
|
||||
/// for the Phase-6 pair shape (exactly two seeds, one of them this node); any other shape
|
||||
/// (single-seed legacy site node, three+ seeds, self absent) returns <see cref="BootstrapRole.Applies"/>
|
||||
/// = false and the caller joins the configured seeds unchanged.
|
||||
/// </summary>
|
||||
/// <param name="seeds">The configured seed URIs (self-first by convention, but order is not relied on here).</param>
|
||||
/// <param name="selfHost">This node's advertised host (<see cref="AkkaClusterOptions.PublicHostname"/>).</param>
|
||||
/// <param name="selfPort">This node's Akka port (<see cref="AkkaClusterOptions.Port"/>).</param>
|
||||
/// <returns>The analyzed role; never null.</returns>
|
||||
public static BootstrapRole Analyze(IReadOnlyList<string> seeds, string selfHost, int selfPort)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(seeds);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(selfHost);
|
||||
|
||||
var na = new BootstrapRole(false, false, null, 0, Array.Empty<string>(), Array.Empty<string>());
|
||||
if (seeds.Count != 2) return na;
|
||||
|
||||
var selfKey = Key(selfHost, selfPort);
|
||||
string? self = null, partner = null;
|
||||
string? partnerHost = null;
|
||||
var partnerPort = 0;
|
||||
|
||||
foreach (var seed in seeds)
|
||||
{
|
||||
if (!TryParse(seed, out var host, out var port)) return na;
|
||||
if (string.Equals(Key(host, port), selfKey, StringComparison.OrdinalIgnoreCase))
|
||||
self = seed;
|
||||
else
|
||||
{
|
||||
partner = seed;
|
||||
partnerHost = host;
|
||||
partnerPort = port;
|
||||
}
|
||||
}
|
||||
|
||||
// Self must be exactly one of the two, and the other must be a distinct partner.
|
||||
if (self is null || partner is null || partnerHost is null) return na;
|
||||
|
||||
var selfFirst = new[] { self, partner };
|
||||
var peerFirst = new[] { partner, self };
|
||||
|
||||
// Deterministic tie-break: the lower canonical address is the preferred founder. Both nodes
|
||||
// compute the same comparison (same two seeds), so exactly one is the founder. Case-INSENSITIVE
|
||||
// to match how self/partner are classified above (and AkkaClusterOptionsValidator.IsSelf):
|
||||
// container/DNS hostnames are conventionally case-insensitive, and a casing difference between
|
||||
// the two sides' seed config must NOT make both think they are the founder (which would reopen
|
||||
// the very split this guard closes).
|
||||
var isFounder = string.Compare(
|
||||
Key(selfHost, selfPort),
|
||||
Key(partnerHost, partnerPort),
|
||||
StringComparison.OrdinalIgnoreCase) < 0;
|
||||
|
||||
return new BootstrapRole(true, isFounder, partnerHost, partnerPort, selfFirst, peerFirst);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The order the higher (non-founder) node uses once it has learned whether its partner is
|
||||
/// reachable: peer-first when the founder is up (join it), self-first when the founder is absent
|
||||
/// (form alone — cold-start-alone preserved).
|
||||
/// </summary>
|
||||
/// <param name="role">The analyzed role (must be applicable and NOT the founder).</param>
|
||||
/// <param name="partnerReachable">Whether the partner's Akka endpoint became reachable within the probe window.</param>
|
||||
/// <returns>The seed order to pass to <c>Cluster.JoinSeedNodes</c>.</returns>
|
||||
public static string[] HigherNodeOrder(BootstrapRole role, bool partnerReachable)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(role);
|
||||
return partnerReachable ? role.PeerFirstOrder : role.SelfFirstOrder;
|
||||
}
|
||||
|
||||
/// <summary>Parses <c>akka.tcp://system@host:port[/...]</c> into host + port.</summary>
|
||||
/// <param name="seed">The seed URI.</param>
|
||||
/// <param name="host">The parsed advertised host.</param>
|
||||
/// <param name="port">The parsed Akka port.</param>
|
||||
/// <returns>True when the seed matched the expected shape.</returns>
|
||||
public static bool TryParse(string? seed, out string host, out int port)
|
||||
{
|
||||
host = string.Empty;
|
||||
port = 0;
|
||||
if (string.IsNullOrWhiteSpace(seed)) return false;
|
||||
|
||||
var m = SeedPattern.Match(seed.Trim());
|
||||
if (!m.Success) return false;
|
||||
if (!int.TryParse(m.Groups["port"].Value, out port)) return false;
|
||||
host = m.Groups["host"].Value;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string Key(string host, int port) => $"{host}:{port}";
|
||||
}
|
||||
@@ -241,7 +241,13 @@ public static class ServiceCollectionExtensions
|
||||
|
||||
return new ClusterOptions
|
||||
{
|
||||
SeedNodes = options.SeedNodes,
|
||||
// When the bootstrap guard is on, DO NOT hand Akka the seed list: Akka.Cluster.Hosting
|
||||
// auto-joins from ClusterOptions.SeedNodes the instant the ActorSystem starts, which is
|
||||
// exactly the self-first race the guard exists to arbitrate. Left empty, the node starts
|
||||
// unjoined and ClusterBootstrapCoordinator picks the order (founder self-first / joiner
|
||||
// peer-first, after a reachability probe) and issues the single Cluster.JoinSeedNodes.
|
||||
// The config seed list is still read — by the coordinator, off AkkaClusterOptions.SeedNodes.
|
||||
SeedNodes = options.BootstrapGuard.Enabled ? Array.Empty<string>() : options.SeedNodes,
|
||||
Roles = options.Roles,
|
||||
SplitBrainResolver = IsKeepOldest(options) ? new KeepOldestOption { DownIfAlone = true } : null,
|
||||
};
|
||||
|
||||
@@ -390,6 +390,16 @@ builder.Services.AddAkka("otopcua", (ab, sp) =>
|
||||
}
|
||||
});
|
||||
|
||||
// Simultaneous-cold-start split-brain guard (dark switch Cluster:BootstrapGuard:Enabled, default off).
|
||||
// Registered AFTER AddAkka so its StartAsync runs once Akka's hosted service has built (and, when the
|
||||
// guard is off, already auto-joined) the ActorSystem. When on, the node started with no config seeds
|
||||
// (BuildClusterOptions) and this coordinator issues the single reachability-gated JoinSeedNodes. It
|
||||
// no-ops when the guard is off, so registering it unconditionally is safe.
|
||||
builder.Services.AddHostedService(sp => new ClusterBootstrapCoordinator(
|
||||
() => sp.GetRequiredService<ActorSystem>(),
|
||||
sp.GetRequiredService<IOptions<AkkaClusterOptions>>(),
|
||||
sp.GetRequiredService<ILogger<ClusterBootstrapCoordinator>>()));
|
||||
|
||||
// Down-if-alone recovery watchdog (#459). Registered AFTER AddAkka so it starts after Akka's own
|
||||
// hosted service has built the ActorSystem; it resolves the system lazily (never at construction) so
|
||||
// it can't race startup. On an unexpected SBR self-down it stops the host so the service supervisor
|
||||
|
||||
Reference in New Issue
Block a user