279d1d0fb1
Closes the residual Phase-7 gap: when BOTH nodes of a 2-node pair cold-start at the same instant, each self-first seed runs FirstSeedNodeProcess, times out waiting for the other, and forms its own 1-node cluster — two Primaries in one pair (the Phase 6 live gate reproduced this reliably on docker). The prior mitigation was operational (staggered start / compose depends_on), which does not exist on production hardware. The guard (dark switch Cluster:BootstrapGuard:Enabled, default OFF) prevents the split without giving up cold-start-alone: - The lower-address node is the preferred founder: self-first, forms immediately. - The higher node probes its partner's Akka port (TCP connect) up to PartnerProbeSeconds: reachable => peer-first (join the founder, never race it); unreachable => self-first (partner is dead, form alone). The order is decided BEFORE the single JoinSeedNodes, from an explicit reachability signal — never re-formed mid-handshake (the retired SelfFormAfter failure mode). When on, Akka gets no config seeds (BuildClusterOptions) and ClusterBootstrapCoordinator drives the join. Review-driven hardening: case-insensitive tie-break (a hostname-casing mismatch would reopen the split); fail-fast validation of the timing knobs; the residual "founder dies in the probe->join window" hang is documented and made operator-visible (warning + a restart recovers). Real-ActorSystem coordinator tests cover the load-bearing higher-node cold-start-alone case; 147/147 Cluster tests pass. Live-gated on docker-dev (site-a = enablement demo, serialization removed, guard on; site-b keeps depends_on-serialization + guard off as the A/B control): simultaneous start -> site-a-1 founds, site-a-2 probes-reachable-joins -> 250/240, NO split; higher-node-alone -> probes dead founder, self-forms -> 250; founder rejoin -> 240. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
118 lines
5.9 KiB
C#
118 lines
5.9 KiB
C#
using Akka.Actor;
|
|
using ZB.MOM.WW.Configuration;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
|
|
|
/// <summary>
|
|
/// Fail-fast startup validator for <see cref="AkkaClusterOptions"/>, built on the shared
|
|
/// <c>ZB.MOM.WW.Configuration</c> <see cref="OptionsValidatorBase{TOptions}"/> and wired through
|
|
/// <c>AddValidatedOptions</c> in <see cref="ServiceCollectionExtensions.AddOtOpcUaCluster"/>.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <b>Self-first seed ordering (decision 2026-07-22).</b> Akka runs
|
|
/// <c>FirstSeedNodeProcess</c> — the only bootstrap path that can form a NEW cluster when no
|
|
/// peer answers <c>InitJoin</c> — exclusively when <c>seed-nodes[0]</c> is this node's own
|
|
/// address. Every other node runs <c>JoinSeedNodeProcess</c> and retries <c>InitJoin</c>
|
|
/// forever. A node that lists its partner first therefore cannot cold-start while that
|
|
/// partner is down, and the failure is <i>silent</i>: the process runs, the port listens, the
|
|
/// node simply never becomes a member. Enforced here so it is loud, at boot, on the node
|
|
/// that has the problem.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>The rule is conditional and must stay that way.</b> It binds only when this node's own
|
|
/// address appears in its own seed list. A driver-only site node is seeded solely by
|
|
/// <c>central-1</c> and is legitimately not a seed of anything; an unconditional "self must
|
|
/// be first" rule would refuse to boot every one of them.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>Identity is the advertised address, never the bind address.</b> In docker-dev
|
|
/// <c>Cluster:Hostname</c> is <c>0.0.0.0</c> and <c>Cluster:PublicHostname</c> is the
|
|
/// container DNS name — the value Akka puts in <c>SelfAddress</c> and the only one a seed URI
|
|
/// can match. Comparing against <c>Hostname</c> would match nothing anywhere, silently exempt
|
|
/// every node, and leave this rule inert. <c>PublicHostname</c> falling back to
|
|
/// <c>Hostname</c> when blank mirrors Akka's own <c>public-hostname</c> semantics.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase<AkkaClusterOptions>
|
|
{
|
|
/// <inheritdoc />
|
|
protected override void Validate(ValidationBuilder builder, AkkaClusterOptions options)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
|
|
ValidateBootstrapGuard(builder, options.BootstrapGuard);
|
|
|
|
var seeds = options.SeedNodes ?? Array.Empty<string>();
|
|
if (seeds.Length == 0)
|
|
{
|
|
// Single-node / dev default: nothing is claimed about ordering, so there is nothing to
|
|
// enforce. (Akka bootstraps from an empty seed list by waiting for an explicit join.)
|
|
return;
|
|
}
|
|
|
|
var selfHost = string.IsNullOrWhiteSpace(options.PublicHostname)
|
|
? options.Hostname
|
|
: options.PublicHostname;
|
|
|
|
var selfIndex = Array.FindIndex(seeds, s => IsSelf(s, selfHost, options.Port));
|
|
if (selfIndex <= 0)
|
|
{
|
|
// -1 = this node is not one of its own seeds (exempt); 0 = correctly ordered.
|
|
return;
|
|
}
|
|
|
|
builder.Add(
|
|
$"Cluster:SeedNodes must list this node itself first: entry {selfIndex} is this node "
|
|
+ $"('{seeds[selfIndex]}') but entry 0 is '{seeds[0]}'. Akka only lets seed-nodes[0] form "
|
|
+ "a new cluster (FirstSeedNodeProcess); every other node retries InitJoin forever, so "
|
|
+ "with the partner listed first this node can never start while its peer is down. Swap "
|
|
+ "the entries — see docs/Redundancy.md → 'Bootstrap: self-first seed ordering'.");
|
|
}
|
|
|
|
/// <summary>
|
|
/// When the bootstrap guard is enabled, its timing knobs must be positive. A zero/negative
|
|
/// <c>PartnerProbeSeconds</c> in particular silently degrades the guard to "never wait, always
|
|
/// conclude the partner is dead" — it would form alone immediately and re-open the very split it
|
|
/// exists to close. Fail-fast at boot rather than producing silence (the same rationale every
|
|
/// sibling options validator in this project cites).
|
|
/// </summary>
|
|
private static void ValidateBootstrapGuard(ValidationBuilder builder, ClusterBootstrapGuardOptions guard)
|
|
{
|
|
if (guard is null || !guard.Enabled) return;
|
|
|
|
if (guard.PartnerProbeSeconds <= 0)
|
|
builder.Add($"Cluster:BootstrapGuard:PartnerProbeSeconds must be > 0 when the guard is enabled (was {guard.PartnerProbeSeconds}).");
|
|
if (guard.PartnerProbeIntervalMs <= 0)
|
|
builder.Add($"Cluster:BootstrapGuard:PartnerProbeIntervalMs must be > 0 when the guard is enabled (was {guard.PartnerProbeIntervalMs}).");
|
|
if (guard.ProbeConnectTimeoutMs <= 0)
|
|
builder.Add($"Cluster:BootstrapGuard:ProbeConnectTimeoutMs must be > 0 when the guard is enabled (was {guard.ProbeConnectTimeoutMs}).");
|
|
}
|
|
|
|
/// <summary>
|
|
/// True when <paramref name="seed"/> addresses this node itself — host AND port.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Host comparison is case-insensitive because DNS names are, but is otherwise exact: Akka does
|
|
/// no DNS canonicalisation either, so <c>central-2</c> and <c>central-2.example.com</c> are
|
|
/// genuinely different seed identities to the cluster. A seed entry Akka itself cannot parse is
|
|
/// treated as "not this node" — reporting it is Akka's job, with Akka's wording, and this rule
|
|
/// must not turn a parse problem into a misleading ordering complaint.
|
|
/// </remarks>
|
|
private static bool IsSelf(string seed, string selfHost, int selfPort)
|
|
{
|
|
Address address;
|
|
try
|
|
{
|
|
address = Address.Parse(seed);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return address.Port == selfPort
|
|
&& string.Equals(address.Host, selfHost, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
}
|