diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index f0abd190..a76868d8 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -516,6 +516,12 @@ services: # FetchAndCache, never gossip; central and site-a share no seed list. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-1:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-a-2:4053" + # Simultaneous-cold-start split-brain guard (SITE-A enablement demo). With the pair no longer + # startup-serialized, this is what stops both self-first seeds forming their own 1-node cluster: + # the lower-address node (site-a-1) founds immediately; the higher (site-a-2) probes site-a-1 and + # joins it when reachable, or forms alone only if site-a-1 is truly down. Akka gets NO config + # seeds when this is on (BuildClusterOptions); ClusterBootstrapCoordinator issues JoinSeedNodes. + Cluster__BootstrapGuard__Enabled: "true" Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see @@ -588,13 +594,11 @@ services: sql: { condition: service_healthy } central-1: { condition: service_started } migrator: { condition: service_completed_successfully } - # Serialize the SITE-A pair's startup so site-a-1 forms the mesh first and site-a-2 JOINS it — - # exactly as central-2 waits on central-1. Both nodes are self-first seeds (seed[0]=self), so BOTH - # run Akka's FirstSeedNodeProcess and, if they start simultaneously, EACH forms its own single-node - # cluster (split brain — observed in the Phase 6 live gate: both logged "JOINING itself ... forming - # a new cluster", both elected themselves Primary → 250/250 instead of 250/240). Waiting on site-a-1 - # gives it the head start to be Up before site-a-2's InitJoin, so site-a-2 joins the existing mesh. - site-a-1: { condition: service_healthy } + # SITE-A is the BOOTSTRAP-GUARD enablement demo: NO startup serialization on the pair — both + # nodes start simultaneously (the exact race that split-brained in the Phase 6 gate). The + # Cluster:BootstrapGuard (enabled in both site-a env blocks) is the ONLY thing preventing the + # split: it makes the lower-address node the founder and the higher-address node probe-then-join. + # site-b keeps the depends_on:service_healthy serialization + guard OFF, as the A/B control. environment: OTOPCUA_ROLES: "driver,cluster-SITE-A" # Per-cluster mesh Phase 4: no ConfigDb string on driver-only site nodes (see site-a-1). @@ -617,6 +621,9 @@ services: # second) — it no longer seeds off central-1. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-2:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-a-1:4053" + # Bootstrap-guard enablement demo — see site-a-1. site-a-2 is the HIGHER address, so the guard + # makes it probe site-a-1 and join it (peer-first) rather than race-form its own cluster. + Cluster__BootstrapGuard__Enabled: "true" Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see diff --git a/docs/Redundancy.md b/docs/Redundancy.md index a276c56e..39550e53 100644 --- a/docs/Redundancy.md +++ b/docs/Redundancy.md @@ -401,11 +401,61 @@ Comparing against `Cluster:Hostname` would be worse than wrong: in docker-dev it no seed URI anywhere, so the rule would silently exempt the entire rig. Sequential recovery is island-free by construction: a `FirstSeedNodeProcess` node self-joins only when **no** -seed answered, so a peer booting after the survivor formed simply joins it as the youngest member. Both -nodes cold-starting *simultaneously* converge on one cluster while they are mutually reachable — the -`InitJoin` handshake resolves it before either self-join deadline expires. The residual risk is a boot-time -**partition** (both cold-starting, mutually unreachable): both form, which is the same dual-active class the -`auto-down` strategy already accepts, with the same recovery (restart one side). +seed answered, so a peer booting after the survivor formed simply joins it as the youngest member. + +**Simultaneous cold-start CAN split — this is the residual gap the bootstrap guard below closes.** When +BOTH nodes start from nothing at the same instant, each runs `FirstSeedNodeProcess`, and if neither has +formed before the other's `seed-node-timeout` expires, EACH self-joins and forms its own single-node +cluster — two Primaries in one pair. On in-process loopback the `InitJoin` handshake usually resolves fast +enough to converge, but the Phase 6 live gate reproduced the split reliably on the docker-dev rig (real +container start timing + DNS): both nodes logged `JOINING itself … forming a new cluster` and both advertised +ServiceLevel 250. Two independent clusters do **not** auto-merge, so the split persists until an operator +restarts one side. + +### Bootstrap guard (`Cluster:BootstrapGuard`, opt-in) + +The guard eliminates the simultaneous-start split without giving up cold-start-alone. It is a **dark switch, +default off** — a node with `Cluster:BootstrapGuard:Enabled=false` (the default) keeps Akka's config-driven +self-first auto-join exactly as above. + +When enabled, the node is given **no config seed nodes** (so Akka does not auto-join), and +`ClusterBootstrapCoordinator` picks the join order from a deterministic address tie-break plus a reachability +probe: + +- The node with the lexicographically **lower** canonical `host:port` is the **preferred founder**: it joins + self-first and forms immediately if no peer answers — no probe, no delay. +- The **higher** node probes its partner's Akka port (a plain TCP connect; the partner binds its port well + before it forms) for up to `PartnerProbeSeconds` (default 25 s): **reachable ⇒ peer-first** (it joins the + founder, never races it); **unreachable after the window ⇒ self-first** (the partner is genuinely down, so + it forms alone — cold-start-alone preserved for the higher node too). + +The order is decided **before** issuing a single `Cluster.JoinSeedNodes`, from an explicit reachability +signal — it never re-decides mid-handshake, the failure mode that retired the `SelfFormAfter` watchdog. + +**Residual trade-off (accepted, operator-visible):** once the higher node commits peer-first it cannot +self-form (Akka's `JoinSeedNodeProcess` retries forever). If the founder dies in the small probe→join window, +the higher node hangs unjoined; the coordinator logs a clear WARNING after a bounded grace, and a **restart +recovers it** (the guard re-runs, finds the founder down, and forms alone). Config keys: + +| Key | Default | Notes | +|---|---|---| +| `Cluster:BootstrapGuard:Enabled` | `false` | Dark switch. Only meaningful on a node that is one of its own two pair seeds; inert elsewhere. | +| `Cluster:BootstrapGuard:PartnerProbeSeconds` | `25` | Higher node's probe window. Must comfortably exceed the founder's process-start-to-Akka-bind time, or a slow-but-alive founder is mistaken for dead and the split re-opens. Validated `> 0` at boot. | +| `Cluster:BootstrapGuard:PartnerProbeIntervalMs` | `500` | Interval between probes. | +| `Cluster:BootstrapGuard:ProbeConnectTimeoutMs` | `1000` | Per-probe TCP connect timeout. | + +On the docker-dev rig the **site-a pair is the enablement demo** (guard on, startup serialization removed so +both nodes cold-start simultaneously); **site-b keeps the `depends_on: service_healthy` startup serialization +with the guard off**, as the A/B control. Either mechanism prevents the split; the guard is the one that also +works on production hardware, where compose `depends_on` does not exist. + +The alternative to the guard is **operational**: stagger the two VMs' service-manager start, or bring the +designated founder VM up first — see the Phase 7 co-located operator runbook +(`docs/plans/2026-07-24-mesh-phase7-failover-drills.md`). + +The residual risk that remains even with the guard is a boot-time **partition** (both cold-starting, mutually +unreachable from the start): both form, which is the same dual-active class the `auto-down` strategy already +accepts, with the same recovery (restart one side). Pinned by `SelfFirstSeedBootstrapTests` (`tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/`), which starts real in-process clusters through the production bootstrap at production failure-detection timings: a lone diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs index 5ca6e1b5..99aad37c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs @@ -79,4 +79,46 @@ public sealed class AkkaClusterOptions /// /// public string SplitBrainResolverStrategy { get; set; } = "auto-down"; + + /// + /// The simultaneous-cold-start split-brain guard (Cluster:BootstrapGuard). Default OFF — a + /// dark switch, so existing deployments and tests keep Akka's config-driven self-first auto-join + /// unchanged. See for the decision logic and + /// ClusterBootstrapCoordinator for the runtime. + /// + public ClusterBootstrapGuardOptions BootstrapGuard { get; set; } = new(); +} + +/// +/// Configuration for the simultaneous-cold-start split-brain guard. When +/// , the node does NOT auto-join from its config seeds; a coordinator picks the +/// join order (founder self-first / joiner peer-first) after a reachability probe. See +/// . +/// +public sealed class ClusterBootstrapGuardOptions +{ + /// + /// Gets or sets whether the guard is active. Default — Akka auto-joins from + /// the config seed list exactly as before. Only meaningful on a node that is one of its own two + /// pair seeds; inert everywhere else. + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets how long the higher-address node probes its partner's Akka endpoint before + /// concluding the partner is dead and forming alone. Must comfortably exceed the partner's + /// worst-case process-start-to-Akka-bind time so a slow-but-alive partner is never mistaken for a + /// dead one (which would re-open the split). Default 25 s. + /// + public int PartnerProbeSeconds { get; set; } = 25; + + /// + /// Gets or sets the interval between partner reachability probes, in milliseconds. Default 500 ms. + /// + public int PartnerProbeIntervalMs { get; set; } = 500; + + /// + /// Gets or sets the per-probe TCP connect timeout, in milliseconds. Default 1000 ms. + /// + public int ProbeConnectTimeoutMs { get; set; } = 1000; } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs index 10a67320..461befa6 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs @@ -41,6 +41,8 @@ public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase(); if (seeds.Length == 0) { @@ -68,6 +70,25 @@ public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase + /// When the bootstrap guard is enabled, its timing knobs must be positive. A zero/negative + /// PartnerProbeSeconds in particular silently degrades the guard to "never wait, always + /// conclude the partner is dead" — it would form alone immediately and re-open the very split it + /// exists to close. Fail-fast at boot rather than producing silence (the same rationale every + /// sibling options validator in this project cites). + /// + private static void ValidateBootstrapGuard(ValidationBuilder builder, ClusterBootstrapGuardOptions guard) + { + if (guard is null || !guard.Enabled) return; + + if (guard.PartnerProbeSeconds <= 0) + builder.Add($"Cluster:BootstrapGuard:PartnerProbeSeconds must be > 0 when the guard is enabled (was {guard.PartnerProbeSeconds})."); + if (guard.PartnerProbeIntervalMs <= 0) + builder.Add($"Cluster:BootstrapGuard:PartnerProbeIntervalMs must be > 0 when the guard is enabled (was {guard.PartnerProbeIntervalMs})."); + if (guard.ProbeConnectTimeoutMs <= 0) + builder.Add($"Cluster:BootstrapGuard:ProbeConnectTimeoutMs must be > 0 when the guard is enabled (was {guard.ProbeConnectTimeoutMs})."); + } + /// /// True when addresses this node itself — host AND port. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapCoordinator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapCoordinator.cs new file mode 100644 index 00000000..cfd0754d --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapCoordinator.cs @@ -0,0 +1,201 @@ +using System.Diagnostics; +using System.Net.Sockets; +using Akka.Actor; +using Akka.Cluster; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Cluster; + +/// +/// Runs the simultaneous-cold-start split-brain guard: when Cluster:BootstrapGuard:Enabled, the +/// node starts with NO config seed nodes (see BuildClusterOptions), and this coordinator picks +/// the join order and issues the single once the +/// ActorSystem is up. See for the decision logic and the +/// split-brain rationale. +/// +/// +/// +/// The founder (lower canonical address) joins its self-first order immediately. The higher +/// node probes its partner's Akka endpoint (a plain TCP connect — the partner has bound its +/// port well before it forms) up to : +/// reachable ⇒ peer-first (join the founder); unreachable ⇒ self-first (partner is dead, form +/// alone). The join runs on a background task so it never blocks host startup, and it is issued +/// exactly once — the coordinator never re-forms a node mid-handshake, the failure mode of the +/// retired SelfFormAfter watchdog. +/// +/// +public sealed class ClusterBootstrapCoordinator : IHostedService +{ + private readonly Func _system; + private readonly AkkaClusterOptions _options; + private readonly ILogger _log; + private readonly CancellationTokenSource _cts = new(); + private Task? _joinTask; + + /// Creates the coordinator. + /// Lazy accessor for the node's ActorSystem (resolved after Akka's hosted service starts it). + /// The bound cluster options (seed list + guard settings). + /// Logger for the bootstrap decision. + public ClusterBootstrapCoordinator( + Func system, IOptions options, ILogger log) + { + _system = system ?? throw new ArgumentNullException(nameof(system)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _log = log ?? throw new ArgumentNullException(nameof(log)); + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (!_options.BootstrapGuard.Enabled) + return Task.CompletedTask; // dark switch off — Akka auto-joined from config seeds already. + + // Fire-and-forget so a higher node with a dead partner (which waits the full probe window) + // never blocks host startup. Exceptions are logged; a failed join leaves the node unjoined, + // which is visible and recoverable, never a silent split. + _joinTask = Task.Run(() => RunAsync(_cts.Token), _cts.Token); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _cts.CancelAsync().ConfigureAwait(false); + if (_joinTask is not null) + { + try { await _joinTask.ConfigureAwait(false); } + catch (OperationCanceledException) { /* expected on shutdown */ } + } + } + + private async Task RunAsync(CancellationToken ct) + { + try + { + var seeds = _options.SeedNodes ?? Array.Empty(); + var role = ClusterBootstrapGuard.Analyze(seeds, _options.PublicHostname, _options.Port); + + string[] order; + var committedPeerFirst = false; + if (!role.Applies) + { + // Not a pair seed (single-seed legacy site node, self absent, malformed, or 3+ seeds): + // join the configured seeds unchanged — the guard arbitrates only the 2-node pair race. + order = seeds; + _log.LogInformation( + "Bootstrap guard: not a pair seed ({SeedCount} seed(s)); joining configured seeds unchanged.", + seeds.Length); + } + else if (role.IsFounder) + { + order = role.SelfFirstOrder; + _log.LogInformation( + "Bootstrap guard: this node is the preferred founder (lower address); joining self-first, forming immediately if no peer answers."); + } + else + { + var reachable = await ProbePartnerAsync(role.PartnerHost!, role.PartnerPort, ct).ConfigureAwait(false); + order = ClusterBootstrapGuard.HigherNodeOrder(role, reachable); + committedPeerFirst = reachable; + _log.LogInformation( + "Bootstrap guard: this node is the higher address; partner {PartnerHost}:{PartnerPort} was {Reachability} within the probe window; joining {Order}.", + role.PartnerHost, role.PartnerPort, reachable ? "REACHABLE" : "UNREACHABLE", + reachable ? "peer-first (join the founder)" : "self-first (partner down, forming alone)"); + } + + if (order.Length == 0) + { + _log.LogWarning("Bootstrap guard: no seed nodes to join; node stays unjoined until a seed is configured."); + return; + } + + var cluster = Akka.Cluster.Cluster.Get(_system()); + var addresses = order.Select(Address.Parse).ToArray(); + cluster.JoinSeedNodes(addresses); + + // Peer-first is a one-way commitment: JoinSeedNodeProcess never self-forms. If the founder + // died in the probe→join window, this node hangs unjoined forever. We do NOT re-form here + // (the retired SelfFormAfter mid-handshake failure mode) — we make the hang operator-visible + // so a restart, which re-runs the guard against the now-dead founder and self-forms, recovers + // it. Nothing is done on the founder / self-first paths, which always self-form on their own. + if (committedPeerFirst) + await WarnIfNotUpAsync(cluster, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Host shutting down before the join completed — nothing to do. + } + catch (Exception ex) + { + _log.LogError(ex, "Bootstrap guard: failed to issue JoinSeedNodes; node remains unjoined."); + } + } + + /// + /// After committing peer-first, waits a bounded time for this node to reach Up and logs a + /// clear warning if it does not — the founder must have died in the probe→join window, leaving this + /// node hung in JoinSeedNodeProcess. Read-only: it never re-forms or re-joins (that is the + /// retired mid-handshake failure mode); it only surfaces the condition so an operator restarts the node. + /// + private async Task WarnIfNotUpAsync(Akka.Cluster.Cluster cluster, CancellationToken ct) + { + // Give the join a generous grace: the founder's own self-form (seed-node-timeout) plus this + // node's join round-trip. Two probe windows is comfortably beyond both. + var grace = TimeSpan.FromSeconds(Math.Max(10, _options.BootstrapGuard.PartnerProbeSeconds * 2)); + var sw = Stopwatch.StartNew(); + while (!ct.IsCancellationRequested && sw.Elapsed < grace) + { + if (cluster.SelfMember.Status == MemberStatus.Up) return; // joined — all good + try { await Task.Delay(TimeSpan.FromSeconds(1), ct).ConfigureAwait(false); } + catch (OperationCanceledException) { return; } + } + + if (!ct.IsCancellationRequested && cluster.SelfMember.Status != MemberStatus.Up) + _log.LogWarning( + "Bootstrap guard: committed peer-first but this node is still {Status} after {Grace}s — its founder likely died in the probe→join window. This node will NOT self-form on its own (peer-first never does). RESTART it to recover: the guard will re-run, find the founder down, and form alone.", + cluster.SelfMember.Status, (int)grace.TotalSeconds); + } + + /// + /// Polls the partner's Akka endpoint with a plain TCP connect until it is reachable or the probe + /// window elapses. Reachable-at-TCP is a sound proxy for "the partner is coming up": a node binds + /// its Akka port near the start of startup, well before it forms or joins a cluster. + /// + private async Task ProbePartnerAsync(string host, int port, CancellationToken ct) + { + var window = TimeSpan.FromSeconds(Math.Max(0, _options.BootstrapGuard.PartnerProbeSeconds)); + var interval = TimeSpan.FromMilliseconds(Math.Max(50, _options.BootstrapGuard.PartnerProbeIntervalMs)); + var sw = Stopwatch.StartNew(); + + while (!ct.IsCancellationRequested) + { + if (await TryConnectAsync(host, port, ct).ConfigureAwait(false)) return true; + if (sw.Elapsed >= window) return false; + try { await Task.Delay(interval, ct).ConfigureAwait(false); } + catch (OperationCanceledException) { return false; } + } + return false; + } + + private async Task TryConnectAsync(string host, int port, CancellationToken ct) + { + try + { + using var client = new TcpClient(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(Math.Max(100, _options.BootstrapGuard.ProbeConnectTimeoutMs)); + await client.ConnectAsync(host, port, cts.Token).ConfigureAwait(false); + return client.Connected; + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; // host shutdown — propagate + } + catch + { + return false; // connect refused / timed out / DNS not resolvable yet — partner not up + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapGuard.cs new file mode 100644 index 00000000..38a69ac1 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapGuard.cs @@ -0,0 +1,153 @@ +using System.Text.RegularExpressions; + +namespace ZB.MOM.WW.OtOpcUa.Cluster; + +/// +/// Pure decision logic for the simultaneous-cold-start split-brain guard. +/// +/// +/// +/// Both nodes of a 2-node pair are self-first seeds so either can cold-start alone when +/// its partner is dead (see ). The cost is that when +/// BOTH cold-start at the same instant, each runs Akka's FirstSeedNodeProcess, times out +/// waiting for the other, and forms its OWN single-node cluster — a split brain (two Primaries +/// in one pair). Two independent clusters do not auto-merge, so the split persists until an +/// operator intervenes. +/// +/// +/// This guard breaks the symmetry deterministically without giving up cold-start-alone. The +/// node with the lexicographically lower canonical address is the preferred +/// founder: it always uses self-first order and forms immediately. The higher node +/// first probes whether its partner's Akka endpoint is reachable (see +/// ): reachable ⇒ peer-first order so it JOINS the +/// founder rather than racing it; unreachable after the probe window ⇒ the partner is genuinely +/// down, so it falls back to self-first and forms alone. Exactly one node founds when both are +/// present; the lower node always founds when it starts alone. +/// +/// +/// Residual trade-off (accepted). Once the higher node observes the partner reachable it +/// commits to peer-first — Akka's JoinSeedNodeProcess retries InitJoin forever and +/// never self-forms. If the founder dies in the small window between the probe succeeding and +/// the join completing, the higher node hangs unjoined until it is restarted (a restart re-runs +/// the guard, finds the founder unreachable, and self-forms). We deliberately do NOT re-decide +/// mid-handshake — that is exactly the retired SelfFormAfter failure mode. Instead +/// makes the hang operator-visible with a warning if +/// the node has not come Up within a bounded time after committing peer-first. +/// +/// +/// This decides the join order BEFORE issuing a single JoinSeedNodes, from an explicit +/// reachability signal — unlike the retired SelfFormAfter watchdog, which fired a +/// Join(self) mid-handshake on a bare timeout and could not tell "no seed answered" from +/// "a join is in flight", islanding a node a failover had just bounced. +/// +/// +public static class ClusterBootstrapGuard +{ + private static readonly Regex SeedPattern = + new(@"^akka(?:\.[a-z0-9]+)?://[^@/]+@(?[^:/]+):(?\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + /// The analyzed bootstrap role of this node within its seed list. + /// True only when the guard engages: exactly two seeds, one of them this node. + /// True when this node is the preferred founder (lower address) — form immediately, no probe. + /// The partner's advertised host (to probe when this node is the higher one); null when not applicable. + /// The partner's Akka port. + /// [self, partner] — self-first; forms a new cluster when no peer answers. + /// [partner, self] — peer-first; joins the partner's cluster, never self-forms while it is up. + public sealed record BootstrapRole( + bool Applies, + bool IsFounder, + string? PartnerHost, + int PartnerPort, + string[] SelfFirstOrder, + string[] PeerFirstOrder); + + /// + /// Analyzes the configured seed list to decide this node's bootstrap role. The guard engages only + /// for the Phase-6 pair shape (exactly two seeds, one of them this node); any other shape + /// (single-seed legacy site node, three+ seeds, self absent) returns + /// = false and the caller joins the configured seeds unchanged. + /// + /// The configured seed URIs (self-first by convention, but order is not relied on here). + /// This node's advertised host (). + /// This node's Akka port (). + /// The analyzed role; never null. + public static BootstrapRole Analyze(IReadOnlyList seeds, string selfHost, int selfPort) + { + ArgumentNullException.ThrowIfNull(seeds); + ArgumentException.ThrowIfNullOrWhiteSpace(selfHost); + + var na = new BootstrapRole(false, false, null, 0, Array.Empty(), Array.Empty()); + if (seeds.Count != 2) return na; + + var selfKey = Key(selfHost, selfPort); + string? self = null, partner = null; + string? partnerHost = null; + var partnerPort = 0; + + foreach (var seed in seeds) + { + if (!TryParse(seed, out var host, out var port)) return na; + if (string.Equals(Key(host, port), selfKey, StringComparison.OrdinalIgnoreCase)) + self = seed; + else + { + partner = seed; + partnerHost = host; + partnerPort = port; + } + } + + // Self must be exactly one of the two, and the other must be a distinct partner. + if (self is null || partner is null || partnerHost is null) return na; + + var selfFirst = new[] { self, partner }; + var peerFirst = new[] { partner, self }; + + // Deterministic tie-break: the lower canonical address is the preferred founder. Both nodes + // compute the same comparison (same two seeds), so exactly one is the founder. Case-INSENSITIVE + // to match how self/partner are classified above (and AkkaClusterOptionsValidator.IsSelf): + // container/DNS hostnames are conventionally case-insensitive, and a casing difference between + // the two sides' seed config must NOT make both think they are the founder (which would reopen + // the very split this guard closes). + var isFounder = string.Compare( + Key(selfHost, selfPort), + Key(partnerHost, partnerPort), + StringComparison.OrdinalIgnoreCase) < 0; + + return new BootstrapRole(true, isFounder, partnerHost, partnerPort, selfFirst, peerFirst); + } + + /// + /// The order the higher (non-founder) node uses once it has learned whether its partner is + /// reachable: peer-first when the founder is up (join it), self-first when the founder is absent + /// (form alone — cold-start-alone preserved). + /// + /// The analyzed role (must be applicable and NOT the founder). + /// Whether the partner's Akka endpoint became reachable within the probe window. + /// The seed order to pass to Cluster.JoinSeedNodes. + public static string[] HigherNodeOrder(BootstrapRole role, bool partnerReachable) + { + ArgumentNullException.ThrowIfNull(role); + return partnerReachable ? role.PeerFirstOrder : role.SelfFirstOrder; + } + + /// Parses akka.tcp://system@host:port[/...] into host + port. + /// The seed URI. + /// The parsed advertised host. + /// The parsed Akka port. + /// True when the seed matched the expected shape. + public static bool TryParse(string? seed, out string host, out int port) + { + host = string.Empty; + port = 0; + if (string.IsNullOrWhiteSpace(seed)) return false; + + var m = SeedPattern.Match(seed.Trim()); + if (!m.Success) return false; + if (!int.TryParse(m.Groups["port"].Value, out port)) return false; + host = m.Groups["host"].Value; + return true; + } + + private static string Key(string host, int port) => $"{host}:{port}"; +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs index 700c8b20..1dad8d14 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs @@ -241,7 +241,13 @@ public static class ServiceCollectionExtensions return new ClusterOptions { - SeedNodes = options.SeedNodes, + // When the bootstrap guard is on, DO NOT hand Akka the seed list: Akka.Cluster.Hosting + // auto-joins from ClusterOptions.SeedNodes the instant the ActorSystem starts, which is + // exactly the self-first race the guard exists to arbitrate. Left empty, the node starts + // unjoined and ClusterBootstrapCoordinator picks the order (founder self-first / joiner + // peer-first, after a reachability probe) and issues the single Cluster.JoinSeedNodes. + // The config seed list is still read — by the coordinator, off AkkaClusterOptions.SeedNodes. + SeedNodes = options.BootstrapGuard.Enabled ? Array.Empty() : options.SeedNodes, Roles = options.Roles, SplitBrainResolver = IsKeepOldest(options) ? new KeepOldestOption { DownIfAlone = true } : null, }; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 8f133bfb..51380bb8 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -390,6 +390,16 @@ builder.Services.AddAkka("otopcua", (ab, sp) => } }); +// Simultaneous-cold-start split-brain guard (dark switch Cluster:BootstrapGuard:Enabled, default off). +// Registered AFTER AddAkka so its StartAsync runs once Akka's hosted service has built (and, when the +// guard is off, already auto-joined) the ActorSystem. When on, the node started with no config seeds +// (BuildClusterOptions) and this coordinator issues the single reachability-gated JoinSeedNodes. It +// no-ops when the guard is off, so registering it unconditionally is safe. +builder.Services.AddHostedService(sp => new ClusterBootstrapCoordinator( + () => sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService>())); + // Down-if-alone recovery watchdog (#459). Registered AFTER AddAkka so it starts after Akka's own // hosted service has built the ActorSystem; it resolves the system lazily (never at construction) so // it can't race startup. On an unexpected SBR self-down it stops the host so the service supervisor diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs index 966b1268..8114898c 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs @@ -174,4 +174,44 @@ public sealed class AkkaClusterOptionsValidatorTests result.Failed.ShouldBeFalse(result.FailureMessage); } + + /// The bootstrap-guard timing knobs default to positive values and pass when enabled. + [Fact] + public void Bootstrap_guard_defaults_pass() + { + var options = CentralNode("site-a-1", Seed("site-a-1"), Seed("site-a-2")); + options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true }; + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } + + /// + /// A zero PartnerProbeSeconds silently degrades the guard to "never wait, form alone + /// immediately" — re-opening the split. It must fail the host at boot, not run degraded. + /// + [Fact] + public void Bootstrap_guard_with_non_positive_probe_seconds_fails() + { + var options = CentralNode("site-a-1", Seed("site-a-1"), Seed("site-a-2")); + options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 }; + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldContain("PartnerProbeSeconds"); + } + + /// The timing knobs are NOT validated when the guard is off — a disabled guard is inert. + [Fact] + public void Bootstrap_guard_disabled_does_not_validate_timings() + { + var options = CentralNode("site-a-1", Seed("site-a-1"), Seed("site-a-2")); + options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = false, PartnerProbeSeconds = 0 }; + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ClusterBootstrapCoordinatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ClusterBootstrapCoordinatorTests.cs new file mode 100644 index 00000000..fd741b0a --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ClusterBootstrapCoordinatorTests.cs @@ -0,0 +1,204 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Actor; +using Akka.Cluster; +using Akka.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Real-ActorSystem tests for the simultaneous-cold-start split-brain guard +/// ( + ), started through +/// the SAME wiring as production ( +/// with Cluster:BootstrapGuard:Enabled + the coordinator hosted service). This subsystem has a +/// history of bugs invisible to pure unit tests, so the load-bearing correctness properties are +/// asserted against running nodes, mirroring . +/// +/// +/// Ephemeral loopback ports are 5-digit, so numeric order equals the guard's ordinal "host:port" +/// string order — Min(port) is the founder, Max(port) is the higher (probing) node. +/// +public sealed class ClusterBootstrapCoordinatorTests +{ + private static int FreePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + /// + /// Starts a node with the bootstrap guard ENABLED, wired exactly as Program.cs does: empty Akka + /// seed list (via BuildClusterOptions) plus the coordinator hosted service that issues the + /// reachability-gated JoinSeedNodes. Seeds are listed self-first (as every shipped config + /// is); the guard, not the order, decides founder vs. joiner. + /// + private static async Task StartGuardedNodeAsync( + string systemName, int selfPort, int peerPort, int probeSeconds = 4) + { + var self = $"akka.tcp://{systemName}@127.0.0.1:{selfPort}"; + var peer = $"akka.tcp://{systemName}@127.0.0.1:{peerPort}"; + var options = new AkkaClusterOptions + { + SystemName = systemName, + Hostname = "127.0.0.1", + PublicHostname = "127.0.0.1", + Port = selfPort, + Roles = new[] { "driver" }, + SeedNodes = new[] { self, peer }, + BootstrapGuard = new ClusterBootstrapGuardOptions + { + Enabled = true, + PartnerProbeSeconds = probeSeconds, + PartnerProbeIntervalMs = 250, + ProbeConnectTimeoutMs = 500, + }, + }; + + var builder = Host.CreateDefaultBuilder(); + builder.ConfigureServices(services => + { + services.AddSingleton>(Options.Create(options)); + services.AddAkka(systemName, (ab, sp) => ab.WithOtOpcUaClusterBootstrap(sp)); + // Mirror Program.cs — the coordinator is what drives the guarded join. + services.AddHostedService(sp => new ClusterBootstrapCoordinator( + () => sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService>())); + }); + + var host = builder.Build(); + await host.StartAsync(); + return host; + } + + private static async Task WaitForUpMembersAsync(IHost host, int expected, TimeSpan timeout) + { + var cluster = Akka.Cluster.Cluster.Get(host.Services.GetRequiredService()); + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected) return true; + await Task.Delay(200); + } + return cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected; + } + + private static async Task StopAsync(IHost host) + { + try { await host.StopAsync(); } + catch (Exception) { /* teardown only */ } + host.Dispose(); + } + + /// + /// The FOUNDER (lower address) with the guard on forms a cluster alone when its partner is dead — + /// exactly as un-guarded self-first does. The guard must not regress the preferred founder. + /// + [Fact] + public async Task Founder_forms_alone_when_partner_is_dead() + { + var low = FreePort(); + var high = FreePort(); + var founderPort = Math.Min(low, high); + var deadPeerPort = Math.Max(low, high); // nothing listening + + var host = await StartGuardedNodeAsync("otopcua-guard-1", founderPort, deadPeerPort); + try + { + (await WaitForUpMembersAsync(host, 1, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("the founder (lower address) must self-form immediately when its partner is down"); + } + finally { await StopAsync(host); } + } + + /// + /// THE LOAD-BEARING CASE. The HIGHER node with the guard on must still cold-start ALONE when its + /// partner is genuinely dead: it probes the (dead) founder, times out, and falls back to self-first. + /// If the guard's peer-first logic were wrong, this node would hang in JoinSeedNodeProcess forever — + /// the very regression a naive "always let the lower node found" rule would introduce. + /// + [Fact] + public async Task Higher_node_forms_alone_when_partner_is_dead_after_probing() + { + var low = FreePort(); + var high = FreePort(); + var higherPort = Math.Max(low, high); + var deadFounderPort = Math.Min(low, high); // the founder is dead + + var host = await StartGuardedNodeAsync("otopcua-guard-2", higherPort, deadFounderPort, probeSeconds: 3); + try + { + // Must come Up AFTER the probe window (~3 s) expires and it falls back to self-first. + (await WaitForUpMembersAsync(host, 1, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("the higher node must fall back to self-first and form alone once the dead partner never answers the probe"); + } + finally { await StopAsync(host); } + } + + /// + /// FALSIFIABILITY CONTROL for the test above: the higher node does NOT form alone during the probe + /// window — it is genuinely waiting/probing, not self-forming immediately (which would mean the + /// guard is inert). If this ever starts seeing a member before the window, the probe is not gating. + /// + [Fact] + public async Task Higher_node_does_not_form_before_the_probe_window_expires() + { + var low = FreePort(); + var high = FreePort(); + var higherPort = Math.Max(low, high); + var deadFounderPort = Math.Min(low, high); + + var host = await StartGuardedNodeAsync("otopcua-guard-3", higherPort, deadFounderPort, probeSeconds: 10); + try + { + // Well within the 10 s probe window: still no cluster, because it is probing the dead founder. + (await WaitForUpMembersAsync(host, 1, TimeSpan.FromSeconds(3))) + .ShouldBeFalse("the higher node must probe for the founder, not self-form immediately"); + } + finally { await StopAsync(host); } + } + + /// + /// The higher node JOINS a live founder rather than forming a second cluster: start the founder, + /// then the higher node — its probe finds the founder reachable, it commits peer-first, and the + /// pair converges to ONE cluster of two. + /// + [Fact] + public async Task Higher_node_joins_a_live_founder() + { + const string systemName = "otopcua-guard-4"; + var low = FreePort(); + var high = FreePort(); + var founderPort = Math.Min(low, high); + var higherPort = Math.Max(low, high); + + var founder = await StartGuardedNodeAsync(systemName, founderPort, higherPort); + IHost? higher = null; + try + { + (await WaitForUpMembersAsync(founder, 1, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("the founder must be Up before the higher node probes it"); + + higher = await StartGuardedNodeAsync(systemName, higherPort, founderPort); + + (await WaitForUpMembersAsync(higher, 2, TimeSpan.FromSeconds(60))) + .ShouldBeTrue("the higher node must join the founder, forming one cluster of two"); + (await WaitForUpMembersAsync(founder, 2, TimeSpan.FromSeconds(60))) + .ShouldBeTrue("the founder must see the higher node as a member of ITS cluster"); + } + finally + { + if (higher is not null) await StopAsync(higher); + await StopAsync(founder); + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ClusterBootstrapGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ClusterBootstrapGuardTests.cs new file mode 100644 index 00000000..d1a92959 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/ClusterBootstrapGuardTests.cs @@ -0,0 +1,137 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Unit tests for — the pure decision core of the +/// simultaneous-cold-start split-brain guard. The runtime probe + JoinSeedNodes wiring lives in +/// ClusterBootstrapCoordinator and is covered by the live gate. +/// +public sealed class ClusterBootstrapGuardTests +{ + private const string A = "akka.tcp://otopcua@site-a-1:4053"; + private const string B = "akka.tcp://otopcua@site-a-2:4053"; + + [Fact] + public void Lower_address_node_is_the_founder_and_needs_no_probe() + { + // site-a-1 < site-a-2, so on site-a-1 the guard makes it the founder. + var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "site-a-1", 4053); + + role.Applies.ShouldBeTrue(); + role.IsFounder.ShouldBeTrue(); + role.SelfFirstOrder.ShouldBe(new[] { A, B }); + } + + [Fact] + public void Higher_address_node_is_not_the_founder_and_targets_the_partner_for_probing() + { + // On site-a-2, the partner is site-a-1 (the lower/founder). + var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053); + + role.Applies.ShouldBeTrue(); + role.IsFounder.ShouldBeFalse(); + role.PartnerHost.ShouldBe("site-a-1"); + role.PartnerPort.ShouldBe(4053); + } + + [Fact] + public void Tie_break_is_symmetric_regardless_of_seed_order() + { + // Both nodes must agree on who founds no matter how each lists its seeds. + var onLower = ClusterBootstrapGuard.Analyze(new[] { A, B }, "site-a-1", 4053); + var onHigher = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053); + + onLower.IsFounder.ShouldBeTrue(); + onHigher.IsFounder.ShouldBeFalse(); + // Exactly one founder. + (onLower.IsFounder ^ onHigher.IsFounder).ShouldBeTrue(); + } + + [Fact] + public void Higher_node_joins_the_founder_when_partner_is_reachable() + { + var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053); + + // Reachable ⇒ peer-first ⇒ JoinSeedNodeProcess ⇒ joins the founder, never self-forms. + ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: true) + .ShouldBe(new[] { A, B }); // partner (site-a-1) first + } + + [Fact] + public void Higher_node_forms_alone_when_partner_is_unreachable() + { + var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053); + + // Unreachable after the window ⇒ partner is dead ⇒ self-first ⇒ cold-start-alone preserved. + ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: false) + .ShouldBe(new[] { B, A }); // self (site-a-2) first + } + + [Fact] + public void Guard_does_not_apply_to_a_single_seed_node() + { + // A driver-only site node seeded solely by central-1 is not a pair seed — guard is inert. + var role = ClusterBootstrapGuard.Analyze(new[] { "akka.tcp://otopcua@central-1:4053" }, "site-x", 4053); + + role.Applies.ShouldBeFalse(); + } + + [Fact] + public void Guard_does_not_apply_when_self_is_absent_from_the_two_seeds() + { + var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "central-1", 4053); + + role.Applies.ShouldBeFalse(); + } + + [Fact] + public void Guard_does_not_apply_to_three_or_more_seeds() + { + var role = ClusterBootstrapGuard.Analyze( + new[] { A, B, "akka.tcp://otopcua@site-a-3:4053" }, "site-a-1", 4053); + + role.Applies.ShouldBeFalse(); + } + + [Fact] + public void Guard_does_not_apply_when_a_seed_is_malformed() + { + var role = ClusterBootstrapGuard.Analyze(new[] { A, "not-a-seed" }, "site-a-1", 4053); + + role.Applies.ShouldBeFalse(); + } + + [Theory] + [InlineData("akka.tcp://otopcua@site-a-1:4053", "site-a-1", 4053)] + [InlineData("akka.tcp://otopcua@10.0.0.5:4053/", "10.0.0.5", 4053)] + [InlineData(" akka.tcp://otopcua@host.lan:1234 ", "host.lan", 1234)] + public void TryParse_extracts_host_and_port(string seed, string expectedHost, int expectedPort) + { + ClusterBootstrapGuard.TryParse(seed, out var host, out var port).ShouldBeTrue(); + host.ShouldBe(expectedHost); + port.ShouldBe(expectedPort); + } + + [Theory] + [InlineData("")] + [InlineData("garbage")] + [InlineData("akka.tcp://otopcua@host")] // no port + public void TryParse_rejects_malformed_seeds(string seed) + { + ClusterBootstrapGuard.TryParse(seed, out _, out _).ShouldBeFalse(); + } + + [Fact] + public void Port_participates_in_the_tie_break_when_hosts_are_equal() + { + // Shared-host loopback pair (test rig): same host, different ports — port breaks the tie. + const string low = "akka.tcp://otopcua@127.0.0.1:4053"; + const string high = "akka.tcp://otopcua@127.0.0.1:4054"; + + ClusterBootstrapGuard.Analyze(new[] { low, high }, "127.0.0.1", 4053).IsFounder.ShouldBeTrue(); + ClusterBootstrapGuard.Analyze(new[] { high, low }, "127.0.0.1", 4054).IsFounder.ShouldBeFalse(); + } +}