279d1d0fb1
Closes the residual Phase-7 gap: when BOTH nodes of a 2-node pair cold-start at the same instant, each self-first seed runs FirstSeedNodeProcess, times out waiting for the other, and forms its own 1-node cluster — two Primaries in one pair (the Phase 6 live gate reproduced this reliably on docker). The prior mitigation was operational (staggered start / compose depends_on), which does not exist on production hardware. The guard (dark switch Cluster:BootstrapGuard:Enabled, default OFF) prevents the split without giving up cold-start-alone: - The lower-address node is the preferred founder: self-first, forms immediately. - The higher node probes its partner's Akka port (TCP connect) up to PartnerProbeSeconds: reachable => peer-first (join the founder, never race it); unreachable => self-first (partner is dead, form alone). The order is decided BEFORE the single JoinSeedNodes, from an explicit reachability signal — never re-formed mid-handshake (the retired SelfFormAfter failure mode). When on, Akka gets no config seeds (BuildClusterOptions) and ClusterBootstrapCoordinator drives the join. Review-driven hardening: case-insensitive tie-break (a hostname-casing mismatch would reopen the split); fail-fast validation of the timing knobs; the residual "founder dies in the probe->join window" hang is documented and made operator-visible (warning + a restart recovers). Real-ActorSystem coordinator tests cover the load-bearing higher-node cold-start-alone case; 147/147 Cluster tests pass. Live-gated on docker-dev (site-a = enablement demo, serialization removed, guard on; site-b keeps depends_on-serialization + guard off as the A/B control): simultaneous start -> site-a-1 founds, site-a-2 probes-reachable-joins -> 250/240, NO split; higher-node-alone -> probes dead founder, self-forms -> 250; founder rejoin -> 240. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
138 lines
5.0 KiB
C#
138 lines
5.0 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Cluster;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
|
|
|
/// <summary>
|
|
/// Unit tests for <see cref="ClusterBootstrapGuard"/> — the pure decision core of the
|
|
/// simultaneous-cold-start split-brain guard. The runtime probe + JoinSeedNodes wiring lives in
|
|
/// <c>ClusterBootstrapCoordinator</c> and is covered by the live gate.
|
|
/// </summary>
|
|
public sealed class ClusterBootstrapGuardTests
|
|
{
|
|
private const string A = "akka.tcp://otopcua@site-a-1:4053";
|
|
private const string B = "akka.tcp://otopcua@site-a-2:4053";
|
|
|
|
[Fact]
|
|
public void Lower_address_node_is_the_founder_and_needs_no_probe()
|
|
{
|
|
// site-a-1 < site-a-2, so on site-a-1 the guard makes it the founder.
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "site-a-1", 4053);
|
|
|
|
role.Applies.ShouldBeTrue();
|
|
role.IsFounder.ShouldBeTrue();
|
|
role.SelfFirstOrder.ShouldBe(new[] { A, B });
|
|
}
|
|
|
|
[Fact]
|
|
public void Higher_address_node_is_not_the_founder_and_targets_the_partner_for_probing()
|
|
{
|
|
// On site-a-2, the partner is site-a-1 (the lower/founder).
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053);
|
|
|
|
role.Applies.ShouldBeTrue();
|
|
role.IsFounder.ShouldBeFalse();
|
|
role.PartnerHost.ShouldBe("site-a-1");
|
|
role.PartnerPort.ShouldBe(4053);
|
|
}
|
|
|
|
[Fact]
|
|
public void Tie_break_is_symmetric_regardless_of_seed_order()
|
|
{
|
|
// Both nodes must agree on who founds no matter how each lists its seeds.
|
|
var onLower = ClusterBootstrapGuard.Analyze(new[] { A, B }, "site-a-1", 4053);
|
|
var onHigher = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053);
|
|
|
|
onLower.IsFounder.ShouldBeTrue();
|
|
onHigher.IsFounder.ShouldBeFalse();
|
|
// Exactly one founder.
|
|
(onLower.IsFounder ^ onHigher.IsFounder).ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Higher_node_joins_the_founder_when_partner_is_reachable()
|
|
{
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053);
|
|
|
|
// Reachable ⇒ peer-first ⇒ JoinSeedNodeProcess ⇒ joins the founder, never self-forms.
|
|
ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: true)
|
|
.ShouldBe(new[] { A, B }); // partner (site-a-1) first
|
|
}
|
|
|
|
[Fact]
|
|
public void Higher_node_forms_alone_when_partner_is_unreachable()
|
|
{
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "site-a-2", 4053);
|
|
|
|
// Unreachable after the window ⇒ partner is dead ⇒ self-first ⇒ cold-start-alone preserved.
|
|
ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: false)
|
|
.ShouldBe(new[] { B, A }); // self (site-a-2) first
|
|
}
|
|
|
|
[Fact]
|
|
public void Guard_does_not_apply_to_a_single_seed_node()
|
|
{
|
|
// A driver-only site node seeded solely by central-1 is not a pair seed — guard is inert.
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { "akka.tcp://otopcua@central-1:4053" }, "site-x", 4053);
|
|
|
|
role.Applies.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void Guard_does_not_apply_when_self_is_absent_from_the_two_seeds()
|
|
{
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "central-1", 4053);
|
|
|
|
role.Applies.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void Guard_does_not_apply_to_three_or_more_seeds()
|
|
{
|
|
var role = ClusterBootstrapGuard.Analyze(
|
|
new[] { A, B, "akka.tcp://otopcua@site-a-3:4053" }, "site-a-1", 4053);
|
|
|
|
role.Applies.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void Guard_does_not_apply_when_a_seed_is_malformed()
|
|
{
|
|
var role = ClusterBootstrapGuard.Analyze(new[] { A, "not-a-seed" }, "site-a-1", 4053);
|
|
|
|
role.Applies.ShouldBeFalse();
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("akka.tcp://otopcua@site-a-1:4053", "site-a-1", 4053)]
|
|
[InlineData("akka.tcp://otopcua@10.0.0.5:4053/", "10.0.0.5", 4053)]
|
|
[InlineData(" akka.tcp://otopcua@host.lan:1234 ", "host.lan", 1234)]
|
|
public void TryParse_extracts_host_and_port(string seed, string expectedHost, int expectedPort)
|
|
{
|
|
ClusterBootstrapGuard.TryParse(seed, out var host, out var port).ShouldBeTrue();
|
|
host.ShouldBe(expectedHost);
|
|
port.ShouldBe(expectedPort);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("")]
|
|
[InlineData("garbage")]
|
|
[InlineData("akka.tcp://otopcua@host")] // no port
|
|
public void TryParse_rejects_malformed_seeds(string seed)
|
|
{
|
|
ClusterBootstrapGuard.TryParse(seed, out _, out _).ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void Port_participates_in_the_tie_break_when_hosts_are_equal()
|
|
{
|
|
// Shared-host loopback pair (test rig): same host, different ports — port breaks the tie.
|
|
const string low = "akka.tcp://otopcua@127.0.0.1:4053";
|
|
const string high = "akka.tcp://otopcua@127.0.0.1:4054";
|
|
|
|
ClusterBootstrapGuard.Analyze(new[] { low, high }, "127.0.0.1", 4053).IsFounder.ShouldBeTrue();
|
|
ClusterBootstrapGuard.Analyze(new[] { high, low }, "127.0.0.1", 4054).IsFounder.ShouldBeFalse();
|
|
}
|
|
}
|