feat(cluster): simultaneous-cold-start split-brain guard (#33)
Port OtOpcUa's bootstrap guard (lmxopcua d1dac87f) to ScadaBridge's BuildHocon bootstrap. Both pair nodes are self-first seeds, so a true simultaneous cold start (shared site power event) races FirstSeedNodeProcess on both and forms two 1-node clusters that never merge. Opt-in dark switch ScadaBridge:Cluster:BootstrapGuard:Enabled (default off, guard-off behavior byte-identical). When on: BuildHocon emits an empty seed list so Akka does not auto-join, and ClusterBootstrapCoordinator (IHostedService, registered in both the Central and Site composition roots) picks the join order from ClusterBootstrapGuard's pure decision core — the lower canonical host:port is the founder (self-first, forms immediately); the higher node TCP-probes the founder up to PartnerProbeSeconds and joins peer-first if reachable, else self-first (cold-start-alone preserved). Decides before a single JoinSeedNodes, never re-forms mid-handshake. Review notes carried over: case-insensitive founder tie-break; fail-fast validation of probe timings when enabled; higher-node-cold-start-alone covered by a real-ActorSystem test. 21 unit + 5 real-cluster tests (incl. the headline both-cold-start-together-form-one-cluster).
This commit is contained in:
Joseph Doherty
+147
@@ -0,0 +1,147 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for <see cref="ClusterBootstrapGuard"/> — the pure decision core of the
|
||||
/// simultaneous-cold-start split-brain guard (Gitea #33). The runtime probe + JoinSeedNodes wiring
|
||||
/// lives in <c>ClusterBootstrapCoordinator</c> and is covered by the real-ActorSystem
|
||||
/// <c>ClusterBootstrapCoordinatorTests</c> in the integration suite.
|
||||
/// </summary>
|
||||
public class ClusterBootstrapGuardTests
|
||||
{
|
||||
private const string A = "akka.tcp://scadabridge@scadabridge-site-a-a:8082";
|
||||
private const string B = "akka.tcp://scadabridge@scadabridge-site-a-b:8082";
|
||||
|
||||
[Fact]
|
||||
public void Lower_address_node_is_the_founder_and_needs_no_probe()
|
||||
{
|
||||
// scadabridge-site-a-a < scadabridge-site-a-b, so on node-a the guard makes it the founder.
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-site-a-a", 8082);
|
||||
|
||||
Assert.True(role.Applies);
|
||||
Assert.True(role.IsFounder);
|
||||
Assert.Equal(new[] { A, B }, role.SelfFirstOrder);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Higher_address_node_is_not_the_founder_and_targets_the_partner_for_probing()
|
||||
{
|
||||
// On node-b the partner is node-a (the lower/founder).
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
Assert.True(role.Applies);
|
||||
Assert.False(role.IsFounder);
|
||||
Assert.Equal("scadabridge-site-a-a", role.PartnerHost);
|
||||
Assert.Equal(8082, role.PartnerPort);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tie_break_is_symmetric_regardless_of_seed_order()
|
||||
{
|
||||
// Both nodes must agree on who founds no matter how each lists its seeds.
|
||||
var onLower = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-site-a-a", 8082);
|
||||
var onHigher = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
Assert.True(onLower.IsFounder);
|
||||
Assert.False(onHigher.IsFounder);
|
||||
// Exactly one founder.
|
||||
Assert.True(onLower.IsFounder ^ onHigher.IsFounder);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tie_break_is_case_insensitive_so_a_casing_mismatch_cannot_make_both_founders()
|
||||
{
|
||||
// The two sides' seed config differ only in host casing (review note 1): if the tie-break
|
||||
// were case-sensitive, both could conclude they are the founder and re-open the split.
|
||||
var lowerSelf = "akka.tcp://scadabridge@SITE-A-A:8082";
|
||||
var higherSelf = "akka.tcp://scadabridge@site-a-b:8082";
|
||||
|
||||
var onLower = ClusterBootstrapGuard.Analyze(new[] { lowerSelf, higherSelf }, "SITE-A-A", 8082);
|
||||
var onHigher = ClusterBootstrapGuard.Analyze(new[] { higherSelf, lowerSelf }, "site-a-b", 8082);
|
||||
|
||||
Assert.True(onLower.IsFounder ^ onHigher.IsFounder);
|
||||
Assert.True(onLower.IsFounder); // "site-a-a" < "site-a-b" ordinal-ignore-case
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Higher_node_joins_the_founder_when_partner_is_reachable()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
// Reachable ⇒ peer-first ⇒ JoinSeedNodeProcess ⇒ joins the founder, never self-forms.
|
||||
Assert.Equal(new[] { A, B }, ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: true)); // partner (node-a) first
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Higher_node_forms_alone_when_partner_is_unreachable()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
// Unreachable after the window ⇒ partner is dead ⇒ self-first ⇒ cold-start-alone preserved.
|
||||
Assert.Equal(new[] { B, A }, ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: false)); // self (node-b) first
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_to_a_single_seed_node()
|
||||
{
|
||||
// A single-node install lists exactly one seed (itself) — not a pair, guard is inert.
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { "akka.tcp://scadabridge@lone:8081" }, "lone", 8081);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_when_self_is_absent_from_the_two_seeds()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-central-a", 8081);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_to_three_or_more_seeds()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(
|
||||
new[] { A, B, "akka.tcp://scadabridge@scadabridge-site-a-c:8082" }, "scadabridge-site-a-a", 8082);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_when_a_seed_is_malformed()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { A, "not-a-seed" }, "scadabridge-site-a-a", 8082);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("akka.tcp://scadabridge@scadabridge-site-a-a:8082", "scadabridge-site-a-a", 8082)]
|
||||
[InlineData("akka.tcp://scadabridge@10.0.0.5:8082/", "10.0.0.5", 8082)]
|
||||
[InlineData(" akka.tcp://scadabridge@host.lan:1234 ", "host.lan", 1234)]
|
||||
public void TryParse_extracts_host_and_port(string seed, string expectedHost, int expectedPort)
|
||||
{
|
||||
Assert.True(ClusterBootstrapGuard.TryParse(seed, out var host, out var port));
|
||||
Assert.Equal(expectedHost, host);
|
||||
Assert.Equal(expectedPort, port);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("garbage")]
|
||||
[InlineData("akka.tcp://scadabridge@host")] // no port
|
||||
public void TryParse_rejects_malformed_seeds(string seed)
|
||||
{
|
||||
Assert.False(ClusterBootstrapGuard.TryParse(seed, out _, out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Port_participates_in_the_tie_break_when_hosts_are_equal()
|
||||
{
|
||||
// Shared-host loopback pair (test rig): same host, different ports — port breaks the tie.
|
||||
const string low = "akka.tcp://scadabridge@127.0.0.1:8081";
|
||||
const string high = "akka.tcp://scadabridge@127.0.0.1:8082";
|
||||
|
||||
Assert.True(ClusterBootstrapGuard.Analyze(new[] { low, high }, "127.0.0.1", 8081).IsFounder);
|
||||
Assert.False(ClusterBootstrapGuard.Analyze(new[] { high, low }, "127.0.0.1", 8082).IsFounder);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user