feat(cluster): simultaneous-cold-start split-brain guard (#33)

Port OtOpcUa's bootstrap guard (lmxopcua d1dac87f) to ScadaBridge's BuildHocon bootstrap. Both pair nodes are self-first seeds, so a true simultaneous cold start (shared site power event) races FirstSeedNodeProcess on both and forms two 1-node clusters that never merge. Opt-in dark switch ScadaBridge:Cluster:BootstrapGuard:Enabled (default off, guard-off behavior byte-identical). When on: BuildHocon emits an empty seed list so Akka does not auto-join, and ClusterBootstrapCoordinator (IHostedService, registered in both the Central and Site composition roots) picks the join order from ClusterBootstrapGuard's pure decision core — the lower canonical host:port is the founder (self-first, forms immediately); the higher node TCP-probes the founder up to PartnerProbeSeconds and joins peer-first if reachable, else self-first (cold-start-alone preserved). Decides before a single JoinSeedNodes, never re-forms mid-handshake. Review notes carried over: case-insensitive founder tie-break; fail-fast validation of probe timings when enabled; higher-node-cold-start-alone covered by a real-ActorSystem test. 21 unit + 5 real-cluster tests (incl. the headline both-cold-start-together-form-one-cluster).
2026-07-24 08:55:48 -04:00
parent bb138d5254
commit 248676ed16
10 changed files with 895 additions and 2 deletions
@@ -0,0 +1,147 @@
+namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
+
+/// <summary>
+/// Unit tests for <see cref="ClusterBootstrapGuard"/> — the pure decision core of the
+/// simultaneous-cold-start split-brain guard (Gitea #33). The runtime probe + JoinSeedNodes wiring
+/// lives in <c>ClusterBootstrapCoordinator</c> and is covered by the real-ActorSystem
+/// <c>ClusterBootstrapCoordinatorTests</c> in the integration suite.
+/// </summary>
+public class ClusterBootstrapGuardTests
+{
+    private const string A = "akka.tcp://scadabridge@scadabridge-site-a-a:8082";
+    private const string B = "akka.tcp://scadabridge@scadabridge-site-a-b:8082";
+
+    [Fact]
+    public void Lower_address_node_is_the_founder_and_needs_no_probe()
+    {
+        // scadabridge-site-a-a < scadabridge-site-a-b, so on node-a the guard makes it the founder.
+        var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-site-a-a", 8082);
+
+        Assert.True(role.Applies);
+        Assert.True(role.IsFounder);
+        Assert.Equal(new[] { A, B }, role.SelfFirstOrder);
+    }
+
+    [Fact]
+    public void Higher_address_node_is_not_the_founder_and_targets_the_partner_for_probing()
+    {
+        // On node-b the partner is node-a (the lower/founder).
+        var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
+
+        Assert.True(role.Applies);
+        Assert.False(role.IsFounder);
+        Assert.Equal("scadabridge-site-a-a", role.PartnerHost);
+        Assert.Equal(8082, role.PartnerPort);
+    }
+
+    [Fact]
+    public void Tie_break_is_symmetric_regardless_of_seed_order()
+    {
+        // Both nodes must agree on who founds no matter how each lists its seeds.
+        var onLower = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-site-a-a", 8082);
+        var onHigher = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
+
+        Assert.True(onLower.IsFounder);
+        Assert.False(onHigher.IsFounder);
+        // Exactly one founder.
+        Assert.True(onLower.IsFounder ^ onHigher.IsFounder);
+    }
+
+    [Fact]
+    public void Tie_break_is_case_insensitive_so_a_casing_mismatch_cannot_make_both_founders()
+    {
+        // The two sides' seed config differ only in host casing (review note 1): if the tie-break
+        // were case-sensitive, both could conclude they are the founder and re-open the split.
+        var lowerSelf = "akka.tcp://scadabridge@SITE-A-A:8082";
+        var higherSelf = "akka.tcp://scadabridge@site-a-b:8082";
+
+        var onLower = ClusterBootstrapGuard.Analyze(new[] { lowerSelf, higherSelf }, "SITE-A-A", 8082);
+        var onHigher = ClusterBootstrapGuard.Analyze(new[] { higherSelf, lowerSelf }, "site-a-b", 8082);
+
+        Assert.True(onLower.IsFounder ^ onHigher.IsFounder);
+        Assert.True(onLower.IsFounder); // "site-a-a" < "site-a-b" ordinal-ignore-case
+    }
+
+    [Fact]
+    public void Higher_node_joins_the_founder_when_partner_is_reachable()
+    {
+        var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
+
+        // Reachable ⇒ peer-first ⇒ JoinSeedNodeProcess ⇒ joins the founder, never self-forms.
+        Assert.Equal(new[] { A, B }, ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: true)); // partner (node-a) first
+    }
+
+    [Fact]
+    public void Higher_node_forms_alone_when_partner_is_unreachable()
+    {
+        var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
+
+        // Unreachable after the window ⇒ partner is dead ⇒ self-first ⇒ cold-start-alone preserved.
+        Assert.Equal(new[] { B, A }, ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: false)); // self (node-b) first
+    }
+
+    [Fact]
+    public void Guard_does_not_apply_to_a_single_seed_node()
+    {
+        // A single-node install lists exactly one seed (itself) — not a pair, guard is inert.
+        var role = ClusterBootstrapGuard.Analyze(new[] { "akka.tcp://scadabridge@lone:8081" }, "lone", 8081);
+
+        Assert.False(role.Applies);
+    }
+
+    [Fact]
+    public void Guard_does_not_apply_when_self_is_absent_from_the_two_seeds()
+    {
+        var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-central-a", 8081);
+
+        Assert.False(role.Applies);
+    }
+
+    [Fact]
+    public void Guard_does_not_apply_to_three_or_more_seeds()
+    {
+        var role = ClusterBootstrapGuard.Analyze(
+            new[] { A, B, "akka.tcp://scadabridge@scadabridge-site-a-c:8082" }, "scadabridge-site-a-a", 8082);
+
+        Assert.False(role.Applies);
+    }
+
+    [Fact]
+    public void Guard_does_not_apply_when_a_seed_is_malformed()
+    {
+        var role = ClusterBootstrapGuard.Analyze(new[] { A, "not-a-seed" }, "scadabridge-site-a-a", 8082);
+
+        Assert.False(role.Applies);
+    }
+
+    [Theory]
+    [InlineData("akka.tcp://scadabridge@scadabridge-site-a-a:8082", "scadabridge-site-a-a", 8082)]
+    [InlineData("akka.tcp://scadabridge@10.0.0.5:8082/", "10.0.0.5", 8082)]
+    [InlineData("  akka.tcp://scadabridge@host.lan:1234  ", "host.lan", 1234)]
+    public void TryParse_extracts_host_and_port(string seed, string expectedHost, int expectedPort)
+    {
+        Assert.True(ClusterBootstrapGuard.TryParse(seed, out var host, out var port));
+        Assert.Equal(expectedHost, host);
+        Assert.Equal(expectedPort, port);
+    }
+
+    [Theory]
+    [InlineData("")]
+    [InlineData("garbage")]
+    [InlineData("akka.tcp://scadabridge@host")] // no port
+    public void TryParse_rejects_malformed_seeds(string seed)
+    {
+        Assert.False(ClusterBootstrapGuard.TryParse(seed, out _, out _));
+    }
+
+    [Fact]
+    public void Port_participates_in_the_tie_break_when_hosts_are_equal()
+    {
+        // Shared-host loopback pair (test rig): same host, different ports — port breaks the tie.
+        const string low = "akka.tcp://scadabridge@127.0.0.1:8081";
+        const string high = "akka.tcp://scadabridge@127.0.0.1:8082";
+
+        Assert.True(ClusterBootstrapGuard.Analyze(new[] { low, high }, "127.0.0.1", 8081).IsFounder);
+        Assert.False(ClusterBootstrapGuard.Analyze(new[] { high, low }, "127.0.0.1", 8082).IsFounder);
+    }
+}
@@ -215,4 +215,68 @@ public class ClusterOptionsValidatorTests
        Assert.Contains("StableAfter", result.FailureMessage);
        Assert.Contains("HeartbeatInterval", result.FailureMessage);
    }
+
+    // ---- BootstrapGuard timing validation (Gitea #33) ----
+
+    [Fact]
+    public void BootstrapGuard_disabled_with_zero_timings_passes_validation()
+    {
+        // The knobs are inert when the guard is off, so a disabled guard must never block a boot —
+        // even if the timing values are nonsense (0 here).
+        var options = ValidOptions();
+        options.BootstrapGuard = new ClusterBootstrapGuardOptions
+        {
+            Enabled = false,
+            PartnerProbeSeconds = 0,
+            PartnerProbeIntervalMs = 0,
+            ProbeConnectTimeoutMs = 0,
+        };
+
+        var result = new ClusterOptionsValidator().Validate(null, options);
+
+        Assert.True(result.Succeeded, result.FailureMessage);
+    }
+
+    [Fact]
+    public void BootstrapGuard_enabled_with_default_timings_passes_validation()
+    {
+        var options = ValidOptions();
+        options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true };
+
+        var result = new ClusterOptionsValidator().Validate(null, options);
+
+        Assert.True(result.Succeeded, result.FailureMessage);
+    }
+
+    [Fact]
+    public void BootstrapGuard_enabled_with_nonpositive_probe_seconds_fails_validation()
+    {
+        // A zero PartnerProbeSeconds silently degrades the guard to "never wait, always conclude the
+        // partner is dead" — the higher node forms alone immediately and re-opens the split. Reject it.
+        var options = ValidOptions();
+        options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 };
+
+        var result = new ClusterOptionsValidator().Validate(null, options);
+
+        Assert.True(result.Failed);
+        Assert.Contains("PartnerProbeSeconds", result.FailureMessage);
+    }
+
+    [Fact]
+    public void BootstrapGuard_enabled_with_nonpositive_interval_or_timeout_fails_validation()
+    {
+        var options = ValidOptions();
+        options.BootstrapGuard = new ClusterBootstrapGuardOptions
+        {
+            Enabled = true,
+            PartnerProbeIntervalMs = 0,
+            ProbeConnectTimeoutMs = -1,
+        };
+
+        var result = new ClusterOptionsValidator().Validate(null, options);
+
+        Assert.True(result.Failed);
+        Assert.Contains("PartnerProbeIntervalMs", result.FailureMessage);
+        Assert.Contains("ProbeConnectTimeoutMs", result.FailureMessage);
+    }
 }