using Akka.Actor; using Akka.Cluster; using Akka.Configuration; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure; using ZB.MOM.WW.ScadaBridge.Host; using ZB.MOM.WW.ScadaBridge.Host.Actors; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; /// /// Real-ActorSystem tests for the simultaneous-cold-start split-brain guard (Gitea #33): /// + , driven exactly as /// production drives them — the node's ActorSystem is built from the production BuildHocon output /// with the guard enabled (which emits an EMPTY seed list, so Akka does not auto-join), then the real /// coordinator issues the reachability-gated JoinSeedNodes. This subsystem has a history of bugs /// invisible to pure unit tests, so the load-bearing correctness properties are asserted against running /// nodes, mirroring . /// /// /// Ephemeral loopback ports share host 127.0.0.1, so the port breaks the guard's ordinal /// host:port tie — Min(port) is the founder, Max(port) is the higher (probing) node. /// public sealed class ClusterBootstrapCoordinatorTests : IAsyncLifetime { private readonly List _systems = new(); private readonly List _coordinators = new(); /// Starts a node with the bootstrap guard ENABLED, wired exactly as the composition roots /// do: the ActorSystem is built with an empty HOCON seed list (guard on) and the real coordinator /// issues the reachability-gated join. Seeds are listed self-first (as every shipped config is); the /// guard, not the order, decides founder vs. joiner. private async Task StartGuardedNodeAsync(int selfPort, int peerPort, int probeSeconds = 4) { var self = $"akka.tcp://scadabridge@127.0.0.1:{selfPort}"; var peer = $"akka.tcp://scadabridge@127.0.0.1:{peerPort}"; var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = selfPort }; var clusterOptions = new ClusterOptions { SeedNodes = new List { self, peer }, BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = probeSeconds, PartnerProbeIntervalMs = 250, ProbeConnectTimeoutMs = 500, }, StableAfter = TimeSpan.FromSeconds(3), HeartbeatInterval = TimeSpan.FromMilliseconds(500), FailureDetectionThreshold = TimeSpan.FromSeconds(2), MinNrOfMembers = 1, }; var hocon = AkkaHostedService.BuildHocon( nodeOptions, clusterOptions, new[] { "Central" }, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3)); var system = ActorSystem.Create("scadabridge", ConfigurationFactory.ParseString(hocon)); _systems.Add(system); var coordinator = new ClusterBootstrapCoordinator( () => system, Options.Create(clusterOptions), Options.Create(nodeOptions), NullLogger.Instance); _coordinators.Add(coordinator); await coordinator.StartAsync(CancellationToken.None); return system; } private static async Task WaitForUpAsync(ActorSystem system, int expected, TimeSpan timeout) { var cluster = Akka.Cluster.Cluster.Get(system); var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected) return true; await Task.Delay(200); } return cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected; } /// /// The FOUNDER (lower address) with the guard on forms a cluster alone when its partner is dead — /// exactly as un-guarded self-first does. The guard must not regress the preferred founder. /// [Fact] public async Task Founder_forms_alone_when_partner_is_dead() { var low = TwoNodeClusterFixture.GetFreeTcpPort(); var high = TwoNodeClusterFixture.GetFreeTcpPort(); var founderPort = Math.Min(low, high); var deadPeerPort = Math.Max(low, high); // nothing listening var system = await StartGuardedNodeAsync(founderPort, deadPeerPort); Assert.True(await WaitForUpAsync(system, 1, TimeSpan.FromSeconds(30)), "the founder (lower address) must self-form immediately when its partner is down"); } /// /// THE LOAD-BEARING CASE. The HIGHER node with the guard on must still cold-start ALONE when its /// partner is genuinely dead: it probes the (dead) founder, times out, and falls back to self-first. /// If the guard's peer-first logic were wrong, this node would hang in JoinSeedNodeProcess forever — /// the very regression a naive "always let the lower node found" rule would introduce. /// [Fact] public async Task Higher_node_forms_alone_when_partner_is_dead_after_probing() { var low = TwoNodeClusterFixture.GetFreeTcpPort(); var high = TwoNodeClusterFixture.GetFreeTcpPort(); var higherPort = Math.Max(low, high); var deadFounderPort = Math.Min(low, high); // the founder is dead var system = await StartGuardedNodeAsync(higherPort, deadFounderPort, probeSeconds: 3); // Must come Up AFTER the probe window (~3s) expires and it falls back to self-first. Assert.True(await WaitForUpAsync(system, 1, TimeSpan.FromSeconds(30)), "the higher node must fall back to self-first and form alone once the dead partner never answers the probe"); } /// /// FALSIFIABILITY CONTROL for the test above: the higher node does NOT form alone during the probe /// window — it is genuinely waiting/probing, not self-forming immediately (which would mean the /// guard is inert). If this ever starts seeing a member before the window, the probe is not gating. /// [Fact] public async Task Higher_node_does_not_form_before_the_probe_window_expires() { var low = TwoNodeClusterFixture.GetFreeTcpPort(); var high = TwoNodeClusterFixture.GetFreeTcpPort(); var higherPort = Math.Max(low, high); var deadFounderPort = Math.Min(low, high); var system = await StartGuardedNodeAsync(higherPort, deadFounderPort, probeSeconds: 10); // Well within the 10s probe window: still no cluster, because it is probing the dead founder. Assert.False(await WaitForUpAsync(system, 1, TimeSpan.FromSeconds(3)), "the higher node must probe for the founder, not self-form immediately"); } /// /// The higher node JOINS a live founder rather than forming a second cluster: start the founder, /// then the higher node — its probe finds the founder reachable, it commits peer-first, and the /// pair converges to ONE cluster of two. /// [Fact] public async Task Higher_node_joins_a_live_founder() { var low = TwoNodeClusterFixture.GetFreeTcpPort(); var high = TwoNodeClusterFixture.GetFreeTcpPort(); var founderPort = Math.Min(low, high); var higherPort = Math.Max(low, high); var founder = await StartGuardedNodeAsync(founderPort, higherPort); Assert.True(await WaitForUpAsync(founder, 1, TimeSpan.FromSeconds(30)), "the founder must be Up before the higher node probes it"); var higher = await StartGuardedNodeAsync(higherPort, founderPort); Assert.True(await WaitForUpAsync(higher, 2, TimeSpan.FromSeconds(60)), "the higher node must join the founder, forming one cluster of two"); Assert.True(await WaitForUpAsync(founder, 2, TimeSpan.FromSeconds(60)), "the founder must see the higher node as a member of ITS cluster"); } /// /// THE HEADLINE #33 CASE. Both nodes cold-start truly simultaneously with the guard on. Without the /// guard each would run FirstSeedNodeProcess, race, and form its own 1-node cluster (split brain). /// With it, the founder forms and the higher node probes-then-joins, so the pair converges to /// EXACTLY ONE cluster of two — no two oldest, no dual singletons. /// [Fact] public async Task Both_nodes_cold_starting_together_form_exactly_one_cluster() { var low = TwoNodeClusterFixture.GetFreeTcpPort(); var high = TwoNodeClusterFixture.GetFreeTcpPort(); var founderPort = Math.Min(low, high); var higherPort = Math.Max(low, high); // Start both without serialization — as a shared power event does. var founderTask = StartGuardedNodeAsync(founderPort, higherPort); var higherTask = StartGuardedNodeAsync(higherPort, founderPort); var founder = await founderTask; var higher = await higherTask; Assert.True(await WaitForUpAsync(founder, 2, TimeSpan.FromSeconds(60)), "the pair must converge to one 2-member cluster, not two 1-member clusters"); Assert.True(await WaitForUpAsync(higher, 2, TimeSpan.FromSeconds(60)), "the higher node must be in the SAME cluster as the founder"); } public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() { foreach (var c in _coordinators) { try { await c.StopAsync(CancellationToken.None); } catch { /* teardown */ } } foreach (var s in _systems) { try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ } } } }