using System.Net; using System.Net.Sockets; using Akka.Actor; using Akka.Cluster; using Akka.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; using Shouldly; using Xunit; namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; /// /// Guards the InitJoin self-form fallback — whether a node whose peer is down at boot ever becomes /// operational. /// /// /// /// Why these tests start real hosts through the production bootstrap. Like /// , the question is what a running node /// actually does, not what an options object holds. Every host here is built with /// — the shipped entry /// point — so a change that stops arming the fallback (or arms it from a place production /// does not reach) turns these red. A test that called /// itself would pin the test's wiring instead. /// /// /// Seed ordering is load-bearing. The node under test lists itself SECOND in its own /// seed list, behind a dead peer port. Akka only lets the FIRST listed seed self-join, so /// nothing but the fallback can bring these nodes Up — which is what makes the negative /// scenarios (disabled window, non-seed node) meaningful rather than vacuous. Each of those /// carries a positive control (an explicit Cluster.Join(SelfAddress)) proving the node /// was formable all along and only the fallback was absent. /// /// public sealed class SelfFormBootstrapTests { private static int FreePort() { var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; listener.Stop(); return port; } private static async Task StartNodeAsync( string systemName, int selfPort, int peerPort, TimeSpan? selfFormAfter, bool selfInSeeds = true) { var options = new AkkaClusterOptions { SystemName = systemName, Hostname = "127.0.0.1", PublicHostname = "127.0.0.1", Port = selfPort, Roles = new[] { "admin", "driver" }, SelfFormAfter = selfFormAfter, // Self listed SECOND, behind a port nothing is listening on: Akka lets only the first // seed self-join, so this node can never form a cluster on its own except via the // fallback. SeedNodes = selfInSeeds ? new[] { $"akka.tcp://{systemName}@127.0.0.1:{peerPort}", $"akka.tcp://{systemName}@127.0.0.1:{selfPort}", } : new[] { $"akka.tcp://{systemName}@127.0.0.1:{peerPort}" }, }; var builder = Host.CreateDefaultBuilder(); builder.ConfigureServices(services => { services.AddSingleton>(Options.Create(options)); services.AddAkka(systemName, (ab, sp) => ab.WithOtOpcUaClusterBootstrap(sp)); }); var host = builder.Build(); await host.StartAsync(); return host; } /// Polls cluster state until it holds at least one Up member, or the deadline passes. private static async Task WaitForUpMemberAsync(ActorSystem system, TimeSpan timeout) { var cluster = Akka.Cluster.Cluster.Get(system); var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) { return true; } await Task.Delay(200); } return cluster.State.Members.Any(m => m.Status == MemberStatus.Up); } private static async Task StopAsync(IHost host) { await host.StopAsync(); host.Dispose(); } /// /// An unconfigured deployment gets the fallback: a pair node that cold-starts while its peer is /// down must come up on its own, not wait for an operator. /// [Fact] public void SelfFormAfter_defaults_to_ten_seconds() { new AkkaClusterOptions().SelfFormAfter.ShouldBe(TimeSpan.FromSeconds(10)); } /// /// The core scenario: a node listed as a non-first seed, booting with its peer dead, becomes Up /// on its own after the window. Without the fallback it loops on InitJoin forever. /// [Fact] public async Task Lone_non_first_seed_self_forms_after_the_window() { var selfPort = FreePort(); var peerPort = FreePort(); var host = await StartNodeAsync( "otopcua-selfform-1", selfPort, peerPort, TimeSpan.FromSeconds(2)); try { var system = host.Services.GetRequiredService(); (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20))) .ShouldBeTrue("a lone non-first seed must self-form and come Up unattended"); } finally { await StopAsync(host); } } /// /// The opt-out is real: with the window unset the node keeps waiting on InitJoin. The positive /// control proves the node was formable and only the fallback was missing — otherwise this test /// would pass just as happily against a node that could never join anything. /// [Fact] public async Task Disabled_fallback_keeps_waiting() { var selfPort = FreePort(); var peerPort = FreePort(); var host = await StartNodeAsync("otopcua-selfform-2", selfPort, peerPort, selfFormAfter: null); try { var system = host.Services.GetRequiredService(); (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(6))) .ShouldBeFalse("with SelfFormAfter unset the node must keep waiting on InitJoin"); // Positive control: the node WAS formable all along. var cluster = Akka.Cluster.Cluster.Get(system); cluster.Join(cluster.SelfAddress); (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20))) .ShouldBeTrue("positive control — an explicit self-join must bring this node Up"); } finally { await StopAsync(host); } } /// /// Regression for the live-gate islanding defect (2026-07-22). A peer that is reachable /// but not yet admitting the join must NOT be treated as "peer down at boot". /// /// /// /// On the docker-dev rig, a node bounced by a manual failover restarted, received /// InitJoinAck from its live peer — the join was in flight and healthy — but did /// not get the Welcome inside the window, because the peer's ring still held the node's /// previous incarnation. The fallback fired on the timer and the node formed a /// second cluster, islanded until an operator restarted it. The original design /// assumed Join(SelfAddress) would be ignored mid-handshake; it is not. /// /// /// The peer here is a bare that accepts connections and speaks /// no Akka at all — reachable at the transport level, never answering a join. That is the /// defect's shape with nothing faked: under the pre-fix code this node self-formed on the /// timer; under the guard it keeps waiting. The positive control then proves the guard is /// a guard and not a disablement: drop the listener and the same node self-forms. /// /// [Fact] public async Task Reachable_peer_suppresses_self_forming_until_it_goes_away() { var selfPort = FreePort(); var peerPort = FreePort(); // A peer that accepts TCP and does nothing else — reachable, but no join will ever complete. var peer = new TcpListener(IPAddress.Loopback, peerPort); peer.Start(); var host = await StartNodeAsync( "otopcua-selfform-4", selfPort, peerPort, TimeSpan.FromSeconds(2)); try { var system = host.Services.GetRequiredService(); // Several windows pass. Pre-fix this self-formed after the first one. (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(10))) .ShouldBeFalse("a reachable peer means a join may be in flight — self-forming here islands this node"); // Positive control: the peer really goes away, and now the fallback does its job. peer.Stop(); (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(30))) .ShouldBeTrue("once the peer is genuinely gone the fallback must still self-form"); } finally { try { peer.Stop(); } catch (SocketException) { /* already stopped by the test body */ } await StopAsync(host); } } /// /// THE island guard. A node that is not in its own seed list — the current docker-dev site-node /// topology, where site-a/site-b list only central-1 — must never self-form: it would island /// itself from the real seeds permanently. Positive control as above. /// [Fact] public async Task Site_node_seeded_only_by_central_never_self_forms() { var selfPort = FreePort(); var peerPort = FreePort(); var host = await StartNodeAsync( "otopcua-selfform-3", selfPort, peerPort, TimeSpan.FromSeconds(1), selfInSeeds: false); try { var system = host.Services.GetRequiredService(); (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(5))) .ShouldBeFalse("a node absent from its own seed list must wait, not island itself"); var cluster = Akka.Cluster.Cluster.Get(system); cluster.Join(cluster.SelfAddress); (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20))) .ShouldBeTrue("positive control — an explicit self-join must bring this node Up"); } finally { await StopAsync(host); } } }