using System.Net;
using System.Net.Sockets;
using Akka.Actor;
using Akka.Cluster;
using Akka.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
///
/// Guards the InitJoin self-form fallback — whether a node whose peer is down at boot ever becomes
/// operational.
///
///
///
/// Why these tests start real hosts through the production bootstrap. Like
/// , the question is what a running node
/// actually does, not what an options object holds. Every host here is built with
/// — the shipped entry
/// point — so a change that stops arming the fallback (or arms it from a place production
/// does not reach) turns these red. A test that called
/// itself would pin the test's wiring instead.
///
///
/// Seed ordering is load-bearing. The node under test lists itself SECOND in its own
/// seed list, behind a dead peer port. Akka only lets the FIRST listed seed self-join, so
/// nothing but the fallback can bring these nodes Up — which is what makes the negative
/// scenarios (disabled window, non-seed node) meaningful rather than vacuous. Each of those
/// carries a positive control (an explicit Cluster.Join(SelfAddress)) proving the node
/// was formable all along and only the fallback was absent.
///
///
public sealed class SelfFormBootstrapTests
{
private static int FreePort()
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
return port;
}
private static async Task StartNodeAsync(
string systemName,
int selfPort,
int peerPort,
TimeSpan? selfFormAfter,
bool selfInSeeds = true)
{
var options = new AkkaClusterOptions
{
SystemName = systemName,
Hostname = "127.0.0.1",
PublicHostname = "127.0.0.1",
Port = selfPort,
Roles = new[] { "admin", "driver" },
SelfFormAfter = selfFormAfter,
// Self listed SECOND, behind a port nothing is listening on: Akka lets only the first
// seed self-join, so this node can never form a cluster on its own except via the
// fallback.
SeedNodes = selfInSeeds
? new[]
{
$"akka.tcp://{systemName}@127.0.0.1:{peerPort}",
$"akka.tcp://{systemName}@127.0.0.1:{selfPort}",
}
: new[] { $"akka.tcp://{systemName}@127.0.0.1:{peerPort}" },
};
var builder = Host.CreateDefaultBuilder();
builder.ConfigureServices(services =>
{
services.AddSingleton>(Options.Create(options));
services.AddAkka(systemName, (ab, sp) => ab.WithOtOpcUaClusterBootstrap(sp));
});
var host = builder.Build();
await host.StartAsync();
return host;
}
/// Polls cluster state until it holds at least one Up member, or the deadline passes.
private static async Task WaitForUpMemberAsync(ActorSystem system, TimeSpan timeout)
{
var cluster = Akka.Cluster.Cluster.Get(system);
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up))
{
return true;
}
await Task.Delay(200);
}
return cluster.State.Members.Any(m => m.Status == MemberStatus.Up);
}
private static async Task StopAsync(IHost host)
{
await host.StopAsync();
host.Dispose();
}
///
/// An unconfigured deployment gets the fallback: a pair node that cold-starts while its peer is
/// down must come up on its own, not wait for an operator.
///
[Fact]
public void SelfFormAfter_defaults_to_ten_seconds()
{
new AkkaClusterOptions().SelfFormAfter.ShouldBe(TimeSpan.FromSeconds(10));
}
///
/// The core scenario: a node listed as a non-first seed, booting with its peer dead, becomes Up
/// on its own after the window. Without the fallback it loops on InitJoin forever.
///
[Fact]
public async Task Lone_non_first_seed_self_forms_after_the_window()
{
var selfPort = FreePort();
var peerPort = FreePort();
var host = await StartNodeAsync(
"otopcua-selfform-1", selfPort, peerPort, TimeSpan.FromSeconds(2));
try
{
var system = host.Services.GetRequiredService();
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20)))
.ShouldBeTrue("a lone non-first seed must self-form and come Up unattended");
}
finally
{
await StopAsync(host);
}
}
///
/// The opt-out is real: with the window unset the node keeps waiting on InitJoin. The positive
/// control proves the node was formable and only the fallback was missing — otherwise this test
/// would pass just as happily against a node that could never join anything.
///
[Fact]
public async Task Disabled_fallback_keeps_waiting()
{
var selfPort = FreePort();
var peerPort = FreePort();
var host = await StartNodeAsync("otopcua-selfform-2", selfPort, peerPort, selfFormAfter: null);
try
{
var system = host.Services.GetRequiredService();
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(6)))
.ShouldBeFalse("with SelfFormAfter unset the node must keep waiting on InitJoin");
// Positive control: the node WAS formable all along.
var cluster = Akka.Cluster.Cluster.Get(system);
cluster.Join(cluster.SelfAddress);
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20)))
.ShouldBeTrue("positive control — an explicit self-join must bring this node Up");
}
finally
{
await StopAsync(host);
}
}
///
/// Regression for the live-gate islanding defect (2026-07-22). A peer that is reachable
/// but not yet admitting the join must NOT be treated as "peer down at boot".
///
///
///
/// On the docker-dev rig, a node bounced by a manual failover restarted, received
/// InitJoinAck from its live peer — the join was in flight and healthy — but did
/// not get the Welcome inside the window, because the peer's ring still held the node's
/// previous incarnation. The fallback fired on the timer and the node formed a
/// second cluster, islanded until an operator restarted it. The original design
/// assumed Join(SelfAddress) would be ignored mid-handshake; it is not.
///
///
/// The peer here is a bare that accepts connections and speaks
/// no Akka at all — reachable at the transport level, never answering a join. That is the
/// defect's shape with nothing faked: under the pre-fix code this node self-formed on the
/// timer; under the guard it keeps waiting. The positive control then proves the guard is
/// a guard and not a disablement: drop the listener and the same node self-forms.
///
///
[Fact]
public async Task Reachable_peer_suppresses_self_forming_until_it_goes_away()
{
var selfPort = FreePort();
var peerPort = FreePort();
// A peer that accepts TCP and does nothing else — reachable, but no join will ever complete.
var peer = new TcpListener(IPAddress.Loopback, peerPort);
peer.Start();
var host = await StartNodeAsync(
"otopcua-selfform-4", selfPort, peerPort, TimeSpan.FromSeconds(2));
try
{
var system = host.Services.GetRequiredService();
// Several windows pass. Pre-fix this self-formed after the first one.
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(10)))
.ShouldBeFalse("a reachable peer means a join may be in flight — self-forming here islands this node");
// Positive control: the peer really goes away, and now the fallback does its job.
peer.Stop();
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(30)))
.ShouldBeTrue("once the peer is genuinely gone the fallback must still self-form");
}
finally
{
try { peer.Stop(); } catch (SocketException) { /* already stopped by the test body */ }
await StopAsync(host);
}
}
///
/// THE island guard. A node that is not in its own seed list — the current docker-dev site-node
/// topology, where site-a/site-b list only central-1 — must never self-form: it would island
/// itself from the real seeds permanently. Positive control as above.
///
[Fact]
public async Task Site_node_seeded_only_by_central_never_self_forms()
{
var selfPort = FreePort();
var peerPort = FreePort();
var host = await StartNodeAsync(
"otopcua-selfform-3", selfPort, peerPort, TimeSpan.FromSeconds(1), selfInSeeds: false);
try
{
var system = host.Services.GetRequiredService();
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(5)))
.ShouldBeFalse("a node absent from its own seed list must wait, not island itself");
var cluster = Akka.Cluster.Cluster.Get(system);
cluster.Join(cluster.SelfAddress);
(await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20)))
.ShouldBeTrue("positive control — an explicit self-join must bring this node Up");
}
finally
{
await StopAsync(host);
}
}
}