279d1d0fb1
Closes the residual Phase-7 gap: when BOTH nodes of a 2-node pair cold-start at the same instant, each self-first seed runs FirstSeedNodeProcess, times out waiting for the other, and forms its own 1-node cluster — two Primaries in one pair (the Phase 6 live gate reproduced this reliably on docker). The prior mitigation was operational (staggered start / compose depends_on), which does not exist on production hardware. The guard (dark switch Cluster:BootstrapGuard:Enabled, default OFF) prevents the split without giving up cold-start-alone: - The lower-address node is the preferred founder: self-first, forms immediately. - The higher node probes its partner's Akka port (TCP connect) up to PartnerProbeSeconds: reachable => peer-first (join the founder, never race it); unreachable => self-first (partner is dead, form alone). The order is decided BEFORE the single JoinSeedNodes, from an explicit reachability signal — never re-formed mid-handshake (the retired SelfFormAfter failure mode). When on, Akka gets no config seeds (BuildClusterOptions) and ClusterBootstrapCoordinator drives the join. Review-driven hardening: case-insensitive tie-break (a hostname-casing mismatch would reopen the split); fail-fast validation of the timing knobs; the residual "founder dies in the probe->join window" hang is documented and made operator-visible (warning + a restart recovers). Real-ActorSystem coordinator tests cover the load-bearing higher-node cold-start-alone case; 147/147 Cluster tests pass. Live-gated on docker-dev (site-a = enablement demo, serialization removed, guard on; site-b keeps depends_on-serialization + guard off as the A/B control): simultaneous start -> site-a-1 founds, site-a-2 probes-reachable-joins -> 250/240, NO split; higher-node-alone -> probes dead founder, self-forms -> 250; founder rejoin -> 240. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
218 lines
9.0 KiB
C#
218 lines
9.0 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
|
|
|
/// <summary>
|
|
/// Guards the self-first seed-ordering rule at the only moment it can still be fixed cheaply: boot.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// The invariant (decision 2026-07-22) is <b>conditional</b>: <i>if</i> a node's own address
|
|
/// appears in its own <see cref="AkkaClusterOptions.SeedNodes"/>, it must be entry 0. Akka
|
|
/// runs <c>FirstSeedNodeProcess</c> — the only path that can form a NEW cluster when no peer
|
|
/// answers <c>InitJoin</c> — exclusively for <c>seed-nodes[0]</c>; every other node retries
|
|
/// <c>InitJoin</c> forever. A node listed second therefore cannot cold-start while its peer
|
|
/// is down, and it fails <i>silently</i>: the process is healthy, the port is open, the node
|
|
/// simply never becomes a cluster member. That is why the rule is enforced loudly here.
|
|
/// </para>
|
|
/// <para>
|
|
/// The conditional form is required, not incidental: a flat "self must be first" rule would
|
|
/// reject every driver-only site node, which is seeded solely by <c>central-1</c> and is
|
|
/// legitimately not a seed of anything —
|
|
/// see <see cref="Node_absent_from_its_own_seed_list_is_exempt"/>.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class AkkaClusterOptionsValidatorTests
|
|
{
|
|
private static AkkaClusterOptions CentralNode(string publicHostname, params string[] seeds) =>
|
|
new()
|
|
{
|
|
// The docker-dev shape: bind to every interface, advertise the container DNS name.
|
|
Hostname = "0.0.0.0",
|
|
PublicHostname = publicHostname,
|
|
Port = 4053,
|
|
Roles = new[] { "admin", "driver" },
|
|
SeedNodes = seeds,
|
|
};
|
|
|
|
private static string Seed(string host, int port = 4053) => $"akka.tcp://otopcua@{host}:{port}";
|
|
|
|
/// <summary>The shipped central-1 / central-2 shape: own address first, partner second.</summary>
|
|
[Fact]
|
|
public void Self_first_seed_order_passes()
|
|
{
|
|
var options = CentralNode("central-2", Seed("central-2"), Seed("central-1"));
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The pre-2026-07-22 ordering — partner first — is the boot-alone outage gap, and must not
|
|
/// start. This is the shape <c>docker-dev</c>'s <c>central-2</c> carried.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Peer_first_seed_order_fails()
|
|
{
|
|
var options = CentralNode("central-2", Seed("central-1"), Seed("central-2"));
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldNotBeNull().ShouldContain("SeedNodes");
|
|
result.FailureMessage.ShouldContain("must list this node itself first");
|
|
}
|
|
|
|
/// <summary>
|
|
/// THE conditional exemption. A driver-only site node lists only <c>central-1</c> — it is not a
|
|
/// seed at all, is never legitimately first, and must pass. An unconditional "self must be
|
|
/// first" rule would refuse to boot every site node in the fleet.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Node_absent_from_its_own_seed_list_is_exempt()
|
|
{
|
|
var options = CentralNode("site-a-1", Seed("central-1"));
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The bind address is NOT the node's identity. In docker-dev <c>Cluster:Hostname</c> is
|
|
/// <c>0.0.0.0</c> and <c>Cluster:PublicHostname</c> is the container name — the value Akka puts
|
|
/// in <c>SelfAddress</c> and therefore the only one a seed entry can match. A validator that
|
|
/// compared against <c>Hostname</c> would find no match anywhere, silently exempt every node,
|
|
/// and pass this misordered config.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Identity_is_the_public_hostname_not_the_bind_address()
|
|
{
|
|
var options = CentralNode("central-2", Seed("central-1"), Seed("central-2"));
|
|
options.Hostname.ShouldBe("0.0.0.0", "the trap only exists when the bind address is a wildcard");
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeTrue("matching on the 0.0.0.0 bind address would make this rule vacuous");
|
|
}
|
|
|
|
/// <summary>
|
|
/// With <c>PublicHostname</c> blank Akka advertises the bind hostname, so that is the identity to
|
|
/// match — otherwise a loopback/bare-metal pair configured that way would be silently exempt.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Identity_falls_back_to_the_bind_hostname_when_no_public_hostname_is_set()
|
|
{
|
|
var options = CentralNode("central-2", Seed("node-a"), Seed("node-b"));
|
|
options.Hostname = "node-b";
|
|
options.PublicHostname = string.Empty;
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldNotBeNull().ShouldContain("must list this node itself first");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Host AND port: a two-node install sharing one hostname and differing only by port (a
|
|
/// loopback/dev pair) is a real topology, and matching on host alone would clear the misordered
|
|
/// node.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Self_match_compares_host_and_port()
|
|
{
|
|
var options = CentralNode("127.0.0.1", Seed("127.0.0.1", 4053), Seed("127.0.0.1", 4054));
|
|
options.Port = 4054;
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeTrue("this node is 127.0.0.1:4054, which is the SECOND seed");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Positive control for the pair above: the same host on the same port really is a match, so the
|
|
/// port comparison cannot be passing merely by never matching anything.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Self_match_on_the_same_host_and_port_passes()
|
|
{
|
|
var options = CentralNode("127.0.0.1", Seed("127.0.0.1", 4054), Seed("127.0.0.1", 4053));
|
|
options.Port = 4054;
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// An empty seed list is the single-node/dev default and says nothing about ordering — the rule
|
|
/// has nothing to bind to and must not invent a failure.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Empty_seed_list_passes()
|
|
{
|
|
var options = CentralNode("central-1");
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A malformed seed URI is Akka's error to report, with Akka's wording. This rule must not
|
|
/// crash on it, and must not turn a parse problem into a misleading "ordering" complaint —
|
|
/// here the ordering is correct and only a later entry is unparseable.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Malformed_seed_entry_does_not_throw()
|
|
{
|
|
var options = CentralNode("central-1", Seed("central-1"), "not-an-akka-address");
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>The bootstrap-guard timing knobs default to positive values and pass when enabled.</summary>
|
|
[Fact]
|
|
public void Bootstrap_guard_defaults_pass()
|
|
{
|
|
var options = CentralNode("site-a-1", Seed("site-a-1"), Seed("site-a-2"));
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true };
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A zero <c>PartnerProbeSeconds</c> silently degrades the guard to "never wait, form alone
|
|
/// immediately" — re-opening the split. It must fail the host at boot, not run degraded.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Bootstrap_guard_with_non_positive_probe_seconds_fails()
|
|
{
|
|
var options = CentralNode("site-a-1", Seed("site-a-1"), Seed("site-a-2"));
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 };
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("PartnerProbeSeconds");
|
|
}
|
|
|
|
/// <summary>The timing knobs are NOT validated when the guard is off — a disabled guard is inert.</summary>
|
|
[Fact]
|
|
public void Bootstrap_guard_disabled_does_not_validate_timings()
|
|
{
|
|
var options = CentralNode("site-a-1", Seed("site-a-1"), Seed("site-a-2"));
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = false, PartnerProbeSeconds = 0 };
|
|
|
|
var result = new AkkaClusterOptionsValidator().Validate(null, options);
|
|
|
|
result.Failed.ShouldBeFalse(result.FailureMessage);
|
|
}
|
|
}
|