feat(cluster): self-first seed ordering closes the boot-alone outage gap
Every node now lists ITSELF as seed-nodes[0] and its partner second. Akka runs FirstSeedNodeProcess -- the only bootstrap path that can form a NEW cluster when no peer answers InitJoin -- exclusively for seed-nodes[0]; every other node runs JoinSeedNodeProcess and retries InitJoin forever. That is why a lone cold-starting central-b never came Up (the "registered outage gap"), and self-first ordering closes it using Akka's own protocol. - 6 node appsettings swapped (the *-node-b configs; the -a nodes were already self-first). All 14 shipped node configs now satisfy the invariant. - StartupValidator enforces it at boot, comparing host AND port -- the invariant fails silently when broken, so it is enforced loudly. NOTE: the gitignored deploy/wonder-app-vd03/ overlay must be reordered before its next deploy or that node will refuse to boot. - SelfFirstSeedBootstrapTests: real in-process clusters at production failure-detection timings, incl. a falsifiability control proving the OLD peer-first ordering never forms. Rejected alternative (implemented, measured, discarded): an external self-form timer calling Cluster.Join(SelfAddress) after a window. It sits outside Akka's join handshake and so cannot tell "no seed answered" from "a seed answered and the join is in flight". On a routine standby restart the peer is alive but the join stalls behind removal of the node's own stale incarnation; a Join(self) during TryingToJoin abandons the in-flight join and forms a second cluster at the same address -- still split after 90s. Docs that claimed self-first ordering was unsafe for simultaneous cold start are corrected: while mutually reachable the InitJoin handshake converges them to one cluster (measured).
This commit is contained in:
@@ -124,6 +124,10 @@ public class StartupValidatorTests
|
||||
{
|
||||
var values = ValidCentralConfig();
|
||||
values["ScadaBridge:Node:RemotingPort"] = port;
|
||||
// The self-first seed rule (2026-07-22) compares host AND port, so this node's own
|
||||
// seed entry moves with its remoting port — otherwise this port-range test would be
|
||||
// asserting against a config that is inconsistent for an unrelated reason.
|
||||
values["ScadaBridge:Cluster:SeedNodes:0"] = $"akka.tcp://scadabridge@central-node1:{port}";
|
||||
var config = BuildConfig(values);
|
||||
|
||||
var ex = Record.Exception(() => StartupValidator.Validate(config));
|
||||
@@ -273,6 +277,52 @@ public class StartupValidatorTests
|
||||
Assert.Contains("SeedNodes must have at least 2 entries", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PeerFirstSeedOrder_FailsValidation()
|
||||
{
|
||||
// Decision 2026-07-22: every node must list ITSELF as seed-nodes[0]. Akka only runs
|
||||
// FirstSeedNodeProcess (the process that can form a new cluster when no peer answers)
|
||||
// when seed-nodes[0] is this node's own address; with the peer first the node can
|
||||
// never cold-start alone — the "registered outage gap".
|
||||
var values = ValidCentralConfig();
|
||||
values["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@central-node2:8081";
|
||||
values["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@central-node1:8081";
|
||||
var config = BuildConfig(values);
|
||||
|
||||
var ex = Assert.Throws<InvalidOperationException>(() => StartupValidator.Validate(config));
|
||||
Assert.Contains("SeedNodes", ex.Message);
|
||||
Assert.Contains("must list this node itself first", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfFirstSeedOrder_OnASiteNode_PassesValidation()
|
||||
{
|
||||
// Positive control for the rule above: the shipped ordering must validate on a Site
|
||||
// node too (the rule is unconditional, not Central-only).
|
||||
var config = BuildConfig(ValidSiteConfig());
|
||||
|
||||
var ex = Record.Exception(() => StartupValidator.Validate(config));
|
||||
|
||||
Assert.Null(ex);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfFirstSeed_MatchedOnHostAndPort_NotJustHost()
|
||||
{
|
||||
// Both nodes of a pair can share a hostname when they differ by port (a two-node
|
||||
// dev/loopback install). The rule must compare host AND port, or such a node passes
|
||||
// while actually being the non-first seed.
|
||||
var values = ValidCentralConfig();
|
||||
values["ScadaBridge:Node:NodeHostname"] = "localhost";
|
||||
values["ScadaBridge:Node:RemotingPort"] = "8082";
|
||||
values["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:8081";
|
||||
values["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:8082";
|
||||
var config = BuildConfig(values);
|
||||
|
||||
var ex = Assert.Throws<InvalidOperationException>(() => StartupValidator.Validate(config));
|
||||
Assert.Contains("must list this node itself first", ex.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("0")]
|
||||
[InlineData("-1")]
|
||||
|
||||
Reference in New Issue
Block a user