diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index b6b33a7d..cdfc663c 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -13,7 +13,9 @@ # central-1, central-2 OTOPCUA_ROLES=admin,driver — the ONLY UI + deploy # singleton, plus the MAIN cluster's OPC UA publishers. # Reachable at http://localhost:9200 (via Traefik). -# central-1 is the Akka seed node; central-2 joins it. +# Both are Akka seed nodes, each listing ITSELF first +# (2026-07-22) so either can cold-start while the other +# is down; the site nodes seed off central-1 only. # site-a-1, site-a-2 OTOPCUA_ROLES=driver — driver-only members of the same # site-b-1, site-b-2 mesh, scoped to SITE-A / SITE-B by ClusterId. They # serve no UI and authenticate no users; the central @@ -136,7 +138,8 @@ services: # ── Central cluster (2-node fused admin+driver) ───────────────────────────── # The only UI + deploy singleton; also the MAIN cluster's OPC UA publishers. - # central-1 seeds the single Akka mesh that every other node joins. + # central-1 seeds the single Akka mesh that every other node joins; central-2 is a seed too + # (listing itself first), so it can form the mesh alone if central-1 is down. central-1: &otopcua-host build: @@ -176,8 +179,13 @@ services: Cluster__Port: "4053" Cluster__PublicHostname: "central-1" # Both redundancy peers are seeds (#459 / ScadaBridge parity) so a restarted node can - # re-join the mesh via EITHER peer, not only central-1. With restart supervision this is - # the 2-node keep-oldest exit-and-rejoin recovery path. + # re-join the mesh via EITHER peer, not only central-1. + # + # ORDER IS LOAD-BEARING (2026-07-22): each node lists ITSELF first. Akka runs + # FirstSeedNodeProcess — the only path that can form a NEW cluster when no peer answers + # InitJoin — exclusively for seed-nodes[0]; every other node retries InitJoin forever. So a + # node listing its partner first cannot cold-start while that partner is down. Enforced at + # boot by AkkaClusterOptionsValidator; see docs/Redundancy.md. Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" Cluster__Roles__0: "admin" @@ -248,9 +256,11 @@ services: Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "central-2" - # Both redundancy peers are seeds (#459 / ScadaBridge parity) — see central-1. - Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" - Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" + # Both redundancy peers are seeds (#459 / ScadaBridge parity) — see central-1. SELF FIRST: + # central-2 lists itself as seed-nodes[0], which is what lets it cold-start while central-1 + # is down (it previously listed central-1 first and simply never came Up in that case). + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-2:4053" + Cluster__SeedNodes__1: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" @@ -314,6 +324,9 @@ services: Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-a-1" + # Site nodes are deliberately NOT seeds — they join the central pair's mesh. The self-first + # seed rule is conditional for exactly this reason (it binds only when a node's own address + # is in its own seed list), so these configs are exempt rather than broken. Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" <<: *secrets-env diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs index 7ad64fe6..5ca6e1b5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptions.cs @@ -20,7 +20,31 @@ public sealed class AkkaClusterOptions /// public string PublicHostname { get; set; } = "127.0.0.1"; - /// Gets or sets the seed nodes for cluster bootstrapping. + /// + /// Gets or sets the seed nodes for cluster bootstrapping. + /// + /// + /// + /// ORDER IS LOAD-BEARING (decision 2026-07-22): a node that is one of its own seeds + /// must list ITSELF first. Akka runs FirstSeedNodeProcess — the only bootstrap + /// path that can form a NEW cluster when no peer answers InitJoin — exclusively + /// when seed-nodes[0] is this node's own address; any other node runs + /// JoinSeedNodeProcess, which retries InitJoin forever and can never form a + /// cluster. So listing both peers does NOT mean either can cold-start alone: a node that + /// lists its partner first never comes Up while that partner is down. Self-first ordering + /// closes that gap inside Akka's own handshake — unlike the retired + /// Cluster:SelfFormAfter watchdog, which sat outside it and could not tell "no + /// seed answered" from "a seed answered and the join is in flight". + /// + /// + /// The rule is conditional: it binds only when this node's own address appears in + /// this list. A node seeded exclusively by someone else — today's docker-dev site nodes, + /// which list only central-1 — is legitimately not a seed and is exempt. Enforced + /// at boot by ; identity is + /// + (what Akka puts in + /// SelfAddress), never the bind address. + /// + /// public string[] SeedNodes { get; set; } = Array.Empty(); /// @@ -55,19 +79,4 @@ public sealed class AkkaClusterOptions /// /// public string SplitBrainResolverStrategy { get; set; } = "auto-down"; - - /// - /// Bootstrap self-form fallback window (decision 2026-07-22, scadaproj/akka_failover.md §6.1). - /// Akka only lets the FIRST listed seed form a new cluster; a non-first seed cold-starting - /// while its peer is down loops on InitJoin forever ("auto-down removes the crash outage, - /// not this one" — docs/Redundancy.md). When this node has waited longer than this window - /// without cluster membership it joins itself — but ONLY if its own address appears in - /// ; a non-seed node (e.g. a site node whose only seed is central-1) - /// stays waiting, because self-forming there creates a permanent island. Default 10s - /// (same-datacenter pair; a live peer answers InitJoin in milliseconds). null or a - /// non-positive value disables the fallback. Accepted trade: both pair nodes cold-starting - /// inside the window while mutually unreachable form two clusters — the same dual-active - /// class the auto-down strategy already accepts, same recovery (restart one side). - /// - public TimeSpan? SelfFormAfter { get; set; } = TimeSpan.FromSeconds(10); } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs new file mode 100644 index 00000000..10a67320 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/AkkaClusterOptionsValidator.cs @@ -0,0 +1,96 @@ +using Akka.Actor; +using ZB.MOM.WW.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Cluster; + +/// +/// Fail-fast startup validator for , built on the shared +/// ZB.MOM.WW.Configuration and wired through +/// AddValidatedOptions in . +/// +/// +/// +/// Self-first seed ordering (decision 2026-07-22). Akka runs +/// FirstSeedNodeProcess — the only bootstrap path that can form a NEW cluster when no +/// peer answers InitJoin — exclusively when seed-nodes[0] is this node's own +/// address. Every other node runs JoinSeedNodeProcess and retries InitJoin +/// forever. A node that lists its partner first therefore cannot cold-start while that +/// partner is down, and the failure is silent: the process runs, the port listens, the +/// node simply never becomes a member. Enforced here so it is loud, at boot, on the node +/// that has the problem. +/// +/// +/// The rule is conditional and must stay that way. It binds only when this node's own +/// address appears in its own seed list. A driver-only site node is seeded solely by +/// central-1 and is legitimately not a seed of anything; an unconditional "self must +/// be first" rule would refuse to boot every one of them. +/// +/// +/// Identity is the advertised address, never the bind address. In docker-dev +/// Cluster:Hostname is 0.0.0.0 and Cluster:PublicHostname is the +/// container DNS name — the value Akka puts in SelfAddress and the only one a seed URI +/// can match. Comparing against Hostname would match nothing anywhere, silently exempt +/// every node, and leave this rule inert. PublicHostname falling back to +/// Hostname when blank mirrors Akka's own public-hostname semantics. +/// +/// +public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase +{ + /// + protected override void Validate(ValidationBuilder builder, AkkaClusterOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var seeds = options.SeedNodes ?? Array.Empty(); + if (seeds.Length == 0) + { + // Single-node / dev default: nothing is claimed about ordering, so there is nothing to + // enforce. (Akka bootstraps from an empty seed list by waiting for an explicit join.) + return; + } + + var selfHost = string.IsNullOrWhiteSpace(options.PublicHostname) + ? options.Hostname + : options.PublicHostname; + + var selfIndex = Array.FindIndex(seeds, s => IsSelf(s, selfHost, options.Port)); + if (selfIndex <= 0) + { + // -1 = this node is not one of its own seeds (exempt); 0 = correctly ordered. + return; + } + + builder.Add( + $"Cluster:SeedNodes must list this node itself first: entry {selfIndex} is this node " + + $"('{seeds[selfIndex]}') but entry 0 is '{seeds[0]}'. Akka only lets seed-nodes[0] form " + + "a new cluster (FirstSeedNodeProcess); every other node retries InitJoin forever, so " + + "with the partner listed first this node can never start while its peer is down. Swap " + + "the entries — see docs/Redundancy.md → 'Bootstrap: self-first seed ordering'."); + } + + /// + /// True when addresses this node itself — host AND port. + /// + /// + /// Host comparison is case-insensitive because DNS names are, but is otherwise exact: Akka does + /// no DNS canonicalisation either, so central-2 and central-2.example.com are + /// genuinely different seed identities to the cluster. A seed entry Akka itself cannot parse is + /// treated as "not this node" — reporting it is Akka's job, with Akka's wording, and this rule + /// must not turn a parse problem into a misleading ordering complaint. + /// + private static bool IsSelf(string seed, string selfHost, int selfPort) + { + Address address; + try + { + address = Address.Parse(seed); + } + catch (Exception) + { + return false; + } + + return address.Port == selfPort + && string.Equals(address.Host, selfHost, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapFallback.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapFallback.cs deleted file mode 100644 index 82a69226..00000000 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterBootstrapFallback.cs +++ /dev/null @@ -1,206 +0,0 @@ -using Akka.Actor; -using Microsoft.Extensions.Logging; - -namespace ZB.MOM.WW.OtOpcUa.Cluster; - -/// -/// InitJoin self-form fallback (decision 2026-07-22, scadaproj/akka_failover.md §6.1). -/// Akka only lets the FIRST listed seed form a NEW cluster; every other node retries InitJoin -/// forever. So "both nodes are seed nodes" () does NOT -/// mean either can cold-start alone — a non-first seed booting while its peer is down waits -/// indefinitely ("auto-down removes the crash outage, not this one" — docs/Redundancy.md). -/// This watchdog waits for membership; on expiry it -/// forms a cluster on itself. -/// -/// Island safety. Fires ONLY when this node's own address is in its own seed list. -/// A node that is not a seed (never legitimately first) must keep waiting: if it self-formed, a -/// later-booting real seed would form a second cluster and the two can never merge. That is the -/// current docker-dev site-node topology — site-a/site-b nodes list only central-1. For nodes that -/// ARE seeds, sequential recovery is island-free — Akka's join protocol prefers an existing cluster -/// (a booting node only self-forms when NO seed answers InitJoin), so a peer booting after this -/// node self-formed simply joins it. -/// -/// Reachability guard (added 2026-07-22 after a live-gate failure). An expired window -/// is NOT sufficient evidence that the peer is down, and joining on the timer alone islands nodes. -/// Drilled on the docker-dev rig: a node bounced by a manual failover restarted, received -/// InitJoinAck from its live peer — a join in flight and healthy — but did not get the -/// Welcome inside the window, because the peer's ring still held the node's previous incarnation -/// (Exiting → Down → Removed). The fallback fired anyway and the node formed a SECOND cluster, -/// islanded until an operator restarted it. So Join(SelfAddress) is not ignored -/// mid-handshake — it wins, which is the opposite of what the original design assumed. Before -/// self-forming, this therefore TCP-probes the other seed addresses: if any accepts a connection -/// the peer is alive, a join is likely already in flight, and the fallback waits another window -/// instead. That is also exactly what the fallback claims to detect — "no seed answered InitJoin -/// (peer down at boot)". -/// -/// Residual risk. Both pair nodes cold-starting inside the window while mutually -/// unreachable (a boot-time partition): both self-form, the same dual-active class the auto-down -/// downing strategy already accepts, with the same recovery (restart one side). -/// -/// Deliberately duplicated from ScadaBridge's equivalent (like the termination watchdogs) — -/// the two repos share the behavior, not a package. -/// -public static class ClusterBootstrapFallback -{ - /// - /// Arms the fallback on a freshly created actor system. Safe to call unconditionally: it no-ops - /// (with an explanatory log) when the fallback is disabled or when this node is not one of its - /// own seeds. - /// - /// The actor system whose cluster membership is being watched. - /// The bound cluster options carrying the window and the seed list. - /// Logger for the armed / inert / self-forming decisions. - /// - /// Overrides the reachability probe (host, port) → reachable. Production passes - /// for the real TCP connect; tests substitute it to drive the guard - /// deterministically. - /// - public static void Arm( - ActorSystem system, - AkkaClusterOptions options, - ILogger logger, - Func>? peerProbe = null) - { - ArgumentNullException.ThrowIfNull(system); - ArgumentNullException.ThrowIfNull(options); - ArgumentNullException.ThrowIfNull(logger); - - if (options.SelfFormAfter is not { } window || window <= TimeSpan.Zero) - { - logger.LogInformation( - "Cluster self-form fallback disabled (SelfFormAfter not set) — a node cold-starting " - + "while its peer is down will wait on InitJoin indefinitely."); - return; - } - - var cluster = Akka.Cluster.Cluster.Get(system); - var self = cluster.SelfAddress; - var isSeed = options.SeedNodes.Any(s => TryParseAddress(s, out var a) && a.Equals(self)); - if (!isSeed) - { - logger.LogInformation( - "Cluster self-form fallback inactive: this node ({Self}) is not in its own seed list " - + "[{Seeds}] — self-forming here would island it from the real seeds.", - self, - string.Join(", ", options.SeedNodes)); - return; - } - - // Every seed that is not this node. These are the addresses whose reachability decides - // whether an expired window means "peer is down" or "peer is alive and we are mid-join". - var peerSeeds = options.SeedNodes - .Select(s => TryParseAddress(s, out var a) ? a : null) - .Where(a => a is not null && !a.Equals(self)) - .Select(a => (Host: a!.Host ?? string.Empty, Port: a!.Port ?? 0)) - .Where(p => !string.IsNullOrWhiteSpace(p.Host) && p.Port > 0) - .ToList(); - - var probe = peerProbe ?? TryConnectAsync; - - var joined = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - cluster.RegisterOnMemberUp(() => joined.TrySetResult()); - - _ = Task.Run(async () => - { - while (true) - { - var winner = await Task.WhenAny(joined.Task, Task.Delay(window)); - if (winner == joined.Task || system.WhenTerminated.IsCompleted) - { - return; - } - - var reachable = await AnyReachableAsync(peerSeeds, probe); - if (reachable is not null) - { - // Do NOT self-form. A reachable peer means either a join already in flight (the - // failure this guard exists for) or a peer about to answer — self-forming here - // creates a second cluster that can never merge. - logger.LogInformation( - "Self-form window ({Window}) expired, but seed peer {Peer} is reachable — a join is " - + "most likely already in flight (a node re-joining after a restart is not admitted " - + "until its previous incarnation is removed). Continuing to wait rather than " - + "forming a second cluster.", - window, - reachable); - continue; - } - - logger.LogWarning( - "No cluster membership after {Window} and no seed peer is reachable — the peer is down " - + "at boot. Self-forming a cluster at {Self} so this node becomes operational; if the " - + "peer was merely partitioned (not dead), the pair is now dual-active — restart one " - + "side after the partition heals (accepted availability-first trade, decision " - + "2026-07-22).", - window, - self); - cluster.Join(self); - return; - } - }); - } - - /// How long a single peer-reachability connect attempt may take. - public static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(2); - - /// - /// Returns the first reachable peer as host:port, or when none - /// answered — including when there are no peer seeds at all (a lone-seed config, where an - /// expired window really does mean this node is on its own). - /// - private static async Task AnyReachableAsync( - IReadOnlyList<(string Host, int Port)> peers, - Func> probe) - { - foreach (var (host, port) in peers) - { - bool ok; - try - { - ok = await probe(host, port); - } - catch (Exception) - { - // A probe that cannot even be attempted (DNS gone, socket exhaustion) is treated as - // unreachable — it must not throw out of the watchdog and disarm the fallback. - ok = false; - } - - if (ok) return $"{host}:{port}"; - } - - return null; - } - - private static async Task TryConnectAsync(string host, int port) - { - using var client = new System.Net.Sockets.TcpClient(); - using var cts = new CancellationTokenSource(ProbeTimeout); - try - { - await client.ConnectAsync(host, port, cts.Token); - return client.Connected; - } - catch (Exception) - { - // Refused, unresolvable, or timed out — all mean "not reachable right now". - return false; - } - } - - private static bool TryParseAddress(string seed, out Address address) - { - try - { - address = Address.Parse(seed); - return true; - } - catch (Exception) - { - // A malformed seed entry must not disarm the fallback; bad seed URIs surface from - // Akka's own bootstrap. - address = Address.AllSystems; - return false; - } - } -} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs index f585c1b6..7e5eaa45 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using Akka.Remote.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; +using ZB.MOM.WW.Configuration; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; namespace ZB.MOM.WW.OtOpcUa.Cluster; @@ -19,13 +20,19 @@ public static class ServiceCollectionExtensions /// configurator via — keeping the entire Akka graph /// under Akka.Hosting's management so cluster singletons land on the same ActorSystem. /// + /// + /// The binding is validated at startup (ValidateOnStart) by + /// , so a seed list that cannot bootstrap this node — + /// self listed behind its partner — fails the host loudly instead of leaving it running but + /// permanently outside the cluster. + /// /// The service collection to configure. /// The application configuration containing cluster options. /// The same service collection, for chaining. public static IServiceCollection AddOtOpcUaCluster(this IServiceCollection services, IConfiguration configuration) { - services.AddOptions() - .Bind(configuration.GetSection(AkkaClusterOptions.SectionName)); + services.AddValidatedOptions( + configuration, AkkaClusterOptions.SectionName); services.AddSingleton(); @@ -92,22 +99,13 @@ public static class ServiceCollectionExtensions // ActorSystem for exactly that reason: the effective value is the only one worth asserting. builder.AddHocon(BuildDowningHocon(options), HoconAddMode.Prepend); - // InitJoin self-form fallback (decision 2026-07-22): registered as an Akka.Hosting startup - // task so every host built through this bootstrap — production AND the test hosts in - // Cluster.Tests — arms it identically. Guarded inside Arm: disabled when SelfFormAfter is - // unset; inert when this node is not in its own seed list (site nodes seeded only by - // central-1 must wait, not island). - // ILoggerFactory is fully qualified: `using Microsoft.Extensions.Logging` would make the - // LogLevel in ConfigureLoggers above ambiguous with Akka.Event.LogLevel. - var fallbackLogger = - (serviceProvider.GetService() - ?? Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance) - .CreateLogger("ZB.MOM.WW.OtOpcUa.Cluster.ClusterBootstrapFallback"); - builder.AddStartup((system, _) => - { - ClusterBootstrapFallback.Arm(system, options, fallbackLogger); - return Task.CompletedTask; - }); + // NOTE (2026-07-22): no bootstrap watchdog is armed here any more. The cold-start-alone gap + // it existed for — Akka lets only seed-nodes[0] form a NEW cluster — is now closed by + // self-first seed ordering (AkkaClusterOptions.SeedNodes), enforced at boot by + // AkkaClusterOptionsValidator. The retired ClusterBootstrapFallback timer sat OUTSIDE Akka's + // join handshake, so it could not distinguish "no seed answered" from "a seed answered and + // the join is in flight", and Cluster.Join(SelfAddress) is not ignored mid-handshake — it + // wins. See docs/Redundancy.md → "Bootstrap: self-first seed ordering". return builder; } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ZB.MOM.WW.OtOpcUa.Cluster.csproj b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ZB.MOM.WW.OtOpcUa.Cluster.csproj index ccfd258f..353c9628 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ZB.MOM.WW.OtOpcUa.Cluster.csproj +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ZB.MOM.WW.OtOpcUa.Cluster.csproj @@ -14,6 +14,10 @@ + + diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs new file mode 100644 index 00000000..966b1268 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/AkkaClusterOptionsValidatorTests.cs @@ -0,0 +1,177 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Guards the self-first seed-ordering rule at the only moment it can still be fixed cheaply: boot. +/// +/// +/// +/// The invariant (decision 2026-07-22) is conditional: if a node's own address +/// appears in its own , it must be entry 0. Akka +/// runs FirstSeedNodeProcess — the only path that can form a NEW cluster when no peer +/// answers InitJoin — exclusively for seed-nodes[0]; every other node retries +/// InitJoin forever. A node listed second therefore cannot cold-start while its peer +/// is down, and it fails silently: the process is healthy, the port is open, the node +/// simply never becomes a cluster member. That is why the rule is enforced loudly here. +/// +/// +/// The conditional form is required, not incidental: a flat "self must be first" rule would +/// reject every driver-only site node, which is seeded solely by central-1 and is +/// legitimately not a seed of anything — +/// see . +/// +/// +public sealed class AkkaClusterOptionsValidatorTests +{ + private static AkkaClusterOptions CentralNode(string publicHostname, params string[] seeds) => + new() + { + // The docker-dev shape: bind to every interface, advertise the container DNS name. + Hostname = "0.0.0.0", + PublicHostname = publicHostname, + Port = 4053, + Roles = new[] { "admin", "driver" }, + SeedNodes = seeds, + }; + + private static string Seed(string host, int port = 4053) => $"akka.tcp://otopcua@{host}:{port}"; + + /// The shipped central-1 / central-2 shape: own address first, partner second. + [Fact] + public void Self_first_seed_order_passes() + { + var options = CentralNode("central-2", Seed("central-2"), Seed("central-1")); + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } + + /// + /// The pre-2026-07-22 ordering — partner first — is the boot-alone outage gap, and must not + /// start. This is the shape docker-dev's central-2 carried. + /// + [Fact] + public void Peer_first_seed_order_fails() + { + var options = CentralNode("central-2", Seed("central-1"), Seed("central-2")); + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull().ShouldContain("SeedNodes"); + result.FailureMessage.ShouldContain("must list this node itself first"); + } + + /// + /// THE conditional exemption. A driver-only site node lists only central-1 — it is not a + /// seed at all, is never legitimately first, and must pass. An unconditional "self must be + /// first" rule would refuse to boot every site node in the fleet. + /// + [Fact] + public void Node_absent_from_its_own_seed_list_is_exempt() + { + var options = CentralNode("site-a-1", Seed("central-1")); + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } + + /// + /// The bind address is NOT the node's identity. In docker-dev Cluster:Hostname is + /// 0.0.0.0 and Cluster:PublicHostname is the container name — the value Akka puts + /// in SelfAddress and therefore the only one a seed entry can match. A validator that + /// compared against Hostname would find no match anywhere, silently exempt every node, + /// and pass this misordered config. + /// + [Fact] + public void Identity_is_the_public_hostname_not_the_bind_address() + { + var options = CentralNode("central-2", Seed("central-1"), Seed("central-2")); + options.Hostname.ShouldBe("0.0.0.0", "the trap only exists when the bind address is a wildcard"); + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeTrue("matching on the 0.0.0.0 bind address would make this rule vacuous"); + } + + /// + /// With PublicHostname blank Akka advertises the bind hostname, so that is the identity to + /// match — otherwise a loopback/bare-metal pair configured that way would be silently exempt. + /// + [Fact] + public void Identity_falls_back_to_the_bind_hostname_when_no_public_hostname_is_set() + { + var options = CentralNode("central-2", Seed("node-a"), Seed("node-b")); + options.Hostname = "node-b"; + options.PublicHostname = string.Empty; + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull().ShouldContain("must list this node itself first"); + } + + /// + /// Host AND port: a two-node install sharing one hostname and differing only by port (a + /// loopback/dev pair) is a real topology, and matching on host alone would clear the misordered + /// node. + /// + [Fact] + public void Self_match_compares_host_and_port() + { + var options = CentralNode("127.0.0.1", Seed("127.0.0.1", 4053), Seed("127.0.0.1", 4054)); + options.Port = 4054; + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeTrue("this node is 127.0.0.1:4054, which is the SECOND seed"); + } + + /// + /// Positive control for the pair above: the same host on the same port really is a match, so the + /// port comparison cannot be passing merely by never matching anything. + /// + [Fact] + public void Self_match_on_the_same_host_and_port_passes() + { + var options = CentralNode("127.0.0.1", Seed("127.0.0.1", 4054), Seed("127.0.0.1", 4053)); + options.Port = 4054; + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } + + /// + /// An empty seed list is the single-node/dev default and says nothing about ordering — the rule + /// has nothing to bind to and must not invent a failure. + /// + [Fact] + public void Empty_seed_list_passes() + { + var options = CentralNode("central-1"); + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } + + /// + /// A malformed seed URI is Akka's error to report, with Akka's wording. This rule must not + /// crash on it, and must not turn a parse problem into a misleading "ordering" complaint — + /// here the ordering is correct and only a later entry is unparseable. + /// + [Fact] + public void Malformed_seed_entry_does_not_throw() + { + var options = CentralNode("central-1", Seed("central-1"), "not-an-akka-address"); + + var result = new AkkaClusterOptionsValidator().Validate(null, options); + + result.Failed.ShouldBeFalse(result.FailureMessage); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SelfFirstSeedBootstrapTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SelfFirstSeedBootstrapTests.cs new file mode 100644 index 00000000..c2d2bb55 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SelfFirstSeedBootstrapTests.cs @@ -0,0 +1,287 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Actor; +using Akka.Cluster; +using Akka.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Guards the self-first seed-ordering invariant (decision 2026-07-22): every node that is a seed +/// of its own mesh lists ITSELF as SeedNodes[0] and its partner second. +/// +/// +/// +/// Why the ordering is the mechanism. Akka runs a different bootstrap process +/// depending on whether seed-nodes[0] is this node's own address. When it is, +/// FirstSeedNodeProcess runs: it InitJoins the other seeds and self-joins once +/// seed-node-timeout (5 s) passes with nobody answering. When it is not, +/// JoinSeedNodeProcess runs, and that process can never form a new cluster — it +/// retries InitJoin forever. That is why "both peers are listed in SeedNodes" never +/// meant "either can cold-start alone", and why a node listed second stayed out of the +/// cluster indefinitely while its peer was down. +/// +/// +/// Why not the self-form watchdog this replaces. ClusterBootstrapFallback +/// waited Cluster:SelfFormAfter for membership and then called +/// Cluster.Join(SelfAddress). A timer outside Akka's join handshake cannot +/// distinguish "no seed answered" from "a seed answered and the join is in flight", and +/// Join(SelfAddress) is not ignored mid-handshake — it wins. The live gate on +/// the docker-dev rig (2026-07-22) caught exactly that: a node bounced by a manual failover +/// restarted, got InitJoinAck from its live peer, but no Welcome inside the window +/// because the peer's ring still held its previous incarnation +/// (Exiting → Down → Removed); the watchdog fired and formed a SECOND +/// cluster. A TCP reachability guard patched that one shape; the race stayed. +/// is that scenario, +/// and self-first ordering has no such race because the decision is made inside the +/// handshake by Akka itself. +/// +/// +/// Why these start real hosts through the production bootstrap. Same reason as +/// : the question is what a running node does. +/// Every node here is built with +/// — the shipped entry +/// point — so these run at the production failure-detection envelope (heartbeat 2 s, +/// acceptable pause 10 s, auto-down 15 s, down-removal-margin 15 s) from +/// Resources/akka.conf rather than at timings invented by the test. +/// +/// +public sealed class SelfFirstSeedBootstrapTests +{ + private static int FreePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + /// + /// Starts a node the way every shipped node config is written: its own address first, the + /// partner second. + /// + /// Actor-system name; also the system part of every seed address. + /// This node's remoting port. + /// The partner's remoting port (may have nothing listening on it). + /// + /// reproduces the pre-2026-07-22 ordering (partner first), which is what + /// makes a falsifiability + /// control rather than a restatement of the fix. + /// + private static async Task StartNodeAsync( + string systemName, + int selfPort, + int peerPort, + bool selfFirst = true) + { + var self = $"akka.tcp://{systemName}@127.0.0.1:{selfPort}"; + var peer = $"akka.tcp://{systemName}@127.0.0.1:{peerPort}"; + var options = new AkkaClusterOptions + { + SystemName = systemName, + Hostname = "127.0.0.1", + PublicHostname = "127.0.0.1", + Port = selfPort, + Roles = new[] { "admin", "driver" }, + SeedNodes = selfFirst ? new[] { self, peer } : new[] { peer, self }, + }; + + var builder = Host.CreateDefaultBuilder(); + builder.ConfigureServices(services => + { + services.AddSingleton>(Options.Create(options)); + services.AddAkka(systemName, (ab, sp) => + { + ab.WithOtOpcUaClusterBootstrap(sp); + + // Crash simulation, test-only: with this off, ActorSystem.Terminate() skips + // CoordinatedShutdown and so gossips no Leave — a process death, not a graceful + // drain. Without it "restart the standby" would degenerate into the easy path where + // the peer had already removed the old member before the new one dialled in. + // Prepend is Akka.Hosting's highest-precedence merge mode (see BuildDowningHocon). + ab.AddHocon( + "akka.coordinated-shutdown.run-by-actor-system-terminate = off", + HoconAddMode.Prepend); + }); + }); + + var host = builder.Build(); + await host.StartAsync(); + return host; + } + + /// Polls until the node sees Up members, or the deadline passes. + private static async Task WaitForUpMembersAsync(IHost host, int expected, TimeSpan timeout) + { + var cluster = Akka.Cluster.Cluster.Get(host.Services.GetRequiredService()); + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected) + { + return true; + } + + await Task.Delay(200); + } + + return cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected; + } + + private static async Task StopAsync(IHost host) + { + try + { + await host.StopAsync(); + } + catch (Exception) + { + // Teardown only. A host whose ActorSystem was terminated by a crash-simulation cannot + // run CoordinatedShutdown again; that must not fail the test that already asserted. + } + + host.Dispose(); + } + + /// + /// The unattended cold-start-alone guarantee: a node whose partner is dead forms the cluster by + /// itself, through Akka's own FirstSeedNodeProcess — no watchdog, no timer, no operator. + /// + [Fact] + public async Task Lone_cold_start_forms_a_cluster_when_the_peer_is_dead() + { + var selfPort = FreePort(); + var deadPeerPort = FreePort(); // nothing is listening there + var host = await StartNodeAsync("otopcua-selffirst-1", selfPort, deadPeerPort); + try + { + // seed-node-timeout is Akka's 5 s default; 30 s is slack for a loaded CI box. + (await WaitForUpMembersAsync(host, 1, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("a self-first seed must form a cluster alone when no peer answers InitJoin"); + } + finally + { + await StopAsync(host); + } + } + + /// + /// FALSIFIABILITY CONTROL for the test above. With the OLD ordering — partner first — the + /// identical scenario never comes Up, because JoinSeedNodeProcess has no self-join path. + /// If this test ever starts passing quickly, the seed ordering has stopped being the thing doing + /// the work and the suite above has gone vacuous. + /// + /// + /// This is also the test that fails while the retired ClusterBootstrapFallback watchdog + /// is still armed: that timer would bring this node Up at SelfFormAfter (10 s) despite the + /// ordering, which is precisely the "something other than Akka's handshake is deciding" that the + /// retirement removes. + /// + [Fact] + public async Task Peer_first_ordering_is_the_outage_gap_and_never_forms() + { + var selfPort = FreePort(); + var deadPeerPort = FreePort(); + var host = await StartNodeAsync("otopcua-selffirst-2", selfPort, deadPeerPort, selfFirst: false); + try + { + // 3x the 5 s seed-node-timeout, and longer than the retired watchdog's 10 s window. + (await WaitForUpMembersAsync(host, 1, TimeSpan.FromSeconds(15))) + .ShouldBeFalse("a node listed behind its peer cannot form a cluster — it InitJoin-loops forever"); + + var cluster = Akka.Cluster.Cluster.Get(host.Services.GetRequiredService()); + cluster.State.Members.ShouldBeEmpty(); + + // Positive control: the node was formable all along; only the ordering blocked it. + cluster.Join(cluster.SelfAddress); + (await WaitForUpMembersAsync(host, 1, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("positive control — an explicit self-join must bring this node Up"); + } + finally + { + await StopAsync(host); + } + } + + /// + /// The case that killed the self-form watchdog (live gate, docker-dev, 2026-07-22): a routine + /// standby restart while the peer is alive must REJOIN, never island. Under self-first ordering + /// the restarted node's FirstSeedNodeProcess only self-joins when no seed answers, so the + /// live peer's InitJoinAck keeps it on the join path however long the peer takes to + /// retire its previous incarnation. + /// + [Fact] + public async Task Restarting_node_rejoins_its_live_peer_instead_of_islanding() + { + const string systemName = "otopcua-selffirst-3"; + var portA = FreePort(); + var portB = FreePort(); + + var nodeA = await StartNodeAsync(systemName, portA, portB); + IHost? nodeB = null; + IHost? nodeB2 = null; + try + { + (await WaitForUpMembersAsync(nodeA, 1, TimeSpan.FromSeconds(30))).ShouldBeTrue(); + + nodeB = await StartNodeAsync(systemName, portB, portA); + (await WaitForUpMembersAsync(nodeA, 2, TimeSpan.FromSeconds(60))) + .ShouldBeTrue("the pair must converge before the restart is meaningful"); + + // Hard-crash B and immediately restart it at the SAME address, as a service restart or + // container recreate does. No Leave gossip: A still holds B's previous incarnation. + await nodeB.Services.GetRequiredService().Terminate() + .WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + nodeB2 = await StartNodeAsync(systemName, portB, portA); + + // ONE cluster of two, never two clusters of one. The budget covers auto-down (15 s) plus + // down-removal-margin (15 s) before the new incarnation is admitted. + (await WaitForUpMembersAsync(nodeB2, 2, TimeSpan.FromSeconds(120))) + .ShouldBeTrue("the restarted node must rejoin its live peer, not form a second cluster"); + (await WaitForUpMembersAsync(nodeA, 2, TimeSpan.FromSeconds(120))) + .ShouldBeTrue("the surviving node must see the restarted peer as a member of ITS cluster"); + } + finally + { + if (nodeB2 is not null) await StopAsync(nodeB2); + if (nodeB is not null) await StopAsync(nodeB); + await StopAsync(nodeA); + } + } + + /// + /// The obvious objection to self-first on BOTH nodes: does a simultaneous cold start produce two + /// clusters? While the two are mutually reachable it does not — each runs + /// FirstSeedNodeProcess, and the InitJoin handshake resolves which one forms before + /// either self-join deadline expires. (A genuine boot-time PARTITION would still split, the same + /// dual-active class the auto-down strategy already accepts, with the same recovery.) + /// + [Fact] + public async Task Both_nodes_cold_starting_together_converge_on_one_cluster() + { + const string systemName = "otopcua-selffirst-4"; + var portA = FreePort(); + var portB = FreePort(); + + var nodeA = await StartNodeAsync(systemName, portA, portB); + var nodeB = await StartNodeAsync(systemName, portB, portA); + try + { + (await WaitForUpMembersAsync(nodeA, 2, TimeSpan.FromSeconds(60))) + .ShouldBeTrue("both nodes cold-starting self-first must converge, not split"); + (await WaitForUpMembersAsync(nodeB, 2, TimeSpan.FromSeconds(60))) + .ShouldBeTrue("both nodes cold-starting self-first must converge, not split"); + } + finally + { + await StopAsync(nodeB); + await StopAsync(nodeA); + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SelfFormBootstrapTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SelfFormBootstrapTests.cs deleted file mode 100644 index 0cbdffcf..00000000 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/SelfFormBootstrapTests.cs +++ /dev/null @@ -1,259 +0,0 @@ -using System.Net; -using System.Net.Sockets; -using Akka.Actor; -using Akka.Cluster; -using Akka.Hosting; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Options; -using Shouldly; -using Xunit; - -namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; - -/// -/// Guards the InitJoin self-form fallback — whether a node whose peer is down at boot ever becomes -/// operational. -/// -/// -/// -/// Why these tests start real hosts through the production bootstrap. Like -/// , the question is what a running node -/// actually does, not what an options object holds. Every host here is built with -/// — the shipped entry -/// point — so a change that stops arming the fallback (or arms it from a place production -/// does not reach) turns these red. A test that called -/// itself would pin the test's wiring instead. -/// -/// -/// Seed ordering is load-bearing. The node under test lists itself SECOND in its own -/// seed list, behind a dead peer port. Akka only lets the FIRST listed seed self-join, so -/// nothing but the fallback can bring these nodes Up — which is what makes the negative -/// scenarios (disabled window, non-seed node) meaningful rather than vacuous. Each of those -/// carries a positive control (an explicit Cluster.Join(SelfAddress)) proving the node -/// was formable all along and only the fallback was absent. -/// -/// -public sealed class SelfFormBootstrapTests -{ - private static int FreePort() - { - var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; - } - - private static async Task StartNodeAsync( - string systemName, - int selfPort, - int peerPort, - TimeSpan? selfFormAfter, - bool selfInSeeds = true) - { - var options = new AkkaClusterOptions - { - SystemName = systemName, - Hostname = "127.0.0.1", - PublicHostname = "127.0.0.1", - Port = selfPort, - Roles = new[] { "admin", "driver" }, - SelfFormAfter = selfFormAfter, - - // Self listed SECOND, behind a port nothing is listening on: Akka lets only the first - // seed self-join, so this node can never form a cluster on its own except via the - // fallback. - SeedNodes = selfInSeeds - ? new[] - { - $"akka.tcp://{systemName}@127.0.0.1:{peerPort}", - $"akka.tcp://{systemName}@127.0.0.1:{selfPort}", - } - : new[] { $"akka.tcp://{systemName}@127.0.0.1:{peerPort}" }, - }; - - var builder = Host.CreateDefaultBuilder(); - builder.ConfigureServices(services => - { - services.AddSingleton>(Options.Create(options)); - services.AddAkka(systemName, (ab, sp) => ab.WithOtOpcUaClusterBootstrap(sp)); - }); - - var host = builder.Build(); - await host.StartAsync(); - return host; - } - - /// Polls cluster state until it holds at least one Up member, or the deadline passes. - private static async Task WaitForUpMemberAsync(ActorSystem system, TimeSpan timeout) - { - var cluster = Akka.Cluster.Cluster.Get(system); - var deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) - { - return true; - } - - await Task.Delay(200); - } - - return cluster.State.Members.Any(m => m.Status == MemberStatus.Up); - } - - private static async Task StopAsync(IHost host) - { - await host.StopAsync(); - host.Dispose(); - } - - /// - /// An unconfigured deployment gets the fallback: a pair node that cold-starts while its peer is - /// down must come up on its own, not wait for an operator. - /// - [Fact] - public void SelfFormAfter_defaults_to_ten_seconds() - { - new AkkaClusterOptions().SelfFormAfter.ShouldBe(TimeSpan.FromSeconds(10)); - } - - /// - /// The core scenario: a node listed as a non-first seed, booting with its peer dead, becomes Up - /// on its own after the window. Without the fallback it loops on InitJoin forever. - /// - [Fact] - public async Task Lone_non_first_seed_self_forms_after_the_window() - { - var selfPort = FreePort(); - var peerPort = FreePort(); - var host = await StartNodeAsync( - "otopcua-selfform-1", selfPort, peerPort, TimeSpan.FromSeconds(2)); - try - { - var system = host.Services.GetRequiredService(); - - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20))) - .ShouldBeTrue("a lone non-first seed must self-form and come Up unattended"); - } - finally - { - await StopAsync(host); - } - } - - /// - /// The opt-out is real: with the window unset the node keeps waiting on InitJoin. The positive - /// control proves the node was formable and only the fallback was missing — otherwise this test - /// would pass just as happily against a node that could never join anything. - /// - [Fact] - public async Task Disabled_fallback_keeps_waiting() - { - var selfPort = FreePort(); - var peerPort = FreePort(); - var host = await StartNodeAsync("otopcua-selfform-2", selfPort, peerPort, selfFormAfter: null); - try - { - var system = host.Services.GetRequiredService(); - - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(6))) - .ShouldBeFalse("with SelfFormAfter unset the node must keep waiting on InitJoin"); - - // Positive control: the node WAS formable all along. - var cluster = Akka.Cluster.Cluster.Get(system); - cluster.Join(cluster.SelfAddress); - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20))) - .ShouldBeTrue("positive control — an explicit self-join must bring this node Up"); - } - finally - { - await StopAsync(host); - } - } - - /// - /// Regression for the live-gate islanding defect (2026-07-22). A peer that is reachable - /// but not yet admitting the join must NOT be treated as "peer down at boot". - /// - /// - /// - /// On the docker-dev rig, a node bounced by a manual failover restarted, received - /// InitJoinAck from its live peer — the join was in flight and healthy — but did - /// not get the Welcome inside the window, because the peer's ring still held the node's - /// previous incarnation. The fallback fired on the timer and the node formed a - /// second cluster, islanded until an operator restarted it. The original design - /// assumed Join(SelfAddress) would be ignored mid-handshake; it is not. - /// - /// - /// The peer here is a bare that accepts connections and speaks - /// no Akka at all — reachable at the transport level, never answering a join. That is the - /// defect's shape with nothing faked: under the pre-fix code this node self-formed on the - /// timer; under the guard it keeps waiting. The positive control then proves the guard is - /// a guard and not a disablement: drop the listener and the same node self-forms. - /// - /// - [Fact] - public async Task Reachable_peer_suppresses_self_forming_until_it_goes_away() - { - var selfPort = FreePort(); - var peerPort = FreePort(); - - // A peer that accepts TCP and does nothing else — reachable, but no join will ever complete. - var peer = new TcpListener(IPAddress.Loopback, peerPort); - peer.Start(); - - var host = await StartNodeAsync( - "otopcua-selfform-4", selfPort, peerPort, TimeSpan.FromSeconds(2)); - try - { - var system = host.Services.GetRequiredService(); - - // Several windows pass. Pre-fix this self-formed after the first one. - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(10))) - .ShouldBeFalse("a reachable peer means a join may be in flight — self-forming here islands this node"); - - // Positive control: the peer really goes away, and now the fallback does its job. - peer.Stop(); - - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(30))) - .ShouldBeTrue("once the peer is genuinely gone the fallback must still self-form"); - } - finally - { - try { peer.Stop(); } catch (SocketException) { /* already stopped by the test body */ } - await StopAsync(host); - } - } - - /// - /// THE island guard. A node that is not in its own seed list — the current docker-dev site-node - /// topology, where site-a/site-b list only central-1 — must never self-form: it would island - /// itself from the real seeds permanently. Positive control as above. - /// - [Fact] - public async Task Site_node_seeded_only_by_central_never_self_forms() - { - var selfPort = FreePort(); - var peerPort = FreePort(); - var host = await StartNodeAsync( - "otopcua-selfform-3", selfPort, peerPort, TimeSpan.FromSeconds(1), selfInSeeds: false); - try - { - var system = host.Services.GetRequiredService(); - - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(5))) - .ShouldBeFalse("a node absent from its own seed list must wait, not island itself"); - - var cluster = Akka.Cluster.Cluster.Get(system); - cluster.Join(cluster.SelfAddress); - (await WaitForUpMemberAsync(system, TimeSpan.FromSeconds(20))) - .ShouldBeTrue("positive control — an explicit self-join must bring this node Up"); - } - finally - { - await StopAsync(host); - } - } -}