From e46c1db5e5135ef0abac4c2554f4f522c336b5fe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 20 Aug 2026 10:04:16 -0400 Subject: [PATCH] =?UTF-8?q?fix(cluster):=20gossiped-role=20predicates=20co?= =?UTF-8?q?mpare=20case-insensitively=20=E2=80=94=20a=20miscased=20peer=20?= =?UTF-8?q?no=20longer=20un-elects=20the=20Primary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/R3-04-one-role-authority-plan.md | 46 ++++ ...3-04-one-role-authority-plan.md.tasks.json | 2 +- .../ClusterRoleInfo.cs | 9 +- .../ZB.MOM.WW.OtOpcUa.Cluster/RoleParser.cs | 11 + .../Components/Pages/Fleet.razor | 5 +- .../ClusterNodeAddressReconcilerActor.cs | 6 +- .../Redundancy/ManualFailoverService.cs | 5 +- .../Redundancy/RedundancyStateActor.cs | 10 +- .../Drivers/DriverHostActor.cs | 7 +- .../ScriptedAlarms/ScriptedAlarmHostActor.cs | 6 +- .../GossipedRoleMatchTests.cs | 54 +++++ .../GossipedRoleCaseInsensitivityTests.cs | 220 ++++++++++++++++++ 12 files changed, 370 insertions(+), 11 deletions(-) create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/GossipedRoleMatchTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/GossipedRoleCaseInsensitivityTests.cs diff --git a/archreview/plans/R3-04-one-role-authority-plan.md b/archreview/plans/R3-04-one-role-authority-plan.md index be92c942..b809adcf 100644 --- a/archreview/plans/R3-04-one-role-authority-plan.md +++ b/archreview/plans/R3-04-one-role-authority-plan.md @@ -456,3 +456,49 @@ T11, T12 (rig config) before T13; T13 last, after everything is merge-ready ``` Overall effort: **~1 dev-day code+unit, plus the rig session.** The three HIGH-RISK slices (T1 identity type, T3 composition root, T8 the guard) each land behind their full project suites before anything stacks on them; T13 is merge-blocking and is only meaningful because T11/T12 gave the rig a node that can actually exhibit the defect. + +--- + +## Execution notes (recorded during T1–T12 implementation, 2026-08-20) + +**T5 — `SelectOldestUpMemberOfRole`/`SelectDriverPrimary` cannot be reached from this repo; the plan +was wrong about this one call site.** `RedundancyStateActor.SelectOldestUpMemberOfRole` +(`RedundancyStateActor.cs:172-178`) is a thin wrapper around `ClusterActiveNode.OldestUpMember` +(`ZB.MOM.WW.Health.Akka`, a separately-versioned NuGet package out of `scadaproj`), which itself +matches via **Akka's own `Member.HasRole`** — an ordinal comparison this repo does not own. This is +structurally identical to the "not ours to change" carve-out the plan's own choice 2 already makes +for `ClusterSingletonOptions.Role` (rows 18/19); the plan simply did not notice the same constraint +applies to `SelectOldestUpMemberOfRole` before naming it as a `GossipedRolesHave` adoption site. + +Consequence: the plan's stated T5 test — "a `RedundancyStateActor` selection case where the member +gossips 'Driver' and a Primary IS elected" — is **not achievable**. What shipped instead +(`tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/GossipedRoleCaseInsensitivityTests.cs`): + +- `GossipedRolesHave` was implemented and adopted at every site that genuinely reads gossiped roles + directly in this repo: `RedundancyStateActor.cs` (`BuildSnapshot`'s own role classification, NOT + the Primary-selection call), `ManualFailoverService.cs` (`OrderedDrivers`), + `ClusterNodeAddressReconcilerActor.cs` (`FilterOwnClusterDriverMembers`), `DriverHostActor.cs` and + `ScriptedAlarmHostActor.cs` (`DriverMemberCount`), and `Fleet.razor` (the bare `"driver"` literals, + now `RoleParser.Driver` — the actual case-insensitivity there comes from `ClusterRoleInfo`'s + `_roleLeaders`/`_membersByRole` dictionaries moving to `OrdinalIgnoreCase`, which + `IClusterRoleInfo.MembersWithRole`/`RoleLeader` read through). +- `A_member_gossiping_a_miscased_driver_role_classifies_Secondary_not_Detached` pins the fix that + **is** achievable: a single member gossiping `"Driver"` now classifies `Secondary` in the + redundancy snapshot (was `Detached` pre-fix) — real value, because `/hosts` and `/fleet` render + this list, even though that member can still never be *named* Primary by itself. +- `Primary_election_itself_remains_ordinal_and_still_elects_no_one_for_an_all_miscased_pair` pins the + **true** (negative) outcome instead of asserting a false positive: a node (or pair) gossiping only + `"Driver"` still elects no Primary, because that decision runs through the vendored ordinal match. + This is a genuine residual — **not closed by R3-04** — worth a follow-up issue against + `ZB.MOM.WW.Health.Akka`'s `ClusterActiveNode.OldestUpMember` (e.g. an optional case-insensitive + role-match parameter) if the rolling-upgrade window this defends against is judged worth closing + fully. In practice T1/T2 already close the *headline* outage the census describes for a + FIXED-build node (a node can no longer gossip a miscased plain role itself — normalization happens + before the gossip handoff), so this residual only bites a peer still running a pre-R3-04 build. +- `ManualFailoverService_recognises_a_miscased_driver_node_as_a_candidate` uses + `GetSnapshot()`/`PrimaryAddress` rather than `FailOverCore(dryRun: true)` as originally sketched — + `FailOverCore` refuses below 2 driver members ("nothing to fail over TO"), which a single-node + fixture can never satisfy; `GetSnapshot()` shares the same `OrderedDrivers` candidate selection + this test is actually about, without needing a second live node. + +No other T1–T5 deviations from the plan. diff --git a/archreview/plans/R3-04-one-role-authority-plan.md.tasks.json b/archreview/plans/R3-04-one-role-authority-plan.md.tasks.json index 9ab49ce5..09c0cfdd 100644 --- a/archreview/plans/R3-04-one-role-authority-plan.md.tasks.json +++ b/archreview/plans/R3-04-one-role-authority-plan.md.tasks.json @@ -35,7 +35,7 @@ { "id": "T5", "subject": "RoleParser.GossipedRolesHave adopted at the six ordinal gossip predicates + ClusterRoleInfo index (Primary election, R2-04 member count, /fleet)", - "status": "pending", + "status": "completed", "blockedBy": [ "T1" ] diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterRoleInfo.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterRoleInfo.cs index 49949497..74151f43 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterRoleInfo.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/ClusterRoleInfo.cs @@ -23,8 +23,13 @@ public sealed class ClusterRoleInfo : IClusterRoleInfo, IDisposable private readonly string? _clusterRole; private readonly string? _clusterId; private readonly object _lock = new(); - private readonly Dictionary _roleLeaders = new(StringComparer.Ordinal); - private readonly Dictionary> _membersByRole = new(StringComparer.Ordinal); + // R3-04 (01/C-9, census row 17): OrdinalIgnoreCase — Akka compares gossiped role names ordinally, + // so a PEER running an older build can still gossip a miscased plain role during a rolling + // upgrade. Keying these by the exact-cased role string would leave MembersWithRole("driver") and + // RoleLeader("driver") blind to a peer that gossips "Driver", the same class of gap #532 closed + // for _localRoles above. + private readonly Dictionary _roleLeaders = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary> _membersByRole = new(StringComparer.OrdinalIgnoreCase); private IActorRef? _subscriber; /// Initializes a new instance of the ClusterRoleInfo class. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/RoleParser.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/RoleParser.cs index f9d58a6b..71cee807 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/RoleParser.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/RoleParser.cs @@ -153,4 +153,15 @@ public static class RoleParser return roles; } + + /// True when a GOSSIPED role list carries , compared + /// case-insensitively. Akka compares role names ordinally and a PEER may be running an older + /// build that gossips a verbatim 'Driver' (archreview 01/C-9), so every predicate over + /// Member.Roles goes through here rather than List.Contains. + /// The gossiped role list (e.g. Member.Roles). + /// The role to test for. + /// True when is present, case-insensitively. + public static bool GossipedRolesHave(IEnumerable? gossipedRoles, string role) => + gossipedRoles is not null + && gossipedRoles.Any(r => string.Equals(r, role, StringComparison.OrdinalIgnoreCase)); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Fleet.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Fleet.razor index dbdf4435..a9053fcb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Fleet.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Fleet.razor @@ -6,6 +6,7 @@ @attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)] @rendermode RenderMode.InteractiveServer @using Microsoft.EntityFrameworkCore +@using ZB.MOM.WW.OtOpcUa.Cluster @using ZB.MOM.WW.OtOpcUa.Commons.Interfaces @using ZB.MOM.WW.OtOpcUa.Configuration @using ZB.MOM.WW.OtOpcUa.Configuration.Entities @@ -146,7 +147,7 @@ else // Union with current Akka driver members so a freshly-joined node that has no // NodeDeploymentState row yet still appears as "waiting". - var akkaDrivers = Cluster.MembersWithRole("driver") + var akkaDrivers = Cluster.MembersWithRole(RoleParser.Driver) .Select(n => n.Value).ToHashSet(StringComparer.OrdinalIgnoreCase); var allNodes = byNode.Keys.Union(akkaDrivers, StringComparer.OrdinalIgnoreCase) .OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList(); @@ -156,7 +157,7 @@ else byNode.TryGetValue(nodeId, out var state); return new NodeRow( NodeId: nodeId, - Roles: akkaDrivers.Contains(nodeId) ? new[] { "driver" } : Array.Empty(), + Roles: akkaDrivers.Contains(nodeId) ? new[] { RoleParser.Driver } : Array.Empty(), Status: state?.Status, StartedAtUtc: state?.StartedAtUtc, AppliedAtUtc: state?.AppliedAtUtc, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs index 61ebbc3d..080684cf 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Fleet/ClusterNodeAddressReconcilerActor.cs @@ -181,10 +181,12 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer public static IReadOnlyList<(string Host, int Port)> FilterOwnClusterDriverMembers( IEnumerable members, string? ownClusterRole) { + // R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased + // 'Driver' (rolling-upgrade window) must not read as a phantom RunningNodeHasNoRow finding. var driverMembers = members - .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole)); + .Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, DriverRole)); if (!string.IsNullOrEmpty(ownClusterRole)) - driverMembers = driverMembers.Where(m => m.Roles.Contains(ownClusterRole)); + driverMembers = driverMembers.Where(m => RoleParser.GossipedRolesHave(m.Roles, ownClusterRole)); return driverMembers .Select(m => (Host: m.Address.Host ?? string.Empty, Port: m.Address.Port ?? 0)) .Where(a => !string.IsNullOrWhiteSpace(a.Host) && a.Port > 0) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs index ddedd053..e5bc3a62 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/ManualFailoverService.cs @@ -3,6 +3,7 @@ using Akka.Actor; using Akka.Cluster; using Microsoft.Extensions.Logging; using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Cluster; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; @@ -122,7 +123,9 @@ public sealed class ManualFailoverService : IManualFailoverService } private static List OrderedDrivers(IEnumerable members) => members - .Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(RedundancyStateActor.DriverRole)) + // R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased + // 'Driver' (rolling-upgrade window) must still be a manual-failover candidate. + .Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, RedundancyStateActor.DriverRole)) .OrderBy(m => m, Member.AgeOrdering) .ToList(); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs index 2c43f34a..61e47ee0 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Redundancy/RedundancyStateActor.cs @@ -190,7 +190,15 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers if (string.IsNullOrWhiteSpace(host)) continue; var isPrimary = driverPrimary is not null && driverPrimary == member.Address; - var role = member.Roles.Contains(DriverRole) + // R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer running an older build + // can still gossip a miscased 'Driver' during a rolling upgrade, and this list is what + // /hosts and /fleet render, so a miscased-but-genuinely-driver member must not read as + // Detached. NOTE: this does NOT reach SelectDriverPrimary/driverPrimary above — that + // delegates to ClusterActiveNode.OldestUpMember (ZB.MOM.WW.Health.Akka), which matches via + // Akka's own ordinal Member.HasRole and is not this repo's to change (see R3-04 execution + // notes). So a member gossiping only 'Driver' now classifies Secondary rather than + // Detached, but a PAIR that BOTH gossip 'Driver' still elects no Primary. + var role = RoleParser.GossipedRolesHave(member.Roles, DriverRole) ? (isPrimary ? CommonsRedundancyRole.Primary : CommonsRedundancyRole.Secondary) : CommonsRedundancyRole.Detached; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index e95ca092..4890d0bc 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -5,6 +5,7 @@ using Akka.Event; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; @@ -1608,8 +1609,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS if (_driverMemberCountProvider is not null) return _driverMemberCountProvider(); try { + // R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased + // 'Driver' (rolling-upgrade window) must still count toward the R2-04 primary-gate member + // count, or a genuine 2-node pair silently takes the single-node allow branch. return Akka.Cluster.Cluster.Get(Context.System).State.Members - .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole)); + .Count(m => m.Status == Akka.Cluster.MemberStatus.Up + && RoleParser.GossipedRolesHave(m.Roles, ServiceCollectionExtensions.DriverRole)); } catch (Exception) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index 3ec6e08c..bc7bfc1b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -1,6 +1,7 @@ using Akka.Actor; using Akka.Cluster.Tools.PublishSubscribe; using Akka.Event; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; using ZB.MOM.WW.OtOpcUa.Commons.Observability; @@ -516,8 +517,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor if (_driverMemberCountProvider is not null) return _driverMemberCountProvider(); try { + // R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased + // 'Driver' (rolling-upgrade window) must still count toward the alerts-emit gate. return Akka.Cluster.Cluster.Get(Context.System).State.Members - .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole)); + .Count(m => m.Status == Akka.Cluster.MemberStatus.Up + && RoleParser.GossipedRolesHave(m.Roles, ServiceCollectionExtensions.DriverRole)); } catch (Exception) { diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/GossipedRoleMatchTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/GossipedRoleMatchTests.cs new file mode 100644 index 00000000..47852a07 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/GossipedRoleMatchTests.cs @@ -0,0 +1,54 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// R3-04 (01/C-9, choice 6): is defence in depth for a +/// PEER running an older build that still gossips a miscased plain role — a locally-authored node +/// can no longer gossip one at all once (T1) is wired in, but Akka +/// compares gossiped role names ordinally, so a rolling-upgrade window can still show a +/// mixed-case peer. +/// +public sealed class GossipedRoleMatchTests +{ + [Fact] + public void Matches_the_canonical_spelling() + { + RoleParser.GossipedRolesHave(["driver"], "driver").ShouldBeTrue(); + } + + [Theory] + [InlineData("Driver")] + [InlineData("DRIVER")] + [InlineData("driVer")] + public void Matches_a_miscased_gossiped_role(string gossiped) + { + RoleParser.GossipedRolesHave([gossiped], "driver").ShouldBeTrue(); + } + + [Fact] + public void Matches_a_miscased_cluster_role() + { + RoleParser.GossipedRolesHave(["cluster-SITE-A"], "cluster-site-a").ShouldBeTrue(); + RoleParser.GossipedRolesHave(["Cluster-Site-A"], "cluster-SITE-A").ShouldBeTrue(); + } + + [Fact] + public void Does_not_match_a_different_role() + { + RoleParser.GossipedRolesHave(["admin"], "driver").ShouldBeFalse(); + } + + [Fact] + public void Null_gossiped_roles_is_false() + { + RoleParser.GossipedRolesHave(null, "driver").ShouldBeFalse(); + } + + [Fact] + public void Empty_gossiped_roles_is_false() + { + RoleParser.GossipedRolesHave([], "driver").ShouldBeFalse(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/GossipedRoleCaseInsensitivityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/GossipedRoleCaseInsensitivityTests.cs new file mode 100644 index 00000000..3cc6c3c9 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/GossipedRoleCaseInsensitivityTests.cs @@ -0,0 +1,220 @@ +using System.Collections.Immutable; +using System.Reflection; +using Akka.Actor; +using Akka.Cluster; +using Akka.Configuration; +using Akka.Util; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R3-04 (01/C-9, census rows 17/20-24/30): a PEER gossiping a miscased plain role (a +/// rolling-upgrade-window peer running an older build — (T1) already +/// means a node on the FIXED build cannot gossip a miscased role itself) must not read as a +/// stranger to the six production predicates this plan names. +/// +/// +/// +/// Deviation from the plan's stated T5 test (recorded per the execution brief): the +/// plan asked for "a RedundancyStateActor selection case where the member gossips +/// 'Driver' and a Primary IS elected". That is not achievable from this repo. +/// / +/// delegate to +/// ZB.MOM.WW.Health.Akka's ClusterActiveNode.OldestUpMember, which matches via +/// Akka's OWN Member.HasRole — an ordinal comparison this repo does not own, the exact +/// same class of "not ours to change" the plan itself already carves out for +/// ClusterSingletonOptions.Role (rows 18/19). +/// below pins that TRUE (negative) outcome instead of asserting a false positive. +/// +/// +/// What R3-04 DOES fix and this file proves: every consumer that reads gossiped roles +/// DIRECTLY (not through that external oldest-member helper) is now case-insensitive, so an +/// individual miscased member is no longer invisible to failover candidacy, address +/// reconciliation, or the redundancy snapshot's own role classification. +/// +/// +public sealed class GossipedRoleCaseInsensitivityTests +{ + // ---- Pure-unit: ClusterNodeAddressReconcilerActor.FilterOwnClusterDriverMembers --------------- + // (public static over IEnumerable — no live cluster needed) + + private static int _memberUid; + + // Akka's Member.Create factories are internal — same reflection idiom as + // ClusterNodeAddressReconcilerTests.UpMember. + private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod( + "Create", + BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public, + binder: null, + types: + [ + typeof(UniqueAddress), typeof(int), typeof(MemberStatus), + typeof(ImmutableHashSet), typeof(AppVersion), + ], + modifiers: null) + ?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found — Akka API changed"); + + private static Member UpMember(string host, int port, params string[] roles) => + (Member)MemberCreate.Invoke(null, + [ + new UniqueAddress(new Address("akka.tcp", "otopcua", host, port), Interlocked.Increment(ref _memberUid)), + 1, + MemberStatus.Up, + roles.ToImmutableHashSet(), + AppVersion.Zero, + ])!; + + [Fact] + public void FilterOwnClusterDriverMembers_recognises_a_miscased_driver_role() + { + var members = new[] { UpMember("site-a-1", 4053, "Driver", "cluster-SITE-A") }; + + var result = ClusterNodeAddressReconcilerActor.FilterOwnClusterDriverMembers(members, "cluster-SITE-A"); + + result.ShouldHaveSingleItem(); + } + + [Fact] + public void FilterOwnClusterDriverMembers_recognises_a_miscased_cluster_role() + { + var members = new[] { UpMember("site-a-1", 4053, "driver", "Cluster-SITE-A") }; + + // The caller's OWN cluster role names the canonical spelling; the member gossips a different + // casing ("Cluster-SITE-A") — GossipedRolesHave must still match the two as the same role. + var result = ClusterNodeAddressReconcilerActor.FilterOwnClusterDriverMembers(members, "cluster-site-a"); + + result.ShouldHaveSingleItem(); + } + + // ---- Live single-node cluster: BuildSnapshot classification + ManualFailoverService ----------- + + private const int Port = 19_560; + + private static Config NodeConfig(int port) => ConfigurationFactory.ParseString($$""" + akka { + loglevel = "WARNING" + actor.provider = "Akka.Cluster.ClusterActorRefProvider, Akka.Cluster" + remote.dot-netty.tcp { + hostname = "127.0.0.1" + port = {{port}} + } + cluster { + seed-nodes = ["akka.tcp://gossip-case-test@127.0.0.1:{{port}}"] + roles = ["Driver"] + min-nr-of-members = 1 + run-coordinated-shutdown-when-down = off + downing-provider-class = "" + } + } + """); + + private static async Task StartSingleNodeAsync() + { + var system = ActorSystem.Create("gossip-case-test", NodeConfig(Port)); + var cluster = Akka.Cluster.Cluster.Get(system); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline) + { + if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) return system; + await Task.Delay(100); + } + + await system.Terminate(); + throw new TimeoutException("node never came Up"); + } + + [Fact] + public async Task A_member_gossiping_a_miscased_driver_role_classifies_Secondary_not_Detached() + { + // Pins the fix that IS achievable: BuildSnapshot's own role classification (member.Roles + // Contains -> GossipedRolesHave) no longer reads a genuinely-driver-but-miscased member as + // Detached, even though (see class remarks) it can never be named Primary by itself. + var system = await StartSingleNodeAsync(); + try + { + var tcs = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + system.ActorOf( + RedundancyStateActor.Props(broadcast: msg => + { + if (msg is RedundancyStateChanged changed) tcs.TrySetResult(changed); + }), + $"redundancy-{Guid.NewGuid():N}"); + + var snapshot = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(15)); + + var node = snapshot.Nodes.ShouldHaveSingleItem(); + node.Role.ShouldBe( + RedundancyRole.Secondary, + "gossiping 'Driver' must not read as Detached — it is a real driver member, just " + + "miscased; BEFORE R3-04 this was Detached (ordinal Contains(\"driver\") on \"Driver\")"); + } + finally + { + await system.Terminate(); + } + } + + /// + /// THE recorded deviation: proves the residual gap is real, not assumed. A pair (here, + /// trivially, a single node) that gossips ONLY 'Driver' still elects no Primary — because that + /// decision runs through Akka's own ordinal Member.HasRole inside the vendored + /// ClusterActiveNode.OldestUpMember, which this repo cannot patch. Tracked as a + /// follow-up against ZB.MOM.WW.Health.Akka, not closed by R3-04. + /// + [Fact] + public async Task Primary_election_itself_remains_ordinal_and_still_elects_no_one_for_an_all_miscased_pair() + { + var system = await StartSingleNodeAsync(); + try + { + var members = Akka.Cluster.Cluster.Get(system).State.Members; + + RedundancyStateActor.SelectOldestUpMemberOfRole(members, "driver").ShouldBeNull( + "known residual (R3-04 execution notes): SelectOldestUpMemberOfRole delegates to " + + "ClusterActiveNode.OldestUpMember -> Akka's own ordinal Member.HasRole, which this " + + "repo does not own — GossipedRolesHave cannot reach inside it"); + } + finally + { + await system.Terminate(); + } + } + + [Fact] + public async Task ManualFailoverService_recognises_a_miscased_driver_node_as_a_candidate() + { + // FailOverCore refuses below 2 driver members (nothing to fail over TO), so this single-node + // fixture exercises GetSnapshot()'s PrimaryAddress instead — which shares the same + // OrderedDrivers candidate selection this test is actually about. + var system = await StartSingleNodeAsync(); + try + { + var service = new ManualFailoverService( + () => system, new RecordingAuditWriter(), NullLogger.Instance); + + var snapshot = service.GetSnapshot(); + + snapshot.PrimaryAddress.ShouldNotBeNull(); + snapshot.PrimaryAddress!.ShouldContain(Port.ToString()); + } + finally + { + await system.Terminate(); + } + } + + private sealed class RecordingAuditWriter : IAuditWriter + { + public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default) => + Task.CompletedTask; + } +}