fix(cluster): gossiped-role predicates compare case-insensitively — a miscased peer no longer un-elects the Primary
This commit is contained in:
Joseph Doherty
+220
@@ -0,0 +1,220 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Reflection;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using Akka.Util;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// R3-04 (01/C-9, census rows 17/20-24/30): a PEER gossiping a miscased plain role (a
|
||||
/// rolling-upgrade-window peer running an older build — <see cref="EffectiveRoles"/> (T1) already
|
||||
/// means a node on the FIXED build cannot gossip a miscased role itself) must not read as a
|
||||
/// stranger to the six production predicates this plan names.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Deviation from the plan's stated T5 test (recorded per the execution brief):</b> the
|
||||
/// plan asked for "a <c>RedundancyStateActor</c> selection case where the member gossips
|
||||
/// 'Driver' and a Primary IS elected". That is not achievable from this repo.
|
||||
/// <see cref="RedundancyStateActor.SelectOldestUpMemberOfRole"/> /
|
||||
/// <see cref="RedundancyStateActor.SelectDriverPrimary"/> delegate to
|
||||
/// <c>ZB.MOM.WW.Health.Akka</c>'s <c>ClusterActiveNode.OldestUpMember</c>, which matches via
|
||||
/// Akka's OWN <c>Member.HasRole</c> — an ordinal comparison this repo does not own, the exact
|
||||
/// same class of "not ours to change" the plan itself already carves out for
|
||||
/// <c>ClusterSingletonOptions.Role</c> (rows 18/19). <see cref="Primary_election_itself_remains_ordinal_and_still_elects_no_one_for_an_all_miscased_pair"/>
|
||||
/// below pins that TRUE (negative) outcome instead of asserting a false positive.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// What R3-04 DOES fix and this file proves: every consumer that reads gossiped roles
|
||||
/// DIRECTLY (not through that external oldest-member helper) is now case-insensitive, so an
|
||||
/// individual miscased member is no longer invisible to failover candidacy, address
|
||||
/// reconciliation, or the redundancy snapshot's own role classification.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class GossipedRoleCaseInsensitivityTests
|
||||
{
|
||||
// ---- Pure-unit: ClusterNodeAddressReconcilerActor.FilterOwnClusterDriverMembers ---------------
|
||||
// (public static over IEnumerable<Member> — no live cluster needed)
|
||||
|
||||
private static int _memberUid;
|
||||
|
||||
// Akka's Member.Create factories are internal — same reflection idiom as
|
||||
// ClusterNodeAddressReconcilerTests.UpMember.
|
||||
private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod(
|
||||
"Create",
|
||||
BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public,
|
||||
binder: null,
|
||||
types:
|
||||
[
|
||||
typeof(UniqueAddress), typeof(int), typeof(MemberStatus),
|
||||
typeof(ImmutableHashSet<string>), typeof(AppVersion),
|
||||
],
|
||||
modifiers: null)
|
||||
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found — Akka API changed");
|
||||
|
||||
private static Member UpMember(string host, int port, params string[] roles) =>
|
||||
(Member)MemberCreate.Invoke(null,
|
||||
[
|
||||
new UniqueAddress(new Address("akka.tcp", "otopcua", host, port), Interlocked.Increment(ref _memberUid)),
|
||||
1,
|
||||
MemberStatus.Up,
|
||||
roles.ToImmutableHashSet(),
|
||||
AppVersion.Zero,
|
||||
])!;
|
||||
|
||||
[Fact]
|
||||
public void FilterOwnClusterDriverMembers_recognises_a_miscased_driver_role()
|
||||
{
|
||||
var members = new[] { UpMember("site-a-1", 4053, "Driver", "cluster-SITE-A") };
|
||||
|
||||
var result = ClusterNodeAddressReconcilerActor.FilterOwnClusterDriverMembers(members, "cluster-SITE-A");
|
||||
|
||||
result.ShouldHaveSingleItem();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterOwnClusterDriverMembers_recognises_a_miscased_cluster_role()
|
||||
{
|
||||
var members = new[] { UpMember("site-a-1", 4053, "driver", "Cluster-SITE-A") };
|
||||
|
||||
// The caller's OWN cluster role names the canonical spelling; the member gossips a different
|
||||
// casing ("Cluster-SITE-A") — GossipedRolesHave must still match the two as the same role.
|
||||
var result = ClusterNodeAddressReconcilerActor.FilterOwnClusterDriverMembers(members, "cluster-site-a");
|
||||
|
||||
result.ShouldHaveSingleItem();
|
||||
}
|
||||
|
||||
// ---- Live single-node cluster: BuildSnapshot classification + ManualFailoverService -----------
|
||||
|
||||
private const int Port = 19_560;
|
||||
|
||||
private static Config NodeConfig(int port) => ConfigurationFactory.ParseString($$"""
|
||||
akka {
|
||||
loglevel = "WARNING"
|
||||
actor.provider = "Akka.Cluster.ClusterActorRefProvider, Akka.Cluster"
|
||||
remote.dot-netty.tcp {
|
||||
hostname = "127.0.0.1"
|
||||
port = {{port}}
|
||||
}
|
||||
cluster {
|
||||
seed-nodes = ["akka.tcp://gossip-case-test@127.0.0.1:{{port}}"]
|
||||
roles = ["Driver"]
|
||||
min-nr-of-members = 1
|
||||
run-coordinated-shutdown-when-down = off
|
||||
downing-provider-class = ""
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
private static async Task<ActorSystem> StartSingleNodeAsync()
|
||||
{
|
||||
var system = ActorSystem.Create("gossip-case-test", NodeConfig(Port));
|
||||
var cluster = Akka.Cluster.Cluster.Get(system);
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) return system;
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
await system.Terminate();
|
||||
throw new TimeoutException("node never came Up");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_member_gossiping_a_miscased_driver_role_classifies_Secondary_not_Detached()
|
||||
{
|
||||
// Pins the fix that IS achievable: BuildSnapshot's own role classification (member.Roles
|
||||
// Contains -> GossipedRolesHave) no longer reads a genuinely-driver-but-miscased member as
|
||||
// Detached, even though (see class remarks) it can never be named Primary by itself.
|
||||
var system = await StartSingleNodeAsync();
|
||||
try
|
||||
{
|
||||
var tcs = new TaskCompletionSource<RedundancyStateChanged>(
|
||||
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
system.ActorOf(
|
||||
RedundancyStateActor.Props(broadcast: msg =>
|
||||
{
|
||||
if (msg is RedundancyStateChanged changed) tcs.TrySetResult(changed);
|
||||
}),
|
||||
$"redundancy-{Guid.NewGuid():N}");
|
||||
|
||||
var snapshot = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(15));
|
||||
|
||||
var node = snapshot.Nodes.ShouldHaveSingleItem();
|
||||
node.Role.ShouldBe(
|
||||
RedundancyRole.Secondary,
|
||||
"gossiping 'Driver' must not read as Detached — it is a real driver member, just "
|
||||
+ "miscased; BEFORE R3-04 this was Detached (ordinal Contains(\"driver\") on \"Driver\")");
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// THE recorded deviation: proves the residual gap is real, not assumed. A pair (here,
|
||||
/// trivially, a single node) that gossips ONLY 'Driver' still elects no Primary — because that
|
||||
/// decision runs through Akka's own ordinal <c>Member.HasRole</c> inside the vendored
|
||||
/// <c>ClusterActiveNode.OldestUpMember</c>, which this repo cannot patch. Tracked as a
|
||||
/// follow-up against <c>ZB.MOM.WW.Health.Akka</c>, not closed by R3-04.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Primary_election_itself_remains_ordinal_and_still_elects_no_one_for_an_all_miscased_pair()
|
||||
{
|
||||
var system = await StartSingleNodeAsync();
|
||||
try
|
||||
{
|
||||
var members = Akka.Cluster.Cluster.Get(system).State.Members;
|
||||
|
||||
RedundancyStateActor.SelectOldestUpMemberOfRole(members, "driver").ShouldBeNull(
|
||||
"known residual (R3-04 execution notes): SelectOldestUpMemberOfRole delegates to "
|
||||
+ "ClusterActiveNode.OldestUpMember -> Akka's own ordinal Member.HasRole, which this "
|
||||
+ "repo does not own — GossipedRolesHave cannot reach inside it");
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ManualFailoverService_recognises_a_miscased_driver_node_as_a_candidate()
|
||||
{
|
||||
// FailOverCore refuses below 2 driver members (nothing to fail over TO), so this single-node
|
||||
// fixture exercises GetSnapshot()'s PrimaryAddress instead — which shares the same
|
||||
// OrderedDrivers candidate selection this test is actually about.
|
||||
var system = await StartSingleNodeAsync();
|
||||
try
|
||||
{
|
||||
var service = new ManualFailoverService(
|
||||
() => system, new RecordingAuditWriter(), NullLogger<ManualFailoverService>.Instance);
|
||||
|
||||
var snapshot = service.GetSnapshot();
|
||||
|
||||
snapshot.PrimaryAddress.ShouldNotBeNull();
|
||||
snapshot.PrimaryAddress!.ShouldContain(Port.ToString());
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingAuditWriter : IAuditWriter
|
||||
{
|
||||
public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default) =>
|
||||
Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user