feat(cluster): opt-in leader-auto-down strategy - in-repo port of Akka AutoDowning (ScadaBridge#106)
Adds LeaderAutoDowningProvider (Host/Actors), a line-for-line port of Akka.NET 1.5.62 AutoDowning/AutoDown/AutoDownBase read from the decompiled assembly: the leader among the reachable members downs a peer unreachable for StableAfter; a non-leader records it as pending and downs it on becoming leader; reachable-again or removed members are forgotten; Down/Exiting members are skipped. Its window is carried under scadabridge.cluster.leader-auto-down.unreachable-after, so no deprecated auto-down-unreachable-after key is emitted and Akka's boot-time "keep-majority will be used instead" warning goes away. DownRemovalMargin reads akka.cluster.down-removal-margin exactly as AutoDowning does (default off). New SplitBrainResolverStrategy value "leader-auto-down", wired through BuildHocon and ClusterOptionsValidator. The default stays "auto-down"; auto-down and keep-oldest HOCON is byte-identical to before. Placed in Host, not ClusterInfrastructure, per the Code Placement rule (ClusterInfrastructure stays Akka-free). Tests: HOCON + validator units; LeaderAutoDownTests ports Akka's AutoDownSpec cases (leader-only, pending-on-leader-change, losing leadership, reachable again, removed, Down/Exiting skipped, single Down per member); LeaderAutoDownFailoverTests clones SbrFailoverTests' oldest-crash and younger-crash cases on a real two-node cluster plus a test that Akka actually loaded the provider and its downing actor. Mutation-checked: a provider that never calls Cluster.Down fails both failover tests; never-downs, leader-check-dropped, reachable-ignored and pending-not-flushed mutations each fail the unit tests. Remaining gate before switching the default: the docker failover drill (DRILL_MODE=active and standby mode) on leader-auto-down. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
+29
@@ -298,4 +298,33 @@ public class ClusterOptionsValidatorTests
|
||||
Assert.Contains("PartnerProbeIntervalMs", result.FailureMessage);
|
||||
Assert.Contains("ProbeConnectTimeoutMs", result.FailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the leader-auto-down strategy (ScadaBridge#106) passes validation, case-insensitively.</summary>
|
||||
/// <param name="strategy">The strategy spelling.</param>
|
||||
[Theory]
|
||||
[InlineData("leader-auto-down")]
|
||||
[InlineData("Leader-Auto-Down")]
|
||||
public void LeaderAutoDownStrategy_Passes(string strategy)
|
||||
{
|
||||
var options = ValidOptions();
|
||||
options.SplitBrainResolverStrategy = strategy;
|
||||
options.DownIfAlone = false; // keep-oldest-only knob; inert here, must not block startup
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the rejection message lists leader-auto-down among the supported strategies.</summary>
|
||||
[Fact]
|
||||
public void UnsupportedStrategy_MessageNamesLeaderAutoDown()
|
||||
{
|
||||
var options = ValidOptions();
|
||||
options.SplitBrainResolverStrategy = "leader-autodown";
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("'leader-auto-down'", result.FailureMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -250,4 +250,76 @@ public class HoconBuilderTests
|
||||
Assert.Equal(1, config.GetInt("akka.cluster.min-nr-of-members"));
|
||||
Assert.True(config.GetBoolean("akka.cluster.run-coordinated-shutdown-when-down"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the leader-auto-down strategy (ScadaBridge#106) installs the in-repo
|
||||
/// provider, carries StableAfter under its own key, and emits neither Akka's deprecated
|
||||
/// auto-down key nor an SBR section.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void BuildHocon_LeaderAutoDownStrategy_EmitsInRepoProvider_AndNoDeprecatedKey()
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = "leader-auto-down";
|
||||
cluster.StableAfter = TimeSpan.FromMilliseconds(15250);
|
||||
|
||||
var hocon = AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
|
||||
|
||||
var config = ConfigurationFactory.ParseString(hocon);
|
||||
Assert.Equal(
|
||||
"ZB.MOM.WW.ScadaBridge.Host.Actors.LeaderAutoDowningProvider, ZB.MOM.WW.ScadaBridge.Host",
|
||||
config.GetString("akka.cluster.downing-provider-class"));
|
||||
// The emitted name must resolve the way Akka resolves it (Type.GetType(name, true)).
|
||||
Assert.Equal(typeof(LeaderAutoDowningProvider),
|
||||
Type.GetType(config.GetString("akka.cluster.downing-provider-class"), throwOnError: true));
|
||||
Assert.Equal(
|
||||
TimeSpan.FromMilliseconds(15250),
|
||||
config.GetTimeSpan(LeaderAutoDowningProvider.UnreachableAfterPath));
|
||||
// No deprecated key (that is what triggers Akka's "keep-majority will be used" warning)
|
||||
// and no SBR section alongside the provider.
|
||||
Assert.False(config.HasPath("akka.cluster.auto-down-unreachable-after"));
|
||||
Assert.False(config.HasPath("akka.cluster.split-brain-resolver.active-strategy"));
|
||||
// Document stays intact after the downing block.
|
||||
Assert.Equal(1, config.GetInt("akka.cluster.min-nr-of-members"));
|
||||
Assert.True(config.GetBoolean("akka.cluster.run-coordinated-shutdown-when-down"));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the leader-auto-down strategy name is case-insensitive.</summary>
|
||||
[Fact]
|
||||
public void BuildHocon_LeaderAutoDownStrategy_IsCaseInsensitive()
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = "Leader-Auto-Down";
|
||||
|
||||
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
|
||||
|
||||
Assert.Equal(LeaderAutoDowningProvider.ProviderClassName,
|
||||
config.GetString("akka.cluster.downing-provider-class"));
|
||||
Assert.Equal(TimeSpan.FromSeconds(15), config.GetTimeSpan(LeaderAutoDowningProvider.UnreachableAfterPath));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the leader-auto-down key is emitted ONLY for that strategy: the existing
|
||||
/// auto-down and keep-oldest documents are unchanged by ScadaBridge#106.
|
||||
/// </summary>
|
||||
/// <param name="strategy">An existing strategy.</param>
|
||||
[Theory]
|
||||
[InlineData("auto-down")]
|
||||
[InlineData("keep-oldest")]
|
||||
public void BuildHocon_OtherStrategies_DoNotEmitLeaderAutoDownKey(string strategy)
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = strategy;
|
||||
|
||||
var hocon = AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
|
||||
|
||||
Assert.DoesNotContain("leader-auto-down", hocon);
|
||||
Assert.DoesNotContain(nameof(LeaderAutoDowningProvider), hocon);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Reflection;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Akka.Util;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#106: the decision logic of <see cref="LeaderAutoDownBase"/> must match Akka.NET
|
||||
/// 1.5.62 <c>AutoDownBase</c> exactly. These cases are the ones Akka's own <c>AutoDownSpec</c>
|
||||
/// pins — cluster events are fed straight to the actor and the <c>Down</c> seam reports to a
|
||||
/// probe, so the leader-only and reachability semantics are proven without a running cluster.
|
||||
/// The two-node real-cluster behaviour is proven separately in IntegrationTests
|
||||
/// (<c>LeaderAutoDownFailoverTests</c>).
|
||||
/// </summary>
|
||||
public class LeaderAutoDownTests : TestKit
|
||||
{
|
||||
private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552);
|
||||
private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552);
|
||||
private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552);
|
||||
|
||||
private sealed record DownCalled(Address Address);
|
||||
|
||||
/// <summary>Test double: <c>Down</c> reports to a probe instead of calling the cluster.</summary>
|
||||
private sealed class TestLeaderAutoDown : LeaderAutoDownBase
|
||||
{
|
||||
private readonly IActorRef _probe;
|
||||
|
||||
public TestLeaderAutoDown(Address selfAddress, TimeSpan unreachableAfter, IActorRef probe)
|
||||
: base(unreachableAfter)
|
||||
{
|
||||
SelfAddress = selfAddress;
|
||||
_probe = probe;
|
||||
}
|
||||
|
||||
public override Address SelfAddress { get; }
|
||||
|
||||
public override IScheduler Scheduler => Context.System.Scheduler;
|
||||
|
||||
public override void Down(Address node)
|
||||
{
|
||||
// Report EVERY call, leader or not: a base that wrongly downs while not leader must
|
||||
// show up as an unexpected message, not be swallowed by a throw-and-restart.
|
||||
_probe.Tell(new DownCalled(node));
|
||||
if (!IsLeader)
|
||||
{
|
||||
_probe.Tell("down-called-while-not-leader");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Akka's <c>Member</c> constructors are internal; tests build members via the internal factory.</summary>
|
||||
private static Member CreateMember(Address address, MemberStatus status)
|
||||
{
|
||||
var create = typeof(Member).GetMethod(
|
||||
"Create",
|
||||
BindingFlags.Static | BindingFlags.NonPublic,
|
||||
new[] { typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet<string>), typeof(AppVersion) })
|
||||
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(…) not found — Akka internals changed.");
|
||||
return (Member)create.Invoke(null, new object[]
|
||||
{
|
||||
new UniqueAddress(address, 1), 1, status, ImmutableHashSet<string>.Empty, AppVersion.Zero,
|
||||
})!;
|
||||
}
|
||||
|
||||
private IActorRef AutoDownActor(TimeSpan unreachableAfter) =>
|
||||
Sys.ActorOf(Props.Create(() => new TestLeaderAutoDown(MemberA, unreachableAfter, TestActor)));
|
||||
|
||||
/// <summary>A non-leader never downs an unreachable member.</summary>
|
||||
[Fact]
|
||||
public void NotLeader_DoesNotDownUnreachable()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
/// <summary>The leader downs an unreachable member immediately when the window is zero.</summary>
|
||||
[Fact]
|
||||
public void Leader_DownsUnreachable_WhenWindowIsZero()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
|
||||
ExpectMsg(new DownCalled(MemberB));
|
||||
}
|
||||
|
||||
/// <summary>The leader downs an unreachable member only after the full window.</summary>
|
||||
[Fact]
|
||||
public void Leader_DownsUnreachable_AfterWindow()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(2));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up)));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(1));
|
||||
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
/// <summary>The initial snapshot drives leadership and schedules members already unreachable.</summary>
|
||||
[Fact]
|
||||
public void CurrentClusterState_Leader_DownsAlreadyUnreachable()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.CurrentClusterState(
|
||||
ImmutableSortedSet<Member>.Empty,
|
||||
ImmutableHashSet.Create(b),
|
||||
ImmutableHashSet<Address>.Empty,
|
||||
MemberA,
|
||||
ImmutableDictionary<string, Address>.Empty));
|
||||
ExpectMsg(new DownCalled(MemberB));
|
||||
}
|
||||
|
||||
/// <summary>A pending member is downed as soon as this node becomes leader.</summary>
|
||||
[Fact]
|
||||
public void BecomingLeader_DownsPendingUnreachable()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
ExpectMsg(new DownCalled(MemberC));
|
||||
}
|
||||
|
||||
/// <summary>A member that timed out while not leader is downed on becoming leader.</summary>
|
||||
[Fact]
|
||||
public void BecomingLeader_AfterWindow_DownsPendingUnreachable()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(1));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(2));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
ExpectMsg(new DownCalled(MemberC));
|
||||
}
|
||||
|
||||
/// <summary>Losing leadership inside the window means no Down when the timer fires.</summary>
|
||||
[Fact]
|
||||
public void LosingLeadership_InsideWindow_DoesNotDown()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(2));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up)));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
/// <summary>A member that becomes reachable again inside the window is not downed.</summary>
|
||||
[Fact]
|
||||
public void ReachableAgain_InsideWindow_DoesNotDown()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(2));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
a.Tell(new ClusterEvent.ReachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
/// <summary>A pending member that becomes reachable again is not downed on becoming leader.</summary>
|
||||
[Fact]
|
||||
public void ReachableAgain_WhilePending_IsForgotten()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
|
||||
var c = CreateMember(MemberC, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.UnreachableMember(c));
|
||||
a.Tell(new ClusterEvent.ReachableMember(c));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
/// <summary>A member removed inside the window is not downed.</summary>
|
||||
[Fact]
|
||||
public void RemovedInsideWindow_DoesNotDown()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(2));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
a.Tell(new ClusterEvent.MemberRemoved(b.Copy(MemberStatus.Removed), MemberStatus.Up));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
/// <summary>Members already Down or Exiting are skipped (Akka's ConvergenceSkipUnreachableWithMemberStatus).</summary>
|
||||
/// <param name="status">The member status.</param>
|
||||
[Theory]
|
||||
[InlineData(MemberStatus.Down)]
|
||||
[InlineData(MemberStatus.Exiting)]
|
||||
public void AlreadyDownOrExiting_IsNotDownedAgain(MemberStatus status)
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, status)));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
/// <summary>A repeated UnreachableMember for an already-scheduled member does not schedule a second Down.</summary>
|
||||
[Fact]
|
||||
public void RepeatedUnreachable_DownsOnce()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(1));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
using Akka.Actor;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#106: the in-repo <see cref="LeaderAutoDowningProvider"/> (strategy
|
||||
/// <c>leader-auto-down</c>) must give a REAL two-node cluster exactly the outcomes Akka's
|
||||
/// <c>AutoDowning</c> (<c>auto-down</c>) gives in <see cref="SbrFailoverTests"/>: a hard crash
|
||||
/// of EITHER node — the oldest included — is downed and removed by the survivor and the
|
||||
/// singleton ends up answering on the survivor. The two failover cases are clones of
|
||||
/// <c>SbrFailoverTests.AutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton</c> and
|
||||
/// <c>…AutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton</c>, differing only in the strategy.
|
||||
/// Under a provider that never downs, the member-removal step times out (the crashed member
|
||||
/// stays Unreachable forever) — that is what gives them teeth.
|
||||
/// </summary>
|
||||
public class LeaderAutoDownFailoverTests
|
||||
{
|
||||
private const string Strategy = "leader-auto-down";
|
||||
|
||||
/// <summary>
|
||||
/// The configured provider is the one Akka actually instantiated and started: the Cluster
|
||||
/// extension holds a <see cref="LeaderAutoDowningProvider"/> and its downing actor is running
|
||||
/// under the cluster daemon on both nodes.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LeaderAutoDown_ProviderIsWhatAkkaLoads_OnBothNodes()
|
||||
{
|
||||
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
|
||||
|
||||
foreach (var node in new[] { cluster.NodeA, cluster.NodeB })
|
||||
{
|
||||
var akkaCluster = Akka.Cluster.Cluster.Get(node);
|
||||
Assert.IsType<LeaderAutoDowningProvider>(akkaCluster.DowningProvider);
|
||||
|
||||
// ClusterCoreDaemon creates the provider's DowningActorProps as "downingProvider".
|
||||
var identity = await node.ActorSelection("/system/cluster/core/daemon/downingProvider")
|
||||
.Ask<ActorIdentity>(new Identify("downing"), TimeSpan.FromSeconds(5));
|
||||
Assert.NotNull(identity.Subject);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// With leader-auto-down, a hard crash of the OLDEST node lets the younger survivor down it
|
||||
/// and take over the singleton — the direction two-node keep-oldest cannot survive.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LeaderAutoDown_HardCrashOfOldestNode_YoungerSurvivorTakesOverSingleton()
|
||||
{
|
||||
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
|
||||
SbrFailoverTests.StartSingleton(cluster.NodeA); // oldest hosts the singleton initially
|
||||
var (_, proxyB) = SbrFailoverTests.StartSingleton(cluster.NodeB);
|
||||
|
||||
var echo = await proxyB.Ask<string>("ping", TimeSpan.FromSeconds(20));
|
||||
Assert.Equal("ping", echo);
|
||||
|
||||
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeA).SelfAddress;
|
||||
await TwoNodeClusterFixture.CrashNode(cluster.NodeA);
|
||||
|
||||
// 1) The younger survivor must DOWN and REMOVE the crashed OLDEST member.
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(
|
||||
cluster.NodeB, victimAddress, TimeSpan.FromSeconds(30));
|
||||
|
||||
// 2) B must still be a functioning cluster member (not self-downed) …
|
||||
var clusterB = Akka.Cluster.Cluster.Get(cluster.NodeB);
|
||||
Assert.False(clusterB.IsTerminated, "survivor's Cluster extension terminated — it downed itself");
|
||||
|
||||
// 3) … and the singleton must migrate to B and answer again.
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
Exception? last = null;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
try
|
||||
{
|
||||
var echo2 = await proxyB.Ask<string>("ping-after-oldest-crash", TimeSpan.FromSeconds(3));
|
||||
Assert.Equal("ping-after-oldest-crash", echo2);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) { last = ex; }
|
||||
}
|
||||
throw new Xunit.Sdk.XunitException(
|
||||
$"Singleton never migrated to the younger survivor after the oldest crashed under leader-auto-down: {last}");
|
||||
}
|
||||
|
||||
/// <summary>With leader-auto-down, a hard crash of the younger node leaves the singleton on the oldest node.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task LeaderAutoDown_HardCrashOfYoungerNode_OldestKeepsSingleton()
|
||||
{
|
||||
await using var cluster = await TwoNodeClusterFixture.StartAsync(strategy: Strategy);
|
||||
var (_, proxyA) = SbrFailoverTests.StartSingleton(cluster.NodeA);
|
||||
SbrFailoverTests.StartSingleton(cluster.NodeB);
|
||||
|
||||
Assert.Equal("ping", await proxyA.Ask<string>("ping", TimeSpan.FromSeconds(20)));
|
||||
|
||||
var victimAddress = Akka.Cluster.Cluster.Get(cluster.NodeB).SelfAddress;
|
||||
await TwoNodeClusterFixture.CrashNode(cluster.NodeB);
|
||||
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(
|
||||
cluster.NodeA, victimAddress, TimeSpan.FromSeconds(30));
|
||||
|
||||
Assert.False(Akka.Cluster.Cluster.Get(cluster.NodeA).IsTerminated,
|
||||
"surviving oldest's Cluster extension terminated — it downed itself");
|
||||
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
Exception? last = null;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
try
|
||||
{
|
||||
var echo2 = await proxyA.Ask<string>("ping-after-crash", TimeSpan.FromSeconds(3));
|
||||
Assert.Equal("ping-after-crash", echo2);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) { last = ex; }
|
||||
}
|
||||
throw new Xunit.Sdk.XunitException(
|
||||
$"Singleton stopped answering on the surviving oldest node under leader-auto-down: {last}");
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ public class SbrFailoverTests
|
||||
public EchoActor() => ReceiveAny(msg => Sender.Tell(msg));
|
||||
}
|
||||
|
||||
private static (IActorRef manager, IActorRef proxy) StartSingleton(ActorSystem sys)
|
||||
internal static (IActorRef manager, IActorRef proxy) StartSingleton(ActorSystem sys)
|
||||
{
|
||||
var manager = sys.ActorOf(ClusterSingletonManager.Props(
|
||||
Props.Create(() => new EchoActor()),
|
||||
|
||||
Reference in New Issue
Block a user