fix(cluster): leader-auto-down restarts a node its peer has shut out; higher address restarts when both are (#132)
A node frozen past failure detection + StableAfter, or both sides of a partition that heals after each downed the other, used to stay a second active node until an operator restarted one: the peer quarantines the removed UID and Akka.Cluster 1.5.62 never acts on ThisActorSystemQuarantinedEvent. ShutOutRestart (child of the LeaderAutoDown downing actor; opt-out via ScadaBridge:Cluster:LeaderAutoDown:RestartWhenShutOut, default on for leader-auto-down only) watches two signals: - S1 ThisActorSystemQuarantinedEvent: the peer quarantined this node. If the peer is still a live member here, only this side can be shut out -> restart now. - S2 AssociationErrorEvent 'has a UID that has been quarantined': the peer still runs the incarnation this node removed. A healed partition yields only S2, on both sides. Wait ShutOutGrace (60s) for a frozen peer to restart itself, then tiebreak. Tiebreak: the higher address (Akka Member.AddressOrdering) restarts, the lower stays and restarts only if still split one grace later. Removed peers are probed (Identify, 5s -> 60s back-off) since nothing else contacts them after mutual removal. Restart = Cluster.Down(self) -> existing self-down path. The Host now exits 75 after an unplanned ActorSystem termination (and sets it on the Windows service before it reports stopped): SCM recovery with failureflag 1 only restarts a service that stops with a non-zero code, and the watchdog used to exit 0. Shipped default strategy stays auto-down. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
+28
@@ -327,4 +327,32 @@ public class ClusterOptionsValidatorTests
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("'leader-auto-down'", result.FailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132: a non-positive shut-out grace is refused under leader-auto-down with the
|
||||
/// restart on (it would make both sides of a healed partition restart together), and ignored
|
||||
/// when the restart is off or another strategy is selected.
|
||||
/// </summary>
|
||||
/// <param name="strategy">The strategy.</param>
|
||||
/// <param name="restart">The shut-out restart switch.</param>
|
||||
/// <param name="expectValid">Whether validation must pass.</param>
|
||||
[Theory]
|
||||
[InlineData("leader-auto-down", true, false)]
|
||||
[InlineData("leader-auto-down", false, true)]
|
||||
[InlineData("auto-down", true, true)]
|
||||
public void LeaderAutoDown_ZeroShutOutGrace(string strategy, bool restart, bool expectValid)
|
||||
{
|
||||
var options = ValidOptions();
|
||||
options.SplitBrainResolverStrategy = strategy;
|
||||
options.LeaderAutoDown = new LeaderAutoDownOptions { RestartWhenShutOut = restart, ShutOutGrace = TimeSpan.Zero };
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.Equal(expectValid, result.Succeeded);
|
||||
if (!expectValid)
|
||||
{
|
||||
Assert.Contains("ShutOutGrace", result.FailureMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -322,4 +322,46 @@ public class HoconBuilderTests
|
||||
Assert.DoesNotContain("leader-auto-down", hocon);
|
||||
Assert.DoesNotContain(nameof(LeaderAutoDowningProvider), hocon);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132: leader-auto-down carries the shut-out restart switch (default ON) and its
|
||||
/// grace into the provider's HOCON block, and the provider reads back exactly those values.
|
||||
/// </summary>
|
||||
/// <param name="enabled">The configured switch.</param>
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void BuildHocon_LeaderAutoDown_EmitsShutOutRestartSettings(bool enabled)
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = "leader-auto-down";
|
||||
cluster.LeaderAutoDown = new LeaderAutoDownOptions
|
||||
{
|
||||
RestartWhenShutOut = enabled,
|
||||
ShutOutGrace = TimeSpan.FromMilliseconds(45500),
|
||||
};
|
||||
|
||||
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
|
||||
|
||||
var settings = ShutOutRestartSettings.FromConfig(config);
|
||||
Assert.Equal(enabled, settings.Enabled);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(45500), settings.Grace);
|
||||
}
|
||||
|
||||
/// <summary>ScadaBridge#132: the shut-out restart is ON by default when leader-auto-down is selected.</summary>
|
||||
[Fact]
|
||||
public void BuildHocon_LeaderAutoDown_ShutOutRestartOnByDefault()
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = "leader-auto-down";
|
||||
|
||||
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
|
||||
|
||||
Assert.True(config.GetBoolean(ShutOutRestartSettings.EnabledPath));
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), config.GetTimeSpan(ShutOutRestartSettings.GracePath));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Reflection;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using Akka.Remote;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Akka.Util;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132: the decision logic of <see cref="ShutOutRestartBase"/>. Cluster events and
|
||||
/// <see cref="ThisActorSystemQuarantinedEvent"/> are fed straight to the actor; the restart and
|
||||
/// probe seams report to the test actor. Covers the single-side shut-out (restart, no tiebreak),
|
||||
/// the tiebreak in both orders, the both-sides case where exactly one of two nodes restarts, the
|
||||
/// winner's grace, and the probing that makes a healed partition produce the event at all. The
|
||||
/// real two-node behaviour is proven in IntegrationTests (<c>ShutOutRestartScenarioTests</c>).
|
||||
/// </summary>
|
||||
public class ShutOutRestartTests : TestKit
|
||||
{
|
||||
/// <summary>INFO so the decision-log assertions see every line.</summary>
|
||||
public ShutOutRestartTests()
|
||||
: base("akka.loglevel = INFO")
|
||||
{
|
||||
}
|
||||
|
||||
// "a" < "b" < "c" by Akka's member address ordering (host, then port).
|
||||
private static readonly Address NodeA = new("akka.tcp", "scadabridge", "a", 8081);
|
||||
private static readonly Address NodeB = new("akka.tcp", "scadabridge", "b", 8081);
|
||||
private static readonly Address Stranger = new("akka.tcp", "scadabridge", "z", 9999);
|
||||
|
||||
private static readonly ShutOutRestartSettings Fast = new(
|
||||
Enabled: true,
|
||||
Grace: TimeSpan.FromSeconds(1.5),
|
||||
ProbeInterval: TimeSpan.FromMilliseconds(200),
|
||||
ProbeMaxInterval: TimeSpan.FromMilliseconds(800));
|
||||
|
||||
private sealed record Restarted(Address Self);
|
||||
|
||||
private sealed record Probed(Address Self, Address Peer);
|
||||
|
||||
/// <summary>Test double: restart and probe report to a probe actor.</summary>
|
||||
private sealed class TestShutOutRestart : ShutOutRestartBase
|
||||
{
|
||||
private readonly IActorRef _probe;
|
||||
private readonly Address[] _seeds;
|
||||
|
||||
public TestShutOutRestart(Address self, Address[] seeds, ShutOutRestartSettings settings, IActorRef probe)
|
||||
: base(settings)
|
||||
{
|
||||
SelfAddress = self;
|
||||
_seeds = seeds;
|
||||
_probe = probe;
|
||||
}
|
||||
|
||||
public override Address SelfAddress { get; }
|
||||
|
||||
public override IScheduler Scheduler => Context.System.Scheduler;
|
||||
|
||||
protected override IEnumerable<Address> SeedNodes => _seeds;
|
||||
|
||||
protected override void RestartSelf() => _probe.Tell(new Restarted(SelfAddress));
|
||||
|
||||
protected override void Probe(Address peer) => _probe.Tell(new Probed(SelfAddress, peer));
|
||||
}
|
||||
|
||||
private static Member CreateMember(Address address, MemberStatus status)
|
||||
{
|
||||
var create = typeof(Member).GetMethod(
|
||||
"Create",
|
||||
BindingFlags.Static | BindingFlags.NonPublic,
|
||||
new[] { typeof(UniqueAddress), typeof(int), typeof(MemberStatus), typeof(ImmutableHashSet<string>), typeof(AppVersion) })
|
||||
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(…) not found — Akka internals changed.");
|
||||
return (Member)create.Invoke(null, new object[]
|
||||
{
|
||||
new UniqueAddress(address, 1), 1, status, ImmutableHashSet<string>.Empty, AppVersion.Zero,
|
||||
})!;
|
||||
}
|
||||
|
||||
private static ClusterEvent.CurrentClusterState StateWith(params Member[] members) =>
|
||||
new(
|
||||
ImmutableSortedSet.CreateRange(members),
|
||||
ImmutableHashSet<Member>.Empty,
|
||||
ImmutableHashSet<Address>.Empty,
|
||||
null,
|
||||
ImmutableDictionary<string, Address>.Empty);
|
||||
|
||||
private IActorRef Node(Address self, Address peer, IActorRef? probe = null, ShutOutRestartSettings? settings = null) =>
|
||||
Sys.ActorOf(Props.Create(() => new TestShutOutRestart(
|
||||
self, new[] { self, peer }, settings ?? Fast, probe ?? TestActor)));
|
||||
|
||||
/// <summary>Both nodes see each other Up, then each removes the other after downing it (the partition outcome).</summary>
|
||||
private static void BothRemovedEachOther(IActorRef node, Address self, Address peer)
|
||||
{
|
||||
node.Tell(StateWith(CreateMember(self, MemberStatus.Up), CreateMember(peer, MemberStatus.Up)));
|
||||
node.Tell(new ClusterEvent.MemberRemoved(CreateMember(peer, MemberStatus.Removed), MemberStatus.Down));
|
||||
}
|
||||
|
||||
private static ThisActorSystemQuarantinedEvent ShutOutBy(Address peer, Address self) => new(self, peer);
|
||||
|
||||
/// <summary>S2 as Akka.Remote 1.5.62 surfaces it: the endpoint writer's InvalidAssociation wrapping the protocol refusal.</summary>
|
||||
private static AssociationErrorEvent StillRunningRemoved(Address peer, Address self) =>
|
||||
new(
|
||||
new Exception(
|
||||
$"Association failed with {peer}",
|
||||
new AggregateException(new Exception("The remote system has a UID that has been quarantined. Association aborted."))),
|
||||
self, peer, inbound: false, Akka.Event.LogLevel.WarningLevel);
|
||||
|
||||
/// <summary>Everything <paramref name="probe"/> receives during <paramref name="window"/>.</summary>
|
||||
private static List<object> Drain(Akka.TestKit.TestProbe probe, TimeSpan window)
|
||||
{
|
||||
var received = new List<object>();
|
||||
var until = DateTime.UtcNow + window;
|
||||
while (true)
|
||||
{
|
||||
var left = until - DateTime.UtcNow;
|
||||
if (left <= TimeSpan.Zero)
|
||||
{
|
||||
return received;
|
||||
}
|
||||
|
||||
if (probe.TryReceiveOne(out var envelope, left))
|
||||
{
|
||||
received.Add(envelope.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single side (the frozen node): the peer is still a live member in this node's view, so only
|
||||
/// this side can be shut out — it restarts at once, whichever address is higher.
|
||||
/// </summary>
|
||||
/// <param name="selfIsA">Whether this node is the lower-address node.</param>
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void ShutOut_PeerStillAMember_RestartsWithoutTiebreak(bool selfIsA)
|
||||
{
|
||||
var (self, peer) = selfIsA ? (NodeA, NodeB) : (NodeB, NodeA);
|
||||
var node = Node(self, peer);
|
||||
node.Tell(StateWith(CreateMember(self, MemberStatus.Up), CreateMember(peer, MemberStatus.Up)));
|
||||
|
||||
EventFilter.Warning(contains: "tiebreak not needed").ExpectOne(() => node.Tell(ShutOutBy(peer, self)));
|
||||
ExpectMsg(new Restarted(self));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
/// <summary>Tiebreak, higher address: both sides removed each other; the higher address restarts at once.</summary>
|
||||
[Fact]
|
||||
public void ShutOut_PeerRemoved_HigherAddress_Restarts()
|
||||
{
|
||||
var node = Node(NodeB, NodeA);
|
||||
BothRemovedEachOther(node, NodeB, NodeA);
|
||||
|
||||
EventFilter.Warning(contains: "Tiebreak says RESTART").ExpectOne(() => node.Tell(ShutOutBy(NodeA, NodeB)));
|
||||
ExpectMsg(new Restarted(NodeB));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tiebreak, lower address: it stays; it keeps probing during the grace, and at the end of the
|
||||
/// grace — the peer silent, i.e. it restarted — it still stays.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ShutOut_PeerRemoved_LowerAddress_Stays_AndStaysAfterGraceWhenPeerWentQuiet()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var node = Node(NodeA, NodeB, probe.Ref);
|
||||
BothRemovedEachOther(node, NodeA, NodeB);
|
||||
|
||||
EventFilter.Warning(contains: "Tiebreak says STAY").ExpectOne(() => node.Tell(ShutOutBy(NodeB, NodeA)));
|
||||
EventFilter.Info(contains: "nothing of its old incarnation has been seen").ExpectOne(TimeSpan.FromSeconds(4), () => { });
|
||||
|
||||
var received = Drain(probe, TimeSpan.FromMilliseconds(500));
|
||||
Assert.Contains(new Probed(NodeA, NodeB), received); // it kept probing its peer
|
||||
Assert.DoesNotContain(new Restarted(NodeA), received); // and never restarted
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tiebreak, lower address, but the peer keeps refusing for the whole grace and never rejoins:
|
||||
/// the lower node was the stale side after all (single-sided with the lower address) and restarts.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ShutOut_PeerRemoved_LowerAddress_StillRefusedAfterGrace_Restarts()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var node = Node(NodeA, NodeB, probe.Ref);
|
||||
BothRemovedEachOther(node, NodeA, NodeB);
|
||||
node.Tell(ShutOutBy(NodeB, NodeA));
|
||||
|
||||
var until = DateTime.UtcNow + Fast.Grace + TimeSpan.FromSeconds(1);
|
||||
var restarted = false;
|
||||
while (DateTime.UtcNow < until && !restarted)
|
||||
{
|
||||
node.Tell(ShutOutBy(NodeB, NodeA)); // every probe is refused
|
||||
await Task.Delay(100);
|
||||
while (probe.HasMessages)
|
||||
{
|
||||
restarted |= probe.ReceiveOne(TimeSpan.Zero) is Restarted;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(restarted, "the lower node never restarted although its peer kept refusing it past the grace");
|
||||
}
|
||||
|
||||
/// <summary>Tiebreak, lower address: the peer rejoins during the grace; the lower node stays.</summary>
|
||||
[Fact]
|
||||
public void ShutOut_PeerRemoved_LowerAddress_PeerRejoins_Stays()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var node = Node(NodeA, NodeB, probe.Ref);
|
||||
BothRemovedEachOther(node, NodeA, NodeB);
|
||||
node.Tell(ShutOutBy(NodeB, NodeA));
|
||||
node.Tell(ShutOutBy(NodeB, NodeA)); // refused right up to the restart of the peer
|
||||
node.Tell(new ClusterEvent.MemberUp(CreateMember(NodeB, MemberStatus.Up)));
|
||||
|
||||
EventFilter.Info(contains: "rejoined").ExpectOne(TimeSpan.FromSeconds(4), () => { });
|
||||
Assert.DoesNotContain(new Restarted(NodeA), Drain(probe, TimeSpan.FromMilliseconds(500)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Both sides shut out (a healed partition): two nodes, each having removed the other, both
|
||||
/// receive the event — exactly ONE restarts, and it is the higher address. Run in both
|
||||
/// delivery orders so the outcome cannot depend on who hears first.
|
||||
/// </summary>
|
||||
/// <param name="higherHearsFirst">Whether the higher-address node receives its event first.</param>
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void BothSidesShutOut_ExactlyOneRestarts_TheHigherAddress(bool higherHearsFirst)
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var a = Node(NodeA, NodeB, probe.Ref);
|
||||
var b = Node(NodeB, NodeA, probe.Ref);
|
||||
BothRemovedEachOther(a, NodeA, NodeB);
|
||||
BothRemovedEachOther(b, NodeB, NodeA);
|
||||
|
||||
if (higherHearsFirst)
|
||||
{
|
||||
b.Tell(ShutOutBy(NodeA, NodeB));
|
||||
a.Tell(ShutOutBy(NodeB, NodeA));
|
||||
}
|
||||
else
|
||||
{
|
||||
a.Tell(ShutOutBy(NodeB, NodeA));
|
||||
b.Tell(ShutOutBy(NodeA, NodeB));
|
||||
}
|
||||
|
||||
// Within the grace, the only restart is B's.
|
||||
var restarts = Drain(probe, Fast.Grace - TimeSpan.FromMilliseconds(300)).OfType<Restarted>().ToList();
|
||||
Assert.Equal(new[] { new Restarted(NodeB) }, restarts);
|
||||
}
|
||||
|
||||
/// <summary>The same event delivered many times restarts once.</summary>
|
||||
[Fact]
|
||||
public void RepeatedShutOut_RestartsOnce()
|
||||
{
|
||||
var node = Node(NodeB, NodeA);
|
||||
node.Tell(StateWith(CreateMember(NodeB, MemberStatus.Up), CreateMember(NodeA, MemberStatus.Up)));
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
node.Tell(ShutOutBy(NodeA, NodeB));
|
||||
}
|
||||
|
||||
ExpectMsg(new Restarted(NodeB));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>A quarantine by an address that is neither a seed nor a member is not this pair's business.</summary>
|
||||
[Fact]
|
||||
public void ShutOutByStranger_IsIgnored()
|
||||
{
|
||||
var node = Node(NodeB, NodeA);
|
||||
node.Tell(StateWith(CreateMember(NodeB, MemberStatus.Up), CreateMember(NodeA, MemberStatus.Up)));
|
||||
|
||||
EventFilter.Info(contains: "not a seed or member of this cluster").ExpectOne(() => node.Tell(ShutOutBy(Stranger, NodeB)));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A peer removed after being DOWNED is probed with a back-off until it is a member again; a
|
||||
/// peer that LEFT (Exiting → Removed) is not probed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void DownedPeer_IsProbedUntilItRejoins_LeftPeerIsNot()
|
||||
{
|
||||
var node = Node(NodeA, NodeB);
|
||||
node.Tell(StateWith(CreateMember(NodeA, MemberStatus.Up), CreateMember(NodeB, MemberStatus.Up)));
|
||||
|
||||
node.Tell(new ClusterEvent.MemberRemoved(CreateMember(NodeB, MemberStatus.Removed), MemberStatus.Exiting));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(600));
|
||||
|
||||
node.Tell(new ClusterEvent.MemberRemoved(CreateMember(NodeB, MemberStatus.Removed), MemberStatus.Down));
|
||||
ExpectMsg(new Probed(NodeA, NodeB), TimeSpan.FromSeconds(1));
|
||||
ExpectMsg(new Probed(NodeA, NodeB), TimeSpan.FromSeconds(1));
|
||||
|
||||
node.Tell(new ClusterEvent.MemberJoined(CreateMember(NodeB, MemberStatus.Joining)));
|
||||
ReceiveWhile<object>(TimeSpan.FromMilliseconds(100), m => m); // a tick already in flight
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(1.5));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// S2 on the higher node (it removed a peer whose old incarnation is still alive): no restart
|
||||
/// during the grace — a frozen peer gets the chance to restart itself — then, still split, the
|
||||
/// tiebreak restarts it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task StillRunningRemoved_HigherAddress_WaitsGrace_ThenRestartsIfStillSplit()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var node = Node(NodeB, NodeA, probe.Ref);
|
||||
BothRemovedEachOther(node, NodeB, NodeA);
|
||||
|
||||
EventFilter.Warning(contains: "Giving it").ExpectOne(() => node.Tell(StillRunningRemoved(NodeA, NodeB)));
|
||||
var started = DateTime.UtcNow;
|
||||
var restartedAt = (DateTime?)null;
|
||||
while (DateTime.UtcNow - started < Fast.Grace + TimeSpan.FromSeconds(1.5) && restartedAt is null)
|
||||
{
|
||||
node.Tell(StillRunningRemoved(NodeA, NodeB)); // every probe still meets the old incarnation
|
||||
await Task.Delay(100);
|
||||
while (probe.HasMessages)
|
||||
{
|
||||
if (probe.ReceiveOne(TimeSpan.Zero) is Restarted)
|
||||
{
|
||||
restartedAt = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assert.NotNull(restartedAt);
|
||||
Assert.True(restartedAt.Value - started >= Fast.Grace - TimeSpan.FromMilliseconds(200),
|
||||
$"restarted after {(restartedAt.Value - started).TotalSeconds:0.0}s, before the {Fast.Grace} grace");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The freeze survivor: S2 (the frozen node's old incarnation is alive), then the frozen node
|
||||
/// restarts itself and its old incarnation goes quiet — the survivor does NOT restart, even
|
||||
/// though it has the higher address.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void StillRunningRemoved_PeerGoesQuiet_NoRestartEvenWhenHigher()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var node = Node(NodeB, NodeA, probe.Ref);
|
||||
BothRemovedEachOther(node, NodeB, NodeA);
|
||||
node.Tell(StillRunningRemoved(NodeA, NodeB));
|
||||
|
||||
EventFilter.Info(contains: "nothing of its old incarnation has been seen").ExpectOne(TimeSpan.FromSeconds(4), () => { });
|
||||
Assert.DoesNotContain(new Restarted(NodeB), Drain(probe, TimeSpan.FromMilliseconds(500)));
|
||||
}
|
||||
|
||||
/// <summary>S2, then the peer's new incarnation joins during the grace: no restart.</summary>
|
||||
[Fact]
|
||||
public void StillRunningRemoved_PeerRejoins_NoRestart()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var node = Node(NodeB, NodeA, probe.Ref);
|
||||
BothRemovedEachOther(node, NodeB, NodeA);
|
||||
node.Tell(StillRunningRemoved(NodeA, NodeB));
|
||||
node.Tell(new ClusterEvent.MemberUp(CreateMember(NodeA, MemberStatus.Up)));
|
||||
node.Tell(StillRunningRemoved(NodeA, NodeB)); // stale evidence from before the restart
|
||||
|
||||
EventFilter.Info(contains: "rejoined").ExpectOne(TimeSpan.FromSeconds(4), () => { });
|
||||
Assert.DoesNotContain(new Restarted(NodeB), Drain(probe, TimeSpan.FromMilliseconds(500)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A healed partition as Akka 1.5.62 really reports it — S2 on BOTH sides, no S1 — ends with
|
||||
/// exactly one restart, the higher address, after the grace; the lower stays.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task BothSidesStillRunningRemoved_ExactlyOneRestarts_TheHigherAddress()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var a = Node(NodeA, NodeB, probe.Ref);
|
||||
var b = Node(NodeB, NodeA, probe.Ref);
|
||||
BothRemovedEachOther(a, NodeA, NodeB);
|
||||
BothRemovedEachOther(b, NodeB, NodeA);
|
||||
|
||||
var restarts = new List<Restarted>();
|
||||
var until = DateTime.UtcNow + Fast.Grace + Fast.Grace - TimeSpan.FromMilliseconds(400);
|
||||
while (DateTime.UtcNow < until)
|
||||
{
|
||||
if (!restarts.Contains(new Restarted(NodeB)))
|
||||
{
|
||||
// Until B's old incarnation is gone, each side's probes meet the other's.
|
||||
a.Tell(StillRunningRemoved(NodeB, NodeA));
|
||||
b.Tell(StillRunningRemoved(NodeA, NodeB));
|
||||
}
|
||||
|
||||
await Task.Delay(100);
|
||||
while (probe.HasMessages)
|
||||
{
|
||||
if (probe.ReceiveOne(TimeSpan.Zero) is Restarted r)
|
||||
{
|
||||
restarts.Add(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(new[] { new Restarted(NodeB) }, restarts);
|
||||
}
|
||||
|
||||
/// <summary>S1 arriving while an S2 grace runs: the peer has quarantined us too, so the tiebreak runs at once.</summary>
|
||||
[Fact]
|
||||
public void StillRunningRemoved_ThenShutOut_TiebreaksImmediately()
|
||||
{
|
||||
var node = Node(NodeB, NodeA);
|
||||
BothRemovedEachOther(node, NodeB, NodeA);
|
||||
node.Tell(StillRunningRemoved(NodeA, NodeB));
|
||||
|
||||
EventFilter.Warning(contains: "Tiebreak says RESTART").ExpectOne(() => node.Tell(ShutOutBy(NodeA, NodeB)));
|
||||
FishForMessage(m => m is Restarted, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
/// <summary>S2 detection matches Akka 1.5.62's two wordings anywhere in the cause chain, and nothing else.</summary>
|
||||
[Fact]
|
||||
public void IsQuarantinedUidRefusal_MatchesAkkaWordingOnly()
|
||||
{
|
||||
Assert.True(ShutOutRestartBase.IsQuarantinedUidRefusal(
|
||||
new Exception("Association failed with x", new AggregateException(
|
||||
new Exception("The remote system has a UID that has been quarantined. Association aborted.")))));
|
||||
Assert.True(ShutOutRestartBase.IsQuarantinedUidRefusal(
|
||||
new Exception("The remote system [akka.tcp://scadabridge@b:8081] has a UID [42] that has been quarantined. Association aborted.")));
|
||||
Assert.False(ShutOutRestartBase.IsQuarantinedUidRefusal(
|
||||
new Exception("Association failed with x", new Exception("Connection refused"))));
|
||||
Assert.False(ShutOutRestartBase.IsQuarantinedUidRefusal(
|
||||
new Exception("The remote system has quarantined this system. No further associations to the remote system are possible until this system is restarted.")));
|
||||
Assert.False(ShutOutRestartBase.IsQuarantinedUidRefusal(null));
|
||||
}
|
||||
|
||||
/// <summary>Settings: defaults when keys are missing; the switch and durations are read when present.</summary>
|
||||
[Fact]
|
||||
public void Settings_FromConfig_DefaultsAndOverrides()
|
||||
{
|
||||
var defaults = ShutOutRestartSettings.FromConfig(ConfigurationFactory.Empty);
|
||||
Assert.Equal(new ShutOutRestartSettings(true, TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(60)), defaults);
|
||||
|
||||
var set = ShutOutRestartSettings.FromConfig(ConfigurationFactory.ParseString($@"
|
||||
{ShutOutRestartSettings.EnabledPath} = off
|
||||
{ShutOutRestartSettings.GracePath} = 2500ms
|
||||
{ShutOutRestartSettings.ProbeIntervalPath} = 1s
|
||||
{ShutOutRestartSettings.ProbeMaxIntervalPath} = 4s"));
|
||||
Assert.Equal(new ShutOutRestartSettings(false, TimeSpan.FromMilliseconds(2500), TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(4)), set);
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,8 @@ public sealed class UnexpectedTerminationTests
|
||||
await Task.Delay(500);
|
||||
|
||||
lifetime.Received(1).StopApplication();
|
||||
// ScadaBridge#132: Program turns this flag into the non-zero SelfDownExitCode.
|
||||
Assert.True(service.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a normal stop does not trigger an application stop.</summary>
|
||||
@@ -84,5 +86,19 @@ public sealed class UnexpectedTerminationTests
|
||||
await Task.Delay(500);
|
||||
|
||||
lifetime.DidNotReceive().StopApplication();
|
||||
Assert.False(service.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132: the self-down exit code is non-zero (Windows service recovery with
|
||||
/// failureflag 1 restarts only on a non-zero exit) and is only pushed into a Windows service
|
||||
/// lifetime — a console / docker lifetime is left alone.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void SelfDownExitCode_IsNonZero_AndOnlyAppliedToAServiceLifetime()
|
||||
{
|
||||
Assert.NotEqual(0, AkkaHostedService.SelfDownExitCode);
|
||||
Assert.False(AkkaHostedService.MarkServiceSelfDownExit(null));
|
||||
Assert.False(AkkaHostedService.MarkServiceSelfDownExit(Substitute.For<IHostLifetime>()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using Akka.Remote;
|
||||
using Akka.Remote.Transport;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132, on real in-process two-node clusters built from the production
|
||||
/// <c>BuildHocon</c> (same harness as <see cref="LeaderAutoDownScenarioTests"/>, with Akka's
|
||||
/// throttler transport to cut and restore the network without killing either node):
|
||||
/// <list type="bullet">
|
||||
/// <item>the FROZEN-node shape — only one side downed the other before the network came back —
|
||||
/// ends with the shut-out node restarting (its <c>ActorSystem</c> terminates) and the other
|
||||
/// staying Up;</item>
|
||||
/// <item>a partition held past the window and then HEALED — each side downed the other — ends
|
||||
/// with exactly one restart, the higher address; a fresh incarnation on that address rejoins the
|
||||
/// survivor as a 2-member cluster, and the survivor is still Up after its grace;</item>
|
||||
/// <item>with the switch OFF, the healed partition keeps both sides up (the #116 gate-4 contract
|
||||
/// before #132) — the positive control that the restart comes from the feature.</item>
|
||||
/// </list>
|
||||
/// "Restart" in-process is the termination of the node's <c>ActorSystem</c>; the Host turns that
|
||||
/// into a process exit (covered by <c>UnexpectedTerminationTests</c>).
|
||||
/// </summary>
|
||||
public sealed class ShutOutRestartScenarioTests : IAsyncLifetime
|
||||
{
|
||||
private static readonly TimeSpan Heartbeat = TimeSpan.FromMilliseconds(500);
|
||||
private static readonly TimeSpan Threshold = TimeSpan.FromSeconds(2);
|
||||
|
||||
private readonly List<ActorSystem> _systems = new();
|
||||
|
||||
private ActorSystem StartNode(
|
||||
int selfPort, int peerPort, TimeSpan stableAfter, bool restartWhenShutOut = true, TimeSpan? grace = null)
|
||||
{
|
||||
const string scheme = "akka.trttl.tcp";
|
||||
var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = selfPort };
|
||||
var clusterOptions = new ClusterOptions
|
||||
{
|
||||
SeedNodes = new List<string>
|
||||
{
|
||||
$"{scheme}://scadabridge@127.0.0.1:{selfPort}",
|
||||
$"{scheme}://scadabridge@127.0.0.1:{peerPort}",
|
||||
},
|
||||
SplitBrainResolverStrategy = "leader-auto-down",
|
||||
StableAfter = stableAfter,
|
||||
HeartbeatInterval = Heartbeat,
|
||||
FailureDetectionThreshold = Threshold,
|
||||
MinNrOfMembers = 1,
|
||||
LeaderAutoDown = new LeaderAutoDownOptions
|
||||
{
|
||||
RestartWhenShutOut = restartWhenShutOut,
|
||||
ShutOutGrace = grace ?? TimeSpan.FromSeconds(20),
|
||||
},
|
||||
};
|
||||
var hocon = AkkaHostedService.BuildHocon(
|
||||
nodeOptions, clusterOptions, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(60));
|
||||
var overrides = "akka.coordinated-shutdown.run-by-actor-system-terminate = off\n"
|
||||
+ "akka.remote.dot-netty.tcp.applied-adapters = [trttl]\n"
|
||||
+ "akka.remote.retry-gate-closed-for = 500ms\n"
|
||||
// Fast probes so a healed partition is noticed within the test's patience.
|
||||
+ "scadabridge.cluster.leader-auto-down.shut-out-probe-interval = 1s\n"
|
||||
+ "scadabridge.cluster.leader-auto-down.shut-out-probe-max-interval = 2s\n";
|
||||
var config = ConfigurationFactory.ParseString(overrides)
|
||||
.WithFallback(ConfigurationFactory.ParseString(hocon));
|
||||
var system = ActorSystem.Create("scadabridge", config);
|
||||
_systems.Add(system);
|
||||
return system;
|
||||
}
|
||||
|
||||
private static Task<bool> Throttle(ActorSystem from, ActorSystem to, ThrottleMode mode) =>
|
||||
((RemoteActorRefProvider)((ExtendedActorSystem)from).Provider).Transport.ManagementCommand(
|
||||
new SetThrottle(Akka.Cluster.Cluster.Get(to).SelfAddress, ThrottleTransportAdapter.Direction.Both, mode));
|
||||
|
||||
private async Task<(ActorSystem Lower, ActorSystem Higher, int LowerPort, int HigherPort)> StartPair(
|
||||
TimeSpan stableLower, TimeSpan stableHigher, bool restartWhenShutOut = true)
|
||||
{
|
||||
var p1 = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var p2 = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var (lowerPort, higherPort) = p1 < p2 ? (p1, p2) : (p2, p1);
|
||||
var lower = StartNode(lowerPort, higherPort, stableLower, restartWhenShutOut);
|
||||
await TwoNodeClusterFixture.WaitForMembersUp(lower, 1, TimeSpan.FromSeconds(30));
|
||||
var higher = StartNode(higherPort, lowerPort, stableHigher, restartWhenShutOut);
|
||||
await TwoNodeClusterFixture.WaitForMembersUp(lower, 2, TimeSpan.FromSeconds(30));
|
||||
await TwoNodeClusterFixture.WaitForMembersUp(higher, 2, TimeSpan.FromSeconds(30));
|
||||
return (lower, higher, lowerPort, higherPort);
|
||||
}
|
||||
|
||||
private static async Task<bool> Terminates(ActorSystem system, TimeSpan within)
|
||||
{
|
||||
var done = await Task.WhenAny(system.WhenTerminated, Task.Delay(within));
|
||||
return done == system.WhenTerminated;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Frozen-node shape: only the LOWER node downs the higher one (the higher one's window is far
|
||||
/// longer than the cut), then the network returns. The higher node — still counting the lower as
|
||||
/// a member — is refused, restarts without a tiebreak; the lower node stays Up. Then the same with
|
||||
/// roles swapped, so the single-side rule is proven independent of the address order.
|
||||
/// </summary>
|
||||
/// <param name="shutOutIsHigher">Whether the shut-out (frozen) node has the higher address.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public async Task SingleSideShutOut_ShutOutNodeRestarts_OtherStaysUp(bool shutOutIsHigher)
|
||||
{
|
||||
var quick = TimeSpan.FromSeconds(2);
|
||||
var never = TimeSpan.FromMinutes(5);
|
||||
var (lower, higher, _, _) = shutOutIsHigher
|
||||
? await StartPair(quick, never)
|
||||
: await StartPair(never, quick);
|
||||
var (survivor, shutOut) = shutOutIsHigher ? (lower, higher) : (higher, lower);
|
||||
|
||||
Assert.True(await Throttle(lower, higher, Blackhole.Instance));
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(survivor, Akka.Cluster.Cluster.Get(shutOut).SelfAddress, TimeSpan.FromSeconds(30));
|
||||
Assert.Contains(Akka.Cluster.Cluster.Get(shutOut).State.Members,
|
||||
m => m.Address == Akka.Cluster.Cluster.Get(survivor).SelfAddress); // it still counts the survivor
|
||||
Assert.True(await Throttle(lower, higher, Unthrottled.Instance));
|
||||
|
||||
Assert.True(await Terminates(shutOut, TimeSpan.FromSeconds(30)),
|
||||
"the shut-out node kept running as a second active node");
|
||||
Assert.False(await Terminates(survivor, TimeSpan.FromSeconds(5)), "the survivor restarted too");
|
||||
Assert.Equal(MemberStatus.Up, Akka.Cluster.Cluster.Get(survivor).SelfMember.Status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A partition past the window, then healed: both sides downed and removed each other, so both
|
||||
/// are refused — exactly ONE restarts, the higher address. A fresh incarnation on the higher
|
||||
/// port (what the service supervisor starts) rejoins the survivor, and the survivor is still Up
|
||||
/// after its grace has run out.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task HealedPartition_BothShutOut_OnlyTheHigherAddressRestarts_AndRejoins()
|
||||
{
|
||||
var stable = TimeSpan.FromSeconds(2);
|
||||
var (lower, higher, lowerPort, higherPort) = await StartPair(stable, stable);
|
||||
var lowerAddress = Akka.Cluster.Cluster.Get(lower).SelfAddress;
|
||||
var higherAddress = Akka.Cluster.Cluster.Get(higher).SelfAddress;
|
||||
|
||||
Assert.True(await Throttle(lower, higher, Blackhole.Instance));
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(lower, higherAddress, TimeSpan.FromSeconds(30));
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(higher, lowerAddress, TimeSpan.FromSeconds(30));
|
||||
Assert.False(lower.WhenTerminated.IsCompleted || higher.WhenTerminated.IsCompleted,
|
||||
"a node restarted while the partition was still in place");
|
||||
Assert.True(await Throttle(lower, higher, Unthrottled.Instance));
|
||||
|
||||
Assert.True(await Terminates(higher, TimeSpan.FromSeconds(30)),
|
||||
"the higher-address node did not restart after the partition healed");
|
||||
Assert.False(lower.WhenTerminated.IsCompleted, "the lower-address node restarted as well");
|
||||
|
||||
// The supervisor's restart: a fresh incarnation on the same address rejoins the survivor.
|
||||
var reborn = StartNode(higherPort, lowerPort, stable);
|
||||
await TwoNodeClusterFixture.WaitForMembersUp(lower, 2, TimeSpan.FromSeconds(60));
|
||||
await TwoNodeClusterFixture.WaitForMembersUp(reborn, 2, TimeSpan.FromSeconds(60));
|
||||
|
||||
// Past the survivor's 20s grace: it saw the peer rejoin, so it stays.
|
||||
Assert.False(await Terminates(lower, TimeSpan.FromSeconds(25)), "the survivor restarted after its peer rejoined");
|
||||
Assert.Equal(2, Akka.Cluster.Cluster.Get(lower).State.Members.Count(m => m.Status == MemberStatus.Up));
|
||||
Assert.Empty(Akka.Cluster.Cluster.Get(lower).State.Unreachable);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Switch OFF: the same healed partition keeps BOTH sides running (the pre-#132 dual-active)
|
||||
/// — the positive control that the restarts above come from the feature.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task HealedPartition_SwitchOff_BothStayUp()
|
||||
{
|
||||
var stable = TimeSpan.FromSeconds(2);
|
||||
var (lower, higher, _, _) = await StartPair(stable, stable, restartWhenShutOut: false);
|
||||
var lowerAddress = Akka.Cluster.Cluster.Get(lower).SelfAddress;
|
||||
var higherAddress = Akka.Cluster.Cluster.Get(higher).SelfAddress;
|
||||
|
||||
Assert.True(await Throttle(lower, higher, Blackhole.Instance));
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(lower, higherAddress, TimeSpan.FromSeconds(30));
|
||||
await TwoNodeClusterFixture.WaitForMemberRemoved(higher, lowerAddress, TimeSpan.FromSeconds(30));
|
||||
Assert.True(await Throttle(lower, higher, Unthrottled.Instance));
|
||||
|
||||
Assert.False(await Terminates(higher, TimeSpan.FromSeconds(15)), "a node restarted with the switch off");
|
||||
Assert.False(lower.WhenTerminated.IsCompleted, "a node restarted with the switch off");
|
||||
}
|
||||
|
||||
/// <summary>Prepares the fixture before the tests run.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
/// <summary>Releases the resources held by this fixture asynchronously.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
foreach (var s in _systems)
|
||||
{
|
||||
try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user