fix(cluster): leader-auto-down restarts a node its peer has shut out; higher address restarts when both are (#132)
A node frozen past failure detection + StableAfter, or both sides of a partition that heals after each downed the other, used to stay a second active node until an operator restarted one: the peer quarantines the removed UID and Akka.Cluster 1.5.62 never acts on ThisActorSystemQuarantinedEvent. ShutOutRestart (child of the LeaderAutoDown downing actor; opt-out via ScadaBridge:Cluster:LeaderAutoDown:RestartWhenShutOut, default on for leader-auto-down only) watches two signals: - S1 ThisActorSystemQuarantinedEvent: the peer quarantined this node. If the peer is still a live member here, only this side can be shut out -> restart now. - S2 AssociationErrorEvent 'has a UID that has been quarantined': the peer still runs the incarnation this node removed. A healed partition yields only S2, on both sides. Wait ShutOutGrace (60s) for a frozen peer to restart itself, then tiebreak. Tiebreak: the higher address (Akka Member.AddressOrdering) restarts, the lower stays and restarts only if still split one grace later. Removed peers are probed (Identify, 5s -> 60s back-off) since nothing else contacts them after mutual removal. Restart = Cluster.Down(self) -> existing self-down path. The Host now exits 75 after an unplanned ActorSystem termination (and sets it on the Windows service before it reports stopped): SCM recovery with failureflag 1 only restarts a service that stops with a non-zero code, and the watchdog used to exit 0. Shipped default strategy stays auto-down. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -322,4 +322,46 @@ public class HoconBuilderTests
|
||||
Assert.DoesNotContain("leader-auto-down", hocon);
|
||||
Assert.DoesNotContain(nameof(LeaderAutoDowningProvider), hocon);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132: leader-auto-down carries the shut-out restart switch (default ON) and its
|
||||
/// grace into the provider's HOCON block, and the provider reads back exactly those values.
|
||||
/// </summary>
|
||||
/// <param name="enabled">The configured switch.</param>
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void BuildHocon_LeaderAutoDown_EmitsShutOutRestartSettings(bool enabled)
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = "leader-auto-down";
|
||||
cluster.LeaderAutoDown = new LeaderAutoDownOptions
|
||||
{
|
||||
RestartWhenShutOut = enabled,
|
||||
ShutOutGrace = TimeSpan.FromMilliseconds(45500),
|
||||
};
|
||||
|
||||
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
|
||||
|
||||
var settings = ShutOutRestartSettings.FromConfig(config);
|
||||
Assert.Equal(enabled, settings.Enabled);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(45500), settings.Grace);
|
||||
}
|
||||
|
||||
/// <summary>ScadaBridge#132: the shut-out restart is ON by default when leader-auto-down is selected.</summary>
|
||||
[Fact]
|
||||
public void BuildHocon_LeaderAutoDown_ShutOutRestartOnByDefault()
|
||||
{
|
||||
var cluster = DefaultCluster();
|
||||
cluster.SplitBrainResolverStrategy = "leader-auto-down";
|
||||
|
||||
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
|
||||
DefaultNode(), cluster, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
|
||||
|
||||
Assert.True(config.GetBoolean(ShutOutRestartSettings.EnabledPath));
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), config.GetTimeSpan(ShutOutRestartSettings.GracePath));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user