feat(cluster): leader-auto-down decision logging + cold-start/flapping/partition/switch-off scenarios (ScadaBridge#106)
Every downing decision of LeaderAutoDowningProvider is now logged under a [leader-auto-down] prefix: start-up snapshot (leader, window, already-unreachable), unreachable observed + its window (Warning), skipped because already Down/Exiting, pending Down cancelled on reachable-again/removed, timed out on a non-leader and kept pending, leadership gained/lost, and DOWNING (Warning) before Cluster.Down. Akka's own AutoDown logged only the final Down. Decision semantics are unchanged. New real two-node scenarios (IntegrationTests/Cluster/LeaderAutoDownScenarioTests), built from production BuildHocon with self-first seeds: - both nodes cold-start together -> one cluster, nobody downed past FD + window; - lone cold start with the peer dead (the down-if-alone cold-recreate shape) stays Up, then the peer cold-starts and joins; - flapping peer via Akka's trttl throttler Blackhole: unreachable shorter than the window, repeated past it in total -> never downed; positive control asserts the cuts really produced unreachability; - partition held past the window -> each side downs the other, both stay Up and leader (the accepted dual-active trade); - switch off: auto-down loads AutoDowning, keep-oldest loads SplitBrainResolverProvider, only leader-auto-down loads the in-repo provider. Unit: flapping never downs; a flapping member that finally stays unreachable is downed exactly once a full window after its LAST unreachable; each decision log line. Mutation-proved: no timer cancel on reachable -> flap-then-stay unit fails; reachable ignored -> both unit flapping tests, the cancel-log test and the real-cluster flapping test fail; DOWNING log dropped -> log test fails; provider never downs -> partition test fails; auto-down miswired to the new provider -> switch-off theory fails. Docker failover drill not run (it would rebuild the shared scadabridge:latest image and reuse the shared rig volumes while another agent works in the repo); filed as deploy gate ScadaBridge#116. Default remains auto-down. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -18,6 +18,12 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
/// </summary>
|
||||
public class LeaderAutoDownTests : TestKit
|
||||
{
|
||||
/// <summary>INFO so the decision-log assertions can see the Info-level decisions.</summary>
|
||||
public LeaderAutoDownTests()
|
||||
: base("akka.loglevel = INFO")
|
||||
{
|
||||
}
|
||||
|
||||
private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552);
|
||||
private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552);
|
||||
private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552);
|
||||
@@ -212,4 +218,107 @@ public class LeaderAutoDownTests : TestKit
|
||||
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A flapping member — unreachable, reachable again inside the window, repeatedly, for far
|
||||
/// longer than one window in total — is never downed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FlappingMember_ReachableAgainInsideEachWindow_IsNeverDowned()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(700));
|
||||
a.Tell(new ClusterEvent.ReachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
|
||||
}
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When a flapping member finally stays unreachable, the window restarts from its LAST
|
||||
/// unreachable — an earlier flap's timer must not fire — and it is downed exactly once.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FlappingMember_ThatStaysUnreachable_IsDownedOneFullWindowAfterLastUnreachable()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
a.Tell(new ClusterEvent.ReachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
a.Tell(new ClusterEvent.ReachableMember(b));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b)); // final: stays unreachable from here
|
||||
|
||||
// The first flap's timer would fire ~0.1s from here; the final window ends ~1.5s from here.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(1100));
|
||||
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(2));
|
||||
ExpectNoMsg(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
/// <summary>The Down decision is logged at Warning, naming the member and the window.</summary>
|
||||
[Fact]
|
||||
public void DecisionLog_Down_IsLoggedAtWarning()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
EventFilter.Warning(contains: $"[leader-auto-down] DOWNING {MemberB}").ExpectOne(() =>
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
|
||||
ExpectMsg(new DownCalled(MemberB));
|
||||
}
|
||||
|
||||
/// <summary>Observing an unreachable member (and its window) is logged at Warning.</summary>
|
||||
[Fact]
|
||||
public void DecisionLog_UnreachableScheduled_IsLoggedAtWarning()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(30));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
EventFilter.Warning(contains: $"{MemberB} is unreachable; it will be downed if still unreachable after").ExpectOne(() =>
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
|
||||
}
|
||||
|
||||
/// <summary>A cancelled Down (member reachable again) is logged.</summary>
|
||||
[Fact]
|
||||
public void DecisionLog_ReachableAgain_CancelIsLogged()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.FromSeconds(30));
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
var b = CreateMember(MemberB, MemberStatus.Up);
|
||||
a.Tell(new ClusterEvent.UnreachableMember(b));
|
||||
EventFilter.Info(contains: $"{MemberB} became reachable again; its pending Down is cancelled").ExpectOne(() =>
|
||||
a.Tell(new ClusterEvent.ReachableMember(b)));
|
||||
}
|
||||
|
||||
/// <summary>A timeout on a non-leader (kept pending) is logged, and so is the later Down on becoming leader.</summary>
|
||||
[Fact]
|
||||
public void DecisionLog_PendingOnNonLeader_ThenDownOnBecomingLeader_AreLogged()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
|
||||
EventFilter.Info(contains: "is not leader; kept pending").ExpectOne(() =>
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))));
|
||||
EventFilter.Warning(contains: $"DOWNING {MemberC}: it timed out as unreachable while this node was not leader").ExpectOne(() =>
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA)));
|
||||
ExpectMsg(new DownCalled(MemberC));
|
||||
}
|
||||
|
||||
/// <summary>Skipping an already Down member is logged.</summary>
|
||||
[Fact]
|
||||
public void DecisionLog_SkippedDownMember_IsLogged()
|
||||
{
|
||||
var a = AutoDownActor(TimeSpan.Zero);
|
||||
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
|
||||
EventFilter.Info(contains: "but already Down; not scheduling a Down").ExpectOne(() =>
|
||||
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Down))));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user