feat(cluster): leader-auto-down decision logging + cold-start/flapping/partition/switch-off scenarios (ScadaBridge#106)

Every downing decision of LeaderAutoDowningProvider is now logged under a
[leader-auto-down] prefix: start-up snapshot (leader, window, already-unreachable),
unreachable observed + its window (Warning), skipped because already Down/Exiting,
pending Down cancelled on reachable-again/removed, timed out on a non-leader and kept
pending, leadership gained/lost, and DOWNING (Warning) before Cluster.Down. Akka's own
AutoDown logged only the final Down. Decision semantics are unchanged.

New real two-node scenarios (IntegrationTests/Cluster/LeaderAutoDownScenarioTests),
built from production BuildHocon with self-first seeds:
- both nodes cold-start together -> one cluster, nobody downed past FD + window;
- lone cold start with the peer dead (the down-if-alone cold-recreate shape) stays Up,
  then the peer cold-starts and joins;
- flapping peer via Akka's trttl throttler Blackhole: unreachable shorter than the
  window, repeated past it in total -> never downed; positive control asserts the cuts
  really produced unreachability;
- partition held past the window -> each side downs the other, both stay Up and
  leader (the accepted dual-active trade);
- switch off: auto-down loads AutoDowning, keep-oldest loads SplitBrainResolverProvider,
  only leader-auto-down loads the in-repo provider.
Unit: flapping never downs; a flapping member that finally stays unreachable is downed
exactly once a full window after its LAST unreachable; each decision log line.

Mutation-proved: no timer cancel on reachable -> flap-then-stay unit fails; reachable
ignored -> both unit flapping tests, the cancel-log test and the real-cluster flapping
test fail; DOWNING log dropped -> log test fails; provider never downs -> partition
test fails; auto-down miswired to the new provider -> switch-off theory fails.

Docker failover drill not run (it would rebuild the shared scadabridge:latest image
and reuse the shared rig volumes while another agent works in the repo); filed as
deploy gate ScadaBridge#116. Default remains auto-down.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 04:35:43 -05:00
parent a13aabc7c3
commit 456c6659e7
6 changed files with 436 additions and 10 deletions
@@ -18,6 +18,12 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// </summary>
public class LeaderAutoDownTests : TestKit
{
/// <summary>INFO so the decision-log assertions can see the Info-level decisions.</summary>
public LeaderAutoDownTests()
: base("akka.loglevel = INFO")
{
}
private static readonly Address MemberA = new("akka.tcp", "sys", "a", 2552);
private static readonly Address MemberB = new("akka.tcp", "sys", "b", 2552);
private static readonly Address MemberC = new("akka.tcp", "sys", "c", 2552);
@@ -212,4 +218,107 @@ public class LeaderAutoDownTests : TestKit
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(3));
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
/// <summary>
/// A flapping member — unreachable, reachable again inside the window, repeatedly, for far
/// longer than one window in total — is never downed.
/// </summary>
[Fact]
public void FlappingMember_ReachableAgainInsideEachWindow_IsNeverDowned()
{
var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
for (var i = 0; i < 5; i++)
{
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(700));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
}
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
/// <summary>
/// When a flapping member finally stays unreachable, the window restarts from its LAST
/// unreachable — an earlier flap's timer must not fire — and it is downed exactly once.
/// </summary>
[Fact]
public void FlappingMember_ThatStaysUnreachable_IsDownedOneFullWindowAfterLastUnreachable()
{
var a = AutoDownActor(TimeSpan.FromMilliseconds(1500));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
a.Tell(new ClusterEvent.UnreachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
a.Tell(new ClusterEvent.ReachableMember(b));
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
a.Tell(new ClusterEvent.UnreachableMember(b)); // final: stays unreachable from here
// The first flap's timer would fire ~0.1s from here; the final window ends ~1.5s from here.
ExpectNoMsg(TimeSpan.FromMilliseconds(1100));
ExpectMsg(new DownCalled(MemberB), TimeSpan.FromSeconds(2));
ExpectNoMsg(TimeSpan.FromSeconds(2));
}
/// <summary>The Down decision is logged at Warning, naming the member and the window.</summary>
[Fact]
public void DecisionLog_Down_IsLoggedAtWarning()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
EventFilter.Warning(contains: $"[leader-auto-down] DOWNING {MemberB}").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
ExpectMsg(new DownCalled(MemberB));
}
/// <summary>Observing an unreachable member (and its window) is logged at Warning.</summary>
[Fact]
public void DecisionLog_UnreachableScheduled_IsLoggedAtWarning()
{
var a = AutoDownActor(TimeSpan.FromSeconds(30));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
EventFilter.Warning(contains: $"{MemberB} is unreachable; it will be downed if still unreachable after").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Up))));
}
/// <summary>A cancelled Down (member reachable again) is logged.</summary>
[Fact]
public void DecisionLog_ReachableAgain_CancelIsLogged()
{
var a = AutoDownActor(TimeSpan.FromSeconds(30));
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
var b = CreateMember(MemberB, MemberStatus.Up);
a.Tell(new ClusterEvent.UnreachableMember(b));
EventFilter.Info(contains: $"{MemberB} became reachable again; its pending Down is cancelled").ExpectOne(() =>
a.Tell(new ClusterEvent.ReachableMember(b)));
}
/// <summary>A timeout on a non-leader (kept pending) is logged, and so is the later Down on becoming leader.</summary>
[Fact]
public void DecisionLog_PendingOnNonLeader_ThenDownOnBecomingLeader_AreLogged()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberB));
EventFilter.Info(contains: "is not leader; kept pending").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberC, MemberStatus.Up))));
EventFilter.Warning(contains: $"DOWNING {MemberC}: it timed out as unreachable while this node was not leader").ExpectOne(() =>
a.Tell(new ClusterEvent.LeaderChanged(MemberA)));
ExpectMsg(new DownCalled(MemberC));
}
/// <summary>Skipping an already Down member is logged.</summary>
[Fact]
public void DecisionLog_SkippedDownMember_IsLogged()
{
var a = AutoDownActor(TimeSpan.Zero);
a.Tell(new ClusterEvent.LeaderChanged(MemberA));
EventFilter.Info(contains: "but already Down; not scheduling a Down").ExpectOne(() =>
a.Tell(new ClusterEvent.UnreachableMember(CreateMember(MemberB, MemberStatus.Down))));
}
}