dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cluster): leader-auto-down decision logging + cold-start/flapping/partition/switch-off scenarios (ScadaBridge#106)

Browse Source 
Every downing decision of LeaderAutoDowningProvider is now logged under a
[leader-auto-down] prefix: start-up snapshot (leader, window, already-unreachable),
unreachable observed + its window (Warning), skipped because already Down/Exiting,
pending Down cancelled on reachable-again/removed, timed out on a non-leader and kept
pending, leadership gained/lost, and DOWNING (Warning) before Cluster.Down. Akka's own
AutoDown logged only the final Down. Decision semantics are unchanged.

New real two-node scenarios (IntegrationTests/Cluster/LeaderAutoDownScenarioTests),
built from production BuildHocon with self-first seeds:
- both nodes cold-start together -> one cluster, nobody downed past FD + window;
- lone cold start with the peer dead (the down-if-alone cold-recreate shape) stays Up,
  then the peer cold-starts and joins;
- flapping peer via Akka's trttl throttler Blackhole: unreachable shorter than the
  window, repeated past it in total -> never downed; positive control asserts the cuts
  really produced unreachability;
- partition held past the window -> each side downs the other, both stay Up and
  leader (the accepted dual-active trade);
- switch off: auto-down loads AutoDowning, keep-oldest loads SplitBrainResolverProvider,
  only leader-auto-down loads the in-repo provider.
Unit: flapping never downs; a flapping member that finally stays unreachable is downed
exactly once a full window after its LAST unreachable; each decision log line.

Mutation-proved: no timer cancel on reachable -> flap-then-stay unit fails; reachable
ignored -> both unit flapping tests, the cancel-log test and the real-cluster flapping
test fail; DOWNING log dropped -> log test fails; provider never downs -> partition
test fails; auto-down miswired to the new provider -> switch-off theory fails.

Docker failover drill not run (it would rebuild the shared scadabridge:latest image
and reuse the shared rig volumes while another agent works in the repo); filed as
deploy gate ScadaBridge#116. Default remains auto-down.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 04:35:43 -05:00
parent a13aabc7c3
commit 456c6659e7
6 changed files with 436 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Host/Actors/LeaderAutoDowningProvider.cs
+58 -8
View File
@@ -119,7 +119,6 @@ public sealed class LeaderAutoDowningProvider : IDowningProvider
internal sealed class LeaderAutoDown : LeaderAutoDownBase
{
private readonly Akka.Cluster.Cluster _cluster;
private readonly ILoggingAdapter _log = Context.GetLogger();
/// <summary>Props for the downing actor.</summary>
/// <param name="unreachableAfter">How long a member must stay unreachable before it is downed.</param>
@@ -165,7 +164,6 @@ internal sealed class LeaderAutoDown : LeaderAutoDownBase
throw new InvalidOperationException("Must be leader to down node");
}
_log.Info("Leader is auto-downing unreachable node [{0}]", node);
_cluster.Down(node);
}
}
@@ -174,6 +172,15 @@ internal sealed class LeaderAutoDown : LeaderAutoDownBase
/// The cluster-free decision logic of <see cref="LeaderAutoDown"/> — port of Akka's internal
/// <c>AutoDownBase</c>, split out (as Akka does) so the leader / reachability semantics can be
/// unit tested by feeding cluster events directly.
/// <para>
/// Every downing decision is logged (ScadaBridge#106) — with the <c>[leader-auto-down]</c>
/// prefix so an operator can grep one node's log for the whole story: a peer observed
/// unreachable (and the window it now has), a peer skipped because it is already Down/Exiting,
/// a pending timer cancelled because the peer became reachable again or was removed, a timer
/// that fired on a non-leader (kept pending), leadership gained/lost, and the actual Down
/// (logged at Warning, before <see cref="Down"/> is called). Akka's own <c>AutoDown</c> logs
/// only the last of these.
/// </para>
/// </summary>
internal abstract class LeaderAutoDownBase : UntypedActor
{
@@ -193,6 +200,9 @@ internal abstract class LeaderAutoDownBase : UntypedActor
ImmutableDictionary<UniqueAddress, ICancelable>.Empty;
private ImmutableHashSet<UniqueAddress> _pendingUnreachable = ImmutableHashSet<UniqueAddress>.Empty;
/// <summary>Decision log; every downing decision is written here.</summary>
private readonly ILoggingAdapter _log = Context.GetLogger();
/// <summary>Creates the base logic.</summary>
/// <param name="unreachableAfter">How long a member must stay unreachable before it is downed.</param>
protected LeaderAutoDownBase(TimeSpan unreachableAfter)
@@ -229,6 +239,10 @@ internal abstract class LeaderAutoDownBase : UntypedActor
{
case ClusterEvent.CurrentClusterState state:
IsLeader = state.Leader != null && state.Leader.Equals(SelfAddress);
_log.Info(
"[leader-auto-down] started on {0}: leader={1} (this node is leader: {2}), unreachable-after={3}, already unreachable=[{4}]",
SelfAddress, state.Leader?.ToString() ?? "none", IsLeader, _unreachableAfter,
string.Join(", ", state.Unreachable.Select(m => m.Address)));
foreach (var member in state.Unreachable)
{
UnreachableMember(member);
@@ -240,19 +254,31 @@ internal abstract class LeaderAutoDownBase : UntypedActor
break;
case ClusterEvent.ReachableMember reachable:
Remove(reachable.Member.UniqueAddress);
Remove(reachable.Member.UniqueAddress, "became reachable again");
break;
case ClusterEvent.MemberRemoved removed:
Remove(removed.Member.UniqueAddress);
Remove(removed.Member.UniqueAddress, "was removed from the cluster");
break;
case ClusterEvent.LeaderChanged leaderChanged:
var wasLeader = IsLeader;
IsLeader = leaderChanged.Leader != null && leaderChanged.Leader.Equals(SelfAddress);
if (wasLeader != IsLeader)
{
_log.Info(
"[leader-auto-down] {0} {1} leadership (leader is now {2}); pending unreachable=[{3}]",
SelfAddress, IsLeader ? "gained" : "lost", leaderChanged.Leader?.ToString() ?? "none",
string.Join(", ", _pendingUnreachable.Select(n => n.Address)));
}
if (IsLeader)
{
foreach (var node in _pendingUnreachable)
{
_log.Warning(
"[leader-auto-down] DOWNING {0}: it timed out as unreachable while this node was not leader, and this node ({1}) is now leader",
node.Address, SelfAddress);
Down(node.Address);
}
@@ -274,7 +300,15 @@ internal abstract class LeaderAutoDownBase : UntypedActor
private void UnreachableMember(Member member)
{
if (!SkipMemberStatus.Contains(member.Status) && !_scheduledUnreachable.ContainsKey(member.UniqueAddress))
if (SkipMemberStatus.Contains(member.Status))
{
_log.Info(
"[leader-auto-down] {0} is unreachable but already {1}; not scheduling a Down",
member.Address, member.Status);
return;
}
if (!_scheduledUnreachable.ContainsKey(member.UniqueAddress))
{
ScheduleUnreachable(member.UniqueAddress);
}
@@ -288,6 +322,10 @@ internal abstract class LeaderAutoDownBase : UntypedActor
return;
}
_log.Warning(
"[leader-auto-down] {0} is unreachable; it will be downed if still unreachable after {1} (this node {2} is leader: {3})",
node.Address, _unreachableAfter, SelfAddress, IsLeader);
var cancelable = Scheduler.ScheduleTellOnceCancelable(
_unreachableAfter, Self, new UnreachableTimeout(node), Self);
_scheduledUnreachable = _scheduledUnreachable.Add(node, cancelable);
@@ -297,19 +335,31 @@ internal abstract class LeaderAutoDownBase : UntypedActor
{
if (IsLeader)
{
_log.Warning(
"[leader-auto-down] DOWNING {0}: unreachable for {1} and this node ({2}) is leader",
node.Address, _unreachableAfter, SelfAddress);
Down(node.Address);
}
else
{
_log.Info(
"[leader-auto-down] {0} unreachable for {1} but this node ({2}) is not leader; kept pending until it becomes leader",
node.Address, _unreachableAfter, SelfAddress);
_pendingUnreachable = _pendingUnreachable.Add(node);
}
}
private void Remove(UniqueAddress node)
private void Remove(UniqueAddress node, string reason)
{
if (_scheduledUnreachable.TryGetValue(node, out var cancelable))
var wasScheduled = _scheduledUnreachable.TryGetValue(node, out var cancelable);
if (wasScheduled)
{
cancelable.Cancel();
cancelable!.Cancel();
}
if (wasScheduled || _pendingUnreachable.Contains(node))
{
_log.Info("[leader-auto-down] {0} {1}; its pending Down is cancelled", node.Address, reason);
}
_scheduledUnreachable = _scheduledUnreachable.Remove(node);