fix(717): Warm crash survivor stays Active through auto-down; verdict reads its own events
Two defects on the crash-failover path, found by the T13 live gate on an isolated docker pair: 1. When auto-down removed the unreachable Primary (~15 s after the survivor promoted), the snapshot's Primary left the unreachable set, so (3') failed and the survivor demoted to Standby (BadOutOfService, ServiceLevel 50) for ~15 s, until its own singleton published after the hand-over retries. A Primary that is Down, or that this node saw removed, now counts as gone (WarmRoleAuthority.SnapshotPrimaryGone). A member that is merely Leaving/Exiting, or that this node never saw, is not. 2. The verdict read Cluster.State / SelfMember (Akka's ClusterReadView, a separate subscriber to the same events), which can lag the event being handled. Live, the survivor stayed Standby 4.3 s after its own UnreachableMember event (promotion waited for the next health tick). The Warm verdict now uses a membership view built from this actor's own member/reachability subscription, falling back to the read view only before the first event. Tests: WarmRoleAuthorityTests.A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active, OpcUaPublishActorWarmTests.The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal (synthetic events the read view never sees). Mutations: read-view-only verdict and the removed arm switched off each fail the actor test. Runtime.Tests 1058 passed, 3 skipped. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -540,9 +540,21 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Receive<SubscribeAck>(_ => { /* PubSub ack */ });
|
||||
// Gitea #717 spec §4.1 / owner answer Q2: under Warm the verdict re-derives on membership and reachability, not
|
||||
// only on snapshots. Nothing subscribes these under Unset/None.
|
||||
Receive<ClusterEvent.IMemberEvent>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.IReachabilityEvent>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.CurrentClusterState>(_ => RecomputeServiceLevel());
|
||||
Receive<ClusterEvent.IMemberEvent>(e =>
|
||||
{
|
||||
TrackMember(e);
|
||||
RecomputeServiceLevel();
|
||||
});
|
||||
Receive<ClusterEvent.IReachabilityEvent>(e =>
|
||||
{
|
||||
TrackReachability(e);
|
||||
RecomputeServiceLevel();
|
||||
});
|
||||
Receive<ClusterEvent.CurrentClusterState>(state =>
|
||||
{
|
||||
ResetWarmView(state);
|
||||
RecomputeServiceLevel();
|
||||
});
|
||||
Receive<WarmRetry>(_ => RecomputeServiceLevel());
|
||||
}
|
||||
|
||||
@@ -1730,6 +1742,61 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
private readonly SnapshotRevisionGate _revisionGate = new();
|
||||
/// <summary>The latest ACCEPTED revisioned snapshot — the only one the Warm verdict reads (spec §4.2).</summary>
|
||||
private RedundancyStateChanged? _acceptedSnapshot;
|
||||
|
||||
// Gitea #717 T13 live gate: the Warm verdict reads membership from THIS actor's own event subscription, not from
|
||||
// Cluster.State / Cluster.SelfMember. Those come from Akka's ClusterReadView, a separate subscriber to the same
|
||||
// events, so when this actor handles an event the read view may not show it yet. Live, that held a crash survivor
|
||||
// Standby for 4.3 s after its own UnreachableMember event (promotion waited for the next health tick), and let a
|
||||
// gracefully leaving Primary keep serving Good ~240 ms after the survivor was already Active.
|
||||
private readonly Dictionary<Address, Member> _warmMembers = [];
|
||||
private readonly HashSet<Address> _warmUnreachable = [];
|
||||
|
||||
// Gitea #717 T13: nodes seen removed from the cluster (and not back since). A snapshot naming one is as gone as an
|
||||
// unreachable one, so the survivor of a crash stays Active through auto-down's removal (WarmRoleAuthority.SnapshotPrimaryGone).
|
||||
private readonly HashSet<NodeId> _removedMembers = [];
|
||||
|
||||
private void TrackMember(ClusterEvent.IMemberEvent e)
|
||||
{
|
||||
var node = WarmRoleAuthority.ToNodeId(e.Member);
|
||||
if (e is ClusterEvent.MemberRemoved)
|
||||
{
|
||||
_warmMembers.Remove(e.Member.Address);
|
||||
_warmUnreachable.Remove(e.Member.Address);
|
||||
_removedMembers.Add(node);
|
||||
return;
|
||||
}
|
||||
|
||||
_warmMembers[e.Member.Address] = e.Member;
|
||||
if (e is ClusterEvent.MemberJoined or ClusterEvent.MemberUp) _removedMembers.Remove(node);
|
||||
}
|
||||
|
||||
private void TrackReachability(ClusterEvent.IReachabilityEvent e)
|
||||
{
|
||||
if (e is ClusterEvent.UnreachableMember u) _warmUnreachable.Add(u.Member.Address);
|
||||
else if (e is ClusterEvent.ReachableMember r) _warmUnreachable.Remove(r.Member.Address);
|
||||
}
|
||||
|
||||
private void ResetWarmView(ClusterEvent.CurrentClusterState state)
|
||||
{
|
||||
_warmMembers.Clear();
|
||||
_warmUnreachable.Clear();
|
||||
foreach (var m in state.Members) _warmMembers[m.Address] = m;
|
||||
foreach (var m in state.Unreachable) _warmUnreachable.Add(m.Address);
|
||||
}
|
||||
|
||||
/// <summary>The Warm view's own member status, falling back to the read view before the first event arrives.</summary>
|
||||
private MemberStatus WarmSelfStatus()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _warmMembers.TryGetValue(_cluster.SelfAddress, out var self) ? self.Status : SafeSelfStatus();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "OpcUaPublish: self address unavailable for the Warm view; using the read view");
|
||||
return SafeSelfStatus();
|
||||
}
|
||||
}
|
||||
/// <summary>The role the data plane is in. A Warm node manager boots Standby, so this starts Standby.</summary>
|
||||
private WarmRole _warmRole = WarmRole.Standby;
|
||||
/// <summary>A promotion whose Active ServiceLevel publish failed (spec §4.7): the data plane is Active and the
|
||||
@@ -1756,7 +1823,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
var self = _localNode!.Value;
|
||||
var entry = _lastSnapshot!.Nodes.FirstOrDefault(n => n.NodeId == self);
|
||||
var now = _utcNow();
|
||||
var memberState = SafeSelfStatus();
|
||||
var memberState = WarmSelfStatus();
|
||||
var listening = EndpointListening();
|
||||
|
||||
bool dbReachable;
|
||||
@@ -1791,11 +1858,23 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
var snapshotPrimary = _acceptedSnapshot?.Nodes.FirstOrDefault(n => n.Role == RedundancyRole.Primary)?.NodeId;
|
||||
try
|
||||
{
|
||||
var state = _cluster.State;
|
||||
var unreachable = state.Unreachable.ToHashSet();
|
||||
oldest = WarmRoleAuthority.OldestUpMember(state.Members, "driver");
|
||||
oldestReachable = WarmRoleAuthority.OldestUpMember(state.Members, "driver", unreachable);
|
||||
primaryUnreachable = snapshotPrimary is { } sp && unreachable.Any(m => WarmRoleAuthority.ToNodeId(m) == sp);
|
||||
IReadOnlyCollection<Member> members;
|
||||
HashSet<Member> unreachable;
|
||||
if (_warmMembers.Count > 0)
|
||||
{
|
||||
members = _warmMembers.Values;
|
||||
unreachable = _warmMembers.Values.Where(m => _warmUnreachable.Contains(m.Address)).ToHashSet();
|
||||
}
|
||||
else
|
||||
{
|
||||
var state = _cluster.State;
|
||||
members = state.Members;
|
||||
unreachable = state.Unreachable.ToHashSet();
|
||||
}
|
||||
|
||||
oldest = WarmRoleAuthority.OldestUpMember(members, "driver");
|
||||
oldestReachable = WarmRoleAuthority.OldestUpMember(members, "driver", unreachable);
|
||||
primaryUnreachable = WarmRoleAuthority.SnapshotPrimaryGone(snapshotPrimary, members, unreachable, _removedMembers);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -102,6 +102,36 @@ public static class WarmRoleAuthority
|
||||
return oldest is null ? null : ToNodeId(oldest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the snapshot's Primary is gone from this node's view, for <see cref="WarmRoleInputs.SnapshotPrimaryUnreachable"/>:
|
||||
/// unreachable, marked <c>Down</c>, or removed from the cluster after this node saw it as a member.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The "removed" arm is the end of the same crash path (T13 live gate, 2026-11-13). Auto-down removes the
|
||||
/// unreachable Primary about 15 s after the survivor promoted, so it leaves the unreachable set. But the
|
||||
/// survivor's own singleton does not publish a snapshot naming it until the hand-over retries time out, about
|
||||
/// 15 s later. Without this arm the survivor demoted for that whole window (BadOutOfService, ServiceLevel 50)
|
||||
/// and then promoted again. A member that is merely Leaving or Exiting is still present and reachable, so a
|
||||
/// graceful hand-over is unaffected, and a node this one never saw as a member is not assumed gone.
|
||||
/// </remarks>
|
||||
/// <param name="snapshotPrimary">The Primary named by the accepted snapshot, or null.</param>
|
||||
/// <param name="members">The current members (this node's own cluster state).</param>
|
||||
/// <param name="unreachable">The members this node sees as unreachable.</param>
|
||||
/// <param name="removed">Nodes this node has seen removed from the cluster (and not rejoin since).</param>
|
||||
/// <returns>True when a named Primary is unreachable, Down, or removed.</returns>
|
||||
public static bool SnapshotPrimaryGone(NodeId? snapshotPrimary, IEnumerable<Member> members, IReadOnlySet<Member> unreachable,
|
||||
IReadOnlySet<NodeId> removed)
|
||||
{
|
||||
if (snapshotPrimary is not { } sp) return false;
|
||||
foreach (var m in members)
|
||||
{
|
||||
if (ToNodeId(m) != sp) continue;
|
||||
return m.Status is MemberStatus.Down or MemberStatus.Removed || unreachable.Contains(m);
|
||||
}
|
||||
|
||||
return removed.Contains(sp);
|
||||
}
|
||||
|
||||
/// <summary>The canonical <c>host:port</c> id of a member (the RedundancyStateActor format).</summary>
|
||||
/// <param name="m">The member.</param>
|
||||
/// <returns>The node id.</returns>
|
||||
|
||||
Reference in New Issue
Block a user