c569f80694
Two defects on the crash-failover path, found by the T13 live gate on an isolated docker pair: 1. When auto-down removed the unreachable Primary (~15 s after the survivor promoted), the snapshot's Primary left the unreachable set, so (3') failed and the survivor demoted to Standby (BadOutOfService, ServiceLevel 50) for ~15 s, until its own singleton published after the hand-over retries. A Primary that is Down, or that this node saw removed, now counts as gone (WarmRoleAuthority.SnapshotPrimaryGone). A member that is merely Leaving/Exiting, or that this node never saw, is not. 2. The verdict read Cluster.State / SelfMember (Akka's ClusterReadView, a separate subscriber to the same events), which can lag the event being handled. Live, the survivor stayed Standby 4.3 s after its own UnreachableMember event (promotion waited for the next health tick). The Warm verdict now uses a membership view built from this actor's own member/reachability subscription, falling back to the read view only before the first event. Tests: WarmRoleAuthorityTests.A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active, OpcUaPublishActorWarmTests.The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal (synthetic events the read view never sees). Mutations: read-view-only verdict and the removed arm switched off each fail the actor test. Runtime.Tests 1058 passed, 3 skipped. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
161 lines
8.4 KiB
C#
161 lines
8.4 KiB
C#
using System.Collections.Immutable;
|
|
using System.Reflection;
|
|
using Akka.Actor;
|
|
using Akka.Cluster;
|
|
using Akka.Util;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
|
|
|
|
/// <summary>Gitea #717 spec §4.1: the Warm verdict truth table, and the late-bound projection's replay (spec §4.7).</summary>
|
|
public sealed class WarmRoleAuthorityTests
|
|
{
|
|
private static readonly NodeId Self = NodeId.Parse("a:1");
|
|
private static readonly NodeId Peer = NodeId.Parse("b:1");
|
|
|
|
private static WarmRoleInputs In(NodeId? oldest = null, NodeId? oldestReachable = null, bool hasSnapshot = true,
|
|
NodeId? snapshotPrimary = null, bool primaryUnreachable = false, bool promote = true, bool up = true, bool listening = true) =>
|
|
new(Self, up, listening, oldest, oldestReachable ?? oldest, hasSnapshot, snapshotPrimary, primaryUnreachable, promote);
|
|
|
|
[Fact]
|
|
public void Active_iff_membership_and_the_accepted_snapshot_both_name_self()
|
|
{
|
|
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self)).Role.ShouldBe(WarmRole.Active);
|
|
WarmRoleAuthority.Evaluate(In(oldest: Peer, snapshotPrimary: Peer)).Role.ShouldBe(WarmRole.Standby);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(true)]
|
|
[InlineData(false)]
|
|
public void Any_disagreement_is_Standby_and_flagged(bool membershipSaysSelf)
|
|
{
|
|
var verdict = WarmRoleAuthority.Evaluate(In(oldest: membershipSaysSelf ? Self : Peer, snapshotPrimary: membershipSaysSelf ? Peer : Self));
|
|
verdict.Role.ShouldBe(WarmRole.Standby);
|
|
verdict.Disagreement.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Boot_with_no_accepted_snapshot_is_Standby_whatever_the_membership()
|
|
{
|
|
var verdict = WarmRoleAuthority.Evaluate(In(oldest: Self, hasSnapshot: false, snapshotPrimary: null));
|
|
verdict.Role.ShouldBe(WarmRole.Standby);
|
|
verdict.Reason.ShouldBe("boot");
|
|
}
|
|
|
|
[Fact]
|
|
public void Not_up_or_endpoint_failed_is_Standby()
|
|
{
|
|
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, up: false)).Role.ShouldBe(WarmRole.Standby);
|
|
WarmRoleAuthority.Evaluate(In(oldest: Self, snapshotPrimary: Self, listening: false)).Role.ShouldBe(WarmRole.Standby);
|
|
}
|
|
|
|
/// <summary>Owner answer Q2: a crashed Primary stays Up-but-unreachable until auto-down; the survivor promotes.</summary>
|
|
[Fact]
|
|
public void The_survivor_promotes_while_the_Primary_is_unreachable_only_with_the_option()
|
|
{
|
|
var crashed = In(oldest: Peer, oldestReachable: Self, snapshotPrimary: Peer, primaryUnreachable: true);
|
|
WarmRoleAuthority.Evaluate(crashed).Role.ShouldBe(WarmRole.Active);
|
|
WarmRoleAuthority.Evaluate(crashed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
|
|
WarmRoleAuthority.Evaluate(crashed with { SnapshotPrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby,
|
|
"a reachable Primary is never overridden");
|
|
WarmRoleAuthority.Evaluate(crashed with { HasAcceptedSnapshot = false, SnapshotPrimary = null }).Role
|
|
.ShouldBe(WarmRole.Standby, "a node that never accepted a snapshot stays Standby");
|
|
}
|
|
|
|
/// <summary>
|
|
/// T13 live gate (2026-11-13): auto-down removes the crashed Primary ~15 s after the survivor promoted, while the
|
|
/// survivor's own singleton has not yet published a snapshot naming it. A Primary that is Down or no longer a
|
|
/// member is as gone as an unreachable one, so the survivor stays Active through the removal. A Primary that is
|
|
/// present, reachable and merely Leaving/Exiting (a graceful hand-over) is not gone.
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active()
|
|
{
|
|
var self = MemberAt(Self);
|
|
var peer = MemberAt(Peer);
|
|
var none = new HashSet<Member>();
|
|
var seenRemoved = new HashSet<NodeId> { Peer };
|
|
var noneRemoved = new HashSet<NodeId>();
|
|
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], new HashSet<Member> { peer }, noneRemoved).ShouldBeTrue("unreachable");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Down)], none, noneRemoved).ShouldBeTrue("downed");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, seenRemoved).ShouldBeTrue("removed by auto-down");
|
|
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, noneRemoved).ShouldBeFalse("never seen as a member: not assumed gone");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], none, seenRemoved).ShouldBeFalse("back as a reachable Up member");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, peer], none, noneRemoved).ShouldBeFalse("a reachable Up Primary is never overridden");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Leaving)], none, noneRemoved).ShouldBeFalse("graceful leave");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self, MemberAt(Peer, MemberStatus.Exiting)], none, noneRemoved).ShouldBeFalse("graceful exit");
|
|
WarmRoleAuthority.SnapshotPrimaryGone(null, [self], none, seenRemoved).ShouldBeFalse("no snapshot Primary");
|
|
|
|
// The live sequence on the survivor: promoted while the Primary was unreachable, then the Primary is removed.
|
|
var removed = In(oldest: Self, oldestReachable: Self, snapshotPrimary: Peer,
|
|
primaryUnreachable: WarmRoleAuthority.SnapshotPrimaryGone(Peer, [self], none, seenRemoved));
|
|
WarmRoleAuthority.Evaluate(removed).Role.ShouldBe(WarmRole.Active);
|
|
WarmRoleAuthority.Evaluate(removed with { PromoteWhilePrimaryUnreachable = false }).Role.ShouldBe(WarmRole.Standby);
|
|
}
|
|
|
|
private static int _memberUid;
|
|
|
|
// Akka's Member.Create factories are internal; same reflection idiom as DriverHostActorHistorizationRoleViewTests.
|
|
private static readonly MethodInfo MemberCreate = typeof(Member).GetMethod(
|
|
"Create",
|
|
BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public,
|
|
binder: null,
|
|
types:
|
|
[
|
|
typeof(UniqueAddress), typeof(int), typeof(MemberStatus),
|
|
typeof(ImmutableHashSet<string>), typeof(AppVersion),
|
|
],
|
|
modifiers: null)
|
|
?? throw new InvalidOperationException("Akka.Cluster.Member.Create(5-arg) not found; Akka API changed");
|
|
|
|
private static Member MemberAt(NodeId node, MemberStatus status = MemberStatus.Up)
|
|
{
|
|
var text = node.ToString();
|
|
var colon = text.LastIndexOf(':');
|
|
return (Member)MemberCreate.Invoke(null,
|
|
[
|
|
new UniqueAddress(
|
|
new Address("akka.tcp", "otopcua", text[..colon], int.Parse(text[(colon + 1)..])),
|
|
Interlocked.Increment(ref _memberUid)),
|
|
1,
|
|
status,
|
|
ImmutableHashSet.Create("driver"),
|
|
AppVersion.Zero,
|
|
])!;
|
|
}
|
|
|
|
[Fact]
|
|
public void A_late_bind_replays_the_retained_state_but_not_the_actions()
|
|
{
|
|
var deferred = new DeferredStandbyProjection();
|
|
deferred.SetDataProjection(false);
|
|
deferred.SetStandbyDelivery(false);
|
|
deferred.SetAlarmSuppression(false);
|
|
deferred.QueueCurrentToLiveItems(false).ShouldBe(0);
|
|
deferred.RefreshAllEventItems().ShouldBe(0);
|
|
|
|
var inner = new RecordingProjection();
|
|
deferred.SetInner(inner);
|
|
inner.Log.ShouldBe(new[] { "proj:False", "delivery:False", "alarms:False" });
|
|
deferred.QueueCurrentToLiveItems(true);
|
|
inner.Log.Last().ShouldBe("queue:True");
|
|
}
|
|
}
|
|
|
|
/// <summary>Records every <see cref="IStandbyProjection"/> step into a shared, ordered log.</summary>
|
|
internal sealed class RecordingProjection(System.Collections.Concurrent.ConcurrentQueue<string>? log = null) : IStandbyProjection
|
|
{
|
|
public System.Collections.Concurrent.ConcurrentQueue<string> Log { get; } = log ?? new();
|
|
public void SetDataProjection(bool projected) => Log.Enqueue($"proj:{projected}");
|
|
public void SetStandbyDelivery(bool standby) => Log.Enqueue($"delivery:{standby}");
|
|
public void SetAlarmSuppression(bool suppressed) => Log.Enqueue($"alarms:{suppressed}");
|
|
public int QueueCurrentToLiveItems(bool projected) { Log.Enqueue($"queue:{projected}"); return 1; }
|
|
public int RefreshAllEventItems() { Log.Enqueue("refresh"); return 1; }
|
|
}
|