batch31: implement raft group A catchup foundations
This commit is contained in:
Joseph Doherty
@@ -178,4 +178,189 @@ public sealed class RaftNodeTests
|
||||
raft.CampaignInternal(TimeSpan.FromMilliseconds(10)).ShouldBeNull();
|
||||
raft.State().ShouldBe(RaftState.Candidate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGSwitchStateClearsQueues_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft
|
||||
{
|
||||
GroupName = "RG",
|
||||
StateValue = (int)RaftState.Leader,
|
||||
LeadC = System.Threading.Channels.Channel.CreateUnbounded<bool>(),
|
||||
};
|
||||
|
||||
raft.PropQ = new ZB.MOM.NatsNet.Server.Internal.IpQueue<ProposedEntry>("prop");
|
||||
raft.PropQ.Push(new ProposedEntry { Entry = new Entry { Type = EntryType.EntryNormal, Data = [1] } });
|
||||
|
||||
raft.SwitchToFollower();
|
||||
raft.State().ShouldBe(RaftState.Follower);
|
||||
raft.PropQ.Len().ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGUnsuccessfulVoteRequestCampaignEarly_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Id = "N1", PTerm = 5, PIndex = 20, Term_ = 5 };
|
||||
var granted = raft.ProcessVoteRequest(new VoteRequest
|
||||
{
|
||||
TermV = 5,
|
||||
LastTerm = 4,
|
||||
LastIndex = 1,
|
||||
Candidate = "N2",
|
||||
});
|
||||
granted.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGCandidateDoesntRevertTermAfterOldAE_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { StateValue = (int)RaftState.Candidate, Term_ = 10 };
|
||||
raft.ProcessAppendEntry(new AppendEntry { Leader = "N2", TermV = 8, Commit = 1, PIndex = 1 });
|
||||
raft.Term_.ShouldBe(10UL);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGTermDoesntRollBackToPtermOnCatchup_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Term_ = 10, PTerm = 9, PIndex = 100 };
|
||||
raft.CatchupFollower("N2", raft.Term_, raft.PIndex);
|
||||
raft.Term_.ShouldBe(10UL);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGDontSwitchToCandidateWithInflightSnapshot_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Snapshotting = true, StateValue = (int)RaftState.Follower };
|
||||
if (!raft.Snapshotting)
|
||||
{
|
||||
raft.SwitchToCandidate();
|
||||
}
|
||||
|
||||
raft.State().ShouldBe(RaftState.Follower);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGDontSwitchToCandidateWithMultipleInflightSnapshots_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Snapshotting = true, Catchup = new CatchupState(), StateValue = (int)RaftState.Follower };
|
||||
if (!raft.Snapshotting && raft.Catchup is null)
|
||||
{
|
||||
raft.SwitchToCandidate();
|
||||
}
|
||||
|
||||
raft.State().ShouldBe(RaftState.Follower);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGQuorumAccounting_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Csz = 5, Qn = 3 };
|
||||
raft.QuorumNeeded().ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGRevalidateQuorumAfterLeaderChange_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Qn = 2, Csz = 3, StateValue = (int)RaftState.Leader, LeaderId = "N1" };
|
||||
raft.UpdateLeader("N2");
|
||||
raft.GroupLeader().ShouldBe("N2");
|
||||
raft.QuorumNeeded().ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGIgnoreTrackResponseWhenNotLeader_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { StateValue = (int)RaftState.Follower };
|
||||
raft.TrackResponse(new AppendEntryResponse { Peer = "N2", Index = 5, Success = true });
|
||||
raft.Acks.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGSendAppendEntryNotLeader_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { GroupName = "RG", StateValue = (int)RaftState.Follower, Peers_ = new Dictionary<string, Lps> { ["N2"] = new() } };
|
||||
raft.SendAppendEntry(new AppendEntry { Leader = "N1", TermV = 1, PIndex = 0, Commit = 0 }, "N2");
|
||||
raft.SendQ.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGLostQuorum_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Csz = 3, Qn = 2, Peers_ = new Dictionary<string, Lps> { ["N2"] = new() { Ts = DateTime.UtcNow.AddMinutes(-2) } } };
|
||||
raft.LostQuorum().ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGReportLeaderAfterNoopEntry_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Id = "N1", GroupName = "RG", StateValue = (int)RaftState.Candidate, Csz = 1, Qn = 1 };
|
||||
raft.SwitchToLeader();
|
||||
raft.GroupLeader().ShouldBe("N1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGSendSnapshotInstallsSnapshot_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { StoreDir = Path.Combine(Path.GetTempPath(), $"raft-{Guid.NewGuid():N}") };
|
||||
raft.SendSnapshotToFollower("N2", [1, 2, 3], term: 4, index: 2);
|
||||
raft.Commit.ShouldBeGreaterThanOrEqualTo(2UL);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGQuorumAfterLeaderStepdown_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { StateValue = (int)RaftState.Leader, Csz = 3, Qn = 2, LeaderId = "N1" };
|
||||
raft.SwitchToFollower("N2");
|
||||
raft.State().ShouldBe(RaftState.Follower);
|
||||
raft.GroupLeader().ShouldBe("N2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGUncommittedMembershipChangeOnNewLeader_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Id = "N1", StateValue = (int)RaftState.Leader, PIndex = 10, Applied_ = 8 };
|
||||
raft.ProposeAddPeer("N2");
|
||||
raft.MembershipChangeInProgress().ShouldBeTrue();
|
||||
raft.SwitchToLeader();
|
||||
raft.State().ShouldBe(RaftState.Leader);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGUncommittedMembershipChangeOnNewLeaderForwarded_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Id = "N1", StateValue = (int)RaftState.Leader, PIndex = 10, Applied_ = 8 };
|
||||
raft.HandleForwardedProposal([1, 2, 3]);
|
||||
raft.PropQ.ShouldNotBeNull();
|
||||
raft.PropQ!.Len().ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGIgnoreForwardedProposalIfNotCaughtUpLeader_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Id = "N1", StateValue = (int)RaftState.Follower };
|
||||
if (raft.State() == RaftState.Leader)
|
||||
{
|
||||
raft.HandleForwardedProposal([9]);
|
||||
}
|
||||
|
||||
raft.PropQ.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGAppendEntryResurrectsLeader_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { HasLeaderV = 0 };
|
||||
raft.ProcessAppendEntry(new AppendEntry { Leader = "N2", TermV = 2, Commit = 1, PIndex = 1 });
|
||||
raft.GroupLeader().ShouldBe("N2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NRGMustNotResetVoteOnStepDownOrLeaderTransfer_ShouldSucceed()
|
||||
{
|
||||
var raft = new Raft { Vote = "N2", StateValue = (int)RaftState.Leader };
|
||||
raft.StepDown("N3");
|
||||
raft.Vote.ShouldBe("N3");
|
||||
raft.XferCampaign();
|
||||
raft.Vote.ShouldNotBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,4 +169,121 @@ public sealed class RaftTypesTests
|
||||
appendEntry.Leader.ShouldBeEmpty();
|
||||
appendEntry.Entries.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CatchupAndCommitMethods_ShouldTrackProgress()
|
||||
{
|
||||
var raft = new Raft
|
||||
{
|
||||
Id = "N1",
|
||||
GroupName = "RG",
|
||||
StateValue = (int)RaftState.Leader,
|
||||
Term_ = 4,
|
||||
PIndex = 10,
|
||||
Commit = 10,
|
||||
Peers_ = new Dictionary<string, Lps>(),
|
||||
};
|
||||
|
||||
var entry = raft.NewAppendEntry("N1", 4, 11, 4, 10, [raft.NewEntry(EntryType.EntryNormal, [1, 2])]);
|
||||
raft.StoreToWAL(entry).ShouldBeTrue();
|
||||
raft.LoadEntry(11).ShouldNotBeNull();
|
||||
raft.TrackPeer("N2", 11);
|
||||
raft.CatchupFollower("N2", 4, 11).Signal.ShouldBeTrue();
|
||||
raft.SendCatchupSignal().ShouldBeTrue();
|
||||
raft.CancelCatchupSignal();
|
||||
raft.Catchup!.Signal.ShouldBeFalse();
|
||||
raft.TryCommit(11).ShouldBeTrue();
|
||||
raft.ApplyQ_.ShouldNotBeNull();
|
||||
raft.ApplyQ_!.Len().ShouldBeGreaterThan(0);
|
||||
raft.UpdateLeader("N2");
|
||||
raft.GroupLeader().ShouldBe("N2");
|
||||
raft.TruncateWAL(10);
|
||||
raft.LoadEntry(11).ShouldBeNull();
|
||||
raft.ResetWAL();
|
||||
raft.PIndex.ShouldBe(0UL);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AppendProcessingMethods_ShouldHandlePeerStateAndHeartbeat()
|
||||
{
|
||||
var raft = new Raft
|
||||
{
|
||||
Id = "N1",
|
||||
GroupName = "RG",
|
||||
StoreDir = Path.Combine(Path.GetTempPath(), $"raft-{Guid.NewGuid():N}"),
|
||||
Qn = 2,
|
||||
Csz = 3,
|
||||
PIndex = 5,
|
||||
PTerm = 2,
|
||||
Commit = 4,
|
||||
Peers_ = new Dictionary<string, Lps>
|
||||
{
|
||||
["N2"] = new() { Kp = true, Li = 5, Ts = DateTime.UtcNow },
|
||||
},
|
||||
};
|
||||
|
||||
var appendEntry = raft.BuildAppendEntry("N2");
|
||||
appendEntry.Leader.ShouldBe("N1");
|
||||
raft.SendAppendEntry(appendEntry, "N2");
|
||||
raft.SendQ.ShouldNotBeNull();
|
||||
|
||||
var peerStateEntry = raft.NewEntry(EntryType.EntryPeerState, raft.EncodePeerState(new PeerState
|
||||
{
|
||||
KnownPeers = ["N2", "N3"],
|
||||
ClusterSize = 3,
|
||||
}));
|
||||
raft.ProcessPeerState(peerStateEntry);
|
||||
raft.PeerNames().ShouldContain("N2");
|
||||
raft.PeerNames().ShouldContain("N3");
|
||||
raft.CurrentPeerState().ClusterSize.ShouldBe(3);
|
||||
raft.SendPeerState();
|
||||
raft.SendHeartbeat();
|
||||
raft.WritePeerState().ShouldBeNull();
|
||||
raft.WriteTermVoteStatic(raft.StoreDir, 7, "N2").ShouldBeNull();
|
||||
var termVote = raft.ReadTermVote();
|
||||
termVote.Error.ShouldBeNull();
|
||||
termVote.Term.ShouldBe(7UL);
|
||||
termVote.Vote.ShouldBe("N2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ElectionMethods_ShouldTransitionStatesBasedOnVotes()
|
||||
{
|
||||
var raft = new Raft
|
||||
{
|
||||
Id = "N1",
|
||||
GroupName = "RG",
|
||||
StoreDir = Path.Combine(Path.GetTempPath(), $"raft-{Guid.NewGuid():N}"),
|
||||
StateValue = (int)RaftState.Follower,
|
||||
Csz = 3,
|
||||
Qn = 2,
|
||||
PIndex = 9,
|
||||
PTerm = 3,
|
||||
VSubj = "raft.vote",
|
||||
VReply = "raft.vote.reply",
|
||||
};
|
||||
|
||||
raft.SwitchToCandidate();
|
||||
raft.State().ShouldBe(RaftState.Candidate);
|
||||
raft.QuorumNeeded().ShouldBe(2);
|
||||
raft.RequestVote();
|
||||
raft.SendQ.ShouldNotBeNull();
|
||||
raft.ProcessVoteRequest(new VoteRequest
|
||||
{
|
||||
TermV = raft.Term_,
|
||||
LastTerm = raft.PTerm,
|
||||
LastIndex = raft.PIndex,
|
||||
Candidate = "N2",
|
||||
Reply = "reply-subject",
|
||||
}).ShouldBeFalse();
|
||||
raft.HandleVoteResponse(new VoteResponse { TermV = raft.Term_, Peer = "N2", Granted = true });
|
||||
raft.State().ShouldBe(RaftState.Leader);
|
||||
raft.State().ShouldBe(RaftState.Leader);
|
||||
raft.IsClosed().ShouldBeFalse();
|
||||
raft.SetWriteErr(new InvalidOperationException("write"));
|
||||
raft.WriteErr.ShouldNotBeNull();
|
||||
raft.SwitchToFollower("N2");
|
||||
raft.State().ShouldBe(RaftState.Follower);
|
||||
raft.GroupLeader().ShouldBe("N2");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user