Files
natsdotnet/tests/NATS.Server.Raft.Tests/Raft/RaftLogReplicationTests.cs
Joseph Doherty edf9ed770e refactor: extract NATS.Server.Raft.Tests project
Move 43 Raft consensus test files (8 root-level + 35 in Raft/ subfolder)
from NATS.Server.Tests into a dedicated NATS.Server.Raft.Tests project.
Update namespaces, add InternalsVisibleTo, and fix timing/exception
handling issues in moved test files.
2026-03-12 15:36:02 -04:00

607 lines
21 KiB
C#

using NATS.Server.Raft;
namespace NATS.Server.Raft.Tests.Raft;
/// <summary>
/// Log replication tests covering leader propose, follower append, commit index advance,
/// log compaction, out-of-order rejection, duplicate detection, heartbeat keepalive,
/// persistence round-trips, and replicator backtrack semantics.
/// Go: TestNRGSimple, TestNRGSnapshotAndRestart, TestNRGHeartbeatOnLeaderChange,
/// TestNRGNoResetOnAppendEntryResponse, TestNRGTermNoDecreaseAfterWALReset,
/// TestNRGWALEntryWithoutQuorumMustTruncate in server/raft_test.go.
/// </summary>
public class RaftLogReplicationTests
{
// -- Helpers (self-contained) --
private static (RaftNode leader, RaftNode[] followers) CreateLeaderWithFollowers(int followerCount)
{
var total = followerCount + 1;
var nodes = Enumerable.Range(1, total)
.Select(i => new RaftNode($"n{i}"))
.ToArray();
foreach (var node in nodes)
node.ConfigureCluster(nodes);
var candidate = nodes[0];
candidate.StartElection(total);
foreach (var voter in nodes.Skip(1))
candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), total);
return (candidate, nodes.Skip(1).ToArray());
}
private static (RaftNode leader, RaftNode[] followers, InMemoryRaftTransport transport) CreateTransportCluster(int size)
{
var transport = new InMemoryRaftTransport();
var nodes = Enumerable.Range(1, size)
.Select(i => new RaftNode($"n{i}", transport))
.ToArray();
foreach (var node in nodes)
{
transport.Register(node);
node.ConfigureCluster(nodes);
}
var candidate = nodes[0];
candidate.StartElection(size);
foreach (var voter in nodes.Skip(1))
candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), size);
return (candidate, nodes.Skip(1).ToArray(), transport);
}
// Go: TestNRGSimple server/raft_test.go:35 — proposeDelta
[Fact]
public async Task Leader_propose_appends_to_log()
{
var (leader, _) = CreateLeaderWithFollowers(2);
var index = await leader.ProposeAsync("set-x-42", default);
index.ShouldBe(1);
leader.Log.Entries.Count.ShouldBe(1);
leader.Log.Entries[0].Command.ShouldBe("set-x-42");
leader.Log.Entries[0].Term.ShouldBe(leader.Term);
}
// Go: TestNRGSimple server/raft_test.go:35
[Fact]
public async Task Leader_propose_multiple_entries_sequential_indices()
{
var (leader, _) = CreateLeaderWithFollowers(2);
var i1 = await leader.ProposeAsync("cmd-1", default);
var i2 = await leader.ProposeAsync("cmd-2", default);
var i3 = await leader.ProposeAsync("cmd-3", default);
i1.ShouldBe(1);
i2.ShouldBe(2);
i3.ShouldBe(3);
leader.Log.Entries.Count.ShouldBe(3);
leader.Log.Entries[0].Index.ShouldBe(1);
leader.Log.Entries[1].Index.ShouldBe(2);
leader.Log.Entries[2].Index.ShouldBe(3);
}
// Go: TestNRGSimple server/raft_test.go:35 — only leader can propose
[Fact]
public async Task Follower_cannot_propose()
{
var (_, followers) = CreateLeaderWithFollowers(2);
var follower = followers[0];
follower.IsLeader.ShouldBeFalse();
await Should.ThrowAsync<InvalidOperationException>(
async () => await follower.ProposeAsync("should-fail", default));
}
// Go: TestNRGSimple server/raft_test.go:35 — state convergence
[Fact]
public async Task Follower_receives_replicated_entry()
{
var (leader, followers) = CreateLeaderWithFollowers(2);
await leader.ProposeAsync("replicated-cmd", default);
// In-process replication: followers should have the entry
foreach (var follower in followers)
{
follower.Log.Entries.Count.ShouldBe(1);
follower.Log.Entries[0].Command.ShouldBe("replicated-cmd");
}
}
// Go: TestNRGSimple server/raft_test.go:35 — commit index advance
[Fact]
public async Task Commit_index_advances_after_quorum()
{
var (leader, followers) = CreateLeaderWithFollowers(2);
await leader.ProposeAsync("committed-entry", default);
// Leader should have advanced applied index
leader.AppliedIndex.ShouldBeGreaterThan(0);
}
// Go: TestNRGSimple server/raft_test.go:35 — all nodes converge
[Fact]
public async Task All_nodes_converge_applied_index()
{
var (leader, followers) = CreateLeaderWithFollowers(2);
var idx = await leader.ProposeAsync("converge-1", default);
await leader.ProposeAsync("converge-2", default);
var finalIdx = await leader.ProposeAsync("converge-3", default);
// All nodes should converge
leader.AppliedIndex.ShouldBeGreaterThanOrEqualTo(finalIdx);
foreach (var follower in followers)
follower.AppliedIndex.ShouldBeGreaterThanOrEqualTo(finalIdx);
}
// Go: appendEntry dedup in server/raft.go
[Fact]
public void Duplicate_replicated_entry_is_deduplicated()
{
var log = new RaftLog();
var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "dedup-test");
log.AppendReplicated(entry);
log.AppendReplicated(entry); // duplicate
log.AppendReplicated(entry); // duplicate
log.Entries.Count.ShouldBe(1);
}
// Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — stale append rejected
[Fact]
public async Task Stale_term_append_rejected()
{
var node = new RaftNode("n1");
node.StartElection(clusterSize: 1);
node.Term.ShouldBe(1);
var staleEntry = new RaftLogEntry(Index: 1, Term: 0, Command: "stale");
await Should.ThrowAsync<InvalidOperationException>(
async () => await node.TryAppendFromLeaderAsync(staleEntry, default));
}
// Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — current term accepted
[Fact]
public async Task Current_term_append_accepted()
{
var node = new RaftNode("n1");
node.TermState.CurrentTerm = 3;
var entry = new RaftLogEntry(Index: 1, Term: 3, Command: "valid");
await node.TryAppendFromLeaderAsync(entry, default);
node.Log.Entries.Count.ShouldBe(1);
node.Log.Entries[0].Command.ShouldBe("valid");
}
// Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — higher term accepted
[Fact]
public async Task Higher_term_append_accepted()
{
var node = new RaftNode("n1");
node.TermState.CurrentTerm = 1;
var entry = new RaftLogEntry(Index: 1, Term: 5, Command: "future");
await node.TryAppendFromLeaderAsync(entry, default);
node.Log.Entries.Count.ShouldBe(1);
}
// Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 — heartbeat keepalive
[Fact]
public void Heartbeat_updates_follower_term()
{
var follower = new RaftNode("f1");
follower.TermState.CurrentTerm = 1;
follower.ReceiveHeartbeat(term: 3);
follower.Term.ShouldBe(3);
follower.Role.ShouldBe(RaftRole.Follower);
}
// Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708
[Fact]
public async Task Heartbeat_via_transport_updates_follower()
{
var transport = new InMemoryRaftTransport();
var leader = new RaftNode("L", transport);
var follower = new RaftNode("F", transport);
transport.Register(leader);
transport.Register(follower);
await transport.AppendHeartbeatAsync("L", ["F"], term: 5, default);
follower.Term.ShouldBe(5);
follower.Role.ShouldBe(RaftRole.Follower);
}
// Go: TestNRGNoResetOnAppendEntryResponse server/raft_test.go:912 — rejection transport
[Fact]
public async Task Propose_without_quorum_does_not_advance_applied_index()
{
var transport = new RejectAllTransport();
var leader = new RaftNode("n1", transport);
var follower1 = new RaftNode("n2", transport);
var follower2 = new RaftNode("n3", transport);
var nodes = new[] { leader, follower1, follower2 };
foreach (var n in nodes)
n.ConfigureCluster(nodes);
leader.StartElection(nodes.Length);
leader.ReceiveVote(new VoteResponse { Granted = true }, nodes.Length);
leader.IsLeader.ShouldBeTrue();
await leader.ProposeAsync("no-quorum-cmd", default);
// No quorum means applied index should not advance
leader.AppliedIndex.ShouldBe(0);
}
// Go: server/raft.go — log append and entries in term
[Fact]
public void Log_entries_preserve_term()
{
var log = new RaftLog();
var e1 = log.Append(term: 1, command: "term1-a");
var e2 = log.Append(term: 1, command: "term1-b");
var e3 = log.Append(term: 2, command: "term2-a");
e1.Term.ShouldBe(1);
e2.Term.ShouldBe(1);
e3.Term.ShouldBe(2);
}
// Go: TestNRGSnapshotAndRestart server/raft_test.go:49 — log persistence
[Fact]
public async Task Log_persist_and_reload()
{
var dir = Path.Combine(Path.GetTempPath(), $"nats-raft-repl-test-{Guid.NewGuid():N}");
var logPath = Path.Combine(dir, "log.json");
try
{
var log = new RaftLog();
log.Append(term: 1, command: "persist-a");
log.Append(term: 2, command: "persist-b");
await log.PersistAsync(logPath, default);
var reloaded = await RaftLog.LoadAsync(logPath, default);
reloaded.Entries.Count.ShouldBe(2);
reloaded.Entries[0].Command.ShouldBe("persist-a");
reloaded.Entries[1].Command.ShouldBe("persist-b");
reloaded.Entries[0].Term.ShouldBe(1);
reloaded.Entries[1].Term.ShouldBe(2);
}
finally
{
if (Directory.Exists(dir))
Directory.Delete(dir, recursive: true);
}
}
// Go: TestNRGSnapshotAndRestart server/raft_test.go:49 — node persistence
[Fact]
public async Task Node_persist_and_reload_state()
{
var dir = Path.Combine(Path.GetTempPath(), $"nats-raft-node-test-{Guid.NewGuid():N}");
try
{
var node = new RaftNode("n1", persistDirectory: dir);
node.StartElection(clusterSize: 1);
node.IsLeader.ShouldBeTrue();
node.Log.Append(term: 1, command: "persist-cmd");
node.AppliedIndex = 1;
await node.PersistAsync(default);
// Create new node and reload
var reloaded = new RaftNode("n1", persistDirectory: dir);
await reloaded.LoadPersistedStateAsync(default);
reloaded.Term.ShouldBe(1);
reloaded.AppliedIndex.ShouldBe(1);
reloaded.Log.Entries.Count.ShouldBe(1);
reloaded.Log.Entries[0].Command.ShouldBe("persist-cmd");
}
finally
{
if (Directory.Exists(dir))
Directory.Delete(dir, recursive: true);
}
}
// Go: BacktrackNextIndex in server/raft.go
[Fact]
public void Backtrack_next_index_decrements_correctly()
{
RaftReplicator.BacktrackNextIndex(5).ShouldBe(4);
RaftReplicator.BacktrackNextIndex(3).ShouldBe(2);
RaftReplicator.BacktrackNextIndex(2).ShouldBe(1);
}
// Go: BacktrackNextIndex in server/raft.go — floor at 1
[Fact]
public void Backtrack_next_index_floor_at_one()
{
RaftReplicator.BacktrackNextIndex(1).ShouldBe(1);
RaftReplicator.BacktrackNextIndex(0).ShouldBe(1);
}
// Go: RaftReplicator in server/raft.go
[Fact]
public void Replicator_returns_count_of_acknowledged_followers()
{
var replicator = new RaftReplicator();
var follower1 = new RaftNode("f1");
var follower2 = new RaftNode("f2");
var followers = new[] { follower1, follower2 };
var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "replicate-me");
var acks = replicator.Replicate(entry, followers);
acks.ShouldBe(2);
follower1.Log.Entries.Count.ShouldBe(1);
follower2.Log.Entries.Count.ShouldBe(1);
}
// Go: RaftReplicator async via transport
[Fact]
public async Task Replicator_async_via_transport()
{
var (leader, followers, transport) = CreateTransportCluster(3);
var entry = leader.Log.Append(leader.Term, "transport-replicate");
var replicator = new RaftReplicator();
var results = await replicator.ReplicateAsync(leader.Id, entry, followers, transport, default);
results.Count.ShouldBe(2);
results.All(r => r.Success).ShouldBeTrue();
foreach (var follower in followers)
follower.Log.Entries.Count.ShouldBe(1);
}
// Go: RaftReplicator with null transport uses direct replication
[Fact]
public async Task Replicator_async_without_transport_uses_direct()
{
var follower1 = new RaftNode("f1");
var follower2 = new RaftNode("f2");
var followers = new[] { follower1, follower2 };
var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "direct");
var replicator = new RaftReplicator();
var results = await replicator.ReplicateAsync("leader", entry, followers, null, default);
results.Count.ShouldBe(2);
results.All(r => r.Success).ShouldBeTrue();
}
// Go: TestNRGSimple server/raft_test.go:35 — 1000 entries
[Fact]
public async Task Many_entries_replicate_correctly()
{
var (leader, followers) = CreateLeaderWithFollowers(2);
for (int i = 0; i < 100; i++)
await leader.ProposeAsync($"batch-{i}", default);
leader.Log.Entries.Count.ShouldBe(100);
leader.AppliedIndex.ShouldBe(100);
foreach (var follower in followers)
follower.Log.Entries.Count.ShouldBe(100);
}
// Go: Log append after snapshot
[Fact]
public void Log_append_after_snapshot_continues_from_snapshot_index()
{
var log = new RaftLog();
log.Append(term: 1, command: "a");
log.Append(term: 1, command: "b");
log.Append(term: 1, command: "c");
log.ReplaceWithSnapshot(new RaftSnapshot
{
LastIncludedIndex = 3,
LastIncludedTerm = 1,
});
log.Entries.Count.ShouldBe(0);
var e = log.Append(term: 2, command: "post-snap");
e.Index.ShouldBe(4);
}
// Go: Empty log loads from nonexistent path
[Fact]
public async Task Load_from_nonexistent_path_returns_empty_log()
{
var path = Path.Combine(Path.GetTempPath(), $"nats-noexist-{Guid.NewGuid():N}", "log.json");
var log = await RaftLog.LoadAsync(path, default);
log.Entries.Count.ShouldBe(0);
}
// Go: TestNRGWALEntryWithoutQuorumMustTruncate server/raft_test.go:1063
[Fact]
public async Task Propose_with_transport_replicates_to_followers()
{
var (leader, followers, transport) = CreateTransportCluster(3);
var idx = await leader.ProposeAsync("transport-cmd", default);
idx.ShouldBe(1);
leader.Log.Entries.Count.ShouldBe(1);
foreach (var follower in followers)
follower.Log.Entries.Count.ShouldBe(1);
}
// Go: ReceiveReplicatedEntry dedup
[Fact]
public void ReceiveReplicatedEntry_deduplicates()
{
var node = new RaftNode("n1");
var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "once");
node.ReceiveReplicatedEntry(entry);
node.ReceiveReplicatedEntry(entry);
node.Log.Entries.Count.ShouldBe(1);
}
// Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 — repeated proposals
[Fact]
public async Task Multiple_proposals_maintain_sequential_applied_index()
{
var (leader, followers) = CreateLeaderWithFollowers(2);
for (int i = 1; i <= 10; i++)
{
var idx = await leader.ProposeAsync($"seq-{i}", default);
idx.ShouldBe(i);
}
leader.AppliedIndex.ShouldBe(10);
leader.Log.Entries.Count.ShouldBe(10);
}
// Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — entries carry correct term
[Fact]
public async Task Proposed_entries_carry_leader_term()
{
var (leader, _) = CreateLeaderWithFollowers(2);
leader.Term.ShouldBe(1);
await leader.ProposeAsync("term-check", default);
leader.Log.Entries[0].Term.ShouldBe(1);
}
// Go: TestNRGNoResetOnAppendEntryResponse server/raft_test.go:912 — partial transport
[Fact]
public async Task Partial_replication_still_commits_with_quorum()
{
var transport = new PartialTransport();
var nodes = Enumerable.Range(1, 3)
.Select(i => new RaftNode($"n{i}", transport))
.ToArray();
foreach (var n in nodes)
{
transport.Register(n);
n.ConfigureCluster(nodes);
}
var candidate = nodes[0];
candidate.StartElection(3);
candidate.ReceiveVote(new VoteResponse { Granted = true }, 3);
candidate.IsLeader.ShouldBeTrue();
// With partial transport, 1 follower succeeds (quorum = 2 including leader)
var idx = await candidate.ProposeAsync("partial-cmd", default);
idx.ShouldBe(1);
candidate.AppliedIndex.ShouldBeGreaterThan(0);
}
// Go: TestNRGSimple server/raft_test.go:35 — follower log matches leader
[Fact]
public async Task Follower_log_matches_leader_log_content()
{
var (leader, followers) = CreateLeaderWithFollowers(2);
await leader.ProposeAsync("alpha", default);
await leader.ProposeAsync("beta", default);
await leader.ProposeAsync("gamma", default);
foreach (var follower in followers)
{
follower.Log.Entries.Count.ShouldBe(leader.Log.Entries.Count);
for (int i = 0; i < leader.Log.Entries.Count; i++)
{
follower.Log.Entries[i].Index.ShouldBe(leader.Log.Entries[i].Index);
follower.Log.Entries[i].Term.ShouldBe(leader.Log.Entries[i].Term);
follower.Log.Entries[i].Command.ShouldBe(leader.Log.Entries[i].Command);
}
}
}
// -- Helper transport that rejects all appends --
private sealed class RejectAllTransport : IRaftTransport
{
public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(
string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
=> Task.FromResult<IReadOnlyList<AppendResult>>(
followerIds.Select(id => new AppendResult { FollowerId = id, Success = false }).ToArray());
public Task<VoteResponse> RequestVoteAsync(
string candidateId, string voterId, VoteRequest request, CancellationToken ct)
=> Task.FromResult(new VoteResponse { Granted = false });
public Task InstallSnapshotAsync(
string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
=> Task.CompletedTask;
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
=> Task.CompletedTask;
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}
// -- Helper transport that succeeds for first follower, fails for rest --
private sealed class PartialTransport : IRaftTransport
{
private readonly Dictionary<string, RaftNode> _nodes = new(StringComparer.Ordinal);
public void Register(RaftNode node) => _nodes[node.Id] = node;
public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(
string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
{
var results = new List<AppendResult>(followerIds.Count);
var first = true;
foreach (var followerId in followerIds)
{
if (first && _nodes.TryGetValue(followerId, out var node))
{
node.ReceiveReplicatedEntry(entry);
results.Add(new AppendResult { FollowerId = followerId, Success = true });
first = false;
}
else
{
results.Add(new AppendResult { FollowerId = followerId, Success = false });
}
}
return Task.FromResult<IReadOnlyList<AppendResult>>(results);
}
public Task<VoteResponse> RequestVoteAsync(
string candidateId, string voterId, VoteRequest request, CancellationToken ct)
=> Task.FromResult(new VoteResponse { Granted = false });
public Task InstallSnapshotAsync(
string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
=> Task.CompletedTask;
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
=> Task.CompletedTask;
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}
}