using NATS.Server.Raft; namespace NATS.Server.Tests.Raft; /// /// Log replication tests covering leader propose, follower append, commit index advance, /// log compaction, out-of-order rejection, duplicate detection, heartbeat keepalive, /// persistence round-trips, and replicator backtrack semantics. /// Go: TestNRGSimple, TestNRGSnapshotAndRestart, TestNRGHeartbeatOnLeaderChange, /// TestNRGNoResetOnAppendEntryResponse, TestNRGTermNoDecreaseAfterWALReset, /// TestNRGWALEntryWithoutQuorumMustTruncate in server/raft_test.go. /// public class RaftLogReplicationTests { // -- Helpers (self-contained) -- private static (RaftNode leader, RaftNode[] followers) CreateLeaderWithFollowers(int followerCount) { var total = followerCount + 1; var nodes = Enumerable.Range(1, total) .Select(i => new RaftNode($"n{i}")) .ToArray(); foreach (var node in nodes) node.ConfigureCluster(nodes); var candidate = nodes[0]; candidate.StartElection(total); foreach (var voter in nodes.Skip(1)) candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), total); return (candidate, nodes.Skip(1).ToArray()); } private static (RaftNode leader, RaftNode[] followers, InMemoryRaftTransport transport) CreateTransportCluster(int size) { var transport = new InMemoryRaftTransport(); var nodes = Enumerable.Range(1, size) .Select(i => new RaftNode($"n{i}", transport)) .ToArray(); foreach (var node in nodes) { transport.Register(node); node.ConfigureCluster(nodes); } var candidate = nodes[0]; candidate.StartElection(size); foreach (var voter in nodes.Skip(1)) candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), size); return (candidate, nodes.Skip(1).ToArray(), transport); } // Go: TestNRGSimple server/raft_test.go:35 — proposeDelta [Fact] public async Task Leader_propose_appends_to_log() { var (leader, _) = CreateLeaderWithFollowers(2); var index = await leader.ProposeAsync("set-x-42", default); index.ShouldBe(1); leader.Log.Entries.Count.ShouldBe(1); leader.Log.Entries[0].Command.ShouldBe("set-x-42"); leader.Log.Entries[0].Term.ShouldBe(leader.Term); } // Go: TestNRGSimple server/raft_test.go:35 [Fact] public async Task Leader_propose_multiple_entries_sequential_indices() { var (leader, _) = CreateLeaderWithFollowers(2); var i1 = await leader.ProposeAsync("cmd-1", default); var i2 = await leader.ProposeAsync("cmd-2", default); var i3 = await leader.ProposeAsync("cmd-3", default); i1.ShouldBe(1); i2.ShouldBe(2); i3.ShouldBe(3); leader.Log.Entries.Count.ShouldBe(3); leader.Log.Entries[0].Index.ShouldBe(1); leader.Log.Entries[1].Index.ShouldBe(2); leader.Log.Entries[2].Index.ShouldBe(3); } // Go: TestNRGSimple server/raft_test.go:35 — only leader can propose [Fact] public async Task Follower_cannot_propose() { var (_, followers) = CreateLeaderWithFollowers(2); var follower = followers[0]; follower.IsLeader.ShouldBeFalse(); await Should.ThrowAsync( async () => await follower.ProposeAsync("should-fail", default)); } // Go: TestNRGSimple server/raft_test.go:35 — state convergence [Fact] public async Task Follower_receives_replicated_entry() { var (leader, followers) = CreateLeaderWithFollowers(2); await leader.ProposeAsync("replicated-cmd", default); // In-process replication: followers should have the entry foreach (var follower in followers) { follower.Log.Entries.Count.ShouldBe(1); follower.Log.Entries[0].Command.ShouldBe("replicated-cmd"); } } // Go: TestNRGSimple server/raft_test.go:35 — commit index advance [Fact] public async Task Commit_index_advances_after_quorum() { var (leader, followers) = CreateLeaderWithFollowers(2); await leader.ProposeAsync("committed-entry", default); // Leader should have advanced applied index leader.AppliedIndex.ShouldBeGreaterThan(0); } // Go: TestNRGSimple server/raft_test.go:35 — all nodes converge [Fact] public async Task All_nodes_converge_applied_index() { var (leader, followers) = CreateLeaderWithFollowers(2); var idx = await leader.ProposeAsync("converge-1", default); await leader.ProposeAsync("converge-2", default); var finalIdx = await leader.ProposeAsync("converge-3", default); // All nodes should converge leader.AppliedIndex.ShouldBeGreaterThanOrEqualTo(finalIdx); foreach (var follower in followers) follower.AppliedIndex.ShouldBeGreaterThanOrEqualTo(finalIdx); } // Go: appendEntry dedup in server/raft.go [Fact] public void Duplicate_replicated_entry_is_deduplicated() { var log = new RaftLog(); var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "dedup-test"); log.AppendReplicated(entry); log.AppendReplicated(entry); // duplicate log.AppendReplicated(entry); // duplicate log.Entries.Count.ShouldBe(1); } // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — stale append rejected [Fact] public async Task Stale_term_append_rejected() { var node = new RaftNode("n1"); node.StartElection(clusterSize: 1); node.Term.ShouldBe(1); var staleEntry = new RaftLogEntry(Index: 1, Term: 0, Command: "stale"); await Should.ThrowAsync( async () => await node.TryAppendFromLeaderAsync(staleEntry, default)); } // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — current term accepted [Fact] public async Task Current_term_append_accepted() { var node = new RaftNode("n1"); node.TermState.CurrentTerm = 3; var entry = new RaftLogEntry(Index: 1, Term: 3, Command: "valid"); await node.TryAppendFromLeaderAsync(entry, default); node.Log.Entries.Count.ShouldBe(1); node.Log.Entries[0].Command.ShouldBe("valid"); } // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — higher term accepted [Fact] public async Task Higher_term_append_accepted() { var node = new RaftNode("n1"); node.TermState.CurrentTerm = 1; var entry = new RaftLogEntry(Index: 1, Term: 5, Command: "future"); await node.TryAppendFromLeaderAsync(entry, default); node.Log.Entries.Count.ShouldBe(1); } // Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 — heartbeat keepalive [Fact] public void Heartbeat_updates_follower_term() { var follower = new RaftNode("f1"); follower.TermState.CurrentTerm = 1; follower.ReceiveHeartbeat(term: 3); follower.Term.ShouldBe(3); follower.Role.ShouldBe(RaftRole.Follower); } // Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 [Fact] public async Task Heartbeat_via_transport_updates_follower() { var transport = new InMemoryRaftTransport(); var leader = new RaftNode("L", transport); var follower = new RaftNode("F", transport); transport.Register(leader); transport.Register(follower); await transport.AppendHeartbeatAsync("L", ["F"], term: 5, default); follower.Term.ShouldBe(5); follower.Role.ShouldBe(RaftRole.Follower); } // Go: TestNRGNoResetOnAppendEntryResponse server/raft_test.go:912 — rejection transport [Fact] public async Task Propose_without_quorum_does_not_advance_applied_index() { var transport = new RejectAllTransport(); var leader = new RaftNode("n1", transport); var follower1 = new RaftNode("n2", transport); var follower2 = new RaftNode("n3", transport); var nodes = new[] { leader, follower1, follower2 }; foreach (var n in nodes) n.ConfigureCluster(nodes); leader.StartElection(nodes.Length); leader.ReceiveVote(new VoteResponse { Granted = true }, nodes.Length); leader.IsLeader.ShouldBeTrue(); await leader.ProposeAsync("no-quorum-cmd", default); // No quorum means applied index should not advance leader.AppliedIndex.ShouldBe(0); } // Go: server/raft.go — log append and entries in term [Fact] public void Log_entries_preserve_term() { var log = new RaftLog(); var e1 = log.Append(term: 1, command: "term1-a"); var e2 = log.Append(term: 1, command: "term1-b"); var e3 = log.Append(term: 2, command: "term2-a"); e1.Term.ShouldBe(1); e2.Term.ShouldBe(1); e3.Term.ShouldBe(2); } // Go: TestNRGSnapshotAndRestart server/raft_test.go:49 — log persistence [Fact] public async Task Log_persist_and_reload() { var dir = Path.Combine(Path.GetTempPath(), $"nats-raft-repl-test-{Guid.NewGuid():N}"); var logPath = Path.Combine(dir, "log.json"); try { var log = new RaftLog(); log.Append(term: 1, command: "persist-a"); log.Append(term: 2, command: "persist-b"); await log.PersistAsync(logPath, default); var reloaded = await RaftLog.LoadAsync(logPath, default); reloaded.Entries.Count.ShouldBe(2); reloaded.Entries[0].Command.ShouldBe("persist-a"); reloaded.Entries[1].Command.ShouldBe("persist-b"); reloaded.Entries[0].Term.ShouldBe(1); reloaded.Entries[1].Term.ShouldBe(2); } finally { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } } // Go: TestNRGSnapshotAndRestart server/raft_test.go:49 — node persistence [Fact] public async Task Node_persist_and_reload_state() { var dir = Path.Combine(Path.GetTempPath(), $"nats-raft-node-test-{Guid.NewGuid():N}"); try { var node = new RaftNode("n1", persistDirectory: dir); node.StartElection(clusterSize: 1); node.IsLeader.ShouldBeTrue(); node.Log.Append(term: 1, command: "persist-cmd"); node.AppliedIndex = 1; await node.PersistAsync(default); // Create new node and reload var reloaded = new RaftNode("n1", persistDirectory: dir); await reloaded.LoadPersistedStateAsync(default); reloaded.Term.ShouldBe(1); reloaded.AppliedIndex.ShouldBe(1); reloaded.Log.Entries.Count.ShouldBe(1); reloaded.Log.Entries[0].Command.ShouldBe("persist-cmd"); } finally { if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); } } // Go: BacktrackNextIndex in server/raft.go [Fact] public void Backtrack_next_index_decrements_correctly() { RaftReplicator.BacktrackNextIndex(5).ShouldBe(4); RaftReplicator.BacktrackNextIndex(3).ShouldBe(2); RaftReplicator.BacktrackNextIndex(2).ShouldBe(1); } // Go: BacktrackNextIndex in server/raft.go — floor at 1 [Fact] public void Backtrack_next_index_floor_at_one() { RaftReplicator.BacktrackNextIndex(1).ShouldBe(1); RaftReplicator.BacktrackNextIndex(0).ShouldBe(1); } // Go: RaftReplicator in server/raft.go [Fact] public void Replicator_returns_count_of_acknowledged_followers() { var replicator = new RaftReplicator(); var follower1 = new RaftNode("f1"); var follower2 = new RaftNode("f2"); var followers = new[] { follower1, follower2 }; var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "replicate-me"); var acks = replicator.Replicate(entry, followers); acks.ShouldBe(2); follower1.Log.Entries.Count.ShouldBe(1); follower2.Log.Entries.Count.ShouldBe(1); } // Go: RaftReplicator async via transport [Fact] public async Task Replicator_async_via_transport() { var (leader, followers, transport) = CreateTransportCluster(3); var entry = leader.Log.Append(leader.Term, "transport-replicate"); var replicator = new RaftReplicator(); var results = await replicator.ReplicateAsync(leader.Id, entry, followers, transport, default); results.Count.ShouldBe(2); results.All(r => r.Success).ShouldBeTrue(); foreach (var follower in followers) follower.Log.Entries.Count.ShouldBe(1); } // Go: RaftReplicator with null transport uses direct replication [Fact] public async Task Replicator_async_without_transport_uses_direct() { var follower1 = new RaftNode("f1"); var follower2 = new RaftNode("f2"); var followers = new[] { follower1, follower2 }; var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "direct"); var replicator = new RaftReplicator(); var results = await replicator.ReplicateAsync("leader", entry, followers, null, default); results.Count.ShouldBe(2); results.All(r => r.Success).ShouldBeTrue(); } // Go: TestNRGSimple server/raft_test.go:35 — 1000 entries [Fact] public async Task Many_entries_replicate_correctly() { var (leader, followers) = CreateLeaderWithFollowers(2); for (int i = 0; i < 100; i++) await leader.ProposeAsync($"batch-{i}", default); leader.Log.Entries.Count.ShouldBe(100); leader.AppliedIndex.ShouldBe(100); foreach (var follower in followers) follower.Log.Entries.Count.ShouldBe(100); } // Go: Log append after snapshot [Fact] public void Log_append_after_snapshot_continues_from_snapshot_index() { var log = new RaftLog(); log.Append(term: 1, command: "a"); log.Append(term: 1, command: "b"); log.Append(term: 1, command: "c"); log.ReplaceWithSnapshot(new RaftSnapshot { LastIncludedIndex = 3, LastIncludedTerm = 1, }); log.Entries.Count.ShouldBe(0); var e = log.Append(term: 2, command: "post-snap"); e.Index.ShouldBe(4); } // Go: Empty log loads from nonexistent path [Fact] public async Task Load_from_nonexistent_path_returns_empty_log() { var path = Path.Combine(Path.GetTempPath(), $"nats-noexist-{Guid.NewGuid():N}", "log.json"); var log = await RaftLog.LoadAsync(path, default); log.Entries.Count.ShouldBe(0); } // Go: TestNRGWALEntryWithoutQuorumMustTruncate server/raft_test.go:1063 [Fact] public async Task Propose_with_transport_replicates_to_followers() { var (leader, followers, transport) = CreateTransportCluster(3); var idx = await leader.ProposeAsync("transport-cmd", default); idx.ShouldBe(1); leader.Log.Entries.Count.ShouldBe(1); foreach (var follower in followers) follower.Log.Entries.Count.ShouldBe(1); } // Go: ReceiveReplicatedEntry dedup [Fact] public void ReceiveReplicatedEntry_deduplicates() { var node = new RaftNode("n1"); var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "once"); node.ReceiveReplicatedEntry(entry); node.ReceiveReplicatedEntry(entry); node.Log.Entries.Count.ShouldBe(1); } // Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 — repeated proposals [Fact] public async Task Multiple_proposals_maintain_sequential_applied_index() { var (leader, followers) = CreateLeaderWithFollowers(2); for (int i = 1; i <= 10; i++) { var idx = await leader.ProposeAsync($"seq-{i}", default); idx.ShouldBe(i); } leader.AppliedIndex.ShouldBe(10); leader.Log.Entries.Count.ShouldBe(10); } // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — entries carry correct term [Fact] public async Task Proposed_entries_carry_leader_term() { var (leader, _) = CreateLeaderWithFollowers(2); leader.Term.ShouldBe(1); await leader.ProposeAsync("term-check", default); leader.Log.Entries[0].Term.ShouldBe(1); } // Go: TestNRGNoResetOnAppendEntryResponse server/raft_test.go:912 — partial transport [Fact] public async Task Partial_replication_still_commits_with_quorum() { var transport = new PartialTransport(); var nodes = Enumerable.Range(1, 3) .Select(i => new RaftNode($"n{i}", transport)) .ToArray(); foreach (var n in nodes) { transport.Register(n); n.ConfigureCluster(nodes); } var candidate = nodes[0]; candidate.StartElection(3); candidate.ReceiveVote(new VoteResponse { Granted = true }, 3); candidate.IsLeader.ShouldBeTrue(); // With partial transport, 1 follower succeeds (quorum = 2 including leader) var idx = await candidate.ProposeAsync("partial-cmd", default); idx.ShouldBe(1); candidate.AppliedIndex.ShouldBeGreaterThan(0); } // Go: TestNRGSimple server/raft_test.go:35 — follower log matches leader [Fact] public async Task Follower_log_matches_leader_log_content() { var (leader, followers) = CreateLeaderWithFollowers(2); await leader.ProposeAsync("alpha", default); await leader.ProposeAsync("beta", default); await leader.ProposeAsync("gamma", default); foreach (var follower in followers) { follower.Log.Entries.Count.ShouldBe(leader.Log.Entries.Count); for (int i = 0; i < leader.Log.Entries.Count; i++) { follower.Log.Entries[i].Index.ShouldBe(leader.Log.Entries[i].Index); follower.Log.Entries[i].Term.ShouldBe(leader.Log.Entries[i].Term); follower.Log.Entries[i].Command.ShouldBe(leader.Log.Entries[i].Command); } } } // -- Helper transport that rejects all appends -- private sealed class RejectAllTransport : IRaftTransport { public Task> AppendEntriesAsync( string leaderId, IReadOnlyList followerIds, RaftLogEntry entry, CancellationToken ct) => Task.FromResult>( followerIds.Select(id => new AppendResult { FollowerId = id, Success = false }).ToArray()); public Task RequestVoteAsync( string candidateId, string voterId, VoteRequest request, CancellationToken ct) => Task.FromResult(new VoteResponse { Granted = false }); public Task InstallSnapshotAsync( string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct) => Task.CompletedTask; } // -- Helper transport that succeeds for first follower, fails for rest -- private sealed class PartialTransport : IRaftTransport { private readonly Dictionary _nodes = new(StringComparer.Ordinal); public void Register(RaftNode node) => _nodes[node.Id] = node; public Task> AppendEntriesAsync( string leaderId, IReadOnlyList followerIds, RaftLogEntry entry, CancellationToken ct) { var results = new List(followerIds.Count); var first = true; foreach (var followerId in followerIds) { if (first && _nodes.TryGetValue(followerId, out var node)) { node.ReceiveReplicatedEntry(entry); results.Add(new AppendResult { FollowerId = followerId, Success = true }); first = false; } else { results.Add(new AppendResult { FollowerId = followerId, Success = false }); } } return Task.FromResult>(results); } public Task RequestVoteAsync( string candidateId, string voterId, VoteRequest request, CancellationToken ct) => Task.FromResult(new VoteResponse { Granted = false }); public Task InstallSnapshotAsync( string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct) => Task.CompletedTask; } }