natsdotnet/tests/NATS.Server.Tests/Raft/RaftLogReplicationTests.cs

using NATS.Server.Raft;

namespace NATS.Server.Tests.Raft;

/// <summary>
/// Log replication tests covering leader propose, follower append, commit index advance,
/// log compaction, out-of-order rejection, duplicate detection, heartbeat keepalive,
/// persistence round-trips, and replicator backtrack semantics.
/// Go: TestNRGSimple, TestNRGSnapshotAndRestart, TestNRGHeartbeatOnLeaderChange,
///     TestNRGNoResetOnAppendEntryResponse, TestNRGTermNoDecreaseAfterWALReset,
///     TestNRGWALEntryWithoutQuorumMustTruncate in server/raft_test.go.
/// </summary>
public class RaftLogReplicationTests
{
    // -- Helpers (self-contained) --

    private static (RaftNode leader, RaftNode[] followers) CreateLeaderWithFollowers(int followerCount)
    {
        var total = followerCount + 1;
        var nodes = Enumerable.Range(1, total)
            .Select(i => new RaftNode($"n{i}"))
            .ToArray();
        foreach (var node in nodes)
            node.ConfigureCluster(nodes);

        var candidate = nodes[0];
        candidate.StartElection(total);
        foreach (var voter in nodes.Skip(1))
            candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), total);

        return (candidate, nodes.Skip(1).ToArray());
    }

    private static (RaftNode leader, RaftNode[] followers, InMemoryRaftTransport transport) CreateTransportCluster(int size)
    {
        var transport = new InMemoryRaftTransport();
        var nodes = Enumerable.Range(1, size)
            .Select(i => new RaftNode($"n{i}", transport))
            .ToArray();
        foreach (var node in nodes)
        {
            transport.Register(node);
            node.ConfigureCluster(nodes);
        }

        var candidate = nodes[0];
        candidate.StartElection(size);
        foreach (var voter in nodes.Skip(1))
            candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), size);

        return (candidate, nodes.Skip(1).ToArray(), transport);
    }

    // Go: TestNRGSimple server/raft_test.go:35 — proposeDelta
    [Fact]
    public async Task Leader_propose_appends_to_log()
    {
        var (leader, _) = CreateLeaderWithFollowers(2);

        var index = await leader.ProposeAsync("set-x-42", default);
        index.ShouldBe(1);
        leader.Log.Entries.Count.ShouldBe(1);
        leader.Log.Entries[0].Command.ShouldBe("set-x-42");
        leader.Log.Entries[0].Term.ShouldBe(leader.Term);
    }

    // Go: TestNRGSimple server/raft_test.go:35
    [Fact]
    public async Task Leader_propose_multiple_entries_sequential_indices()
    {
        var (leader, _) = CreateLeaderWithFollowers(2);

        var i1 = await leader.ProposeAsync("cmd-1", default);
        var i2 = await leader.ProposeAsync("cmd-2", default);
        var i3 = await leader.ProposeAsync("cmd-3", default);

        i1.ShouldBe(1);
        i2.ShouldBe(2);
        i3.ShouldBe(3);

        leader.Log.Entries.Count.ShouldBe(3);
        leader.Log.Entries[0].Index.ShouldBe(1);
        leader.Log.Entries[1].Index.ShouldBe(2);
        leader.Log.Entries[2].Index.ShouldBe(3);
    }

    // Go: TestNRGSimple server/raft_test.go:35 — only leader can propose
    [Fact]
    public async Task Follower_cannot_propose()
    {
        var (_, followers) = CreateLeaderWithFollowers(2);

        var follower = followers[0];
        follower.IsLeader.ShouldBeFalse();

        await Should.ThrowAsync<InvalidOperationException>(
            async () => await follower.ProposeAsync("should-fail", default));
    }

    // Go: TestNRGSimple server/raft_test.go:35 — state convergence
    [Fact]
    public async Task Follower_receives_replicated_entry()
    {
        var (leader, followers) = CreateLeaderWithFollowers(2);

        await leader.ProposeAsync("replicated-cmd", default);

        // In-process replication: followers should have the entry
        foreach (var follower in followers)
        {
            follower.Log.Entries.Count.ShouldBe(1);
            follower.Log.Entries[0].Command.ShouldBe("replicated-cmd");
        }
    }

    // Go: TestNRGSimple server/raft_test.go:35 — commit index advance
    [Fact]
    public async Task Commit_index_advances_after_quorum()
    {
        var (leader, followers) = CreateLeaderWithFollowers(2);

        await leader.ProposeAsync("committed-entry", default);

        // Leader should have advanced applied index
        leader.AppliedIndex.ShouldBeGreaterThan(0);
    }

    // Go: TestNRGSimple server/raft_test.go:35 — all nodes converge
    [Fact]
    public async Task All_nodes_converge_applied_index()
    {
        var (leader, followers) = CreateLeaderWithFollowers(2);

        var idx = await leader.ProposeAsync("converge-1", default);
        await leader.ProposeAsync("converge-2", default);
        var finalIdx = await leader.ProposeAsync("converge-3", default);

        // All nodes should converge
        leader.AppliedIndex.ShouldBeGreaterThanOrEqualTo(finalIdx);
        foreach (var follower in followers)
            follower.AppliedIndex.ShouldBeGreaterThanOrEqualTo(finalIdx);
    }

    // Go: appendEntry dedup in server/raft.go
    [Fact]
    public void Duplicate_replicated_entry_is_deduplicated()
    {
        var log = new RaftLog();
        var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "dedup-test");

        log.AppendReplicated(entry);
        log.AppendReplicated(entry); // duplicate
        log.AppendReplicated(entry); // duplicate

        log.Entries.Count.ShouldBe(1);
    }

    // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — stale append rejected
    [Fact]
    public async Task Stale_term_append_rejected()
    {
        var node = new RaftNode("n1");
        node.StartElection(clusterSize: 1);
        node.Term.ShouldBe(1);

        var staleEntry = new RaftLogEntry(Index: 1, Term: 0, Command: "stale");
        await Should.ThrowAsync<InvalidOperationException>(
            async () => await node.TryAppendFromLeaderAsync(staleEntry, default));
    }

    // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — current term accepted
    [Fact]
    public async Task Current_term_append_accepted()
    {
        var node = new RaftNode("n1");
        node.TermState.CurrentTerm = 3;

        var entry = new RaftLogEntry(Index: 1, Term: 3, Command: "valid");
        await node.TryAppendFromLeaderAsync(entry, default);

        node.Log.Entries.Count.ShouldBe(1);
        node.Log.Entries[0].Command.ShouldBe("valid");
    }

    // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — higher term accepted
    [Fact]
    public async Task Higher_term_append_accepted()
    {
        var node = new RaftNode("n1");
        node.TermState.CurrentTerm = 1;

        var entry = new RaftLogEntry(Index: 1, Term: 5, Command: "future");
        await node.TryAppendFromLeaderAsync(entry, default);

        node.Log.Entries.Count.ShouldBe(1);
    }

    // Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 — heartbeat keepalive
    [Fact]
    public void Heartbeat_updates_follower_term()
    {
        var follower = new RaftNode("f1");
        follower.TermState.CurrentTerm = 1;

        follower.ReceiveHeartbeat(term: 3);
        follower.Term.ShouldBe(3);
        follower.Role.ShouldBe(RaftRole.Follower);
    }

    // Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708
    [Fact]
    public async Task Heartbeat_via_transport_updates_follower()
    {
        var transport = new InMemoryRaftTransport();
        var leader = new RaftNode("L", transport);
        var follower = new RaftNode("F", transport);
        transport.Register(leader);
        transport.Register(follower);

        await transport.AppendHeartbeatAsync("L", ["F"], term: 5, default);

        follower.Term.ShouldBe(5);
        follower.Role.ShouldBe(RaftRole.Follower);
    }

    // Go: TestNRGNoResetOnAppendEntryResponse server/raft_test.go:912 — rejection transport
    [Fact]
    public async Task Propose_without_quorum_does_not_advance_applied_index()
    {
        var transport = new RejectAllTransport();
        var leader = new RaftNode("n1", transport);
        var follower1 = new RaftNode("n2", transport);
        var follower2 = new RaftNode("n3", transport);
        var nodes = new[] { leader, follower1, follower2 };
        foreach (var n in nodes)
            n.ConfigureCluster(nodes);

        leader.StartElection(nodes.Length);
        leader.ReceiveVote(new VoteResponse { Granted = true }, nodes.Length);
        leader.IsLeader.ShouldBeTrue();

        await leader.ProposeAsync("no-quorum-cmd", default);

        // No quorum means applied index should not advance
        leader.AppliedIndex.ShouldBe(0);
    }

    // Go: server/raft.go — log append and entries in term
    [Fact]
    public void Log_entries_preserve_term()
    {
        var log = new RaftLog();
        var e1 = log.Append(term: 1, command: "term1-a");
        var e2 = log.Append(term: 1, command: "term1-b");
        var e3 = log.Append(term: 2, command: "term2-a");

        e1.Term.ShouldBe(1);
        e2.Term.ShouldBe(1);
        e3.Term.ShouldBe(2);
    }

    // Go: TestNRGSnapshotAndRestart server/raft_test.go:49 — log persistence
    [Fact]
    public async Task Log_persist_and_reload()
    {
        var dir = Path.Combine(Path.GetTempPath(), $"nats-raft-repl-test-{Guid.NewGuid():N}");
        var logPath = Path.Combine(dir, "log.json");

        try
        {
            var log = new RaftLog();
            log.Append(term: 1, command: "persist-a");
            log.Append(term: 2, command: "persist-b");

            await log.PersistAsync(logPath, default);

            var reloaded = await RaftLog.LoadAsync(logPath, default);
            reloaded.Entries.Count.ShouldBe(2);
            reloaded.Entries[0].Command.ShouldBe("persist-a");
            reloaded.Entries[1].Command.ShouldBe("persist-b");
            reloaded.Entries[0].Term.ShouldBe(1);
            reloaded.Entries[1].Term.ShouldBe(2);
        }
        finally
        {
            if (Directory.Exists(dir))
                Directory.Delete(dir, recursive: true);
        }
    }

    // Go: TestNRGSnapshotAndRestart server/raft_test.go:49 — node persistence
    [Fact]
    public async Task Node_persist_and_reload_state()
    {
        var dir = Path.Combine(Path.GetTempPath(), $"nats-raft-node-test-{Guid.NewGuid():N}");

        try
        {
            var node = new RaftNode("n1", persistDirectory: dir);
            node.StartElection(clusterSize: 1);
            node.IsLeader.ShouldBeTrue();

            node.Log.Append(term: 1, command: "persist-cmd");
            node.AppliedIndex = 1;

            await node.PersistAsync(default);

            // Create new node and reload
            var reloaded = new RaftNode("n1", persistDirectory: dir);
            await reloaded.LoadPersistedStateAsync(default);

            reloaded.Term.ShouldBe(1);
            reloaded.AppliedIndex.ShouldBe(1);
            reloaded.Log.Entries.Count.ShouldBe(1);
            reloaded.Log.Entries[0].Command.ShouldBe("persist-cmd");
        }
        finally
        {
            if (Directory.Exists(dir))
                Directory.Delete(dir, recursive: true);
        }
    }

    // Go: BacktrackNextIndex in server/raft.go
    [Fact]
    public void Backtrack_next_index_decrements_correctly()
    {
        RaftReplicator.BacktrackNextIndex(5).ShouldBe(4);
        RaftReplicator.BacktrackNextIndex(3).ShouldBe(2);
        RaftReplicator.BacktrackNextIndex(2).ShouldBe(1);
    }

    // Go: BacktrackNextIndex in server/raft.go — floor at 1
    [Fact]
    public void Backtrack_next_index_floor_at_one()
    {
        RaftReplicator.BacktrackNextIndex(1).ShouldBe(1);
        RaftReplicator.BacktrackNextIndex(0).ShouldBe(1);
    }

    // Go: RaftReplicator in server/raft.go
    [Fact]
    public void Replicator_returns_count_of_acknowledged_followers()
    {
        var replicator = new RaftReplicator();
        var follower1 = new RaftNode("f1");
        var follower2 = new RaftNode("f2");
        var followers = new[] { follower1, follower2 };

        var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "replicate-me");
        var acks = replicator.Replicate(entry, followers);

        acks.ShouldBe(2);
        follower1.Log.Entries.Count.ShouldBe(1);
        follower2.Log.Entries.Count.ShouldBe(1);
    }

    // Go: RaftReplicator async via transport
    [Fact]
    public async Task Replicator_async_via_transport()
    {
        var (leader, followers, transport) = CreateTransportCluster(3);

        var entry = leader.Log.Append(leader.Term, "transport-replicate");
        var replicator = new RaftReplicator();
        var results = await replicator.ReplicateAsync(leader.Id, entry, followers, transport, default);

        results.Count.ShouldBe(2);
        results.All(r => r.Success).ShouldBeTrue();

        foreach (var follower in followers)
            follower.Log.Entries.Count.ShouldBe(1);
    }

    // Go: RaftReplicator with null transport uses direct replication
    [Fact]
    public async Task Replicator_async_without_transport_uses_direct()
    {
        var follower1 = new RaftNode("f1");
        var follower2 = new RaftNode("f2");
        var followers = new[] { follower1, follower2 };

        var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "direct");
        var replicator = new RaftReplicator();
        var results = await replicator.ReplicateAsync("leader", entry, followers, null, default);

        results.Count.ShouldBe(2);
        results.All(r => r.Success).ShouldBeTrue();
    }

    // Go: TestNRGSimple server/raft_test.go:35 — 1000 entries
    [Fact]
    public async Task Many_entries_replicate_correctly()
    {
        var (leader, followers) = CreateLeaderWithFollowers(2);

        for (int i = 0; i < 100; i++)
            await leader.ProposeAsync($"batch-{i}", default);

        leader.Log.Entries.Count.ShouldBe(100);
        leader.AppliedIndex.ShouldBe(100);

        foreach (var follower in followers)
            follower.Log.Entries.Count.ShouldBe(100);
    }

    // Go: Log append after snapshot
    [Fact]
    public void Log_append_after_snapshot_continues_from_snapshot_index()
    {
        var log = new RaftLog();
        log.Append(term: 1, command: "a");
        log.Append(term: 1, command: "b");
        log.Append(term: 1, command: "c");

        log.ReplaceWithSnapshot(new RaftSnapshot
        {
            LastIncludedIndex = 3,
            LastIncludedTerm = 1,
        });

        log.Entries.Count.ShouldBe(0);

        var e = log.Append(term: 2, command: "post-snap");
        e.Index.ShouldBe(4);
    }

    // Go: Empty log loads from nonexistent path
    [Fact]
    public async Task Load_from_nonexistent_path_returns_empty_log()
    {
        var path = Path.Combine(Path.GetTempPath(), $"nats-noexist-{Guid.NewGuid():N}", "log.json");
        var log = await RaftLog.LoadAsync(path, default);
        log.Entries.Count.ShouldBe(0);
    }

    // Go: TestNRGWALEntryWithoutQuorumMustTruncate server/raft_test.go:1063
    [Fact]
    public async Task Propose_with_transport_replicates_to_followers()
    {
        var (leader, followers, transport) = CreateTransportCluster(3);

        var idx = await leader.ProposeAsync("transport-cmd", default);
        idx.ShouldBe(1);

        leader.Log.Entries.Count.ShouldBe(1);
        foreach (var follower in followers)
            follower.Log.Entries.Count.ShouldBe(1);
    }

    // Go: ReceiveReplicatedEntry dedup
    [Fact]
    public void ReceiveReplicatedEntry_deduplicates()
    {
        var node = new RaftNode("n1");
        var entry = new RaftLogEntry(Index: 1, Term: 1, Command: "once");

        node.ReceiveReplicatedEntry(entry);
        node.ReceiveReplicatedEntry(entry);

        node.Log.Entries.Count.ShouldBe(1);
    }

    // Go: TestNRGHeartbeatOnLeaderChange server/raft_test.go:708 — repeated proposals
    [Fact]
    public async Task Multiple_proposals_maintain_sequential_applied_index()
    {
        var (leader, followers) = CreateLeaderWithFollowers(2);

        for (int i = 1; i <= 10; i++)
        {
            var idx = await leader.ProposeAsync($"seq-{i}", default);
            idx.ShouldBe(i);
        }

        leader.AppliedIndex.ShouldBe(10);
        leader.Log.Entries.Count.ShouldBe(10);
    }

    // Go: TestNRGTermNoDecreaseAfterWALReset server/raft_test.go:1156 — entries carry correct term
    [Fact]
    public async Task Proposed_entries_carry_leader_term()
    {
        var (leader, _) = CreateLeaderWithFollowers(2);
        leader.Term.ShouldBe(1);

        await leader.ProposeAsync("term-check", default);

        leader.Log.Entries[0].Term.ShouldBe(1);
    }

    // Go: TestNRGNoResetOnAppendEntryResponse server/raft_test.go:912 — partial transport
    [Fact]
    public async Task Partial_replication_still_commits_with_quorum()
    {
        var transport = new PartialTransport();
        var nodes = Enumerable.Range(1, 3)
            .Select(i => new RaftNode($"n{i}", transport))
            .ToArray();
        foreach (var n in nodes)
        {
            transport.Register(n);
            n.ConfigureCluster(nodes);
        }

        var candidate = nodes[0];
        candidate.StartElection(3);
        candidate.ReceiveVote(new VoteResponse { Granted = true }, 3);
        candidate.IsLeader.ShouldBeTrue();

        // With partial transport, 1 follower succeeds (quorum = 2 including leader)
        var idx = await candidate.ProposeAsync("partial-cmd", default);
        idx.ShouldBe(1);
        candidate.AppliedIndex.ShouldBeGreaterThan(0);
    }

    // Go: TestNRGSimple server/raft_test.go:35 — follower log matches leader
    [Fact]
    public async Task Follower_log_matches_leader_log_content()
    {
        var (leader, followers) = CreateLeaderWithFollowers(2);

        await leader.ProposeAsync("alpha", default);
        await leader.ProposeAsync("beta", default);
        await leader.ProposeAsync("gamma", default);

        foreach (var follower in followers)
        {
            follower.Log.Entries.Count.ShouldBe(leader.Log.Entries.Count);
            for (int i = 0; i < leader.Log.Entries.Count; i++)
            {
                follower.Log.Entries[i].Index.ShouldBe(leader.Log.Entries[i].Index);
                follower.Log.Entries[i].Term.ShouldBe(leader.Log.Entries[i].Term);
                follower.Log.Entries[i].Command.ShouldBe(leader.Log.Entries[i].Command);
            }
        }
    }

    // -- Helper transport that rejects all appends --

    private sealed class RejectAllTransport : IRaftTransport
    {
        public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(
            string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
            => Task.FromResult<IReadOnlyList<AppendResult>>(
                followerIds.Select(id => new AppendResult { FollowerId = id, Success = false }).ToArray());

        public Task<VoteResponse> RequestVoteAsync(
            string candidateId, string voterId, VoteRequest request, CancellationToken ct)
            => Task.FromResult(new VoteResponse { Granted = false });

        public Task InstallSnapshotAsync(
            string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
            => Task.CompletedTask;

        public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
            => Task.CompletedTask;
    }

    // -- Helper transport that succeeds for first follower, fails for rest --

    private sealed class PartialTransport : IRaftTransport
    {
        private readonly Dictionary<string, RaftNode> _nodes = new(StringComparer.Ordinal);

        public void Register(RaftNode node) => _nodes[node.Id] = node;

        public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(
            string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
        {
            var results = new List<AppendResult>(followerIds.Count);
            var first = true;
            foreach (var followerId in followerIds)
            {
                if (first && _nodes.TryGetValue(followerId, out var node))
                {
                    node.ReceiveReplicatedEntry(entry);
                    results.Add(new AppendResult { FollowerId = followerId, Success = true });
                    first = false;
                }
                else
                {
                    results.Add(new AppendResult { FollowerId = followerId, Success = false });
                }
            }
            return Task.FromResult<IReadOnlyList<AppendResult>>(results);
        }

        public Task<VoteResponse> RequestVoteAsync(
            string candidateId, string voterId, VoteRequest request, CancellationToken ct)
            => Task.FromResult(new VoteResponse { Granted = false });

        public Task InstallSnapshotAsync(
            string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
            => Task.CompletedTask;

        public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
            => Task.CompletedTask;
    }
}