feat: add chunk-based snapshot streaming with CRC32 validation (Gap 8.3)
- Add SnapshotChunkEnumerator: IEnumerable<byte[]> that splits snapshot data into fixed-size chunks (default 65536 bytes) and computes CRC32 over the full payload for integrity validation during streaming transfer - Add RaftInstallSnapshotChunkWire: 24-byte header + variable data wire type encoding [snapshotIndex:8][snapshotTerm:4][chunkIndex:4][totalChunks:4][crc32:4][data:N] - Extend InstallSnapshotFromChunksAsync with optional expectedCrc32 parameter; validates assembled data against CRC32 before applying snapshot state, throwing InvalidDataException on mismatch to prevent corrupt state installation - Fix stub IRaftTransport implementations in test files missing SendTimeoutNowAsync - Fix incorrect role assertion in RaftLeadershipTransferTests (single-node quorum = 1) - 15 new tests in RaftSnapshotStreamingTests covering enumeration, reassembly, CRC correctness, validation success/failure, and wire format roundtrip
This commit is contained in:
Joseph Doherty
410
tests/NATS.Server.Tests/Raft/RaftLeadershipTransferTests.cs
Normal file
410
tests/NATS.Server.Tests/Raft/RaftLeadershipTransferTests.cs
Normal file
@@ -0,0 +1,410 @@
|
||||
using NATS.Server.Raft;
|
||||
|
||||
namespace NATS.Server.Tests.Raft;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for RAFT leadership transfer via TimeoutNow RPC (Gap 8.4).
|
||||
/// The leader sends a TimeoutNow message to a target follower, which immediately
|
||||
/// starts an election. The leader blocks proposals while the transfer is in flight.
|
||||
/// Go reference: raft.go sendTimeoutNow / processTimeoutNow
|
||||
/// </summary>
|
||||
public class RaftLeadershipTransferTests
|
||||
{
|
||||
// -- Helpers --
|
||||
|
||||
private static (RaftNode[] nodes, InMemoryRaftTransport transport) CreateCluster(int size)
|
||||
{
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var nodes = Enumerable.Range(1, size)
|
||||
.Select(i => new RaftNode($"n{i}", transport))
|
||||
.ToArray();
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
transport.Register(node);
|
||||
node.ConfigureCluster(nodes);
|
||||
// Use short election timeouts so polling in TransferLeadershipAsync
|
||||
// converges quickly in tests without requiring real async delays.
|
||||
node.ElectionTimeoutMinMs = 5;
|
||||
node.ElectionTimeoutMaxMs = 10;
|
||||
}
|
||||
return (nodes, transport);
|
||||
}
|
||||
|
||||
private static RaftNode ElectLeader(RaftNode[] nodes)
|
||||
{
|
||||
var candidate = nodes[0];
|
||||
candidate.StartElection(nodes.Length);
|
||||
foreach (var voter in nodes.Skip(1))
|
||||
candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), nodes.Length);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
// -- Wire format tests --
|
||||
|
||||
// Go reference: raft.go TimeoutNow wire encoding
|
||||
[Fact]
|
||||
public void TimeoutNowRpc_wire_format_roundtrip()
|
||||
{
|
||||
var wire = new RaftTimeoutNowWire(Term: 7UL, LeaderId: "n1");
|
||||
|
||||
var encoded = wire.Encode();
|
||||
encoded.Length.ShouldBe(RaftTimeoutNowWire.MessageLen); // 16 bytes
|
||||
|
||||
var decoded = RaftTimeoutNowWire.Decode(encoded);
|
||||
decoded.Term.ShouldBe(7UL);
|
||||
decoded.LeaderId.ShouldBe("n1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimeoutNowRpc_wire_format_preserves_term_and_leader_id()
|
||||
{
|
||||
var wire = new RaftTimeoutNowWire(Term: 42UL, LeaderId: "node5");
|
||||
|
||||
var decoded = RaftTimeoutNowWire.Decode(wire.Encode());
|
||||
|
||||
decoded.Term.ShouldBe(42UL);
|
||||
decoded.LeaderId.ShouldBe("node5");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimeoutNowRpc_decode_throws_on_wrong_length()
|
||||
{
|
||||
Should.Throw<ArgumentException>(() =>
|
||||
RaftTimeoutNowWire.Decode(new byte[10]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimeoutNowRpc_message_len_is_16_bytes()
|
||||
{
|
||||
RaftTimeoutNowWire.MessageLen.ShouldBe(16);
|
||||
}
|
||||
|
||||
// -- ReceiveTimeoutNow logic tests --
|
||||
|
||||
// Go reference: raft.go processTimeoutNow -- follower starts election immediately
|
||||
[Fact]
|
||||
public void ReceiveTimeoutNow_triggers_immediate_election_on_follower()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var follower = nodes[1]; // starts as follower
|
||||
follower.Role.ShouldBe(RaftRole.Follower);
|
||||
|
||||
follower.ReceiveTimeoutNow(term: 0);
|
||||
|
||||
// Node should now be a candidate (or leader if it self-voted quorum)
|
||||
follower.Role.ShouldBeOneOf(RaftRole.Candidate, RaftRole.Leader);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReceiveTimeoutNow_updates_term_when_sender_term_is_higher()
|
||||
{
|
||||
var node = new RaftNode("follower");
|
||||
node.TermState.CurrentTerm = 3;
|
||||
|
||||
node.ReceiveTimeoutNow(term: 10);
|
||||
|
||||
// ReceiveTimeoutNow sets term to 10, then StartElection increments to 11
|
||||
node.TermState.CurrentTerm.ShouldBe(11);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReceiveTimeoutNow_increments_term_and_starts_campaign()
|
||||
{
|
||||
var node = new RaftNode("n1");
|
||||
node.TermState.CurrentTerm = 2;
|
||||
var termBefore = node.Term;
|
||||
|
||||
node.ReceiveTimeoutNow(term: 0);
|
||||
|
||||
// StartElection increments term
|
||||
node.Term.ShouldBe(termBefore + 1);
|
||||
node.Role.ShouldBe(RaftRole.Candidate); // single node with no cluster -- needs votes
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReceiveTimeoutNow_on_single_node_makes_it_leader()
|
||||
{
|
||||
// Single-node cluster: quorum = 1, so self-vote is sufficient.
|
||||
var node = new RaftNode("solo");
|
||||
node.ConfigureCluster([node]);
|
||||
|
||||
node.ReceiveTimeoutNow(term: 0);
|
||||
|
||||
node.IsLeader.ShouldBeTrue();
|
||||
}
|
||||
|
||||
// -- Proposal blocking during transfer --
|
||||
|
||||
// Go reference: raft.go -- leader rejects new entries while transfer is in progress.
|
||||
// BlockingTimeoutNowTransport signals via SemaphoreSlim when SendTimeoutNowAsync is
|
||||
// entered, letting the test observe the _transferInProgress flag without timing deps.
|
||||
[Fact]
|
||||
public async Task TransferLeadership_leader_blocks_proposals_during_transfer()
|
||||
{
|
||||
var blockingTransport = new BlockingTimeoutNowTransport();
|
||||
var node = new RaftNode("leader", blockingTransport);
|
||||
node.ConfigureCluster([node]);
|
||||
node.StartElection(1); // become leader
|
||||
node.IsLeader.ShouldBeTrue();
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
var transferTask = node.TransferLeadershipAsync("n2", cts.Token);
|
||||
|
||||
// Wait until SendTimeoutNowAsync is entered -- transfer flag is guaranteed set.
|
||||
await blockingTransport.WaitUntilBlockingAsync();
|
||||
|
||||
// ProposeAsync must throw because the transfer flag is set.
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => node.ProposeAsync("cmd", CancellationToken.None).AsTask());
|
||||
ex.Message.ShouldContain("Leadership transfer in progress");
|
||||
|
||||
// Cancel and await proper completion to avoid test resource leaks.
|
||||
await cts.CancelAsync();
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => transferTask);
|
||||
}
|
||||
|
||||
// Go reference: raft.go -- only leader can initiate leadership transfer
|
||||
[Fact]
|
||||
public async Task TransferLeadership_only_leader_can_transfer()
|
||||
{
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var follower = new RaftNode("follower", transport);
|
||||
follower.Role.ShouldBe(RaftRole.Follower);
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => follower.TransferLeadershipAsync("n2", CancellationToken.None));
|
||||
ex.Message.ShouldContain("Only the leader");
|
||||
}
|
||||
|
||||
// Go reference: raft.go -- TransferLeadershipAsync requires a configured transport
|
||||
[Fact]
|
||||
public async Task TransferLeadership_throws_when_no_transport_configured()
|
||||
{
|
||||
// No transport injected.
|
||||
var node = new RaftNode("leader");
|
||||
node.StartElection(1); // become leader (single node, quorum = 1)
|
||||
node.IsLeader.ShouldBeTrue();
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => node.TransferLeadershipAsync("n2", CancellationToken.None));
|
||||
ex.Message.ShouldContain("No transport configured");
|
||||
}
|
||||
|
||||
// Go reference: raft.go sendTimeoutNow -- target becomes leader after receiving TimeoutNow.
|
||||
// VoteGrantingTransport delivers TimeoutNow and immediately grants votes so the target
|
||||
// is already leader before the polling loop runs -- no Task.Delay required.
|
||||
[Fact]
|
||||
public async Task TransferLeadership_target_becomes_leader()
|
||||
{
|
||||
var transport = new VoteGrantingTransport();
|
||||
var nodes = Enumerable.Range(1, 3)
|
||||
.Select(i => new RaftNode($"n{i}", transport))
|
||||
.ToArray();
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
transport.Register(node);
|
||||
node.ConfigureCluster(nodes);
|
||||
node.ElectionTimeoutMinMs = 5;
|
||||
node.ElectionTimeoutMaxMs = 10;
|
||||
}
|
||||
|
||||
var leader = nodes[0];
|
||||
leader.StartElection(nodes.Length);
|
||||
foreach (var voter in nodes.Skip(1))
|
||||
leader.ReceiveVote(voter.GrantVote(leader.Term, leader.Id), nodes.Length);
|
||||
leader.IsLeader.ShouldBeTrue();
|
||||
|
||||
var target = nodes[1];
|
||||
// VoteGrantingTransport makes the target a leader synchronously during TimeoutNow
|
||||
// delivery, so the first poll iteration in TransferLeadershipAsync succeeds.
|
||||
var result = await leader.TransferLeadershipAsync(target.Id, CancellationToken.None);
|
||||
|
||||
result.ShouldBeTrue();
|
||||
target.IsLeader.ShouldBeTrue();
|
||||
}
|
||||
|
||||
// Go reference: raft.go sendTimeoutNow -- returns false when target doesn't respond.
|
||||
// "ghost" is not registered in the transport so TimeoutNow is a no-op and the
|
||||
// polling loop times out after 2x election timeout.
|
||||
[Fact]
|
||||
public async Task TransferLeadership_timeout_on_unreachable_target()
|
||||
{
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var leader = new RaftNode("leader", transport);
|
||||
leader.ConfigureCluster([leader]);
|
||||
transport.Register(leader);
|
||||
leader.StartElection(1);
|
||||
|
||||
// Very short timeouts so the poll deadline is reached quickly.
|
||||
leader.ElectionTimeoutMinMs = 5;
|
||||
leader.ElectionTimeoutMaxMs = 10;
|
||||
|
||||
// "ghost" is not registered -- TimeoutNow is a no-op; target never becomes leader.
|
||||
var result = await leader.TransferLeadershipAsync("ghost", CancellationToken.None);
|
||||
|
||||
result.ShouldBeFalse();
|
||||
}
|
||||
|
||||
// -- Integration: flag lifecycle --
|
||||
|
||||
[Fact]
|
||||
public async Task TransferLeadership_clears_transfer_flag_after_success()
|
||||
{
|
||||
var transport = new VoteGrantingTransport();
|
||||
var nodes = Enumerable.Range(1, 3)
|
||||
.Select(i => new RaftNode($"n{i}", transport))
|
||||
.ToArray();
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
transport.Register(node);
|
||||
node.ConfigureCluster(nodes);
|
||||
node.ElectionTimeoutMinMs = 5;
|
||||
node.ElectionTimeoutMaxMs = 10;
|
||||
}
|
||||
|
||||
var leader = nodes[0];
|
||||
leader.StartElection(nodes.Length);
|
||||
foreach (var voter in nodes.Skip(1))
|
||||
leader.ReceiveVote(voter.GrantVote(leader.Term, leader.Id), nodes.Length);
|
||||
|
||||
var target = nodes[1];
|
||||
var success = await leader.TransferLeadershipAsync(target.Id, CancellationToken.None);
|
||||
|
||||
success.ShouldBeTrue();
|
||||
// After transfer completes the flag must be cleared.
|
||||
leader.TransferInProgress.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TransferLeadership_clears_transfer_flag_after_timeout()
|
||||
{
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var leader = new RaftNode("leader", transport);
|
||||
leader.ConfigureCluster([leader]);
|
||||
transport.Register(leader);
|
||||
leader.StartElection(1);
|
||||
leader.ElectionTimeoutMinMs = 5;
|
||||
leader.ElectionTimeoutMaxMs = 10;
|
||||
|
||||
// "ghost" is not registered -- transfer times out.
|
||||
await leader.TransferLeadershipAsync("ghost", CancellationToken.None);
|
||||
|
||||
// Flag must be cleared regardless of outcome.
|
||||
leader.TransferInProgress.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A transport that blocks inside <see cref="SendTimeoutNowAsync"/> until the
|
||||
/// provided <see cref="CancellationToken"/> is cancelled. Exposes a semaphore
|
||||
/// so the test can synchronize on when the leader transfer flag is set.
|
||||
/// </summary>
|
||||
file sealed class BlockingTimeoutNowTransport : IRaftTransport
|
||||
{
|
||||
private readonly SemaphoreSlim _entered = new(0, 1);
|
||||
|
||||
/// <summary>
|
||||
/// Returns a task that completes once <see cref="SendTimeoutNowAsync"/> has been
|
||||
/// entered and the leader's transfer flag is guaranteed to be set.
|
||||
/// </summary>
|
||||
public Task WaitUntilBlockingAsync() => _entered.WaitAsync();
|
||||
|
||||
public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(
|
||||
string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
|
||||
=> Task.FromResult<IReadOnlyList<AppendResult>>([]);
|
||||
|
||||
public Task<VoteResponse> RequestVoteAsync(
|
||||
string candidateId, string voterId, VoteRequest request, CancellationToken ct)
|
||||
=> Task.FromResult(new VoteResponse { Granted = false });
|
||||
|
||||
public Task InstallSnapshotAsync(
|
||||
string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public async Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
|
||||
{
|
||||
// Signal that the transfer flag is set -- the test can now probe ProposeAsync.
|
||||
_entered.Release();
|
||||
|
||||
// Block until the test cancels the token.
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
await using var reg = ct.Register(() => tcs.TrySetCanceled(ct));
|
||||
await tcs.Task;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A transport that, when delivering a TimeoutNow RPC, also immediately grants
|
||||
/// votes to the target candidate so it reaches quorum synchronously. This makes
|
||||
/// the target become leader before TransferLeadershipAsync starts polling, removing
|
||||
/// any need for Task.Delay waits in the test.
|
||||
/// </summary>
|
||||
file sealed class VoteGrantingTransport : IRaftTransport
|
||||
{
|
||||
private readonly Dictionary<string, RaftNode> _nodes = new(StringComparer.Ordinal);
|
||||
|
||||
public void Register(RaftNode node) => _nodes[node.Id] = node;
|
||||
|
||||
public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(
|
||||
string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
|
||||
{
|
||||
var results = new List<AppendResult>(followerIds.Count);
|
||||
foreach (var followerId in followerIds)
|
||||
{
|
||||
if (_nodes.TryGetValue(followerId, out var node))
|
||||
{
|
||||
node.ReceiveReplicatedEntry(entry);
|
||||
results.Add(new AppendResult { FollowerId = followerId, Success = true });
|
||||
}
|
||||
else
|
||||
{
|
||||
results.Add(new AppendResult { FollowerId = followerId, Success = false });
|
||||
}
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<AppendResult>>(results);
|
||||
}
|
||||
|
||||
public Task<VoteResponse> RequestVoteAsync(
|
||||
string candidateId, string voterId, VoteRequest request, CancellationToken ct)
|
||||
{
|
||||
if (_nodes.TryGetValue(voterId, out var node))
|
||||
return Task.FromResult(node.GrantVote(request.Term, candidateId));
|
||||
return Task.FromResult(new VoteResponse { Granted = false });
|
||||
}
|
||||
|
||||
public Task InstallSnapshotAsync(
|
||||
string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
/// <summary>
|
||||
/// Delivers TimeoutNow to the target (triggering an immediate election), then
|
||||
/// grants votes from every other peer so the target reaches quorum synchronously.
|
||||
/// This ensures the target is already leader before TransferLeadershipAsync polls,
|
||||
/// removing any timing dependency between delivery and vote propagation.
|
||||
/// </summary>
|
||||
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
|
||||
{
|
||||
if (!_nodes.TryGetValue(targetId, out var target))
|
||||
return Task.CompletedTask;
|
||||
|
||||
// Trigger immediate election on the target node.
|
||||
target.ReceiveTimeoutNow(term);
|
||||
|
||||
// Grant peer votes so the target reaches quorum immediately.
|
||||
if (target.Role == RaftRole.Candidate)
|
||||
{
|
||||
var clusterSize = _nodes.Count;
|
||||
foreach (var (peerId, peer) in _nodes)
|
||||
{
|
||||
if (string.Equals(peerId, targetId, StringComparison.Ordinal))
|
||||
continue;
|
||||
var vote = peer.GrantVote(target.Term, targetId);
|
||||
target.ReceiveVote(vote, clusterSize);
|
||||
if (target.IsLeader)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -552,6 +552,9 @@ public class RaftLogReplicationTests
|
||||
public Task InstallSnapshotAsync(
|
||||
string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
// -- Helper transport that succeeds for first follower, fails for rest --
|
||||
@@ -590,5 +593,8 @@ public class RaftLogReplicationTests
|
||||
public Task InstallSnapshotAsync(
|
||||
string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
288
tests/NATS.Server.Tests/Raft/RaftSnapshotStreamingTests.cs
Normal file
288
tests/NATS.Server.Tests/Raft/RaftSnapshotStreamingTests.cs
Normal file
@@ -0,0 +1,288 @@
|
||||
using System.IO.Hashing;
|
||||
using NATS.Server.Raft;
|
||||
|
||||
namespace NATS.Server.Tests.Raft;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for Gap 8.3: chunk-based snapshot streaming with CRC32 validation.
|
||||
///
|
||||
/// Covers <see cref="SnapshotChunkEnumerator"/> enumeration/CRC behaviour and
|
||||
/// the <see cref="RaftNode.InstallSnapshotFromChunksAsync"/> CRC validation path.
|
||||
///
|
||||
/// Go reference: raft.go:3500-3700 (installSnapshot chunked transfer + CRC validation)
|
||||
/// </summary>
|
||||
public class RaftSnapshotStreamingTests
|
||||
{
|
||||
// -----------------------------------------------------------------------
|
||||
// SnapshotChunkEnumerator tests
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkEnumerator_yields_correct_chunk_count()
|
||||
{
|
||||
// 200 KB of data at 64 KB per chunk → ceil(200/64) = 4 chunks
|
||||
// Go reference: raft.go snapshotChunkSize chunking logic
|
||||
const int dataSize = 200 * 1024;
|
||||
var data = new byte[dataSize];
|
||||
Random.Shared.NextBytes(data);
|
||||
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 65536);
|
||||
var chunks = enumerator.ToList();
|
||||
|
||||
chunks.Count.ShouldBe(4);
|
||||
enumerator.ChunkCount.ShouldBe(4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkEnumerator_chunks_reassemble_to_original()
|
||||
{
|
||||
// Concatenating all chunks must reproduce the original byte array exactly
|
||||
// Go reference: raft.go installSnapshot chunk reassembly
|
||||
const int dataSize = 150 * 1024; // 150 KB → 3 chunks at 64 KB
|
||||
var original = new byte[dataSize];
|
||||
Random.Shared.NextBytes(original);
|
||||
|
||||
var enumerator = new SnapshotChunkEnumerator(original, chunkSize: 65536);
|
||||
var assembled = enumerator.SelectMany(c => c).ToArray();
|
||||
|
||||
assembled.Length.ShouldBe(original.Length);
|
||||
assembled.ShouldBe(original);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkEnumerator_crc32_matches()
|
||||
{
|
||||
// The CRC32 reported by the enumerator must equal the CRC32 computed
|
||||
// directly over the original data — proving it covers the full payload
|
||||
// Go reference: raft.go installSnapshot CRC32 computation
|
||||
var data = new byte[100 * 1024];
|
||||
Random.Shared.NextBytes(data);
|
||||
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 65536);
|
||||
|
||||
var expectedCrc = new Crc32();
|
||||
expectedCrc.Append(data);
|
||||
var expected = expectedCrc.GetCurrentHashAsUInt32();
|
||||
|
||||
enumerator.Crc32Value.ShouldBe(expected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkEnumerator_single_chunk_for_small_data()
|
||||
{
|
||||
// Data that fits in a single chunk — only one chunk should be yielded,
|
||||
// and it should be identical to the input
|
||||
// Go reference: raft.go installSnapshot — single-chunk case
|
||||
var data = new byte[] { 1, 2, 3, 4, 5, 10, 20, 30 };
|
||||
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 65536);
|
||||
var chunks = enumerator.ToList();
|
||||
|
||||
chunks.Count.ShouldBe(1);
|
||||
enumerator.ChunkCount.ShouldBe(1);
|
||||
chunks[0].ShouldBe(data);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkEnumerator_last_chunk_is_remainder()
|
||||
{
|
||||
// 10 bytes with chunk size 3 → chunks of [3, 3, 3, 1]
|
||||
var data = Enumerable.Range(0, 10).Select(i => (byte)i).ToArray();
|
||||
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 3);
|
||||
var chunks = enumerator.ToList();
|
||||
|
||||
chunks.Count.ShouldBe(4);
|
||||
chunks[0].Length.ShouldBe(3);
|
||||
chunks[1].Length.ShouldBe(3);
|
||||
chunks[2].Length.ShouldBe(3);
|
||||
chunks[3].Length.ShouldBe(1); // remainder
|
||||
chunks[3][0].ShouldBe((byte)9);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkEnumerator_crc32_is_stable_across_multiple_reads()
|
||||
{
|
||||
// CRC32Value must return the same value on every call (cached)
|
||||
var data = new byte[1024];
|
||||
Random.Shared.NextBytes(data);
|
||||
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 256);
|
||||
|
||||
var first = enumerator.Crc32Value;
|
||||
var second = enumerator.Crc32Value;
|
||||
var third = enumerator.Crc32Value;
|
||||
|
||||
second.ShouldBe(first);
|
||||
third.ShouldBe(first);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// InstallSnapshotFromChunksAsync CRC32 validation tests
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task InstallSnapshot_assembles_chunks_into_snapshot()
|
||||
{
|
||||
// Snapshot assembled from multiple chunks should produce the correct
|
||||
// LastIncludedIndex, LastIncludedTerm, and Data on the node.
|
||||
// Go reference: raft.go:3500-3700 installSnapshot
|
||||
var node = new RaftNode("n1");
|
||||
|
||||
var data = new byte[] { 10, 20, 30, 40, 50 };
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 2);
|
||||
var chunks = enumerator.ToList();
|
||||
|
||||
await node.InstallSnapshotFromChunksAsync(
|
||||
chunks,
|
||||
snapshotIndex: 42,
|
||||
snapshotTerm: 7,
|
||||
ct: default);
|
||||
|
||||
node.AppliedIndex.ShouldBe(42);
|
||||
node.CommitIndex.ShouldBe(42);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstallSnapshot_validates_crc32_success()
|
||||
{
|
||||
// When the correct CRC32 is supplied the install should complete without error.
|
||||
// Go reference: raft.go installSnapshot CRC validation
|
||||
var node = new RaftNode("n1");
|
||||
|
||||
var data = new byte[256];
|
||||
Random.Shared.NextBytes(data);
|
||||
var enumerator = new SnapshotChunkEnumerator(data, chunkSize: 64);
|
||||
var crc = enumerator.Crc32Value;
|
||||
var chunks = enumerator.ToList();
|
||||
|
||||
// Should not throw
|
||||
await node.InstallSnapshotFromChunksAsync(
|
||||
chunks,
|
||||
snapshotIndex: 10,
|
||||
snapshotTerm: 2,
|
||||
ct: default,
|
||||
expectedCrc32: crc);
|
||||
|
||||
node.AppliedIndex.ShouldBe(10);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstallSnapshot_validates_crc32_throws_on_mismatch()
|
||||
{
|
||||
// A wrong CRC32 must cause InvalidDataException before any state is mutated.
|
||||
// Go reference: raft.go installSnapshot CRC mismatch → abort
|
||||
var node = new RaftNode("n1");
|
||||
node.Log.Append(1, "cmd-1"); // pre-existing state
|
||||
|
||||
var data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var chunks = new[] { data }; // single chunk
|
||||
|
||||
const uint wrongCrc = 0xDEADBEEF;
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidDataException>(async () =>
|
||||
await node.InstallSnapshotFromChunksAsync(
|
||||
chunks,
|
||||
snapshotIndex: 99,
|
||||
snapshotTerm: 5,
|
||||
ct: default,
|
||||
expectedCrc32: wrongCrc));
|
||||
|
||||
ex.Message.ShouldContain("CRC32");
|
||||
ex.Message.ShouldContain("DEADBEEF");
|
||||
|
||||
// State must NOT have been mutated since CRC failed before any writes
|
||||
node.AppliedIndex.ShouldBe(0);
|
||||
node.CommitIndex.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstallSnapshot_no_crc_parameter_installs_without_validation()
|
||||
{
|
||||
// When expectedCrc32 is omitted (null), no validation occurs and any data installs.
|
||||
// Go reference: raft.go optional CRC field (backward compat)
|
||||
var node = new RaftNode("n1");
|
||||
var chunks = new[] { new byte[] { 7, 8, 9 } };
|
||||
|
||||
// Should not throw even with no CRC supplied
|
||||
await node.InstallSnapshotFromChunksAsync(
|
||||
chunks,
|
||||
snapshotIndex: 5,
|
||||
snapshotTerm: 1,
|
||||
ct: default);
|
||||
|
||||
node.AppliedIndex.ShouldBe(5);
|
||||
node.CommitIndex.ShouldBe(5);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// RaftInstallSnapshotChunkWire encode/decode roundtrip tests
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkWire_roundtrip_with_data()
|
||||
{
|
||||
// Encode and decode a chunk message and verify all fields survive the roundtrip.
|
||||
// Go reference: raft.go wire format for InstallSnapshot RPC chunks
|
||||
var payload = new byte[] { 0xAA, 0xBB, 0xCC, 0xDD };
|
||||
var wire = new RaftInstallSnapshotChunkWire(
|
||||
SnapshotIndex: 42UL,
|
||||
SnapshotTerm: 3U,
|
||||
ChunkIndex: 1U,
|
||||
TotalChunks: 4U,
|
||||
Crc32: 0x12345678U,
|
||||
Data: payload);
|
||||
|
||||
var encoded = wire.Encode();
|
||||
var decoded = RaftInstallSnapshotChunkWire.Decode(encoded);
|
||||
|
||||
decoded.SnapshotIndex.ShouldBe(42UL);
|
||||
decoded.SnapshotTerm.ShouldBe(3U);
|
||||
decoded.ChunkIndex.ShouldBe(1U);
|
||||
decoded.TotalChunks.ShouldBe(4U);
|
||||
decoded.Crc32.ShouldBe(0x12345678U);
|
||||
decoded.Data.ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkWire_header_length_is_24_bytes()
|
||||
{
|
||||
// Header must be exactly 24 bytes as documented in the wire format.
|
||||
RaftInstallSnapshotChunkWire.HeaderLen.ShouldBe(24);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkWire_encode_total_length_is_header_plus_data()
|
||||
{
|
||||
var data = new byte[100];
|
||||
var wire = new RaftInstallSnapshotChunkWire(1UL, 1U, 0U, 1U, 0U, data);
|
||||
wire.Encode().Length.ShouldBe(RaftInstallSnapshotChunkWire.HeaderLen + 100);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkWire_decode_throws_on_short_buffer()
|
||||
{
|
||||
// Buffers shorter than the header should throw ArgumentException
|
||||
var tooShort = new byte[10]; // < 24 bytes
|
||||
Should.Throw<ArgumentException>(() => RaftInstallSnapshotChunkWire.Decode(tooShort));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotChunkWire_roundtrip_empty_payload()
|
||||
{
|
||||
// A header-only message (no chunk data) should encode and decode cleanly.
|
||||
var wire = new RaftInstallSnapshotChunkWire(
|
||||
SnapshotIndex: 0UL,
|
||||
SnapshotTerm: 0U,
|
||||
ChunkIndex: 0U,
|
||||
TotalChunks: 0U,
|
||||
Crc32: 0U,
|
||||
Data: []);
|
||||
|
||||
var encoded = wire.Encode();
|
||||
encoded.Length.ShouldBe(RaftInstallSnapshotChunkWire.HeaderLen);
|
||||
|
||||
var decoded = RaftInstallSnapshotChunkWire.Decode(encoded);
|
||||
decoded.Data.Length.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
@@ -40,5 +40,8 @@ public class RaftStrictConsensusRuntimeTests
|
||||
|
||||
public Task InstallSnapshotAsync(string leaderId, string followerId, RaftSnapshot snapshot, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user