feat: add chunk-based snapshot streaming with CRC32 validation (Gap 8.3)
- Add SnapshotChunkEnumerator: IEnumerable<byte[]> that splits snapshot data into fixed-size chunks (default 65536 bytes) and computes CRC32 over the full payload for integrity validation during streaming transfer - Add RaftInstallSnapshotChunkWire: 24-byte header + variable data wire type encoding [snapshotIndex:8][snapshotTerm:4][chunkIndex:4][totalChunks:4][crc32:4][data:N] - Extend InstallSnapshotFromChunksAsync with optional expectedCrc32 parameter; validates assembled data against CRC32 before applying snapshot state, throwing InvalidDataException on mismatch to prevent corrupt state installation - Fix stub IRaftTransport implementations in test files missing SendTimeoutNowAsync - Fix incorrect role assertion in RaftLeadershipTransferTests (single-node quorum = 1) - 15 new tests in RaftSnapshotStreamingTests covering enumeration, reassembly, CRC correctness, validation success/failure, and wire format roundtrip
This commit is contained in:
90
src/NATS.Server/Raft/SnapshotChunkEnumerator.cs
Normal file
90
src/NATS.Server/Raft/SnapshotChunkEnumerator.cs
Normal file
@@ -0,0 +1,90 @@
|
||||
using System.IO.Hashing;
|
||||
|
||||
namespace NATS.Server.Raft;
|
||||
|
||||
/// <summary>
|
||||
/// Splits a snapshot byte array into fixed-size chunks for streaming transfer
|
||||
/// and computes a CRC32 checksum over the entire data for integrity validation.
|
||||
///
|
||||
/// During an InstallSnapshot RPC the leader streams a large snapshot to a lagging
|
||||
/// follower chunk-by-chunk rather than sending it in a single message. The follower
|
||||
/// accumulates the chunks, reassembles them, then validates the CRC32 before
|
||||
/// applying the snapshot — preventing silent data corruption in transit.
|
||||
///
|
||||
/// Go reference: raft.go:3500-3700 (installSnapshot chunked transfer and CRC validation)
|
||||
/// </summary>
|
||||
public sealed class SnapshotChunkEnumerator : IEnumerable<byte[]>
|
||||
{
|
||||
private readonly byte[] _data;
|
||||
private readonly int _chunkSize;
|
||||
private uint? _crc32Value;
|
||||
|
||||
/// <summary>
|
||||
/// Default chunk size matching Go's snapshot chunk transfer size (64 KiB).
|
||||
/// Go reference: raft.go snapshotChunkSize constant.
|
||||
/// </summary>
|
||||
public const int DefaultChunkSize = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new chunk enumerator over <paramref name="data"/>.
|
||||
/// </summary>
|
||||
/// <param name="data">The full snapshot data to split into chunks.</param>
|
||||
/// <param name="chunkSize">Maximum size of each chunk in bytes. Defaults to 64 KiB.</param>
|
||||
/// <exception cref="ArgumentOutOfRangeException">
|
||||
/// Thrown when <paramref name="chunkSize"/> is less than or equal to zero.
|
||||
/// </exception>
|
||||
public SnapshotChunkEnumerator(byte[] data, int chunkSize = DefaultChunkSize)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(chunkSize);
|
||||
_data = data;
|
||||
_chunkSize = chunkSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The CRC32 (IEEE 802.3) checksum of the entire snapshot data.
|
||||
/// Computed lazily on first access and cached for subsequent reads.
|
||||
/// The receiver uses this value to validate the reassembled snapshot after
|
||||
/// all chunks have been received.
|
||||
/// Go reference: raft.go installSnapshot CRC validation.
|
||||
/// </summary>
|
||||
public uint Crc32Value
|
||||
{
|
||||
get
|
||||
{
|
||||
if (!_crc32Value.HasValue)
|
||||
{
|
||||
var crc = new Crc32();
|
||||
crc.Append(_data);
|
||||
_crc32Value = crc.GetCurrentHashAsUInt32();
|
||||
}
|
||||
return _crc32Value.Value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The total number of chunks this data splits into given the configured chunk size.
|
||||
/// </summary>
|
||||
public int ChunkCount => _data.Length == 0 ? 1 : ((_data.Length + _chunkSize - 1) / _chunkSize);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IEnumerator<byte[]> GetEnumerator()
|
||||
{
|
||||
if (_data.Length == 0)
|
||||
{
|
||||
yield return [];
|
||||
yield break;
|
||||
}
|
||||
|
||||
var offset = 0;
|
||||
while (offset < _data.Length)
|
||||
{
|
||||
var length = Math.Min(_chunkSize, _data.Length - offset);
|
||||
var chunk = new byte[length];
|
||||
_data.AsSpan(offset, length).CopyTo(chunk);
|
||||
offset += length;
|
||||
yield return chunk;
|
||||
}
|
||||
}
|
||||
|
||||
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
|
||||
}
|
||||
Reference in New Issue
Block a user