7e0bed2447
- Add SnapshotChunkEnumerator: IEnumerable<byte[]> that splits snapshot data into fixed-size chunks (default 65536 bytes) and computes CRC32 over the full payload for integrity validation during streaming transfer - Add RaftInstallSnapshotChunkWire: 24-byte header + variable data wire type encoding [snapshotIndex:8][snapshotTerm:4][chunkIndex:4][totalChunks:4][crc32:4][data:N] - Extend InstallSnapshotFromChunksAsync with optional expectedCrc32 parameter; validates assembled data against CRC32 before applying snapshot state, throwing InvalidDataException on mismatch to prevent corrupt state installation - Fix stub IRaftTransport implementations in test files missing SendTimeoutNowAsync - Fix incorrect role assertion in RaftLeadershipTransferTests (single-node quorum = 1) - 15 new tests in RaftSnapshotStreamingTests covering enumeration, reassembly, CRC correctness, validation success/failure, and wire format roundtrip
91 lines
3.1 KiB
C#
91 lines
3.1 KiB
C#
using System.IO.Hashing;
|
|
|
|
namespace NATS.Server.Raft;
|
|
|
|
/// <summary>
|
|
/// Splits a snapshot byte array into fixed-size chunks for streaming transfer
|
|
/// and computes a CRC32 checksum over the entire data for integrity validation.
|
|
///
|
|
/// During an InstallSnapshot RPC the leader streams a large snapshot to a lagging
|
|
/// follower chunk-by-chunk rather than sending it in a single message. The follower
|
|
/// accumulates the chunks, reassembles them, then validates the CRC32 before
|
|
/// applying the snapshot — preventing silent data corruption in transit.
|
|
///
|
|
/// Go reference: raft.go:3500-3700 (installSnapshot chunked transfer and CRC validation)
|
|
/// </summary>
|
|
public sealed class SnapshotChunkEnumerator : IEnumerable<byte[]>
|
|
{
|
|
private readonly byte[] _data;
|
|
private readonly int _chunkSize;
|
|
private uint? _crc32Value;
|
|
|
|
/// <summary>
|
|
/// Default chunk size matching Go's snapshot chunk transfer size (64 KiB).
|
|
/// Go reference: raft.go snapshotChunkSize constant.
|
|
/// </summary>
|
|
public const int DefaultChunkSize = 65536;
|
|
|
|
/// <summary>
|
|
/// Creates a new chunk enumerator over <paramref name="data"/>.
|
|
/// </summary>
|
|
/// <param name="data">The full snapshot data to split into chunks.</param>
|
|
/// <param name="chunkSize">Maximum size of each chunk in bytes. Defaults to 64 KiB.</param>
|
|
/// <exception cref="ArgumentOutOfRangeException">
|
|
/// Thrown when <paramref name="chunkSize"/> is less than or equal to zero.
|
|
/// </exception>
|
|
public SnapshotChunkEnumerator(byte[] data, int chunkSize = DefaultChunkSize)
|
|
{
|
|
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(chunkSize);
|
|
_data = data;
|
|
_chunkSize = chunkSize;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The CRC32 (IEEE 802.3) checksum of the entire snapshot data.
|
|
/// Computed lazily on first access and cached for subsequent reads.
|
|
/// The receiver uses this value to validate the reassembled snapshot after
|
|
/// all chunks have been received.
|
|
/// Go reference: raft.go installSnapshot CRC validation.
|
|
/// </summary>
|
|
public uint Crc32Value
|
|
{
|
|
get
|
|
{
|
|
if (!_crc32Value.HasValue)
|
|
{
|
|
var crc = new Crc32();
|
|
crc.Append(_data);
|
|
_crc32Value = crc.GetCurrentHashAsUInt32();
|
|
}
|
|
return _crc32Value.Value;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The total number of chunks this data splits into given the configured chunk size.
|
|
/// </summary>
|
|
public int ChunkCount => _data.Length == 0 ? 1 : ((_data.Length + _chunkSize - 1) / _chunkSize);
|
|
|
|
/// <inheritdoc />
|
|
public IEnumerator<byte[]> GetEnumerator()
|
|
{
|
|
if (_data.Length == 0)
|
|
{
|
|
yield return [];
|
|
yield break;
|
|
}
|
|
|
|
var offset = 0;
|
|
while (offset < _data.Length)
|
|
{
|
|
var length = Math.Min(_chunkSize, _data.Length - offset);
|
|
var chunk = new byte[length];
|
|
_data.AsSpan(offset, length).CopyTo(chunk);
|
|
offset += length;
|
|
yield return chunk;
|
|
}
|
|
}
|
|
|
|
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
|
|
}
|