feat(storage): add MsgBlock block-based message storage unit
MsgBlock is the unit of storage in the file store — a single append-only block file containing sequentially written binary message records. Blocks are sealed (read-only) when they reach a configurable byte-size limit. Key features: - Write: appends MessageRecord-encoded messages with auto-incrementing sequence numbers and configurable first sequence offset - Read: positional I/O via RandomAccess.Read for concurrent reader safety - Delete: soft-delete with on-disk persistence (re-encodes flags byte + checksum so deletions survive recovery) - Recovery: rebuilds in-memory index by scanning block file using MessageRecord.MeasureRecord for record boundary detection - Thread safety: ReaderWriterLockSlim allows concurrent reads during writes Also adds MessageRecord.MeasureRecord() — computes a record's byte length by parsing varint field headers without full decode, needed for sequential record scanning during block recovery. Reference: golang/nats-server/server/filestore.go:217-267 (msgBlock struct) 12 tests covering write, read, delete, seal, recovery, concurrency, and custom sequence offsets.
This commit is contained in:
@@ -215,6 +215,38 @@ public sealed class MessageRecord
|
|||||||
throw new InvalidDataException("Varint is truncated.");
|
throw new InvalidDataException("Varint is truncated.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Measures the total byte length of the first record in a buffer without fully decoding it.
|
||||||
|
/// This parses the varint-encoded field lengths to compute the record size.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="data">Buffer that starts with a record (may contain trailing data).</param>
|
||||||
|
/// <returns>The total byte length of the first record.</returns>
|
||||||
|
/// <exception cref="InvalidDataException">If the buffer is too short to contain a valid record header.</exception>
|
||||||
|
public static int MeasureRecord(ReadOnlySpan<byte> data)
|
||||||
|
{
|
||||||
|
if (data.Length < 1 + 3 + TrailerSize)
|
||||||
|
throw new InvalidDataException("Buffer too short to contain a record.");
|
||||||
|
|
||||||
|
var offset = 1; // flags byte
|
||||||
|
|
||||||
|
// Subject length
|
||||||
|
var (subjectLen, subjectLenBytes) = ReadVarint(data[offset..]);
|
||||||
|
offset += subjectLenBytes + (int)subjectLen;
|
||||||
|
|
||||||
|
// Headers length
|
||||||
|
var (headersLen, headersLenBytes) = ReadVarint(data[offset..]);
|
||||||
|
offset += headersLenBytes + (int)headersLen;
|
||||||
|
|
||||||
|
// Payload length
|
||||||
|
var (payloadLen, payloadLenBytes) = ReadVarint(data[offset..]);
|
||||||
|
offset += payloadLenBytes + (int)payloadLen;
|
||||||
|
|
||||||
|
// Trailer: sequence(8) + timestamp(8) + checksum(8)
|
||||||
|
offset += TrailerSize;
|
||||||
|
|
||||||
|
return offset;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns the number of bytes needed to encode a varint.
|
/// Returns the number of bytes needed to encode a varint.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
358
src/NATS.Server/JetStream/Storage/MsgBlock.cs
Normal file
358
src/NATS.Server/JetStream/Storage/MsgBlock.cs
Normal file
@@ -0,0 +1,358 @@
|
|||||||
|
// Reference: golang/nats-server/server/filestore.go:217-267 (msgBlock struct)
|
||||||
|
// Go block write: filestore.go:6700-6760 (writeMsgRecord / writeMsgRecordLocked)
|
||||||
|
// Go block load: filestore.go:8140-8260 (loadMsgs / msgFromBufEx)
|
||||||
|
// Go deletion: filestore.go dmap (avl.SequenceSet) for soft-deletes
|
||||||
|
// Go sealing: filestore.go rbytes check — block rolls when rbytes >= maxBytes
|
||||||
|
//
|
||||||
|
// MsgBlock is the unit of storage in the file store. Messages are appended
|
||||||
|
// sequentially as binary records (using MessageRecord). Blocks are sealed
|
||||||
|
// (read-only) when they reach a configurable size limit.
|
||||||
|
|
||||||
|
using Microsoft.Win32.SafeHandles;
|
||||||
|
|
||||||
|
namespace NATS.Server.JetStream.Storage;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A block of messages stored in a single append-only file on disk.
|
||||||
|
/// This is the unit of storage in the file store. Messages are appended
|
||||||
|
/// sequentially as binary records. Blocks become sealed (read-only) when
|
||||||
|
/// they reach a configurable byte-size limit.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class MsgBlock : IDisposable
|
||||||
|
{
|
||||||
|
private readonly FileStream _file;
|
||||||
|
private readonly SafeFileHandle _handle;
|
||||||
|
private readonly Dictionary<ulong, (long Offset, int Length)> _index = new();
|
||||||
|
private readonly HashSet<ulong> _deleted = new();
|
||||||
|
private readonly long _maxBytes;
|
||||||
|
private readonly ReaderWriterLockSlim _lock = new();
|
||||||
|
private long _writeOffset; // Tracks the append position independently of FileStream.Position
|
||||||
|
private ulong _nextSequence;
|
||||||
|
private ulong _firstSequence;
|
||||||
|
private ulong _lastSequence;
|
||||||
|
private ulong _totalWritten; // Total records written (including later-deleted)
|
||||||
|
private bool _disposed;
|
||||||
|
|
||||||
|
private MsgBlock(FileStream file, int blockId, long maxBytes, ulong firstSequence)
|
||||||
|
{
|
||||||
|
_file = file;
|
||||||
|
_handle = file.SafeFileHandle;
|
||||||
|
BlockId = blockId;
|
||||||
|
_maxBytes = maxBytes;
|
||||||
|
_firstSequence = firstSequence;
|
||||||
|
_nextSequence = firstSequence;
|
||||||
|
_writeOffset = file.Length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Block identifier.</summary>
|
||||||
|
public int BlockId { get; }
|
||||||
|
|
||||||
|
/// <summary>First sequence number in this block.</summary>
|
||||||
|
public ulong FirstSequence
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try { return _firstSequence; }
|
||||||
|
finally { _lock.ExitReadLock(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Last sequence number written.</summary>
|
||||||
|
public ulong LastSequence
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try { return _lastSequence; }
|
||||||
|
finally { _lock.ExitReadLock(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Total messages excluding deleted.</summary>
|
||||||
|
public ulong MessageCount
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try { return _totalWritten - (ulong)_deleted.Count; }
|
||||||
|
finally { _lock.ExitReadLock(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Count of soft-deleted messages.</summary>
|
||||||
|
public ulong DeletedCount
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try { return (ulong)_deleted.Count; }
|
||||||
|
finally { _lock.ExitReadLock(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Total bytes written to block file.</summary>
|
||||||
|
public long BytesUsed
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try { return _writeOffset; }
|
||||||
|
finally { _lock.ExitReadLock(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>True when BytesUsed >= maxBytes (block is full).</summary>
|
||||||
|
public bool IsSealed
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try { return _writeOffset >= _maxBytes; }
|
||||||
|
finally { _lock.ExitReadLock(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new empty block file.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="blockId">Block identifier.</param>
|
||||||
|
/// <param name="directoryPath">Directory to store the block file.</param>
|
||||||
|
/// <param name="maxBytes">Size limit before sealing.</param>
|
||||||
|
/// <param name="firstSequence">First sequence number (default 1).</param>
|
||||||
|
/// <returns>A new <see cref="MsgBlock"/> ready for writes.</returns>
|
||||||
|
public static MsgBlock Create(int blockId, string directoryPath, long maxBytes, ulong firstSequence = 1)
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(directoryPath);
|
||||||
|
var filePath = BlockFilePath(directoryPath, blockId);
|
||||||
|
var file = new FileStream(filePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read);
|
||||||
|
return new MsgBlock(file, blockId, maxBytes, firstSequence);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Recovers a block from an existing file, rebuilding the in-memory index.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="blockId">Block identifier.</param>
|
||||||
|
/// <param name="directoryPath">Directory containing the block file.</param>
|
||||||
|
/// <returns>A recovered <see cref="MsgBlock"/>.</returns>
|
||||||
|
public static MsgBlock Recover(int blockId, string directoryPath)
|
||||||
|
{
|
||||||
|
var filePath = BlockFilePath(directoryPath, blockId);
|
||||||
|
var file = new FileStream(filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read);
|
||||||
|
|
||||||
|
// We don't know maxBytes from the file alone — use long.MaxValue so
|
||||||
|
// the recovered block is effectively unsealed. The caller can re-create
|
||||||
|
// with proper limits if needed.
|
||||||
|
var block = new MsgBlock(file, blockId, long.MaxValue, firstSequence: 0);
|
||||||
|
block.RebuildIndex();
|
||||||
|
return block;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Appends a message to the block.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="subject">NATS subject.</param>
|
||||||
|
/// <param name="headers">Optional message headers.</param>
|
||||||
|
/// <param name="payload">Message body payload.</param>
|
||||||
|
/// <returns>The assigned sequence number.</returns>
|
||||||
|
/// <exception cref="InvalidOperationException">Block is sealed.</exception>
|
||||||
|
public ulong Write(string subject, ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
|
{
|
||||||
|
_lock.EnterWriteLock();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (_writeOffset >= _maxBytes)
|
||||||
|
throw new InvalidOperationException("Block is sealed; cannot write new messages.");
|
||||||
|
|
||||||
|
var sequence = _nextSequence;
|
||||||
|
var record = new MessageRecord
|
||||||
|
{
|
||||||
|
Sequence = sequence,
|
||||||
|
Subject = subject,
|
||||||
|
Headers = headers,
|
||||||
|
Payload = payload,
|
||||||
|
Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L,
|
||||||
|
Deleted = false,
|
||||||
|
};
|
||||||
|
|
||||||
|
var encoded = MessageRecord.Encode(record);
|
||||||
|
var offset = _writeOffset;
|
||||||
|
|
||||||
|
// Write at the current append offset using positional I/O
|
||||||
|
RandomAccess.Write(_handle, encoded, offset);
|
||||||
|
_writeOffset = offset + encoded.Length;
|
||||||
|
|
||||||
|
_index[sequence] = (offset, encoded.Length);
|
||||||
|
|
||||||
|
if (_totalWritten == 0)
|
||||||
|
_firstSequence = sequence;
|
||||||
|
|
||||||
|
_lastSequence = sequence;
|
||||||
|
_nextSequence = sequence + 1;
|
||||||
|
_totalWritten++;
|
||||||
|
|
||||||
|
return sequence;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.ExitWriteLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads a message by sequence number. Uses positional I/O
|
||||||
|
/// (<see cref="RandomAccess.Read"/>) so concurrent readers don't
|
||||||
|
/// interfere with each other or the writer's append position.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="sequence">The sequence number to read.</param>
|
||||||
|
/// <returns>The decoded record, or null if not found or deleted.</returns>
|
||||||
|
public MessageRecord? Read(ulong sequence)
|
||||||
|
{
|
||||||
|
_lock.EnterReadLock();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (_deleted.Contains(sequence))
|
||||||
|
return null;
|
||||||
|
|
||||||
|
if (!_index.TryGetValue(sequence, out var entry))
|
||||||
|
return null;
|
||||||
|
|
||||||
|
var buffer = new byte[entry.Length];
|
||||||
|
RandomAccess.Read(_handle, buffer, entry.Offset);
|
||||||
|
|
||||||
|
return MessageRecord.Decode(buffer);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.ExitReadLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Soft-deletes a message by sequence number. Re-encodes the record on disk
|
||||||
|
/// with the deleted flag set (and updated checksum) so the deletion survives recovery.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="sequence">The sequence number to delete.</param>
|
||||||
|
/// <returns>True if the message was deleted; false if already deleted or not found.</returns>
|
||||||
|
public bool Delete(ulong sequence)
|
||||||
|
{
|
||||||
|
_lock.EnterWriteLock();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (!_index.TryGetValue(sequence, out var entry))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (!_deleted.Add(sequence))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
// Read the existing record, re-encode with Deleted flag, write back in-place.
|
||||||
|
// The encoded size doesn't change (only flags byte + checksum differ).
|
||||||
|
var buffer = new byte[entry.Length];
|
||||||
|
RandomAccess.Read(_handle, buffer, entry.Offset);
|
||||||
|
var record = MessageRecord.Decode(buffer);
|
||||||
|
|
||||||
|
var deletedRecord = new MessageRecord
|
||||||
|
{
|
||||||
|
Sequence = record.Sequence,
|
||||||
|
Subject = record.Subject,
|
||||||
|
Headers = record.Headers,
|
||||||
|
Payload = record.Payload,
|
||||||
|
Timestamp = record.Timestamp,
|
||||||
|
Deleted = true,
|
||||||
|
};
|
||||||
|
|
||||||
|
var encoded = MessageRecord.Encode(deletedRecord);
|
||||||
|
RandomAccess.Write(_handle, encoded, entry.Offset);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.ExitWriteLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Flushes any buffered writes to disk.
|
||||||
|
/// </summary>
|
||||||
|
public void Flush()
|
||||||
|
{
|
||||||
|
_lock.EnterWriteLock();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_file.Flush(flushToDisk: true);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.ExitWriteLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Closes the file handle and releases resources.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
return;
|
||||||
|
_disposed = true;
|
||||||
|
|
||||||
|
_lock.EnterWriteLock();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_file.Flush();
|
||||||
|
_file.Dispose();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.ExitWriteLock();
|
||||||
|
}
|
||||||
|
|
||||||
|
_lock.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Rebuilds the in-memory index by scanning all records in the block file.
|
||||||
|
/// Uses <see cref="MessageRecord.MeasureRecord"/> to determine each record's
|
||||||
|
/// size before decoding, so trailing data from subsequent records doesn't
|
||||||
|
/// corrupt the checksum validation.
|
||||||
|
/// </summary>
|
||||||
|
private void RebuildIndex()
|
||||||
|
{
|
||||||
|
var fileLength = _file.Length;
|
||||||
|
long offset = 0;
|
||||||
|
ulong count = 0;
|
||||||
|
|
||||||
|
while (offset < fileLength)
|
||||||
|
{
|
||||||
|
// Read remaining bytes from current offset using positional I/O
|
||||||
|
var remaining = (int)(fileLength - offset);
|
||||||
|
var buffer = new byte[remaining];
|
||||||
|
RandomAccess.Read(_handle, buffer, offset);
|
||||||
|
|
||||||
|
// Measure the first record's length, then decode only that slice
|
||||||
|
var recordLength = MessageRecord.MeasureRecord(buffer);
|
||||||
|
var record = MessageRecord.Decode(buffer.AsSpan(0, recordLength));
|
||||||
|
|
||||||
|
_index[record.Sequence] = (offset, recordLength);
|
||||||
|
|
||||||
|
if (record.Deleted)
|
||||||
|
_deleted.Add(record.Sequence);
|
||||||
|
|
||||||
|
if (count == 0)
|
||||||
|
_firstSequence = record.Sequence;
|
||||||
|
|
||||||
|
_lastSequence = record.Sequence;
|
||||||
|
_nextSequence = record.Sequence + 1;
|
||||||
|
count++;
|
||||||
|
|
||||||
|
offset += recordLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
_totalWritten = count;
|
||||||
|
_writeOffset = offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string BlockFilePath(string directoryPath, int blockId)
|
||||||
|
=> Path.Combine(directoryPath, $"{blockId:D6}.blk");
|
||||||
|
}
|
||||||
263
tests/NATS.Server.Tests/JetStream/Storage/MsgBlockTests.cs
Normal file
263
tests/NATS.Server.Tests/JetStream/Storage/MsgBlockTests.cs
Normal file
@@ -0,0 +1,263 @@
|
|||||||
|
// Reference: golang/nats-server/server/filestore.go:217-267 (msgBlock struct)
|
||||||
|
// Go block write: filestore.go:6700-6760 (writeMsgRecord)
|
||||||
|
// Go block load: filestore.go:8140-8260 (loadMsgs / msgFromBufEx)
|
||||||
|
// Go deletion: filestore.go dmap (avl.SequenceSet) for soft-deletes
|
||||||
|
//
|
||||||
|
// These tests verify the .NET MsgBlock abstraction — a block of messages stored
|
||||||
|
// in a single append-only file on disk, with in-memory index and soft-delete support.
|
||||||
|
|
||||||
|
using System.Text;
|
||||||
|
using NATS.Server.JetStream.Storage;
|
||||||
|
|
||||||
|
namespace NATS.Server.Tests.JetStream.Storage;
|
||||||
|
|
||||||
|
public sealed class MsgBlockTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly string _testDir;
|
||||||
|
|
||||||
|
public MsgBlockTests()
|
||||||
|
{
|
||||||
|
_testDir = Path.Combine(Path.GetTempPath(), $"msgblock_test_{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_testDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
try { Directory.Delete(_testDir, recursive: true); }
|
||||||
|
catch { /* best effort cleanup */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: writeMsgRecord — single message write returns first sequence
|
||||||
|
[Fact]
|
||||||
|
public void Write_SingleMessage_ReturnsSequence()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
var seq = block.Write("foo.bar", ReadOnlyMemory<byte>.Empty, "hello"u8.ToArray());
|
||||||
|
seq.ShouldBe(1UL);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: writeMsgRecord — sequential writes increment sequence
|
||||||
|
[Fact]
|
||||||
|
public void Write_MultipleMessages_IncrementsSequence()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
var s1 = block.Write("a", ReadOnlyMemory<byte>.Empty, "one"u8.ToArray());
|
||||||
|
var s2 = block.Write("b", ReadOnlyMemory<byte>.Empty, "two"u8.ToArray());
|
||||||
|
var s3 = block.Write("c", ReadOnlyMemory<byte>.Empty, "three"u8.ToArray());
|
||||||
|
|
||||||
|
s1.ShouldBe(1UL);
|
||||||
|
s2.ShouldBe(2UL);
|
||||||
|
s3.ShouldBe(3UL);
|
||||||
|
block.MessageCount.ShouldBe(3UL);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: loadMsgs / msgFromBufEx — read back by sequence number
|
||||||
|
[Fact]
|
||||||
|
public void Read_BySequence_ReturnsMessage()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
block.Write("test.subject", ReadOnlyMemory<byte>.Empty, "payload data"u8.ToArray());
|
||||||
|
|
||||||
|
var record = block.Read(1);
|
||||||
|
record.ShouldNotBeNull();
|
||||||
|
record.Sequence.ShouldBe(1UL);
|
||||||
|
record.Subject.ShouldBe("test.subject");
|
||||||
|
Encoding.UTF8.GetString(record.Payload.Span).ShouldBe("payload data");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: loadMsgs — reading a non-existent sequence returns nil
|
||||||
|
[Fact]
|
||||||
|
public void Read_NonexistentSequence_ReturnsNull()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
block.Write("a", ReadOnlyMemory<byte>.Empty, "data"u8.ToArray());
|
||||||
|
|
||||||
|
var record = block.Read(999);
|
||||||
|
record.ShouldBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: filestore.go rbytes check — block seals when size exceeds maxBytes
|
||||||
|
[Fact]
|
||||||
|
public void IsSealed_ReturnsTrueWhenFull()
|
||||||
|
{
|
||||||
|
// Use a very small maxBytes so the block seals quickly.
|
||||||
|
// A single record with subject "a", empty headers, and 32-byte payload is ~61 bytes.
|
||||||
|
// Set maxBytes to 50 so one write seals the block.
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 50);
|
||||||
|
|
||||||
|
var payload = new byte[32];
|
||||||
|
block.Write("a", ReadOnlyMemory<byte>.Empty, payload);
|
||||||
|
block.IsSealed.ShouldBeTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: filestore.go errNoRoom — cannot write to sealed block
|
||||||
|
[Fact]
|
||||||
|
public void Write_ThrowsWhenSealed()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 50);
|
||||||
|
block.Write("a", ReadOnlyMemory<byte>.Empty, new byte[32]);
|
||||||
|
block.IsSealed.ShouldBeTrue();
|
||||||
|
|
||||||
|
Should.Throw<InvalidOperationException>(() =>
|
||||||
|
block.Write("b", ReadOnlyMemory<byte>.Empty, "more"u8.ToArray()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: dmap soft-delete — deleted message reads back as null
|
||||||
|
[Fact]
|
||||||
|
public void Delete_MarksSequenceAsDeleted()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
block.Write("a", ReadOnlyMemory<byte>.Empty, "data"u8.ToArray());
|
||||||
|
|
||||||
|
block.Delete(1).ShouldBeTrue();
|
||||||
|
block.Read(1).ShouldBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: dmap — MessageCount reflects only non-deleted messages
|
||||||
|
[Fact]
|
||||||
|
public void Delete_DecreasesMessageCount()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
block.Write("a", ReadOnlyMemory<byte>.Empty, "one"u8.ToArray());
|
||||||
|
block.Write("b", ReadOnlyMemory<byte>.Empty, "two"u8.ToArray());
|
||||||
|
block.Write("c", ReadOnlyMemory<byte>.Empty, "three"u8.ToArray());
|
||||||
|
|
||||||
|
block.MessageCount.ShouldBe(3UL);
|
||||||
|
block.DeletedCount.ShouldBe(0UL);
|
||||||
|
|
||||||
|
block.Delete(2).ShouldBeTrue();
|
||||||
|
|
||||||
|
block.MessageCount.ShouldBe(2UL);
|
||||||
|
block.DeletedCount.ShouldBe(1UL);
|
||||||
|
|
||||||
|
// Double delete returns false
|
||||||
|
block.Delete(2).ShouldBeFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: recovery path — rebuild index from existing block file
|
||||||
|
[Fact]
|
||||||
|
public void Recover_RebuildsIndexFromFile()
|
||||||
|
{
|
||||||
|
// Write messages and dispose
|
||||||
|
using (var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024))
|
||||||
|
{
|
||||||
|
block.Write("a.b", ReadOnlyMemory<byte>.Empty, "first"u8.ToArray());
|
||||||
|
block.Write("c.d", ReadOnlyMemory<byte>.Empty, "second"u8.ToArray());
|
||||||
|
block.Write("e.f", ReadOnlyMemory<byte>.Empty, "third"u8.ToArray());
|
||||||
|
block.Flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recover and verify all messages readable
|
||||||
|
using var recovered = MsgBlock.Recover(0, _testDir);
|
||||||
|
recovered.MessageCount.ShouldBe(3UL);
|
||||||
|
recovered.FirstSequence.ShouldBe(1UL);
|
||||||
|
recovered.LastSequence.ShouldBe(3UL);
|
||||||
|
|
||||||
|
var r1 = recovered.Read(1);
|
||||||
|
r1.ShouldNotBeNull();
|
||||||
|
r1.Subject.ShouldBe("a.b");
|
||||||
|
Encoding.UTF8.GetString(r1.Payload.Span).ShouldBe("first");
|
||||||
|
|
||||||
|
var r2 = recovered.Read(2);
|
||||||
|
r2.ShouldNotBeNull();
|
||||||
|
r2.Subject.ShouldBe("c.d");
|
||||||
|
|
||||||
|
var r3 = recovered.Read(3);
|
||||||
|
r3.ShouldNotBeNull();
|
||||||
|
r3.Subject.ShouldBe("e.f");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: recovery with dmap — deleted records still show as null after recovery
|
||||||
|
[Fact]
|
||||||
|
public void Recover_PreservesDeletedState()
|
||||||
|
{
|
||||||
|
// Write messages, delete one, flush and dispose
|
||||||
|
using (var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024))
|
||||||
|
{
|
||||||
|
block.Write("a", ReadOnlyMemory<byte>.Empty, "one"u8.ToArray());
|
||||||
|
block.Write("b", ReadOnlyMemory<byte>.Empty, "two"u8.ToArray());
|
||||||
|
block.Write("c", ReadOnlyMemory<byte>.Empty, "three"u8.ToArray());
|
||||||
|
block.Delete(2);
|
||||||
|
block.Flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recover — seq 2 should still be deleted
|
||||||
|
using var recovered = MsgBlock.Recover(0, _testDir);
|
||||||
|
recovered.MessageCount.ShouldBe(2UL);
|
||||||
|
recovered.DeletedCount.ShouldBe(1UL);
|
||||||
|
|
||||||
|
recovered.Read(1).ShouldNotBeNull();
|
||||||
|
recovered.Read(2).ShouldBeNull();
|
||||||
|
recovered.Read(3).ShouldNotBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: sync.RWMutex on msgBlock — concurrent reads during writes should not crash
|
||||||
|
[Fact]
|
||||||
|
public async Task ConcurrentReads_DuringWrite()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
|
||||||
|
|
||||||
|
// Pre-populate some messages
|
||||||
|
for (var i = 0; i < 10; i++)
|
||||||
|
block.Write($"subj.{i}", ReadOnlyMemory<byte>.Empty, Encoding.UTF8.GetBytes($"msg-{i}"));
|
||||||
|
|
||||||
|
// Run concurrent reads and writes
|
||||||
|
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
|
||||||
|
var exceptions = new List<Exception>();
|
||||||
|
|
||||||
|
var writerTask = Task.Run(() =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.Token.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
block.Write("concurrent", ReadOnlyMemory<byte>.Empty, "data"u8.ToArray());
|
||||||
|
}
|
||||||
|
catch (InvalidOperationException)
|
||||||
|
{
|
||||||
|
// Block sealed — expected eventually
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex) { lock (exceptions) { exceptions.Add(ex); } }
|
||||||
|
});
|
||||||
|
|
||||||
|
var readerTasks = Enumerable.Range(0, 4).Select(t => Task.Run(() =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!cts.Token.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
for (ulong seq = 1; seq <= 10; seq++)
|
||||||
|
_ = block.Read(seq);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex) { lock (exceptions) { exceptions.Add(ex); } }
|
||||||
|
})).ToArray();
|
||||||
|
|
||||||
|
await Task.WhenAll([writerTask, .. readerTasks]).WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
exceptions.ShouldBeEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go: msgBlock first/last — custom firstSequence offsets sequence numbering
|
||||||
|
[Fact]
|
||||||
|
public void Write_WithCustomFirstSequence()
|
||||||
|
{
|
||||||
|
using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024, firstSequence: 100);
|
||||||
|
var s1 = block.Write("x", ReadOnlyMemory<byte>.Empty, "a"u8.ToArray());
|
||||||
|
var s2 = block.Write("y", ReadOnlyMemory<byte>.Empty, "b"u8.ToArray());
|
||||||
|
|
||||||
|
s1.ShouldBe(100UL);
|
||||||
|
s2.ShouldBe(101UL);
|
||||||
|
block.FirstSequence.ShouldBe(100UL);
|
||||||
|
block.LastSequence.ShouldBe(101UL);
|
||||||
|
|
||||||
|
var r = block.Read(100);
|
||||||
|
r.ShouldNotBeNull();
|
||||||
|
r.Subject.ShouldBe("x");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user