feat(storage): rewrite FileStore to use block-based MsgBlock storage
Replace JSONL persistence with real MsgBlock-based block files (.blk). FileStore now acts as a block manager that creates, seals, and rotates MsgBlocks while maintaining an in-memory cache for fast reads/queries. Key changes: - AppendAsync writes transformed payloads to MsgBlock via WriteAt - Block rotation occurs when active block reaches size limit - Recovery scans .blk files and rebuilds in-memory state from records - Legacy JSONL migration: existing messages.jsonl data is automatically converted to block files on first open, then JSONL is deleted - PurgeAsync disposes and deletes all block files - RewriteBlocks rebuilds blocks from cache (used by trim/restore) - InvalidDataException propagates during recovery (wrong encryption key) MsgBlock.WriteAt added to support explicit sequence numbers and timestamps, needed when rewriting blocks with non-contiguous sequences (after removes). Tests updated: - New FileStoreBlockTests.cs with 9 tests for block-specific behavior - JetStreamFileStoreCompressionEncryptionParityTests updated to read FSV1 magic from .blk files instead of messages.jsonl - JetStreamFileStoreDurabilityParityTests updated to verify .blk files instead of index.manifest.json All 3,562 tests pass (3,535 passed + 27 skipped, 0 failures).
This commit is contained in:
@@ -149,7 +149,7 @@ public sealed class MsgBlock : IDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends a message to the block.
|
||||
/// Appends a message to the block with an auto-assigned sequence number.
|
||||
/// </summary>
|
||||
/// <param name="subject">NATS subject.</param>
|
||||
/// <param name="headers">Optional message headers.</param>
|
||||
@@ -199,6 +199,56 @@ public sealed class MsgBlock : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends a message to the block with an explicit sequence number and timestamp.
|
||||
/// Used by FileStore when rewriting blocks from the in-memory cache where
|
||||
/// sequences may have gaps (from prior removals).
|
||||
/// </summary>
|
||||
/// <param name="sequence">Explicit sequence number to assign.</param>
|
||||
/// <param name="subject">NATS subject.</param>
|
||||
/// <param name="headers">Optional message headers.</param>
|
||||
/// <param name="payload">Message body payload.</param>
|
||||
/// <param name="timestamp">Timestamp in Unix nanoseconds.</param>
|
||||
/// <exception cref="InvalidOperationException">Block is sealed.</exception>
|
||||
public void WriteAt(ulong sequence, string subject, ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload, long timestamp)
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
if (_writeOffset >= _maxBytes)
|
||||
throw new InvalidOperationException("Block is sealed; cannot write new messages.");
|
||||
|
||||
var record = new MessageRecord
|
||||
{
|
||||
Sequence = sequence,
|
||||
Subject = subject,
|
||||
Headers = headers,
|
||||
Payload = payload,
|
||||
Timestamp = timestamp,
|
||||
Deleted = false,
|
||||
};
|
||||
|
||||
var encoded = MessageRecord.Encode(record);
|
||||
var offset = _writeOffset;
|
||||
|
||||
RandomAccess.Write(_handle, encoded, offset);
|
||||
_writeOffset = offset + encoded.Length;
|
||||
|
||||
_index[sequence] = (offset, encoded.Length);
|
||||
|
||||
if (_totalWritten == 0)
|
||||
_firstSequence = sequence;
|
||||
|
||||
_lastSequence = sequence;
|
||||
_nextSequence = Math.Max(_nextSequence, sequence + 1);
|
||||
_totalWritten++;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a message by sequence number. Uses positional I/O
|
||||
/// (<see cref="RandomAccess.Read"/>) so concurrent readers don't
|
||||
|
||||
Reference in New Issue
Block a user