2816e8f048
Replace JSONL persistence with real MsgBlock-based block files (.blk). FileStore now acts as a block manager that creates, seals, and rotates MsgBlocks while maintaining an in-memory cache for fast reads/queries. Key changes: - AppendAsync writes transformed payloads to MsgBlock via WriteAt - Block rotation occurs when active block reaches size limit - Recovery scans .blk files and rebuilds in-memory state from records - Legacy JSONL migration: existing messages.jsonl data is automatically converted to block files on first open, then JSONL is deleted - PurgeAsync disposes and deletes all block files - RewriteBlocks rebuilds blocks from cache (used by trim/restore) - InvalidDataException propagates during recovery (wrong encryption key) MsgBlock.WriteAt added to support explicit sequence numbers and timestamps, needed when rewriting blocks with non-contiguous sequences (after removes). Tests updated: - New FileStoreBlockTests.cs with 9 tests for block-specific behavior - JetStreamFileStoreCompressionEncryptionParityTests updated to read FSV1 magic from .blk files instead of messages.jsonl - JetStreamFileStoreDurabilityParityTests updated to verify .blk files instead of index.manifest.json All 3,562 tests pass (3,535 passed + 27 skipped, 0 failures).
59 lines
2.1 KiB
C#
59 lines
2.1 KiB
C#
using System.Text;
|
|
using NATS.Server.JetStream.Storage;
|
|
|
|
namespace NATS.Server.Tests;
|
|
|
|
public class JetStreamFileStoreCompressionEncryptionParityTests
|
|
{
|
|
[Fact]
|
|
public async Task Compression_and_encryption_roundtrip_is_versioned_and_detects_wrong_key_corruption()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), $"nats-js-fs-crypto-{Guid.NewGuid():N}");
|
|
var options = new FileStoreOptions
|
|
{
|
|
Directory = dir,
|
|
EnableCompression = true,
|
|
EnableEncryption = true,
|
|
EncryptionKey = [1, 2, 3, 4],
|
|
};
|
|
|
|
try
|
|
{
|
|
ulong sequence;
|
|
await using (var store = new FileStore(options))
|
|
{
|
|
sequence = await store.AppendAsync("orders.created", Encoding.UTF8.GetBytes("payload"), default);
|
|
var loaded = await store.LoadAsync(sequence, default);
|
|
loaded.ShouldNotBeNull();
|
|
Encoding.UTF8.GetString(loaded.Payload.ToArray()).ShouldBe("payload");
|
|
}
|
|
|
|
// Block-based storage: read the .blk file to verify FSV1 envelope.
|
|
var blkFiles = Directory.GetFiles(dir, "*.blk");
|
|
blkFiles.Length.ShouldBeGreaterThan(0);
|
|
|
|
// Read the first record from the block file and verify FSV1 magic in payload.
|
|
var blkBytes = File.ReadAllBytes(blkFiles[0]);
|
|
var record = MessageRecord.Decode(blkBytes.AsSpan(0, MessageRecord.MeasureRecord(blkBytes)));
|
|
var persisted = record.Payload.ToArray();
|
|
persisted.Take(4).SequenceEqual("FSV1"u8.ToArray()).ShouldBeTrue();
|
|
|
|
Should.Throw<InvalidDataException>(() =>
|
|
{
|
|
_ = new FileStore(new FileStoreOptions
|
|
{
|
|
Directory = dir,
|
|
EnableCompression = true,
|
|
EnableEncryption = true,
|
|
EncryptionKey = [9, 9, 9, 9],
|
|
});
|
|
});
|
|
}
|
|
finally
|
|
{
|
|
if (Directory.Exists(dir))
|
|
Directory.Delete(dir, recursive: true);
|
|
}
|
|
}
|
|
}
|