diff --git a/Directory.Packages.props b/Directory.Packages.props index f9229d7..1949ded 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -26,6 +26,9 @@ + + + diff --git a/src/NATS.Server/JetStream/Storage/S2Codec.cs b/src/NATS.Server/JetStream/Storage/S2Codec.cs new file mode 100644 index 0000000..c827840 --- /dev/null +++ b/src/NATS.Server/JetStream/Storage/S2Codec.cs @@ -0,0 +1,111 @@ +// Reference: golang/nats-server/server/filestore.go +// Go uses S2 (Snappy variant) compression throughout FileStore: +// - msgCompress / msgDecompress (filestore.go ~line 840) +// - compressBlock / decompressBlock for block-level data +// S2 is faster than Deflate and produces comparable ratios for binary payloads. +// IronSnappy provides Snappy-format encode/decode, which is compatible with +// the Go snappy package used by the S2 library for block compression. + +using IronSnappy; + +namespace NATS.Server.JetStream.Storage; + +/// +/// S2/Snappy codec for FileStore payload compression, mirroring the Go +/// implementation which uses github.com/klauspost/compress/s2. +/// +internal static class S2Codec +{ + /// + /// Compresses using Snappy block format. + /// Returns the compressed bytes, which may be longer than the input for + /// very small payloads (Snappy does not guarantee compression for tiny inputs). + /// + public static byte[] Compress(ReadOnlySpan data) + { + if (data.IsEmpty) + return []; + + return Snappy.Encode(data); + } + + /// + /// Decompresses Snappy-compressed . + /// + /// If the data is not valid Snappy. + public static byte[] Decompress(ReadOnlySpan data) + { + if (data.IsEmpty) + return []; + + return Snappy.Decode(data); + } + + /// + /// Compresses only the body portion of , leaving the + /// last bytes uncompressed (appended verbatim). + /// + /// + /// In the Go FileStore the trailing bytes of a stored record can be a raw + /// checksum that is not part of the compressed payload. This helper mirrors + /// that separation (filestore.go msgCompress, where the CRC lives outside + /// the S2 frame). + /// + public static byte[] CompressWithTrailingChecksum(ReadOnlySpan data, int checksumSize) + { + if (checksumSize < 0) + throw new ArgumentOutOfRangeException(nameof(checksumSize)); + + if (data.IsEmpty) + return []; + + if (checksumSize == 0) + return Compress(data); + + if (checksumSize >= data.Length) + { + // Nothing to compress — return a copy as-is (checksum covers everything). + return data.ToArray(); + } + + var body = data[..^checksumSize]; + var checksum = data[^checksumSize..]; + + var compressedBody = Compress(body); + var result = new byte[compressedBody.Length + checksumSize]; + compressedBody.CopyTo(result.AsSpan()); + checksum.CopyTo(result.AsSpan(compressedBody.Length)); + return result; + } + + /// + /// Decompresses only the body portion of , treating + /// the last bytes as a raw (uncompressed) checksum. + /// + public static byte[] DecompressWithTrailingChecksum(ReadOnlySpan data, int checksumSize) + { + if (checksumSize < 0) + throw new ArgumentOutOfRangeException(nameof(checksumSize)); + + if (data.IsEmpty) + return []; + + if (checksumSize == 0) + return Decompress(data); + + if (checksumSize >= data.Length) + { + // Nothing was compressed — return a copy as-is. + return data.ToArray(); + } + + var compressedBody = data[..^checksumSize]; + var checksum = data[^checksumSize..]; + + var decompressedBody = Decompress(compressedBody); + var result = new byte[decompressedBody.Length + checksumSize]; + decompressedBody.CopyTo(result.AsSpan()); + checksum.CopyTo(result.AsSpan(decompressedBody.Length)); + return result; + } +} diff --git a/src/NATS.Server/NATS.Server.csproj b/src/NATS.Server/NATS.Server.csproj index 390f283..11bedd2 100644 --- a/src/NATS.Server/NATS.Server.csproj +++ b/src/NATS.Server/NATS.Server.csproj @@ -4,6 +4,7 @@ + diff --git a/tests/NATS.Server.Tests/JetStream/Storage/AeadEncryptorTests.cs b/tests/NATS.Server.Tests/JetStream/Storage/AeadEncryptorTests.cs new file mode 100644 index 0000000..ed80da0 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStream/Storage/AeadEncryptorTests.cs @@ -0,0 +1,200 @@ +// Reference: golang/nats-server/server/filestore.go +// Go FileStore uses ChaCha20-Poly1305 and AES-256-GCM for block encryption: +// - StoreCipher=ChaCha → ChaCha20-Poly1305 (filestore.go ~line 300) +// - StoreCipher=AES → AES-256-GCM (filestore.go ~line 310) +// Wire format: [12:nonce][16:tag][N:ciphertext] + +using System.Security.Cryptography; +using NATS.Server.JetStream.Storage; + +namespace NATS.Server.Tests.JetStream.Storage; + +public sealed class AeadEncryptorTests +{ + // 32-byte (256-bit) test key. + private static byte[] TestKey => "nats-aead-test-key-for-32bytes!!"u8.ToArray(); + + // Go: TestFileStoreEncrypted server/filestore_test.go:4204 (ChaCha permutation) + [Fact] + public void ChaCha_encrypt_decrypt_round_trips() + { + var plaintext = "Hello, ChaCha20-Poly1305!"u8.ToArray(); + var key = TestKey; + + var encrypted = AeadEncryptor.Encrypt(plaintext, key, StoreCipher.ChaCha); + var decrypted = AeadEncryptor.Decrypt(encrypted, key, StoreCipher.ChaCha); + + decrypted.ShouldBe(plaintext); + } + + // Go: TestFileStoreEncrypted server/filestore_test.go:4204 (AES permutation) + [Fact] + public void AesGcm_encrypt_decrypt_round_trips() + { + var plaintext = "Hello, AES-256-GCM!"u8.ToArray(); + var key = TestKey; + + var encrypted = AeadEncryptor.Encrypt(plaintext, key, StoreCipher.Aes); + var decrypted = AeadEncryptor.Decrypt(encrypted, key, StoreCipher.Aes); + + decrypted.ShouldBe(plaintext); + } + + [Fact] + public void ChaCha_empty_plaintext_round_trips() + { + var encrypted = AeadEncryptor.Encrypt([], TestKey, StoreCipher.ChaCha); + var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha); + decrypted.ShouldBeEmpty(); + } + + [Fact] + public void AesGcm_empty_plaintext_round_trips() + { + var encrypted = AeadEncryptor.Encrypt([], TestKey, StoreCipher.Aes); + var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.Aes); + decrypted.ShouldBeEmpty(); + } + + [Fact] + public void ChaCha_encrypted_blob_has_correct_overhead() + { + var plaintext = new byte[100]; + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + + // Expected: nonce (12) + tag (16) + ciphertext (100) = 128 + encrypted.Length.ShouldBe(AeadEncryptor.NonceSize + AeadEncryptor.TagSize + plaintext.Length); + } + + [Fact] + public void AesGcm_encrypted_blob_has_correct_overhead() + { + var plaintext = new byte[100]; + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes); + + // Expected: nonce (12) + tag (16) + ciphertext (100) = 128 + encrypted.Length.ShouldBe(AeadEncryptor.NonceSize + AeadEncryptor.TagSize + plaintext.Length); + } + + // Go: TestFileStoreRestoreEncryptedWithNoKeyFuncFails filestore_test.go:5134 + [Fact] + public void ChaCha_wrong_key_throws_CryptographicException() + { + var plaintext = "secret data"u8.ToArray(); + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + + var wrongKey = "wrong-key-wrong-key-wrong-key!!!"u8.ToArray(); + Should.Throw( + () => AeadEncryptor.Decrypt(encrypted, wrongKey, StoreCipher.ChaCha)); + } + + [Fact] + public void AesGcm_wrong_key_throws_CryptographicException() + { + var plaintext = "secret data"u8.ToArray(); + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes); + + var wrongKey = "wrong-key-wrong-key-wrong-key!!!"u8.ToArray(); + Should.Throw( + () => AeadEncryptor.Decrypt(encrypted, wrongKey, StoreCipher.Aes)); + } + + [Fact] + public void ChaCha_tampered_ciphertext_throws_CryptographicException() + { + var plaintext = "tamper me"u8.ToArray(); + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + + // Flip a bit in the ciphertext portion (after nonce+tag). + encrypted[^1] ^= 0xFF; + + Should.Throw( + () => AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha)); + } + + [Fact] + public void AesGcm_tampered_ciphertext_throws_CryptographicException() + { + var plaintext = "tamper me"u8.ToArray(); + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes); + + // Flip a bit in the ciphertext portion. + encrypted[^1] ^= 0xFF; + + Should.Throw( + () => AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.Aes)); + } + + [Fact] + public void ChaCha_tampered_tag_throws_CryptographicException() + { + var plaintext = "tamper tag"u8.ToArray(); + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + + // Flip a bit in the tag (bytes 12-27). + encrypted[AeadEncryptor.NonceSize] ^= 0xFF; + + Should.Throw( + () => AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha)); + } + + [Fact] + public void Key_shorter_than_32_bytes_throws_ArgumentException() + { + var shortKey = new byte[16]; + Should.Throw( + () => AeadEncryptor.Encrypt("data"u8.ToArray(), shortKey, StoreCipher.ChaCha)); + } + + [Fact] + public void Key_longer_than_32_bytes_throws_ArgumentException() + { + var longKey = new byte[64]; + Should.Throw( + () => AeadEncryptor.Encrypt("data"u8.ToArray(), longKey, StoreCipher.ChaCha)); + } + + [Fact] + public void Decrypt_data_too_short_throws_ArgumentException() + { + // Less than nonce (12) + tag (16) = 28 bytes minimum. + var tooShort = new byte[10]; + Should.Throw( + () => AeadEncryptor.Decrypt(tooShort, TestKey, StoreCipher.ChaCha)); + } + + [Fact] + public void ChaCha_each_encrypt_produces_different_ciphertext() + { + // Nonce is random per call so ciphertexts differ even for same plaintext. + var plaintext = "same plaintext"u8.ToArray(); + var enc1 = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + var enc2 = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + + enc1.ShouldNotBe(enc2); + } + + [Fact] + public void ChaCha_large_payload_round_trips() + { + var plaintext = new byte[64 * 1024]; // 64 KB + Random.Shared.NextBytes(plaintext); + + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha); + var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha); + + decrypted.ShouldBe(plaintext); + } + + [Fact] + public void AesGcm_large_payload_round_trips() + { + var plaintext = new byte[64 * 1024]; // 64 KB + Random.Shared.NextBytes(plaintext); + + var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes); + var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.Aes); + + decrypted.ShouldBe(plaintext); + } +} diff --git a/tests/NATS.Server.Tests/JetStream/Storage/FileStoreV2Tests.cs b/tests/NATS.Server.Tests/JetStream/Storage/FileStoreV2Tests.cs new file mode 100644 index 0000000..f1a28c5 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStream/Storage/FileStoreV2Tests.cs @@ -0,0 +1,475 @@ +// Reference: golang/nats-server/server/filestore_test.go +// Tests ported from: TestFileStoreEncrypted (AES + ChaCha permutations), +// testFileStoreAllPermutations (S2 + cipher cross product), +// TestFileStoreS2Compression (filestore_test.go:4180), +// TestFileStoreEncryptedChaChaCipher (filestore_test.go:4250) +// +// The Go server runs testFileStoreAllPermutations which exercises all +// combinations of {NoCompression, S2Compression} x {NoCipher, ChaCha, AES}. +// These tests cover the FSV2 envelope path added in Task 4. + +using System.Text; +using NATS.Server.JetStream.Storage; + +namespace NATS.Server.Tests.JetStream.Storage; + +public sealed class FileStoreV2Tests : IDisposable +{ + private readonly string _dir; + + public FileStoreV2Tests() + { + _dir = Path.Combine(Path.GetTempPath(), $"nats-js-fs-v2-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_dir); + } + + public void Dispose() + { + if (Directory.Exists(_dir)) + Directory.Delete(_dir, recursive: true); + } + + // 32-byte key for AEAD ciphers. + private static byte[] Key32 => "nats-v2-test-key-exactly-32-bytes"u8[..32].ToArray(); + + private FileStore CreateStore(string sub, FileStoreOptions options) + { + options.Directory = Path.Combine(_dir, sub); + return new FileStore(options); + } + + // ------------------------------------------------------------------------- + // S2 compression (no encryption) — FSV2 envelope + // ------------------------------------------------------------------------- + + // Go: TestFileStoreS2Compression filestore_test.go:4180 + [Fact] + public async Task S2_compression_store_and_load() + { + await using var store = CreateStore("s2-basic", new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + }); + + var payload = "Hello, S2!"u8.ToArray(); + for (var i = 1; i <= 10; i++) + { + var seq = await store.AppendAsync("foo", payload, default); + seq.ShouldBe((ulong)i); + } + + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)10); + + var msg = await store.LoadAsync(5, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(payload); + } + + [Fact] + public async Task S2_compression_store_and_recover() + { + const string sub = "s2-recover"; + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + })) + { + for (var i = 0; i < 50; i++) + await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"msg-{i:D4}"), default); + } + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + })) + { + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)50); + + var msg = await store.LoadAsync(25, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("msg-0024")); + } + } + + // ------------------------------------------------------------------------- + // ChaCha20-Poly1305 encryption (no compression) — FSV2 envelope + // ------------------------------------------------------------------------- + + // Go: TestFileStoreEncryptedChaChaCipher filestore_test.go:4250 + [Fact] + public async Task ChaCha_encryption_store_and_load() + { + await using var store = CreateStore("chacha-basic", new FileStoreOptions + { + Cipher = StoreCipher.ChaCha, + EncryptionKey = Key32, + }); + + var payload = "aes ftw"u8.ToArray(); + for (var i = 0; i < 50; i++) + await store.AppendAsync("foo", payload, default); + + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)50); + + var msg = await store.LoadAsync(10, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(payload); + } + + [Fact] + public async Task ChaCha_encryption_store_and_recover() + { + const string sub = "chacha-recover"; + var key = Key32; + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Cipher = StoreCipher.ChaCha, + EncryptionKey = key, + })) + { + for (var i = 0; i < 50; i++) + await store.AppendAsync("foo", "chacha secret"u8.ToArray(), default); + } + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Cipher = StoreCipher.ChaCha, + EncryptionKey = key, + })) + { + var msg = await store.LoadAsync(10, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe("chacha secret"u8.ToArray()); + + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)50); + } + } + + // ------------------------------------------------------------------------- + // AES-256-GCM encryption (no compression) — FSV2 envelope + // ------------------------------------------------------------------------- + + // Go: TestFileStoreEncrypted (AES permutation) filestore_test.go:4204 + [Fact] + public async Task AesGcm_encryption_store_and_load() + { + await using var store = CreateStore("aes-basic", new FileStoreOptions + { + Cipher = StoreCipher.Aes, + EncryptionKey = Key32, + }); + + var payload = "aes-gcm secret"u8.ToArray(); + for (var i = 0; i < 50; i++) + await store.AppendAsync("foo", payload, default); + + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)50); + + var msg = await store.LoadAsync(25, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(payload); + } + + [Fact] + public async Task AesGcm_encryption_store_and_recover() + { + const string sub = "aes-recover"; + var key = Key32; + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Cipher = StoreCipher.Aes, + EncryptionKey = key, + })) + { + for (var i = 0; i < 50; i++) + await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"aes-{i:D4}"), default); + } + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Cipher = StoreCipher.Aes, + EncryptionKey = key, + })) + { + var msg = await store.LoadAsync(30, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("aes-0029")); + + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)50); + } + } + + // ------------------------------------------------------------------------- + // S2 + ChaCha combined — FSV2 envelope + // ------------------------------------------------------------------------- + + [Fact] + public async Task S2_and_ChaCha_combined_round_trip() + { + await using var store = CreateStore("s2-chacha", new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + Cipher = StoreCipher.ChaCha, + EncryptionKey = Key32, + }); + + var payload = "S2 + ChaCha combined payload"u8.ToArray(); + for (var i = 0; i < 20; i++) + await store.AppendAsync("foo", payload, default); + + for (ulong i = 1; i <= 20; i++) + { + var msg = await store.LoadAsync(i, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(payload); + } + } + + [Fact] + public async Task S2_and_AesGcm_combined_round_trip() + { + await using var store = CreateStore("s2-aes", new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + Cipher = StoreCipher.Aes, + EncryptionKey = Key32, + }); + + var payload = "S2 + AES-GCM combined payload"u8.ToArray(); + for (var i = 0; i < 20; i++) + await store.AppendAsync("bar", payload, default); + + for (ulong i = 1; i <= 20; i++) + { + var msg = await store.LoadAsync(i, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(payload); + } + } + + [Fact] + public async Task S2_and_ChaCha_combined_store_and_recover() + { + const string sub = "s2-chacha-recover"; + var key = Key32; + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + Cipher = StoreCipher.ChaCha, + EncryptionKey = key, + })) + { + for (var i = 0; i < 40; i++) + await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"s2-chacha-{i:D3}"), default); + } + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + Cipher = StoreCipher.ChaCha, + EncryptionKey = key, + })) + { + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)40); + + var msg = await store.LoadAsync(20, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("s2-chacha-019")); + } + } + + // ------------------------------------------------------------------------- + // Legacy FSV1 data still readable after upgrade + // ------------------------------------------------------------------------- + + // Go: backward-compat requirement — existing FSV1 files must still load + [Fact] + public async Task Legacy_FSV1_deflate_compression_still_readable() + { + const string sub = "fsv1-compress-legacy"; + + // Write with legacy Deflate (EnableCompression=true, no enum set). + await using (var store = CreateStore(sub, new FileStoreOptions + { + EnableCompression = true, + })) + { + await store.AppendAsync("foo", "legacy deflate"u8.ToArray(), default); + } + + // Reopen with same options — must read back correctly. + await using (var store = CreateStore(sub, new FileStoreOptions + { + EnableCompression = true, + })) + { + var msg = await store.LoadAsync(1, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe("legacy deflate"u8.ToArray()); + } + } + + [Fact] + public async Task Legacy_FSV1_xor_encryption_still_readable() + { + const string sub = "fsv1-encrypt-legacy"; + var key = "legacy-xor-key-16bytes!"u8.ToArray(); + + // Write with legacy XOR (EnableEncryption=true, no cipher enum set). + await using (var store = CreateStore(sub, new FileStoreOptions + { + EnableEncryption = true, + EncryptionKey = key, + })) + { + await store.AppendAsync("foo", "legacy xor encrypted"u8.ToArray(), default); + } + + // Reopen with same options — must read back correctly. + await using (var store = CreateStore(sub, new FileStoreOptions + { + EnableEncryption = true, + EncryptionKey = key, + })) + { + var msg = await store.LoadAsync(1, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe("legacy xor encrypted"u8.ToArray()); + } + } + + // ------------------------------------------------------------------------- + // All 6 permutations: {NoCipher, ChaCha, AesGcm} x {NoCompression, S2} + // Go: testFileStoreAllPermutations (filestore_test.go:98) + // ------------------------------------------------------------------------- + + [Theory] + [InlineData(StoreCipher.NoCipher, StoreCompression.NoCompression)] + [InlineData(StoreCipher.NoCipher, StoreCompression.S2Compression)] + [InlineData(StoreCipher.ChaCha, StoreCompression.NoCompression)] + [InlineData(StoreCipher.ChaCha, StoreCompression.S2Compression)] + [InlineData(StoreCipher.Aes, StoreCompression.NoCompression)] + [InlineData(StoreCipher.Aes, StoreCompression.S2Compression)] + public async Task All_permutations_store_and_load(StoreCipher cipher, StoreCompression compression) + { + var sub = $"perm-{cipher}-{compression}"; + var key = cipher == StoreCipher.NoCipher ? null : Key32; + + var payload = Encoding.UTF8.GetBytes($"payload for {cipher}+{compression}"); + + await using var store = CreateStore(sub, new FileStoreOptions + { + Cipher = cipher, + Compression = compression, + EncryptionKey = key, + }); + + for (var i = 0; i < 10; i++) + await store.AppendAsync("test", payload, default); + + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)10); + + for (ulong i = 1; i <= 10; i++) + { + var msg = await store.LoadAsync(i, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(payload); + } + } + + [Theory] + [InlineData(StoreCipher.NoCipher, StoreCompression.NoCompression)] + [InlineData(StoreCipher.NoCipher, StoreCompression.S2Compression)] + [InlineData(StoreCipher.ChaCha, StoreCompression.NoCompression)] + [InlineData(StoreCipher.ChaCha, StoreCompression.S2Compression)] + [InlineData(StoreCipher.Aes, StoreCompression.NoCompression)] + [InlineData(StoreCipher.Aes, StoreCompression.S2Compression)] + public async Task All_permutations_store_and_recover(StoreCipher cipher, StoreCompression compression) + { + var sub = $"perm-recover-{cipher}-{compression}"; + var key = cipher == StoreCipher.NoCipher ? null : Key32; + + // Write phase. + await using (var store = CreateStore(sub, new FileStoreOptions { Cipher = cipher, Compression = compression, EncryptionKey = key })) + { + for (var i = 0; i < 20; i++) + await store.AppendAsync("x", Encoding.UTF8.GetBytes($"msg-{i:D3}"), default); + } + + // Reopen and verify. + await using (var store = CreateStore(sub, new FileStoreOptions { Cipher = cipher, Compression = compression, EncryptionKey = key })) + { + var state = await store.GetStateAsync(default); + state.Messages.ShouldBe((ulong)20); + + var msg = await store.LoadAsync(10, default); + msg.ShouldNotBeNull(); + msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("msg-009")); + } + } + + // ------------------------------------------------------------------------- + // FSV2 data is not plaintext on disk + // ------------------------------------------------------------------------- + + [Fact] + public async Task S2_data_differs_from_plaintext_on_disk() + { + var sub = "s2-disk"; + var dir = Path.Combine(_dir, sub); + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Compression = StoreCompression.S2Compression, + })) + { + await store.AppendAsync("foo", "AAAAAAAAAAAAAAAAAAAAAAAAA"u8.ToArray(), default); + } + + var dataFile = Path.Combine(dir, "messages.jsonl"); + if (File.Exists(dataFile)) + { + var raw = File.ReadAllText(dataFile); + // The payload is base64-encoded in the JSONL file. + // "FSV2" (0x46 0x53 0x56 0x32) base64-encodes to "RlNWMg". + // FSV1 encodes as "RlNWMQ". Verify FSV2 is used, not FSV1. + raw.ShouldContain("RlNWMg"); + raw.ShouldNotContain("RlNWMQ"); + } + } + + [Fact] + public async Task ChaCha_encrypted_data_not_plaintext_on_disk() + { + var sub = "chacha-disk"; + var dir = Path.Combine(_dir, sub); + + await using (var store = CreateStore(sub, new FileStoreOptions + { + Cipher = StoreCipher.ChaCha, + EncryptionKey = Key32, + })) + { + await store.AppendAsync("foo", "THIS IS SENSITIVE DATA"u8.ToArray(), default); + } + + var dataFile = Path.Combine(dir, "messages.jsonl"); + if (File.Exists(dataFile)) + { + var raw = File.ReadAllText(dataFile); + raw.ShouldNotContain("THIS IS SENSITIVE DATA"); + } + } +} diff --git a/tests/NATS.Server.Tests/JetStream/Storage/S2CodecTests.cs b/tests/NATS.Server.Tests/JetStream/Storage/S2CodecTests.cs new file mode 100644 index 0000000..f6b8a81 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStream/Storage/S2CodecTests.cs @@ -0,0 +1,154 @@ +// Reference: golang/nats-server/server/filestore.go +// Go uses S2/Snappy compression throughout FileStore: +// - msgCompress / msgDecompress (filestore.go ~line 840) +// - compressBlock / decompressBlock for block-level data +// These tests verify the .NET S2Codec helper used in the FSV2 envelope path. + +using NATS.Server.JetStream.Storage; +using System.Text; + +namespace NATS.Server.Tests.JetStream.Storage; + +public sealed class S2CodecTests +{ + // Go: TestFileStoreBasics (S2 permutation) filestore_test.go:86 + [Fact] + public void Compress_then_decompress_round_trips() + { + var original = "Hello, NATS JetStream S2 compression!"u8.ToArray(); + + var compressed = S2Codec.Compress(original); + var restored = S2Codec.Decompress(compressed); + + restored.ShouldBe(original); + } + + [Fact] + public void Compress_empty_returns_empty() + { + var compressed = S2Codec.Compress([]); + compressed.ShouldBeEmpty(); + } + + [Fact] + public void Decompress_empty_returns_empty() + { + var decompressed = S2Codec.Decompress([]); + decompressed.ShouldBeEmpty(); + } + + [Fact] + public void Compress_large_highly_compressible_payload() + { + // 1 MB of repeated 'A' — highly compressible. + var original = new byte[1024 * 1024]; + Array.Fill(original, (byte)'A'); + + var compressed = S2Codec.Compress(original); + var restored = S2Codec.Decompress(compressed); + + // S2/Snappy should compress this well. + compressed.Length.ShouldBeLessThan(original.Length); + restored.ShouldBe(original); + } + + [Fact] + public void Compress_large_incompressible_payload_round_trips() + { + // 1 MB of random data — not compressible, but must still round-trip. + var original = new byte[1024 * 1024]; + Random.Shared.NextBytes(original); + + var compressed = S2Codec.Compress(original); + var restored = S2Codec.Decompress(compressed); + + restored.ShouldBe(original); + } + + [Fact] + public void Compress_single_byte_round_trips() + { + var original = new byte[] { 0x42 }; + var compressed = S2Codec.Compress(original); + var restored = S2Codec.Decompress(compressed); + restored.ShouldBe(original); + } + + [Fact] + public void Compress_binary_all_byte_values_round_trips() + { + var original = new byte[256]; + for (var i = 0; i < 256; i++) + original[i] = (byte)i; + + var compressed = S2Codec.Compress(original); + var restored = S2Codec.Decompress(compressed); + restored.ShouldBe(original); + } + + // Go: msgCompress with trailing CRC (filestore.go ~line 840) — the checksum + // lives outside the S2 frame so only the body is compressed. + [Fact] + public void CompressWithTrailingChecksum_preserves_last_n_bytes_uncompressed() + { + const int checksumSize = 8; + var body = Encoding.UTF8.GetBytes("NATS payload body that should be compressed"); + var checksum = new byte[checksumSize]; + Random.Shared.NextBytes(checksum); + + var input = body.Concat(checksum).ToArray(); + + var result = S2Codec.CompressWithTrailingChecksum(input, checksumSize); + + // Last checksumSize bytes must be verbatim. + var resultChecksum = result[^checksumSize..]; + resultChecksum.ShouldBe(checksum); + } + + [Fact] + public void CompressWithTrailingChecksum_zero_checksum_compresses_all() + { + var data = "Hello, no checksum"u8.ToArray(); + var result = S2Codec.CompressWithTrailingChecksum(data, 0); + var restored = S2Codec.Decompress(result); + restored.ShouldBe(data); + } + + [Fact] + public void DecompressWithTrailingChecksum_round_trips() + { + const int checksumSize = 8; + var body = new byte[512]; + Random.Shared.NextBytes(body); + var checksum = new byte[checksumSize]; + Random.Shared.NextBytes(checksum); + + var input = body.Concat(checksum).ToArray(); + + var compressed = S2Codec.CompressWithTrailingChecksum(input, checksumSize); + var restored = S2Codec.DecompressWithTrailingChecksum(compressed, checksumSize); + + restored.ShouldBe(input); + } + + [Fact] + public void CompressWithTrailingChecksum_empty_input_returns_empty() + { + var result = S2Codec.CompressWithTrailingChecksum([], 0); + result.ShouldBeEmpty(); + } + + [Fact] + public void CompressWithTrailingChecksum_negative_size_throws() + { + Should.Throw( + () => S2Codec.CompressWithTrailingChecksum([1, 2, 3], -1)); + } + + [Fact] + public void DecompressWithTrailingChecksum_negative_size_throws() + { + Should.Throw( + () => S2Codec.DecompressWithTrailingChecksum([1, 2, 3], -1)); + } +}