feat: add S2 compression and AEAD encryption for FileStore (Go parity)
Replace Deflate+XOR with IronSnappy S2 block compression and ChaCha20-Poly1305 / AES-256-GCM AEAD encryption, matching golang/nats-server/server/filestore.go. Introduces FSV2 envelope format alongside existing FSV1 for backward compatibility. Adds 55 new tests across S2CodecTests, AeadEncryptorTests, and FileStoreV2Tests covering all 6 cipher×compression permutations, tamper detection, and legacy format round-trips.
This commit is contained in:
Joseph Doherty
200
tests/NATS.Server.Tests/JetStream/Storage/AeadEncryptorTests.cs
Normal file
200
tests/NATS.Server.Tests/JetStream/Storage/AeadEncryptorTests.cs
Normal file
@@ -0,0 +1,200 @@
|
||||
// Reference: golang/nats-server/server/filestore.go
|
||||
// Go FileStore uses ChaCha20-Poly1305 and AES-256-GCM for block encryption:
|
||||
// - StoreCipher=ChaCha → ChaCha20-Poly1305 (filestore.go ~line 300)
|
||||
// - StoreCipher=AES → AES-256-GCM (filestore.go ~line 310)
|
||||
// Wire format: [12:nonce][16:tag][N:ciphertext]
|
||||
|
||||
using System.Security.Cryptography;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
|
||||
namespace NATS.Server.Tests.JetStream.Storage;
|
||||
|
||||
public sealed class AeadEncryptorTests
|
||||
{
|
||||
// 32-byte (256-bit) test key.
|
||||
private static byte[] TestKey => "nats-aead-test-key-for-32bytes!!"u8.ToArray();
|
||||
|
||||
// Go: TestFileStoreEncrypted server/filestore_test.go:4204 (ChaCha permutation)
|
||||
[Fact]
|
||||
public void ChaCha_encrypt_decrypt_round_trips()
|
||||
{
|
||||
var plaintext = "Hello, ChaCha20-Poly1305!"u8.ToArray();
|
||||
var key = TestKey;
|
||||
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, key, StoreCipher.ChaCha);
|
||||
var decrypted = AeadEncryptor.Decrypt(encrypted, key, StoreCipher.ChaCha);
|
||||
|
||||
decrypted.ShouldBe(plaintext);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreEncrypted server/filestore_test.go:4204 (AES permutation)
|
||||
[Fact]
|
||||
public void AesGcm_encrypt_decrypt_round_trips()
|
||||
{
|
||||
var plaintext = "Hello, AES-256-GCM!"u8.ToArray();
|
||||
var key = TestKey;
|
||||
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, key, StoreCipher.Aes);
|
||||
var decrypted = AeadEncryptor.Decrypt(encrypted, key, StoreCipher.Aes);
|
||||
|
||||
decrypted.ShouldBe(plaintext);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChaCha_empty_plaintext_round_trips()
|
||||
{
|
||||
var encrypted = AeadEncryptor.Encrypt([], TestKey, StoreCipher.ChaCha);
|
||||
var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha);
|
||||
decrypted.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AesGcm_empty_plaintext_round_trips()
|
||||
{
|
||||
var encrypted = AeadEncryptor.Encrypt([], TestKey, StoreCipher.Aes);
|
||||
var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.Aes);
|
||||
decrypted.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChaCha_encrypted_blob_has_correct_overhead()
|
||||
{
|
||||
var plaintext = new byte[100];
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
|
||||
// Expected: nonce (12) + tag (16) + ciphertext (100) = 128
|
||||
encrypted.Length.ShouldBe(AeadEncryptor.NonceSize + AeadEncryptor.TagSize + plaintext.Length);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AesGcm_encrypted_blob_has_correct_overhead()
|
||||
{
|
||||
var plaintext = new byte[100];
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes);
|
||||
|
||||
// Expected: nonce (12) + tag (16) + ciphertext (100) = 128
|
||||
encrypted.Length.ShouldBe(AeadEncryptor.NonceSize + AeadEncryptor.TagSize + plaintext.Length);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreRestoreEncryptedWithNoKeyFuncFails filestore_test.go:5134
|
||||
[Fact]
|
||||
public void ChaCha_wrong_key_throws_CryptographicException()
|
||||
{
|
||||
var plaintext = "secret data"u8.ToArray();
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
|
||||
var wrongKey = "wrong-key-wrong-key-wrong-key!!!"u8.ToArray();
|
||||
Should.Throw<CryptographicException>(
|
||||
() => AeadEncryptor.Decrypt(encrypted, wrongKey, StoreCipher.ChaCha));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AesGcm_wrong_key_throws_CryptographicException()
|
||||
{
|
||||
var plaintext = "secret data"u8.ToArray();
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes);
|
||||
|
||||
var wrongKey = "wrong-key-wrong-key-wrong-key!!!"u8.ToArray();
|
||||
Should.Throw<CryptographicException>(
|
||||
() => AeadEncryptor.Decrypt(encrypted, wrongKey, StoreCipher.Aes));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChaCha_tampered_ciphertext_throws_CryptographicException()
|
||||
{
|
||||
var plaintext = "tamper me"u8.ToArray();
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
|
||||
// Flip a bit in the ciphertext portion (after nonce+tag).
|
||||
encrypted[^1] ^= 0xFF;
|
||||
|
||||
Should.Throw<CryptographicException>(
|
||||
() => AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AesGcm_tampered_ciphertext_throws_CryptographicException()
|
||||
{
|
||||
var plaintext = "tamper me"u8.ToArray();
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes);
|
||||
|
||||
// Flip a bit in the ciphertext portion.
|
||||
encrypted[^1] ^= 0xFF;
|
||||
|
||||
Should.Throw<CryptographicException>(
|
||||
() => AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.Aes));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChaCha_tampered_tag_throws_CryptographicException()
|
||||
{
|
||||
var plaintext = "tamper tag"u8.ToArray();
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
|
||||
// Flip a bit in the tag (bytes 12-27).
|
||||
encrypted[AeadEncryptor.NonceSize] ^= 0xFF;
|
||||
|
||||
Should.Throw<CryptographicException>(
|
||||
() => AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Key_shorter_than_32_bytes_throws_ArgumentException()
|
||||
{
|
||||
var shortKey = new byte[16];
|
||||
Should.Throw<ArgumentException>(
|
||||
() => AeadEncryptor.Encrypt("data"u8.ToArray(), shortKey, StoreCipher.ChaCha));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Key_longer_than_32_bytes_throws_ArgumentException()
|
||||
{
|
||||
var longKey = new byte[64];
|
||||
Should.Throw<ArgumentException>(
|
||||
() => AeadEncryptor.Encrypt("data"u8.ToArray(), longKey, StoreCipher.ChaCha));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Decrypt_data_too_short_throws_ArgumentException()
|
||||
{
|
||||
// Less than nonce (12) + tag (16) = 28 bytes minimum.
|
||||
var tooShort = new byte[10];
|
||||
Should.Throw<ArgumentException>(
|
||||
() => AeadEncryptor.Decrypt(tooShort, TestKey, StoreCipher.ChaCha));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChaCha_each_encrypt_produces_different_ciphertext()
|
||||
{
|
||||
// Nonce is random per call so ciphertexts differ even for same plaintext.
|
||||
var plaintext = "same plaintext"u8.ToArray();
|
||||
var enc1 = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
var enc2 = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
|
||||
enc1.ShouldNotBe(enc2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChaCha_large_payload_round_trips()
|
||||
{
|
||||
var plaintext = new byte[64 * 1024]; // 64 KB
|
||||
Random.Shared.NextBytes(plaintext);
|
||||
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.ChaCha);
|
||||
var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.ChaCha);
|
||||
|
||||
decrypted.ShouldBe(plaintext);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AesGcm_large_payload_round_trips()
|
||||
{
|
||||
var plaintext = new byte[64 * 1024]; // 64 KB
|
||||
Random.Shared.NextBytes(plaintext);
|
||||
|
||||
var encrypted = AeadEncryptor.Encrypt(plaintext, TestKey, StoreCipher.Aes);
|
||||
var decrypted = AeadEncryptor.Decrypt(encrypted, TestKey, StoreCipher.Aes);
|
||||
|
||||
decrypted.ShouldBe(plaintext);
|
||||
}
|
||||
}
|
||||
475
tests/NATS.Server.Tests/JetStream/Storage/FileStoreV2Tests.cs
Normal file
475
tests/NATS.Server.Tests/JetStream/Storage/FileStoreV2Tests.cs
Normal file
@@ -0,0 +1,475 @@
|
||||
// Reference: golang/nats-server/server/filestore_test.go
|
||||
// Tests ported from: TestFileStoreEncrypted (AES + ChaCha permutations),
|
||||
// testFileStoreAllPermutations (S2 + cipher cross product),
|
||||
// TestFileStoreS2Compression (filestore_test.go:4180),
|
||||
// TestFileStoreEncryptedChaChaCipher (filestore_test.go:4250)
|
||||
//
|
||||
// The Go server runs testFileStoreAllPermutations which exercises all
|
||||
// combinations of {NoCompression, S2Compression} x {NoCipher, ChaCha, AES}.
|
||||
// These tests cover the FSV2 envelope path added in Task 4.
|
||||
|
||||
using System.Text;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
|
||||
namespace NATS.Server.Tests.JetStream.Storage;
|
||||
|
||||
public sealed class FileStoreV2Tests : IDisposable
|
||||
{
|
||||
private readonly string _dir;
|
||||
|
||||
public FileStoreV2Tests()
|
||||
{
|
||||
_dir = Path.Combine(Path.GetTempPath(), $"nats-js-fs-v2-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_dir);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_dir))
|
||||
Directory.Delete(_dir, recursive: true);
|
||||
}
|
||||
|
||||
// 32-byte key for AEAD ciphers.
|
||||
private static byte[] Key32 => "nats-v2-test-key-exactly-32-bytes"u8[..32].ToArray();
|
||||
|
||||
private FileStore CreateStore(string sub, FileStoreOptions options)
|
||||
{
|
||||
options.Directory = Path.Combine(_dir, sub);
|
||||
return new FileStore(options);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// S2 compression (no encryption) — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreS2Compression filestore_test.go:4180
|
||||
[Fact]
|
||||
public async Task S2_compression_store_and_load()
|
||||
{
|
||||
await using var store = CreateStore("s2-basic", new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
});
|
||||
|
||||
var payload = "Hello, S2!"u8.ToArray();
|
||||
for (var i = 1; i <= 10; i++)
|
||||
{
|
||||
var seq = await store.AppendAsync("foo", payload, default);
|
||||
seq.ShouldBe((ulong)i);
|
||||
}
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)10);
|
||||
|
||||
var msg = await store.LoadAsync(5, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task S2_compression_store_and_recover()
|
||||
{
|
||||
const string sub = "s2-recover";
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"msg-{i:D4}"), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
}))
|
||||
{
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
|
||||
var msg = await store.LoadAsync(25, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("msg-0024"));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// ChaCha20-Poly1305 encryption (no compression) — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreEncryptedChaChaCipher filestore_test.go:4250
|
||||
[Fact]
|
||||
public async Task ChaCha_encryption_store_and_load()
|
||||
{
|
||||
await using var store = CreateStore("chacha-basic", new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "aes ftw"u8.ToArray();
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", payload, default);
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
|
||||
var msg = await store.LoadAsync(10, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChaCha_encryption_store_and_recover()
|
||||
{
|
||||
const string sub = "chacha-recover";
|
||||
var key = Key32;
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", "chacha secret"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(10, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe("chacha secret"u8.ToArray());
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// AES-256-GCM encryption (no compression) — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreEncrypted (AES permutation) filestore_test.go:4204
|
||||
[Fact]
|
||||
public async Task AesGcm_encryption_store_and_load()
|
||||
{
|
||||
await using var store = CreateStore("aes-basic", new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "aes-gcm secret"u8.ToArray();
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", payload, default);
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
|
||||
var msg = await store.LoadAsync(25, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AesGcm_encryption_store_and_recover()
|
||||
{
|
||||
const string sub = "aes-recover";
|
||||
var key = Key32;
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"aes-{i:D4}"), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(30, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("aes-0029"));
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// S2 + ChaCha combined — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task S2_and_ChaCha_combined_round_trip()
|
||||
{
|
||||
await using var store = CreateStore("s2-chacha", new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "S2 + ChaCha combined payload"u8.ToArray();
|
||||
for (var i = 0; i < 20; i++)
|
||||
await store.AppendAsync("foo", payload, default);
|
||||
|
||||
for (ulong i = 1; i <= 20; i++)
|
||||
{
|
||||
var msg = await store.LoadAsync(i, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task S2_and_AesGcm_combined_round_trip()
|
||||
{
|
||||
await using var store = CreateStore("s2-aes", new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "S2 + AES-GCM combined payload"u8.ToArray();
|
||||
for (var i = 0; i < 20; i++)
|
||||
await store.AppendAsync("bar", payload, default);
|
||||
|
||||
for (ulong i = 1; i <= 20; i++)
|
||||
{
|
||||
var msg = await store.LoadAsync(i, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task S2_and_ChaCha_combined_store_and_recover()
|
||||
{
|
||||
const string sub = "s2-chacha-recover";
|
||||
var key = Key32;
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 40; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"s2-chacha-{i:D3}"), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)40);
|
||||
|
||||
var msg = await store.LoadAsync(20, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("s2-chacha-019"));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Legacy FSV1 data still readable after upgrade
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: backward-compat requirement — existing FSV1 files must still load
|
||||
[Fact]
|
||||
public async Task Legacy_FSV1_deflate_compression_still_readable()
|
||||
{
|
||||
const string sub = "fsv1-compress-legacy";
|
||||
|
||||
// Write with legacy Deflate (EnableCompression=true, no enum set).
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableCompression = true,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "legacy deflate"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
// Reopen with same options — must read back correctly.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableCompression = true,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(1, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe("legacy deflate"u8.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Legacy_FSV1_xor_encryption_still_readable()
|
||||
{
|
||||
const string sub = "fsv1-encrypt-legacy";
|
||||
var key = "legacy-xor-key-16bytes!"u8.ToArray();
|
||||
|
||||
// Write with legacy XOR (EnableEncryption=true, no cipher enum set).
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableEncryption = true,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "legacy xor encrypted"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
// Reopen with same options — must read back correctly.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableEncryption = true,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(1, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe("legacy xor encrypted"u8.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// All 6 permutations: {NoCipher, ChaCha, AesGcm} x {NoCompression, S2}
|
||||
// Go: testFileStoreAllPermutations (filestore_test.go:98)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.S2Compression)]
|
||||
public async Task All_permutations_store_and_load(StoreCipher cipher, StoreCompression compression)
|
||||
{
|
||||
var sub = $"perm-{cipher}-{compression}";
|
||||
var key = cipher == StoreCipher.NoCipher ? null : Key32;
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes($"payload for {cipher}+{compression}");
|
||||
|
||||
await using var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = cipher,
|
||||
Compression = compression,
|
||||
EncryptionKey = key,
|
||||
});
|
||||
|
||||
for (var i = 0; i < 10; i++)
|
||||
await store.AppendAsync("test", payload, default);
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)10);
|
||||
|
||||
for (ulong i = 1; i <= 10; i++)
|
||||
{
|
||||
var msg = await store.LoadAsync(i, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.S2Compression)]
|
||||
public async Task All_permutations_store_and_recover(StoreCipher cipher, StoreCompression compression)
|
||||
{
|
||||
var sub = $"perm-recover-{cipher}-{compression}";
|
||||
var key = cipher == StoreCipher.NoCipher ? null : Key32;
|
||||
|
||||
// Write phase.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions { Cipher = cipher, Compression = compression, EncryptionKey = key }))
|
||||
{
|
||||
for (var i = 0; i < 20; i++)
|
||||
await store.AppendAsync("x", Encoding.UTF8.GetBytes($"msg-{i:D3}"), default);
|
||||
}
|
||||
|
||||
// Reopen and verify.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions { Cipher = cipher, Compression = compression, EncryptionKey = key }))
|
||||
{
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)20);
|
||||
|
||||
var msg = await store.LoadAsync(10, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("msg-009"));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// FSV2 data is not plaintext on disk
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task S2_data_differs_from_plaintext_on_disk()
|
||||
{
|
||||
var sub = "s2-disk";
|
||||
var dir = Path.Combine(_dir, sub);
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "AAAAAAAAAAAAAAAAAAAAAAAAA"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
var dataFile = Path.Combine(dir, "messages.jsonl");
|
||||
if (File.Exists(dataFile))
|
||||
{
|
||||
var raw = File.ReadAllText(dataFile);
|
||||
// The payload is base64-encoded in the JSONL file.
|
||||
// "FSV2" (0x46 0x53 0x56 0x32) base64-encodes to "RlNWMg".
|
||||
// FSV1 encodes as "RlNWMQ". Verify FSV2 is used, not FSV1.
|
||||
raw.ShouldContain("RlNWMg");
|
||||
raw.ShouldNotContain("RlNWMQ");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChaCha_encrypted_data_not_plaintext_on_disk()
|
||||
{
|
||||
var sub = "chacha-disk";
|
||||
var dir = Path.Combine(_dir, sub);
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = Key32,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "THIS IS SENSITIVE DATA"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
var dataFile = Path.Combine(dir, "messages.jsonl");
|
||||
if (File.Exists(dataFile))
|
||||
{
|
||||
var raw = File.ReadAllText(dataFile);
|
||||
raw.ShouldNotContain("THIS IS SENSITIVE DATA");
|
||||
}
|
||||
}
|
||||
}
|
||||
154
tests/NATS.Server.Tests/JetStream/Storage/S2CodecTests.cs
Normal file
154
tests/NATS.Server.Tests/JetStream/Storage/S2CodecTests.cs
Normal file
@@ -0,0 +1,154 @@
|
||||
// Reference: golang/nats-server/server/filestore.go
|
||||
// Go uses S2/Snappy compression throughout FileStore:
|
||||
// - msgCompress / msgDecompress (filestore.go ~line 840)
|
||||
// - compressBlock / decompressBlock for block-level data
|
||||
// These tests verify the .NET S2Codec helper used in the FSV2 envelope path.
|
||||
|
||||
using NATS.Server.JetStream.Storage;
|
||||
using System.Text;
|
||||
|
||||
namespace NATS.Server.Tests.JetStream.Storage;
|
||||
|
||||
public sealed class S2CodecTests
|
||||
{
|
||||
// Go: TestFileStoreBasics (S2 permutation) filestore_test.go:86
|
||||
[Fact]
|
||||
public void Compress_then_decompress_round_trips()
|
||||
{
|
||||
var original = "Hello, NATS JetStream S2 compression!"u8.ToArray();
|
||||
|
||||
var compressed = S2Codec.Compress(original);
|
||||
var restored = S2Codec.Decompress(compressed);
|
||||
|
||||
restored.ShouldBe(original);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Compress_empty_returns_empty()
|
||||
{
|
||||
var compressed = S2Codec.Compress([]);
|
||||
compressed.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Decompress_empty_returns_empty()
|
||||
{
|
||||
var decompressed = S2Codec.Decompress([]);
|
||||
decompressed.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Compress_large_highly_compressible_payload()
|
||||
{
|
||||
// 1 MB of repeated 'A' — highly compressible.
|
||||
var original = new byte[1024 * 1024];
|
||||
Array.Fill(original, (byte)'A');
|
||||
|
||||
var compressed = S2Codec.Compress(original);
|
||||
var restored = S2Codec.Decompress(compressed);
|
||||
|
||||
// S2/Snappy should compress this well.
|
||||
compressed.Length.ShouldBeLessThan(original.Length);
|
||||
restored.ShouldBe(original);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Compress_large_incompressible_payload_round_trips()
|
||||
{
|
||||
// 1 MB of random data — not compressible, but must still round-trip.
|
||||
var original = new byte[1024 * 1024];
|
||||
Random.Shared.NextBytes(original);
|
||||
|
||||
var compressed = S2Codec.Compress(original);
|
||||
var restored = S2Codec.Decompress(compressed);
|
||||
|
||||
restored.ShouldBe(original);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Compress_single_byte_round_trips()
|
||||
{
|
||||
var original = new byte[] { 0x42 };
|
||||
var compressed = S2Codec.Compress(original);
|
||||
var restored = S2Codec.Decompress(compressed);
|
||||
restored.ShouldBe(original);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Compress_binary_all_byte_values_round_trips()
|
||||
{
|
||||
var original = new byte[256];
|
||||
for (var i = 0; i < 256; i++)
|
||||
original[i] = (byte)i;
|
||||
|
||||
var compressed = S2Codec.Compress(original);
|
||||
var restored = S2Codec.Decompress(compressed);
|
||||
restored.ShouldBe(original);
|
||||
}
|
||||
|
||||
// Go: msgCompress with trailing CRC (filestore.go ~line 840) — the checksum
|
||||
// lives outside the S2 frame so only the body is compressed.
|
||||
[Fact]
|
||||
public void CompressWithTrailingChecksum_preserves_last_n_bytes_uncompressed()
|
||||
{
|
||||
const int checksumSize = 8;
|
||||
var body = Encoding.UTF8.GetBytes("NATS payload body that should be compressed");
|
||||
var checksum = new byte[checksumSize];
|
||||
Random.Shared.NextBytes(checksum);
|
||||
|
||||
var input = body.Concat(checksum).ToArray();
|
||||
|
||||
var result = S2Codec.CompressWithTrailingChecksum(input, checksumSize);
|
||||
|
||||
// Last checksumSize bytes must be verbatim.
|
||||
var resultChecksum = result[^checksumSize..];
|
||||
resultChecksum.ShouldBe(checksum);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompressWithTrailingChecksum_zero_checksum_compresses_all()
|
||||
{
|
||||
var data = "Hello, no checksum"u8.ToArray();
|
||||
var result = S2Codec.CompressWithTrailingChecksum(data, 0);
|
||||
var restored = S2Codec.Decompress(result);
|
||||
restored.ShouldBe(data);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecompressWithTrailingChecksum_round_trips()
|
||||
{
|
||||
const int checksumSize = 8;
|
||||
var body = new byte[512];
|
||||
Random.Shared.NextBytes(body);
|
||||
var checksum = new byte[checksumSize];
|
||||
Random.Shared.NextBytes(checksum);
|
||||
|
||||
var input = body.Concat(checksum).ToArray();
|
||||
|
||||
var compressed = S2Codec.CompressWithTrailingChecksum(input, checksumSize);
|
||||
var restored = S2Codec.DecompressWithTrailingChecksum(compressed, checksumSize);
|
||||
|
||||
restored.ShouldBe(input);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompressWithTrailingChecksum_empty_input_returns_empty()
|
||||
{
|
||||
var result = S2Codec.CompressWithTrailingChecksum([], 0);
|
||||
result.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompressWithTrailingChecksum_negative_size_throws()
|
||||
{
|
||||
Should.Throw<ArgumentOutOfRangeException>(
|
||||
() => S2Codec.CompressWithTrailingChecksum([1, 2, 3], -1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecompressWithTrailingChecksum_negative_size_throws()
|
||||
{
|
||||
Should.Throw<ArgumentOutOfRangeException>(
|
||||
() => S2Codec.DecompressWithTrailingChecksum([1, 2, 3], -1));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user