refactor: extract NATS.Server.JetStream.Tests project
Move 225 JetStream-related test files from NATS.Server.Tests into a dedicated NATS.Server.JetStream.Tests project. This includes root-level JetStream*.cs files, storage test files (FileStore, MemStore, StreamStoreContract), and the full JetStream/ subfolder tree (Api, Cluster, Consumers, MirrorSource, Snapshots, Storage, Streams). Updated all namespaces, added InternalsVisibleTo, registered in the solution file, and added the JETSTREAM_INTEGRATION_MATRIX define.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,475 @@
|
||||
// Reference: golang/nats-server/server/filestore_test.go
|
||||
// Tests ported from: TestFileStoreEncrypted (AES + ChaCha permutations),
|
||||
// testFileStoreAllPermutations (S2 + cipher cross product),
|
||||
// TestFileStoreS2Compression (filestore_test.go:4180),
|
||||
// TestFileStoreEncryptedChaChaCipher (filestore_test.go:4250)
|
||||
//
|
||||
// The Go server runs testFileStoreAllPermutations which exercises all
|
||||
// combinations of {NoCompression, S2Compression} x {NoCipher, ChaCha, AES}.
|
||||
// These tests cover the FSV2 envelope path added in Task 4.
|
||||
|
||||
using System.Text;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
|
||||
namespace NATS.Server.JetStream.Tests.JetStream.Storage;
|
||||
|
||||
public sealed class FileStoreV2Tests : IDisposable
|
||||
{
|
||||
private readonly string _dir;
|
||||
|
||||
public FileStoreV2Tests()
|
||||
{
|
||||
_dir = Path.Combine(Path.GetTempPath(), $"nats-js-fs-v2-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_dir);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_dir))
|
||||
Directory.Delete(_dir, recursive: true);
|
||||
}
|
||||
|
||||
// 32-byte key for AEAD ciphers.
|
||||
private static byte[] Key32 => "nats-v2-test-key-exactly-32-bytes"u8[..32].ToArray();
|
||||
|
||||
private FileStore CreateStore(string sub, FileStoreOptions options)
|
||||
{
|
||||
options.Directory = Path.Combine(_dir, sub);
|
||||
return new FileStore(options);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// S2 compression (no encryption) — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreS2Compression filestore_test.go:4180
|
||||
[Fact]
|
||||
public async Task S2_compression_store_and_load()
|
||||
{
|
||||
await using var store = CreateStore("s2-basic", new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
});
|
||||
|
||||
var payload = "Hello, S2!"u8.ToArray();
|
||||
for (var i = 1; i <= 10; i++)
|
||||
{
|
||||
var seq = await store.AppendAsync("foo", payload, default);
|
||||
seq.ShouldBe((ulong)i);
|
||||
}
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)10);
|
||||
|
||||
var msg = await store.LoadAsync(5, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task S2_compression_store_and_recover()
|
||||
{
|
||||
const string sub = "s2-recover";
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"msg-{i:D4}"), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
}))
|
||||
{
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
|
||||
var msg = await store.LoadAsync(25, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("msg-0024"));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// ChaCha20-Poly1305 encryption (no compression) — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreEncryptedChaChaCipher filestore_test.go:4250
|
||||
[Fact]
|
||||
public async Task ChaCha_encryption_store_and_load()
|
||||
{
|
||||
await using var store = CreateStore("chacha-basic", new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "aes ftw"u8.ToArray();
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", payload, default);
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
|
||||
var msg = await store.LoadAsync(10, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChaCha_encryption_store_and_recover()
|
||||
{
|
||||
const string sub = "chacha-recover";
|
||||
var key = Key32;
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", "chacha secret"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(10, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe("chacha secret"u8.ToArray());
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// AES-256-GCM encryption (no compression) — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreEncrypted (AES permutation) filestore_test.go:4204
|
||||
[Fact]
|
||||
public async Task AesGcm_encryption_store_and_load()
|
||||
{
|
||||
await using var store = CreateStore("aes-basic", new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "aes-gcm secret"u8.ToArray();
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", payload, default);
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
|
||||
var msg = await store.LoadAsync(25, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AesGcm_encryption_store_and_recover()
|
||||
{
|
||||
const string sub = "aes-recover";
|
||||
var key = Key32;
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 50; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"aes-{i:D4}"), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(30, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("aes-0029"));
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)50);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// S2 + ChaCha combined — FSV2 envelope
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task S2_and_ChaCha_combined_round_trip()
|
||||
{
|
||||
await using var store = CreateStore("s2-chacha", new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "S2 + ChaCha combined payload"u8.ToArray();
|
||||
for (var i = 0; i < 20; i++)
|
||||
await store.AppendAsync("foo", payload, default);
|
||||
|
||||
for (ulong i = 1; i <= 20; i++)
|
||||
{
|
||||
var msg = await store.LoadAsync(i, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task S2_and_AesGcm_combined_round_trip()
|
||||
{
|
||||
await using var store = CreateStore("s2-aes", new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.Aes,
|
||||
EncryptionKey = Key32,
|
||||
});
|
||||
|
||||
var payload = "S2 + AES-GCM combined payload"u8.ToArray();
|
||||
for (var i = 0; i < 20; i++)
|
||||
await store.AppendAsync("bar", payload, default);
|
||||
|
||||
for (ulong i = 1; i <= 20; i++)
|
||||
{
|
||||
var msg = await store.LoadAsync(i, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task S2_and_ChaCha_combined_store_and_recover()
|
||||
{
|
||||
const string sub = "s2-chacha-recover";
|
||||
var key = Key32;
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
for (var i = 0; i < 40; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"s2-chacha-{i:D3}"), default);
|
||||
}
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)40);
|
||||
|
||||
var msg = await store.LoadAsync(20, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("s2-chacha-019"));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Legacy FSV1 data still readable after upgrade
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: backward-compat requirement — existing FSV1 files must still load
|
||||
[Fact]
|
||||
public async Task Legacy_FSV1_deflate_compression_still_readable()
|
||||
{
|
||||
const string sub = "fsv1-compress-legacy";
|
||||
|
||||
// Write with legacy Deflate (EnableCompression=true, no enum set).
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableCompression = true,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "legacy deflate"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
// Reopen with same options — must read back correctly.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableCompression = true,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(1, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe("legacy deflate"u8.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Legacy_FSV1_xor_encryption_still_readable()
|
||||
{
|
||||
const string sub = "fsv1-encrypt-legacy";
|
||||
var key = "legacy-xor-key-16bytes!"u8.ToArray();
|
||||
|
||||
// Write with legacy XOR (EnableEncryption=true, no cipher enum set).
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableEncryption = true,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "legacy xor encrypted"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
// Reopen with same options — must read back correctly.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
EnableEncryption = true,
|
||||
EncryptionKey = key,
|
||||
}))
|
||||
{
|
||||
var msg = await store.LoadAsync(1, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe("legacy xor encrypted"u8.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// All 6 permutations: {NoCipher, ChaCha, AesGcm} x {NoCompression, S2}
|
||||
// Go: testFileStoreAllPermutations (filestore_test.go:98)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.S2Compression)]
|
||||
public async Task All_permutations_store_and_load(StoreCipher cipher, StoreCompression compression)
|
||||
{
|
||||
var sub = $"perm-{cipher}-{compression}";
|
||||
var key = cipher == StoreCipher.NoCipher ? null : Key32;
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes($"payload for {cipher}+{compression}");
|
||||
|
||||
await using var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = cipher,
|
||||
Compression = compression,
|
||||
EncryptionKey = key,
|
||||
});
|
||||
|
||||
for (var i = 0; i < 10; i++)
|
||||
await store.AppendAsync("test", payload, default);
|
||||
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)10);
|
||||
|
||||
for (ulong i = 1; i <= 10; i++)
|
||||
{
|
||||
var msg = await store.LoadAsync(i, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(payload);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.NoCipher, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.ChaCha, StoreCompression.S2Compression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.NoCompression)]
|
||||
[InlineData(StoreCipher.Aes, StoreCompression.S2Compression)]
|
||||
public async Task All_permutations_store_and_recover(StoreCipher cipher, StoreCompression compression)
|
||||
{
|
||||
var sub = $"perm-recover-{cipher}-{compression}";
|
||||
var key = cipher == StoreCipher.NoCipher ? null : Key32;
|
||||
|
||||
// Write phase.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions { Cipher = cipher, Compression = compression, EncryptionKey = key }))
|
||||
{
|
||||
for (var i = 0; i < 20; i++)
|
||||
await store.AppendAsync("x", Encoding.UTF8.GetBytes($"msg-{i:D3}"), default);
|
||||
}
|
||||
|
||||
// Reopen and verify.
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions { Cipher = cipher, Compression = compression, EncryptionKey = key }))
|
||||
{
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe((ulong)20);
|
||||
|
||||
var msg = await store.LoadAsync(10, default);
|
||||
msg.ShouldNotBeNull();
|
||||
msg!.Payload.ToArray().ShouldBe(Encoding.UTF8.GetBytes("msg-009"));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// FSV2 data is not plaintext on disk
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task S2_data_differs_from_plaintext_on_disk()
|
||||
{
|
||||
var sub = "s2-disk";
|
||||
var dir = Path.Combine(_dir, sub);
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Compression = StoreCompression.S2Compression,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "AAAAAAAAAAAAAAAAAAAAAAAAA"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
var dataFile = Path.Combine(dir, "messages.jsonl");
|
||||
if (File.Exists(dataFile))
|
||||
{
|
||||
var raw = File.ReadAllText(dataFile);
|
||||
// The payload is base64-encoded in the JSONL file.
|
||||
// "FSV2" (0x46 0x53 0x56 0x32) base64-encodes to "RlNWMg".
|
||||
// FSV1 encodes as "RlNWMQ". Verify FSV2 is used, not FSV1.
|
||||
raw.ShouldContain("RlNWMg");
|
||||
raw.ShouldNotContain("RlNWMQ");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChaCha_encrypted_data_not_plaintext_on_disk()
|
||||
{
|
||||
var sub = "chacha-disk";
|
||||
var dir = Path.Combine(_dir, sub);
|
||||
|
||||
await using (var store = CreateStore(sub, new FileStoreOptions
|
||||
{
|
||||
Cipher = StoreCipher.ChaCha,
|
||||
EncryptionKey = Key32,
|
||||
}))
|
||||
{
|
||||
await store.AppendAsync("foo", "THIS IS SENSITIVE DATA"u8.ToArray(), default);
|
||||
}
|
||||
|
||||
var dataFile = Path.Combine(dir, "messages.jsonl");
|
||||
if (File.Exists(dataFile))
|
||||
{
|
||||
var raw = File.ReadAllText(dataFile);
|
||||
raw.ShouldNotContain("THIS IS SENSITIVE DATA");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user