refactor: extract NATS.Server.JetStream.Tests project
Move 225 JetStream-related test files from NATS.Server.Tests into a dedicated NATS.Server.JetStream.Tests project. This includes root-level JetStream*.cs files, storage test files (FileStore, MemStore, StreamStoreContract), and the full JetStream/ subfolder tree (Api, Cluster, Consumers, MirrorSource, Snapshots, Storage, Streams). Updated all namespaces, added InternalsVisibleTo, registered in the solution file, and added the JETSTREAM_INTEGRATION_MATRIX define.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,419 @@
|
||||
// Reference: golang/nats-server/server/filestore_test.go
|
||||
// Tests ported from: TestFileStorePurgeEx, TestFileStorePurgeExWithSubject,
|
||||
// TestFileStorePurgeExKeepOneBug, TestFileStoreCompact, TestFileStoreStreamTruncate,
|
||||
// TestFileStoreState, TestFileStoreFilteredState, TestFileStoreSubjectsState,
|
||||
// TestFileStoreGetSeqFromTime
|
||||
|
||||
using System.Text;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
|
||||
namespace NATS.Server.JetStream.Tests.JetStream.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for FileStore tombstone tracking and purge operations:
|
||||
/// PurgeEx, Compact, Truncate, FilteredState, SubjectsState, SubjectsTotals,
|
||||
/// State (with deleted sequences), and GetSeqFromTime.
|
||||
/// Reference: golang/nats-server/server/filestore_test.go
|
||||
/// </summary>
|
||||
public sealed class FileStorePurgeBlockTests : IDisposable
|
||||
{
|
||||
private readonly string _dir;
|
||||
|
||||
public FileStorePurgeBlockTests()
|
||||
{
|
||||
_dir = Path.Combine(Path.GetTempPath(), $"nats-js-purgeblock-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_dir);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_dir))
|
||||
Directory.Delete(_dir, recursive: true);
|
||||
}
|
||||
|
||||
private FileStore CreateStore(string subdirectory, FileStoreOptions? options = null)
|
||||
{
|
||||
var dir = Path.Combine(_dir, subdirectory);
|
||||
var opts = options ?? new FileStoreOptions();
|
||||
opts.Directory = dir;
|
||||
return new FileStore(opts);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// PurgeEx tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStorePurgeExWithSubject — filestore_test.go:~867
|
||||
[Fact]
|
||||
public async Task PurgeEx_BySubject_RemovesMatchingMessages()
|
||||
{
|
||||
await using var store = CreateStore("purgex-subject");
|
||||
|
||||
// Store 5 messages on "foo" and 5 on "bar"
|
||||
for (var i = 0; i < 5; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"foo-{i}"), default);
|
||||
for (var i = 0; i < 5; i++)
|
||||
await store.AppendAsync("bar", Encoding.UTF8.GetBytes($"bar-{i}"), default);
|
||||
|
||||
var stateBeforePurge = await store.GetStateAsync(default);
|
||||
stateBeforePurge.Messages.ShouldBe(10UL);
|
||||
|
||||
// Purge all "foo" messages (seq=0 means no upper limit; keep=0 means keep none)
|
||||
var purged = store.PurgeEx("foo", 0, 0);
|
||||
purged.ShouldBe(5UL);
|
||||
|
||||
// Only "bar" messages remain
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe(5UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.All(m => m.Subject == "bar").ShouldBeTrue();
|
||||
}
|
||||
|
||||
// Go: TestFileStorePurgeExKeepOneBug — filestore_test.go:~910
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithKeep_RetainsNewestMessages()
|
||||
{
|
||||
await using var store = CreateStore("purgex-keep");
|
||||
|
||||
// Store 10 messages on "events"
|
||||
for (var i = 0; i < 10; i++)
|
||||
await store.AppendAsync("events", Encoding.UTF8.GetBytes($"msg-{i}"), default);
|
||||
|
||||
// Purge keeping the 3 newest
|
||||
var purged = store.PurgeEx("events", 0, 3);
|
||||
purged.ShouldBe(7UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(3);
|
||||
|
||||
// The retained messages should be the 3 highest sequences (8, 9, 10)
|
||||
var seqs = remaining.Select(m => m.Sequence).OrderBy(s => s).ToArray();
|
||||
seqs[0].ShouldBe(8UL);
|
||||
seqs[1].ShouldBe(9UL);
|
||||
seqs[2].ShouldBe(10UL);
|
||||
}
|
||||
|
||||
// Go: TestFileStorePurgeEx — filestore_test.go:~855
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithSeqLimit_OnlyPurgesBelowSequence()
|
||||
{
|
||||
await using var store = CreateStore("purgex-seqlimit");
|
||||
|
||||
// Store 10 messages on "data"
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("data", Encoding.UTF8.GetBytes($"d{i}"), default);
|
||||
|
||||
// Purge "data" messages with seq <= 5 (keep=0)
|
||||
var purged = store.PurgeEx("data", 5, 0);
|
||||
purged.ShouldBe(5UL);
|
||||
|
||||
// Messages 6-10 should remain
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(5);
|
||||
remaining.Min(m => m.Sequence).ShouldBe(6UL);
|
||||
remaining.Max(m => m.Sequence).ShouldBe(10UL);
|
||||
}
|
||||
|
||||
// Go: PurgeEx with wildcard subject — filestore_test.go:~867
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithWildcardSubject_RemovesAllMatchingSubjects()
|
||||
{
|
||||
await using var store = CreateStore("purgex-wildcard");
|
||||
|
||||
await store.AppendAsync("foo.a", "m1"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.b", "m2"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar.a", "m3"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.c", "m4"u8.ToArray(), default);
|
||||
|
||||
var purged = store.PurgeEx("foo.*", 0, 0);
|
||||
purged.ShouldBe(3UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(1);
|
||||
remaining[0].Subject.ShouldBe("bar.a");
|
||||
}
|
||||
|
||||
// Go: PurgeEx with > wildcard — filestore_test.go:~867
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithGtWildcard_RemovesAllMatchingSubjects()
|
||||
{
|
||||
await using var store = CreateStore("purgex-gt-wildcard");
|
||||
|
||||
await store.AppendAsync("a.b.c", "m1"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.b.d", "m2"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.x", "m3"u8.ToArray(), default);
|
||||
await store.AppendAsync("b.x", "m4"u8.ToArray(), default);
|
||||
|
||||
var purged = store.PurgeEx("a.>", 0, 0);
|
||||
purged.ShouldBe(3UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(1);
|
||||
remaining[0].Subject.ShouldBe("b.x");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Compact tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreCompact — filestore_test.go:~964
|
||||
[Fact]
|
||||
public async Task Compact_RemovesMessagesBeforeSequence()
|
||||
{
|
||||
await using var store = CreateStore("compact-basic");
|
||||
|
||||
// Store 10 messages
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("test", Encoding.UTF8.GetBytes($"msg{i}"), default);
|
||||
|
||||
// Compact to remove messages with seq < 5 (removes 1, 2, 3, 4)
|
||||
var removed = store.Compact(5);
|
||||
removed.ShouldBe(4UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(6); // 5-10
|
||||
|
||||
remaining.Min(m => m.Sequence).ShouldBe(5UL);
|
||||
remaining.Max(m => m.Sequence).ShouldBe(10UL);
|
||||
|
||||
// Sequence 1-4 should no longer be loadable
|
||||
(await store.LoadAsync(1, default)).ShouldBeNull();
|
||||
(await store.LoadAsync(4, default)).ShouldBeNull();
|
||||
|
||||
// Sequence 5 should still exist
|
||||
(await store.LoadAsync(5, default)).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Truncate tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreStreamTruncate — filestore_test.go:~1035
|
||||
[Fact]
|
||||
public async Task Truncate_RemovesMessagesAfterSequence()
|
||||
{
|
||||
await using var store = CreateStore("truncate-basic");
|
||||
|
||||
// Store 10 messages
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("stream", Encoding.UTF8.GetBytes($"m{i}"), default);
|
||||
|
||||
// Truncate at seq=5 (removes 6, 7, 8, 9, 10)
|
||||
store.Truncate(5);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(5); // 1-5
|
||||
|
||||
remaining.Min(m => m.Sequence).ShouldBe(1UL);
|
||||
remaining.Max(m => m.Sequence).ShouldBe(5UL);
|
||||
|
||||
// Messages 6-10 should be gone
|
||||
(await store.LoadAsync(6, default)).ShouldBeNull();
|
||||
(await store.LoadAsync(10, default)).ShouldBeNull();
|
||||
|
||||
// Message 5 should still exist
|
||||
(await store.LoadAsync(5, default)).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// FilteredState tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreFilteredState — filestore_test.go:~1200
|
||||
[Fact]
|
||||
public async Task FilteredState_ReturnsCorrectState()
|
||||
{
|
||||
await using var store = CreateStore("filteredstate");
|
||||
|
||||
// Store 5 messages on "orders" and 5 on "invoices"
|
||||
for (var i = 1; i <= 5; i++)
|
||||
await store.AppendAsync("orders", Encoding.UTF8.GetBytes($"o{i}"), default);
|
||||
for (var i = 1; i <= 5; i++)
|
||||
await store.AppendAsync("invoices", Encoding.UTF8.GetBytes($"inv{i}"), default);
|
||||
|
||||
// FilteredState for "orders" from seq=1
|
||||
var ordersState = store.FilteredState(1, "orders");
|
||||
ordersState.Msgs.ShouldBe(5UL);
|
||||
ordersState.First.ShouldBe(1UL);
|
||||
ordersState.Last.ShouldBe(5UL);
|
||||
|
||||
// FilteredState for "invoices" from seq=1
|
||||
var invoicesState = store.FilteredState(1, "invoices");
|
||||
invoicesState.Msgs.ShouldBe(5UL);
|
||||
invoicesState.First.ShouldBe(6UL);
|
||||
invoicesState.Last.ShouldBe(10UL);
|
||||
|
||||
// FilteredState from seq=7 (only 4 invoices remain)
|
||||
var lateInvoices = store.FilteredState(7, "invoices");
|
||||
lateInvoices.Msgs.ShouldBe(4UL);
|
||||
lateInvoices.First.ShouldBe(7UL);
|
||||
lateInvoices.Last.ShouldBe(10UL);
|
||||
|
||||
// No match for non-existent subject
|
||||
var noneState = store.FilteredState(1, "orders.unknown");
|
||||
noneState.Msgs.ShouldBe(0UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SubjectsState tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreSubjectsState — filestore_test.go:~1266
|
||||
[Fact]
|
||||
public async Task SubjectsState_ReturnsPerSubjectState()
|
||||
{
|
||||
await using var store = CreateStore("subjectsstate");
|
||||
|
||||
await store.AppendAsync("a.1", "msg"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.2", "msg"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.1", "msg"u8.ToArray(), default);
|
||||
await store.AppendAsync("b.1", "msg"u8.ToArray(), default);
|
||||
|
||||
var state = store.SubjectsState("a.>");
|
||||
|
||||
state.ShouldContainKey("a.1");
|
||||
state.ShouldContainKey("a.2");
|
||||
state.ShouldNotContainKey("b.1");
|
||||
|
||||
state["a.1"].Msgs.ShouldBe(2UL);
|
||||
state["a.1"].First.ShouldBe(1UL);
|
||||
state["a.1"].Last.ShouldBe(3UL);
|
||||
|
||||
state["a.2"].Msgs.ShouldBe(1UL);
|
||||
state["a.2"].First.ShouldBe(2UL);
|
||||
state["a.2"].Last.ShouldBe(2UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SubjectsTotals tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreSubjectsTotals — filestore_test.go:~1300
|
||||
[Fact]
|
||||
public async Task SubjectsTotals_ReturnsPerSubjectCounts()
|
||||
{
|
||||
await using var store = CreateStore("subjectstotals");
|
||||
|
||||
await store.AppendAsync("x.1", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("x.1", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("x.2", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("y.1", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("x.3", "m"u8.ToArray(), default);
|
||||
|
||||
var totals = store.SubjectsTotals("x.*");
|
||||
|
||||
totals.ShouldContainKey("x.1");
|
||||
totals.ShouldContainKey("x.2");
|
||||
totals.ShouldContainKey("x.3");
|
||||
totals.ShouldNotContainKey("y.1");
|
||||
|
||||
totals["x.1"].ShouldBe(2UL);
|
||||
totals["x.2"].ShouldBe(1UL);
|
||||
totals["x.3"].ShouldBe(1UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// State (with deleted sequences) tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreState — filestore_test.go:~420
|
||||
[Fact]
|
||||
public async Task State_IncludesDeletedSequences()
|
||||
{
|
||||
await using var store = CreateStore("state-deleted");
|
||||
|
||||
// Store 10 messages
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("events", Encoding.UTF8.GetBytes($"e{i}"), default);
|
||||
|
||||
// Remove messages 3, 5, 7
|
||||
await store.RemoveAsync(3, default);
|
||||
await store.RemoveAsync(5, default);
|
||||
await store.RemoveAsync(7, default);
|
||||
|
||||
var state = store.State();
|
||||
|
||||
state.Msgs.ShouldBe(7UL);
|
||||
state.FirstSeq.ShouldBe(1UL);
|
||||
state.LastSeq.ShouldBe(10UL);
|
||||
state.NumDeleted.ShouldBe(3);
|
||||
|
||||
state.Deleted.ShouldNotBeNull();
|
||||
state.Deleted!.ShouldContain(3UL);
|
||||
state.Deleted.ShouldContain(5UL);
|
||||
state.Deleted.ShouldContain(7UL);
|
||||
state.Deleted.Length.ShouldBe(3);
|
||||
|
||||
// NumSubjects: all messages are on "events"
|
||||
state.NumSubjects.ShouldBe(1);
|
||||
state.Subjects.ShouldNotBeNull();
|
||||
state.Subjects!["events"].ShouldBe(7UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GetSeqFromTime tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreGetSeqFromTime — filestore_test.go:~1570
|
||||
[Fact]
|
||||
public async Task GetSeqFromTime_ReturnsCorrectSequence()
|
||||
{
|
||||
await using var store = CreateStore("getseqfromtime");
|
||||
|
||||
// Store 5 messages; we'll query by the timestamp of the 3rd message
|
||||
var timestamps = new List<DateTime>();
|
||||
for (var i = 1; i <= 5; i++)
|
||||
{
|
||||
await store.AppendAsync("time.test", Encoding.UTF8.GetBytes($"t{i}"), default);
|
||||
var msgs = await store.ListAsync(default);
|
||||
timestamps.Add(msgs[^1].TimestampUtc);
|
||||
// Small delay to ensure distinct timestamps
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
// Query for first seq at or after the timestamp of msg 3
|
||||
var targetTime = timestamps[2]; // timestamp of sequence 3
|
||||
var seq = store.GetSeqFromTime(targetTime);
|
||||
seq.ShouldBe(3UL);
|
||||
|
||||
// Query with a time before all messages: should return 1
|
||||
var beforeAll = timestamps[0].AddMilliseconds(-100);
|
||||
store.GetSeqFromTime(beforeAll).ShouldBe(1UL);
|
||||
|
||||
// Query with a time after all messages: should return last+1
|
||||
var afterAll = timestamps[^1].AddSeconds(1);
|
||||
store.GetSeqFromTime(afterAll).ShouldBe(6UL); // _last + 1
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// MsgBlock enhancements
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: filestore.go dmap — soft-delete tracking and enumeration
|
||||
[Fact]
|
||||
public async Task MsgBlock_IsDeleted_AndEnumerateNonDeleted_Work()
|
||||
{
|
||||
await using var store = CreateStore("block-enumerate");
|
||||
|
||||
// Store 5 messages on 2 subjects
|
||||
await store.AppendAsync("a.1", "m1"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.2", "m2"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.1", "m3"u8.ToArray(), default);
|
||||
await store.AppendAsync("b.1", "m4"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.2", "m5"u8.ToArray(), default);
|
||||
|
||||
// Delete sequences 2 and 4
|
||||
await store.RemoveAsync(2, default);
|
||||
await store.RemoveAsync(4, default);
|
||||
|
||||
// Verify the state after deletion
|
||||
var all = await store.ListAsync(default);
|
||||
all.Count.ShouldBe(3);
|
||||
all.Select(m => m.Sequence).ShouldBe([1UL, 3UL, 5UL]);
|
||||
|
||||
// FilteredState should only see non-deleted
|
||||
var aState = store.FilteredState(1, "a.1");
|
||||
aState.Msgs.ShouldBe(2UL); // sequences 1 and 3
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user