feat(storage): add tombstone tracking and purge operations (Go parity)
Implement PurgeEx, Compact, Truncate, FilteredState, SubjectsState, SubjectsTotals, State, FastState, GetSeqFromTime on FileStore. Add MsgBlock.IsDeleted, DeletedSequences, EnumerateNonDeleted. Includes wildcard subject support via SubjectMatch for all filtered operations.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,419 @@
|
||||
// Reference: golang/nats-server/server/filestore_test.go
|
||||
// Tests ported from: TestFileStorePurgeEx, TestFileStorePurgeExWithSubject,
|
||||
// TestFileStorePurgeExKeepOneBug, TestFileStoreCompact, TestFileStoreStreamTruncate,
|
||||
// TestFileStoreState, TestFileStoreFilteredState, TestFileStoreSubjectsState,
|
||||
// TestFileStoreGetSeqFromTime
|
||||
|
||||
using System.Text;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
|
||||
namespace NATS.Server.Tests.JetStream.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for FileStore tombstone tracking and purge operations:
|
||||
/// PurgeEx, Compact, Truncate, FilteredState, SubjectsState, SubjectsTotals,
|
||||
/// State (with deleted sequences), and GetSeqFromTime.
|
||||
/// Reference: golang/nats-server/server/filestore_test.go
|
||||
/// </summary>
|
||||
public sealed class FileStorePurgeBlockTests : IDisposable
|
||||
{
|
||||
private readonly string _dir;
|
||||
|
||||
public FileStorePurgeBlockTests()
|
||||
{
|
||||
_dir = Path.Combine(Path.GetTempPath(), $"nats-js-purgeblock-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_dir);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_dir))
|
||||
Directory.Delete(_dir, recursive: true);
|
||||
}
|
||||
|
||||
private FileStore CreateStore(string subdirectory, FileStoreOptions? options = null)
|
||||
{
|
||||
var dir = Path.Combine(_dir, subdirectory);
|
||||
var opts = options ?? new FileStoreOptions();
|
||||
opts.Directory = dir;
|
||||
return new FileStore(opts);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// PurgeEx tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStorePurgeExWithSubject — filestore_test.go:~867
|
||||
[Fact]
|
||||
public async Task PurgeEx_BySubject_RemovesMatchingMessages()
|
||||
{
|
||||
await using var store = CreateStore("purgex-subject");
|
||||
|
||||
// Store 5 messages on "foo" and 5 on "bar"
|
||||
for (var i = 0; i < 5; i++)
|
||||
await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"foo-{i}"), default);
|
||||
for (var i = 0; i < 5; i++)
|
||||
await store.AppendAsync("bar", Encoding.UTF8.GetBytes($"bar-{i}"), default);
|
||||
|
||||
var stateBeforePurge = await store.GetStateAsync(default);
|
||||
stateBeforePurge.Messages.ShouldBe(10UL);
|
||||
|
||||
// Purge all "foo" messages (seq=0 means no upper limit; keep=0 means keep none)
|
||||
var purged = store.PurgeEx("foo", 0, 0);
|
||||
purged.ShouldBe(5UL);
|
||||
|
||||
// Only "bar" messages remain
|
||||
var state = await store.GetStateAsync(default);
|
||||
state.Messages.ShouldBe(5UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.All(m => m.Subject == "bar").ShouldBeTrue();
|
||||
}
|
||||
|
||||
// Go: TestFileStorePurgeExKeepOneBug — filestore_test.go:~910
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithKeep_RetainsNewestMessages()
|
||||
{
|
||||
await using var store = CreateStore("purgex-keep");
|
||||
|
||||
// Store 10 messages on "events"
|
||||
for (var i = 0; i < 10; i++)
|
||||
await store.AppendAsync("events", Encoding.UTF8.GetBytes($"msg-{i}"), default);
|
||||
|
||||
// Purge keeping the 3 newest
|
||||
var purged = store.PurgeEx("events", 0, 3);
|
||||
purged.ShouldBe(7UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(3);
|
||||
|
||||
// The retained messages should be the 3 highest sequences (8, 9, 10)
|
||||
var seqs = remaining.Select(m => m.Sequence).OrderBy(s => s).ToArray();
|
||||
seqs[0].ShouldBe(8UL);
|
||||
seqs[1].ShouldBe(9UL);
|
||||
seqs[2].ShouldBe(10UL);
|
||||
}
|
||||
|
||||
// Go: TestFileStorePurgeEx — filestore_test.go:~855
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithSeqLimit_OnlyPurgesBelowSequence()
|
||||
{
|
||||
await using var store = CreateStore("purgex-seqlimit");
|
||||
|
||||
// Store 10 messages on "data"
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("data", Encoding.UTF8.GetBytes($"d{i}"), default);
|
||||
|
||||
// Purge "data" messages with seq <= 5 (keep=0)
|
||||
var purged = store.PurgeEx("data", 5, 0);
|
||||
purged.ShouldBe(5UL);
|
||||
|
||||
// Messages 6-10 should remain
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(5);
|
||||
remaining.Min(m => m.Sequence).ShouldBe(6UL);
|
||||
remaining.Max(m => m.Sequence).ShouldBe(10UL);
|
||||
}
|
||||
|
||||
// Go: PurgeEx with wildcard subject — filestore_test.go:~867
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithWildcardSubject_RemovesAllMatchingSubjects()
|
||||
{
|
||||
await using var store = CreateStore("purgex-wildcard");
|
||||
|
||||
await store.AppendAsync("foo.a", "m1"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.b", "m2"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar.a", "m3"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.c", "m4"u8.ToArray(), default);
|
||||
|
||||
var purged = store.PurgeEx("foo.*", 0, 0);
|
||||
purged.ShouldBe(3UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(1);
|
||||
remaining[0].Subject.ShouldBe("bar.a");
|
||||
}
|
||||
|
||||
// Go: PurgeEx with > wildcard — filestore_test.go:~867
|
||||
[Fact]
|
||||
public async Task PurgeEx_WithGtWildcard_RemovesAllMatchingSubjects()
|
||||
{
|
||||
await using var store = CreateStore("purgex-gt-wildcard");
|
||||
|
||||
await store.AppendAsync("a.b.c", "m1"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.b.d", "m2"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.x", "m3"u8.ToArray(), default);
|
||||
await store.AppendAsync("b.x", "m4"u8.ToArray(), default);
|
||||
|
||||
var purged = store.PurgeEx("a.>", 0, 0);
|
||||
purged.ShouldBe(3UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(1);
|
||||
remaining[0].Subject.ShouldBe("b.x");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Compact tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreCompact — filestore_test.go:~964
|
||||
[Fact]
|
||||
public async Task Compact_RemovesMessagesBeforeSequence()
|
||||
{
|
||||
await using var store = CreateStore("compact-basic");
|
||||
|
||||
// Store 10 messages
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("test", Encoding.UTF8.GetBytes($"msg{i}"), default);
|
||||
|
||||
// Compact to remove messages with seq < 5 (removes 1, 2, 3, 4)
|
||||
var removed = store.Compact(5);
|
||||
removed.ShouldBe(4UL);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(6); // 5-10
|
||||
|
||||
remaining.Min(m => m.Sequence).ShouldBe(5UL);
|
||||
remaining.Max(m => m.Sequence).ShouldBe(10UL);
|
||||
|
||||
// Sequence 1-4 should no longer be loadable
|
||||
(await store.LoadAsync(1, default)).ShouldBeNull();
|
||||
(await store.LoadAsync(4, default)).ShouldBeNull();
|
||||
|
||||
// Sequence 5 should still exist
|
||||
(await store.LoadAsync(5, default)).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Truncate tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreStreamTruncate — filestore_test.go:~1035
|
||||
[Fact]
|
||||
public async Task Truncate_RemovesMessagesAfterSequence()
|
||||
{
|
||||
await using var store = CreateStore("truncate-basic");
|
||||
|
||||
// Store 10 messages
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("stream", Encoding.UTF8.GetBytes($"m{i}"), default);
|
||||
|
||||
// Truncate at seq=5 (removes 6, 7, 8, 9, 10)
|
||||
store.Truncate(5);
|
||||
|
||||
var remaining = await store.ListAsync(default);
|
||||
remaining.Count.ShouldBe(5); // 1-5
|
||||
|
||||
remaining.Min(m => m.Sequence).ShouldBe(1UL);
|
||||
remaining.Max(m => m.Sequence).ShouldBe(5UL);
|
||||
|
||||
// Messages 6-10 should be gone
|
||||
(await store.LoadAsync(6, default)).ShouldBeNull();
|
||||
(await store.LoadAsync(10, default)).ShouldBeNull();
|
||||
|
||||
// Message 5 should still exist
|
||||
(await store.LoadAsync(5, default)).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// FilteredState tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreFilteredState — filestore_test.go:~1200
|
||||
[Fact]
|
||||
public async Task FilteredState_ReturnsCorrectState()
|
||||
{
|
||||
await using var store = CreateStore("filteredstate");
|
||||
|
||||
// Store 5 messages on "orders" and 5 on "invoices"
|
||||
for (var i = 1; i <= 5; i++)
|
||||
await store.AppendAsync("orders", Encoding.UTF8.GetBytes($"o{i}"), default);
|
||||
for (var i = 1; i <= 5; i++)
|
||||
await store.AppendAsync("invoices", Encoding.UTF8.GetBytes($"inv{i}"), default);
|
||||
|
||||
// FilteredState for "orders" from seq=1
|
||||
var ordersState = store.FilteredState(1, "orders");
|
||||
ordersState.Msgs.ShouldBe(5UL);
|
||||
ordersState.First.ShouldBe(1UL);
|
||||
ordersState.Last.ShouldBe(5UL);
|
||||
|
||||
// FilteredState for "invoices" from seq=1
|
||||
var invoicesState = store.FilteredState(1, "invoices");
|
||||
invoicesState.Msgs.ShouldBe(5UL);
|
||||
invoicesState.First.ShouldBe(6UL);
|
||||
invoicesState.Last.ShouldBe(10UL);
|
||||
|
||||
// FilteredState from seq=7 (only 4 invoices remain)
|
||||
var lateInvoices = store.FilteredState(7, "invoices");
|
||||
lateInvoices.Msgs.ShouldBe(4UL);
|
||||
lateInvoices.First.ShouldBe(7UL);
|
||||
lateInvoices.Last.ShouldBe(10UL);
|
||||
|
||||
// No match for non-existent subject
|
||||
var noneState = store.FilteredState(1, "orders.unknown");
|
||||
noneState.Msgs.ShouldBe(0UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SubjectsState tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreSubjectsState — filestore_test.go:~1266
|
||||
[Fact]
|
||||
public async Task SubjectsState_ReturnsPerSubjectState()
|
||||
{
|
||||
await using var store = CreateStore("subjectsstate");
|
||||
|
||||
await store.AppendAsync("a.1", "msg"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.2", "msg"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.1", "msg"u8.ToArray(), default);
|
||||
await store.AppendAsync("b.1", "msg"u8.ToArray(), default);
|
||||
|
||||
var state = store.SubjectsState("a.>");
|
||||
|
||||
state.ShouldContainKey("a.1");
|
||||
state.ShouldContainKey("a.2");
|
||||
state.ShouldNotContainKey("b.1");
|
||||
|
||||
state["a.1"].Msgs.ShouldBe(2UL);
|
||||
state["a.1"].First.ShouldBe(1UL);
|
||||
state["a.1"].Last.ShouldBe(3UL);
|
||||
|
||||
state["a.2"].Msgs.ShouldBe(1UL);
|
||||
state["a.2"].First.ShouldBe(2UL);
|
||||
state["a.2"].Last.ShouldBe(2UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SubjectsTotals tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreSubjectsTotals — filestore_test.go:~1300
|
||||
[Fact]
|
||||
public async Task SubjectsTotals_ReturnsPerSubjectCounts()
|
||||
{
|
||||
await using var store = CreateStore("subjectstotals");
|
||||
|
||||
await store.AppendAsync("x.1", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("x.1", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("x.2", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("y.1", "m"u8.ToArray(), default);
|
||||
await store.AppendAsync("x.3", "m"u8.ToArray(), default);
|
||||
|
||||
var totals = store.SubjectsTotals("x.*");
|
||||
|
||||
totals.ShouldContainKey("x.1");
|
||||
totals.ShouldContainKey("x.2");
|
||||
totals.ShouldContainKey("x.3");
|
||||
totals.ShouldNotContainKey("y.1");
|
||||
|
||||
totals["x.1"].ShouldBe(2UL);
|
||||
totals["x.2"].ShouldBe(1UL);
|
||||
totals["x.3"].ShouldBe(1UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// State (with deleted sequences) tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreState — filestore_test.go:~420
|
||||
[Fact]
|
||||
public async Task State_IncludesDeletedSequences()
|
||||
{
|
||||
await using var store = CreateStore("state-deleted");
|
||||
|
||||
// Store 10 messages
|
||||
for (var i = 1; i <= 10; i++)
|
||||
await store.AppendAsync("events", Encoding.UTF8.GetBytes($"e{i}"), default);
|
||||
|
||||
// Remove messages 3, 5, 7
|
||||
await store.RemoveAsync(3, default);
|
||||
await store.RemoveAsync(5, default);
|
||||
await store.RemoveAsync(7, default);
|
||||
|
||||
var state = store.State();
|
||||
|
||||
state.Msgs.ShouldBe(7UL);
|
||||
state.FirstSeq.ShouldBe(1UL);
|
||||
state.LastSeq.ShouldBe(10UL);
|
||||
state.NumDeleted.ShouldBe(3);
|
||||
|
||||
state.Deleted.ShouldNotBeNull();
|
||||
state.Deleted!.ShouldContain(3UL);
|
||||
state.Deleted.ShouldContain(5UL);
|
||||
state.Deleted.ShouldContain(7UL);
|
||||
state.Deleted.Length.ShouldBe(3);
|
||||
|
||||
// NumSubjects: all messages are on "events"
|
||||
state.NumSubjects.ShouldBe(1);
|
||||
state.Subjects.ShouldNotBeNull();
|
||||
state.Subjects!["events"].ShouldBe(7UL);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// GetSeqFromTime tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: TestFileStoreGetSeqFromTime — filestore_test.go:~1570
|
||||
[Fact]
|
||||
public async Task GetSeqFromTime_ReturnsCorrectSequence()
|
||||
{
|
||||
await using var store = CreateStore("getseqfromtime");
|
||||
|
||||
// Store 5 messages; we'll query by the timestamp of the 3rd message
|
||||
var timestamps = new List<DateTime>();
|
||||
for (var i = 1; i <= 5; i++)
|
||||
{
|
||||
await store.AppendAsync("time.test", Encoding.UTF8.GetBytes($"t{i}"), default);
|
||||
var msgs = await store.ListAsync(default);
|
||||
timestamps.Add(msgs[^1].TimestampUtc);
|
||||
// Small delay to ensure distinct timestamps
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
// Query for first seq at or after the timestamp of msg 3
|
||||
var targetTime = timestamps[2]; // timestamp of sequence 3
|
||||
var seq = store.GetSeqFromTime(targetTime);
|
||||
seq.ShouldBe(3UL);
|
||||
|
||||
// Query with a time before all messages: should return 1
|
||||
var beforeAll = timestamps[0].AddMilliseconds(-100);
|
||||
store.GetSeqFromTime(beforeAll).ShouldBe(1UL);
|
||||
|
||||
// Query with a time after all messages: should return last+1
|
||||
var afterAll = timestamps[^1].AddSeconds(1);
|
||||
store.GetSeqFromTime(afterAll).ShouldBe(6UL); // _last + 1
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// MsgBlock enhancements
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Go: filestore.go dmap — soft-delete tracking and enumeration
|
||||
[Fact]
|
||||
public async Task MsgBlock_IsDeleted_AndEnumerateNonDeleted_Work()
|
||||
{
|
||||
await using var store = CreateStore("block-enumerate");
|
||||
|
||||
// Store 5 messages on 2 subjects
|
||||
await store.AppendAsync("a.1", "m1"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.2", "m2"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.1", "m3"u8.ToArray(), default);
|
||||
await store.AppendAsync("b.1", "m4"u8.ToArray(), default);
|
||||
await store.AppendAsync("a.2", "m5"u8.ToArray(), default);
|
||||
|
||||
// Delete sequences 2 and 4
|
||||
await store.RemoveAsync(2, default);
|
||||
await store.RemoveAsync(4, default);
|
||||
|
||||
// Verify the state after deletion
|
||||
var all = await store.ListAsync(default);
|
||||
all.Count.ShouldBe(3);
|
||||
all.Select(m => m.Sequence).ShouldBe([1UL, 3UL, 5UL]);
|
||||
|
||||
// FilteredState should only see non-deleted
|
||||
var aState = store.FilteredState(1, "a.1");
|
||||
aState.Msgs.ShouldBe(2UL); // sequences 1 and 3
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user