natsdotnet/tests/NATS.Server.JetStream.Tests/JetStream/Storage/FileStorePurgeBlockTests.cs

// Reference: golang/nats-server/server/filestore_test.go
// Tests ported from: TestFileStorePurgeEx, TestFileStorePurgeExWithSubject,
//   TestFileStorePurgeExKeepOneBug, TestFileStoreCompact, TestFileStoreStreamTruncate,
//   TestFileStoreState, TestFileStoreFilteredState, TestFileStoreSubjectsState,
//   TestFileStoreGetSeqFromTime

using System.Text;
using NATS.Server.JetStream.Storage;

namespace NATS.Server.JetStream.Tests.JetStream.Storage;

/// <summary>
/// Tests for FileStore tombstone tracking and purge operations:
/// PurgeEx, Compact, Truncate, FilteredState, SubjectsState, SubjectsTotals,
/// State (with deleted sequences), and GetSeqFromTime.
/// Reference: golang/nats-server/server/filestore_test.go
/// </summary>
public sealed class FileStorePurgeBlockTests : IDisposable
{
    private readonly string _dir;

    public FileStorePurgeBlockTests()
    {
        _dir = Path.Combine(Path.GetTempPath(), $"nats-js-purgeblock-{Guid.NewGuid():N}");
        Directory.CreateDirectory(_dir);
    }

    public void Dispose()
    {
        if (Directory.Exists(_dir))
            Directory.Delete(_dir, recursive: true);
    }

    private FileStore CreateStore(string subdirectory, FileStoreOptions? options = null)
    {
        var dir = Path.Combine(_dir, subdirectory);
        var opts = options ?? new FileStoreOptions();
        opts.Directory = dir;
        return new FileStore(opts);
    }

    // -------------------------------------------------------------------------
    // PurgeEx tests
    // -------------------------------------------------------------------------

    // Go: TestFileStorePurgeExWithSubject — filestore_test.go:~867
    [Fact]
    public async Task PurgeEx_BySubject_RemovesMatchingMessages()
    {
        await using var store = CreateStore("purgex-subject");

        // Store 5 messages on "foo" and 5 on "bar"
        for (var i = 0; i < 5; i++)
            await store.AppendAsync("foo", Encoding.UTF8.GetBytes($"foo-{i}"), default);
        for (var i = 0; i < 5; i++)
            await store.AppendAsync("bar", Encoding.UTF8.GetBytes($"bar-{i}"), default);

        var stateBeforePurge = await store.GetStateAsync(default);
        stateBeforePurge.Messages.ShouldBe(10UL);

        // Purge all "foo" messages (seq=0 means no upper limit; keep=0 means keep none)
        var purged = store.PurgeEx("foo", 0, 0);
        purged.ShouldBe(5UL);

        // Only "bar" messages remain
        var state = await store.GetStateAsync(default);
        state.Messages.ShouldBe(5UL);

        var remaining = await store.ListAsync(default);
        remaining.All(m => m.Subject == "bar").ShouldBeTrue();
    }

    // Go: TestFileStorePurgeExKeepOneBug — filestore_test.go:~910
    [Fact]
    public async Task PurgeEx_WithKeep_RetainsNewestMessages()
    {
        await using var store = CreateStore("purgex-keep");

        // Store 10 messages on "events"
        for (var i = 0; i < 10; i++)
            await store.AppendAsync("events", Encoding.UTF8.GetBytes($"msg-{i}"), default);

        // Purge keeping the 3 newest
        var purged = store.PurgeEx("events", 0, 3);
        purged.ShouldBe(7UL);

        var remaining = await store.ListAsync(default);
        remaining.Count.ShouldBe(3);

        // The retained messages should be the 3 highest sequences (8, 9, 10)
        var seqs = remaining.Select(m => m.Sequence).OrderBy(s => s).ToArray();
        seqs[0].ShouldBe(8UL);
        seqs[1].ShouldBe(9UL);
        seqs[2].ShouldBe(10UL);
    }

    // Go: TestFileStorePurgeEx — filestore_test.go:~855
    [Fact]
    public async Task PurgeEx_WithSeqLimit_OnlyPurgesBelowSequence()
    {
        await using var store = CreateStore("purgex-seqlimit");

        // Store 10 messages on "data"
        for (var i = 1; i <= 10; i++)
            await store.AppendAsync("data", Encoding.UTF8.GetBytes($"d{i}"), default);

        // Purge "data" messages with seq <= 5 (keep=0)
        var purged = store.PurgeEx("data", 5, 0);
        purged.ShouldBe(5UL);

        // Messages 6-10 should remain
        var remaining = await store.ListAsync(default);
        remaining.Count.ShouldBe(5);
        remaining.Min(m => m.Sequence).ShouldBe(6UL);
        remaining.Max(m => m.Sequence).ShouldBe(10UL);
    }

    // Go: PurgeEx with wildcard subject — filestore_test.go:~867
    [Fact]
    public async Task PurgeEx_WithWildcardSubject_RemovesAllMatchingSubjects()
    {
        await using var store = CreateStore("purgex-wildcard");

        await store.AppendAsync("foo.a", "m1"u8.ToArray(), default);
        await store.AppendAsync("foo.b", "m2"u8.ToArray(), default);
        await store.AppendAsync("bar.a", "m3"u8.ToArray(), default);
        await store.AppendAsync("foo.c", "m4"u8.ToArray(), default);

        var purged = store.PurgeEx("foo.*", 0, 0);
        purged.ShouldBe(3UL);

        var remaining = await store.ListAsync(default);
        remaining.Count.ShouldBe(1);
        remaining[0].Subject.ShouldBe("bar.a");
    }

    // Go: PurgeEx with > wildcard — filestore_test.go:~867
    [Fact]
    public async Task PurgeEx_WithGtWildcard_RemovesAllMatchingSubjects()
    {
        await using var store = CreateStore("purgex-gt-wildcard");

        await store.AppendAsync("a.b.c", "m1"u8.ToArray(), default);
        await store.AppendAsync("a.b.d", "m2"u8.ToArray(), default);
        await store.AppendAsync("a.x", "m3"u8.ToArray(), default);
        await store.AppendAsync("b.x", "m4"u8.ToArray(), default);

        var purged = store.PurgeEx("a.>", 0, 0);
        purged.ShouldBe(3UL);

        var remaining = await store.ListAsync(default);
        remaining.Count.ShouldBe(1);
        remaining[0].Subject.ShouldBe("b.x");
    }

    // -------------------------------------------------------------------------
    // Compact tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreCompact — filestore_test.go:~964
    [Fact]
    public async Task Compact_RemovesMessagesBeforeSequence()
    {
        await using var store = CreateStore("compact-basic");

        // Store 10 messages
        for (var i = 1; i <= 10; i++)
            await store.AppendAsync("test", Encoding.UTF8.GetBytes($"msg{i}"), default);

        // Compact to remove messages with seq < 5 (removes 1, 2, 3, 4)
        var removed = store.Compact(5);
        removed.ShouldBe(4UL);

        var remaining = await store.ListAsync(default);
        remaining.Count.ShouldBe(6); // 5-10

        remaining.Min(m => m.Sequence).ShouldBe(5UL);
        remaining.Max(m => m.Sequence).ShouldBe(10UL);

        // Sequence 1-4 should no longer be loadable
        (await store.LoadAsync(1, default)).ShouldBeNull();
        (await store.LoadAsync(4, default)).ShouldBeNull();

        // Sequence 5 should still exist
        (await store.LoadAsync(5, default)).ShouldNotBeNull();
    }

    // -------------------------------------------------------------------------
    // Truncate tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreStreamTruncate — filestore_test.go:~1035
    [Fact]
    public async Task Truncate_RemovesMessagesAfterSequence()
    {
        await using var store = CreateStore("truncate-basic");

        // Store 10 messages
        for (var i = 1; i <= 10; i++)
            await store.AppendAsync("stream", Encoding.UTF8.GetBytes($"m{i}"), default);

        // Truncate at seq=5 (removes 6, 7, 8, 9, 10)
        store.Truncate(5);

        var remaining = await store.ListAsync(default);
        remaining.Count.ShouldBe(5); // 1-5

        remaining.Min(m => m.Sequence).ShouldBe(1UL);
        remaining.Max(m => m.Sequence).ShouldBe(5UL);

        // Messages 6-10 should be gone
        (await store.LoadAsync(6, default)).ShouldBeNull();
        (await store.LoadAsync(10, default)).ShouldBeNull();

        // Message 5 should still exist
        (await store.LoadAsync(5, default)).ShouldNotBeNull();
    }

    // -------------------------------------------------------------------------
    // FilteredState tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreFilteredState — filestore_test.go:~1200
    [Fact]
    public async Task FilteredState_ReturnsCorrectState()
    {
        await using var store = CreateStore("filteredstate");

        // Store 5 messages on "orders" and 5 on "invoices"
        for (var i = 1; i <= 5; i++)
            await store.AppendAsync("orders", Encoding.UTF8.GetBytes($"o{i}"), default);
        for (var i = 1; i <= 5; i++)
            await store.AppendAsync("invoices", Encoding.UTF8.GetBytes($"inv{i}"), default);

        // FilteredState for "orders" from seq=1
        var ordersState = store.FilteredState(1, "orders");
        ordersState.Msgs.ShouldBe(5UL);
        ordersState.First.ShouldBe(1UL);
        ordersState.Last.ShouldBe(5UL);

        // FilteredState for "invoices" from seq=1
        var invoicesState = store.FilteredState(1, "invoices");
        invoicesState.Msgs.ShouldBe(5UL);
        invoicesState.First.ShouldBe(6UL);
        invoicesState.Last.ShouldBe(10UL);

        // FilteredState from seq=7 (only 4 invoices remain)
        var lateInvoices = store.FilteredState(7, "invoices");
        lateInvoices.Msgs.ShouldBe(4UL);
        lateInvoices.First.ShouldBe(7UL);
        lateInvoices.Last.ShouldBe(10UL);

        // No match for non-existent subject
        var noneState = store.FilteredState(1, "orders.unknown");
        noneState.Msgs.ShouldBe(0UL);
    }

    // -------------------------------------------------------------------------
    // SubjectsState tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreSubjectsState — filestore_test.go:~1266
    [Fact]
    public async Task SubjectsState_ReturnsPerSubjectState()
    {
        await using var store = CreateStore("subjectsstate");

        await store.AppendAsync("a.1", "msg"u8.ToArray(), default);
        await store.AppendAsync("a.2", "msg"u8.ToArray(), default);
        await store.AppendAsync("a.1", "msg"u8.ToArray(), default);
        await store.AppendAsync("b.1", "msg"u8.ToArray(), default);

        var state = store.SubjectsState("a.>");

        state.ShouldContainKey("a.1");
        state.ShouldContainKey("a.2");
        state.ShouldNotContainKey("b.1");

        state["a.1"].Msgs.ShouldBe(2UL);
        state["a.1"].First.ShouldBe(1UL);
        state["a.1"].Last.ShouldBe(3UL);

        state["a.2"].Msgs.ShouldBe(1UL);
        state["a.2"].First.ShouldBe(2UL);
        state["a.2"].Last.ShouldBe(2UL);
    }

    // -------------------------------------------------------------------------
    // SubjectsTotals tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreSubjectsTotals — filestore_test.go:~1300
    [Fact]
    public async Task SubjectsTotals_ReturnsPerSubjectCounts()
    {
        await using var store = CreateStore("subjectstotals");

        await store.AppendAsync("x.1", "m"u8.ToArray(), default);
        await store.AppendAsync("x.1", "m"u8.ToArray(), default);
        await store.AppendAsync("x.2", "m"u8.ToArray(), default);
        await store.AppendAsync("y.1", "m"u8.ToArray(), default);
        await store.AppendAsync("x.3", "m"u8.ToArray(), default);

        var totals = store.SubjectsTotals("x.*");

        totals.ShouldContainKey("x.1");
        totals.ShouldContainKey("x.2");
        totals.ShouldContainKey("x.3");
        totals.ShouldNotContainKey("y.1");

        totals["x.1"].ShouldBe(2UL);
        totals["x.2"].ShouldBe(1UL);
        totals["x.3"].ShouldBe(1UL);
    }

    // -------------------------------------------------------------------------
    // State (with deleted sequences) tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreState — filestore_test.go:~420
    [Fact]
    public async Task State_IncludesDeletedSequences()
    {
        await using var store = CreateStore("state-deleted");

        // Store 10 messages
        for (var i = 1; i <= 10; i++)
            await store.AppendAsync("events", Encoding.UTF8.GetBytes($"e{i}"), default);

        // Remove messages 3, 5, 7
        await store.RemoveAsync(3, default);
        await store.RemoveAsync(5, default);
        await store.RemoveAsync(7, default);

        var state = store.State();

        state.Msgs.ShouldBe(7UL);
        state.FirstSeq.ShouldBe(1UL);
        state.LastSeq.ShouldBe(10UL);
        state.NumDeleted.ShouldBe(3);

        state.Deleted.ShouldNotBeNull();
        state.Deleted!.ShouldContain(3UL);
        state.Deleted.ShouldContain(5UL);
        state.Deleted.ShouldContain(7UL);
        state.Deleted.Length.ShouldBe(3);

        // NumSubjects: all messages are on "events"
        state.NumSubjects.ShouldBe(1);
        state.Subjects.ShouldNotBeNull();
        state.Subjects!["events"].ShouldBe(7UL);
    }

    // -------------------------------------------------------------------------
    // GetSeqFromTime tests
    // -------------------------------------------------------------------------

    // Go: TestFileStoreGetSeqFromTime — filestore_test.go:~1570
    [Fact]
    public async Task GetSeqFromTime_ReturnsCorrectSequence()
    {
        await using var store = CreateStore("getseqfromtime");

        // Store 5 messages; we'll query by the timestamp of the 3rd message
        var timestamps = new List<DateTime>();
        for (var i = 1; i <= 5; i++)
        {
            await store.AppendAsync("time.test", Encoding.UTF8.GetBytes($"t{i}"), default);
            var msgs = await store.ListAsync(default);
            timestamps.Add(msgs[^1].TimestampUtc);
            // Small delay to ensure distinct timestamps
            await Task.Delay(5);
        }

        // Query for first seq at or after the timestamp of msg 3
        var targetTime = timestamps[2]; // timestamp of sequence 3
        var seq = store.GetSeqFromTime(targetTime);
        seq.ShouldBe(3UL);

        // Query with a time before all messages: should return 1
        var beforeAll = timestamps[0].AddMilliseconds(-100);
        store.GetSeqFromTime(beforeAll).ShouldBe(1UL);

        // Query with a time after all messages: should return last+1
        var afterAll = timestamps[^1].AddSeconds(1);
        store.GetSeqFromTime(afterAll).ShouldBe(6UL); // _last + 1
    }

    // -------------------------------------------------------------------------
    // MsgBlock enhancements
    // -------------------------------------------------------------------------

    // Go: filestore.go dmap — soft-delete tracking and enumeration
    [Fact]
    public async Task MsgBlock_IsDeleted_AndEnumerateNonDeleted_Work()
    {
        await using var store = CreateStore("block-enumerate");

        // Store 5 messages on 2 subjects
        await store.AppendAsync("a.1", "m1"u8.ToArray(), default);
        await store.AppendAsync("a.2", "m2"u8.ToArray(), default);
        await store.AppendAsync("a.1", "m3"u8.ToArray(), default);
        await store.AppendAsync("b.1", "m4"u8.ToArray(), default);
        await store.AppendAsync("a.2", "m5"u8.ToArray(), default);

        // Delete sequences 2 and 4
        await store.RemoveAsync(2, default);
        await store.RemoveAsync(4, default);

        // Verify the state after deletion
        var all = await store.ListAsync(default);
        all.Count.ShouldBe(3);
        all.Select(m => m.Sequence).ShouldBe([1UL, 3UL, 5UL]);

        // FilteredState should only see non-deleted
        var aState = store.FilteredState(1, "a.1");
        aState.Msgs.ShouldBe(2UL); // sequences 1 and 3
    }
}