natsdotnet/tests/NATS.Server.Tests/JetStream/Storage/MsgBlockTests.cs

// Reference: golang/nats-server/server/filestore.go:217-267 (msgBlock struct)
// Go block write: filestore.go:6700-6760 (writeMsgRecord)
// Go block load: filestore.go:8140-8260 (loadMsgs / msgFromBufEx)
// Go deletion: filestore.go dmap (avl.SequenceSet) for soft-deletes
//
// These tests verify the .NET MsgBlock abstraction — a block of messages stored
// in a single append-only file on disk, with in-memory index and soft-delete support.

using System.Text;
using NATS.Server.JetStream.Storage;

namespace NATS.Server.Tests.JetStream.Storage;

public sealed class MsgBlockTests : IDisposable
{
    private readonly string _testDir;

    public MsgBlockTests()
    {
        _testDir = Path.Combine(Path.GetTempPath(), $"msgblock_test_{Guid.NewGuid():N}");
        Directory.CreateDirectory(_testDir);
    }

    public void Dispose()
    {
        try { Directory.Delete(_testDir, recursive: true); }
        catch { /* best effort cleanup */ }
    }

    // Go: writeMsgRecord — single message write returns first sequence
    [Fact]
    public void Write_SingleMessage_ReturnsSequence()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
        var seq = block.Write("foo.bar", ReadOnlyMemory<byte>.Empty, "hello"u8.ToArray());
        seq.ShouldBe(1UL);
    }

    // Go: writeMsgRecord — sequential writes increment sequence
    [Fact]
    public void Write_MultipleMessages_IncrementsSequence()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
        var s1 = block.Write("a", ReadOnlyMemory<byte>.Empty, "one"u8.ToArray());
        var s2 = block.Write("b", ReadOnlyMemory<byte>.Empty, "two"u8.ToArray());
        var s3 = block.Write("c", ReadOnlyMemory<byte>.Empty, "three"u8.ToArray());

        s1.ShouldBe(1UL);
        s2.ShouldBe(2UL);
        s3.ShouldBe(3UL);
        block.MessageCount.ShouldBe(3UL);
    }

    // Go: loadMsgs / msgFromBufEx — read back by sequence number
    [Fact]
    public void Read_BySequence_ReturnsMessage()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
        block.Write("test.subject", ReadOnlyMemory<byte>.Empty, "payload data"u8.ToArray());

        var record = block.Read(1);
        record.ShouldNotBeNull();
        record.Sequence.ShouldBe(1UL);
        record.Subject.ShouldBe("test.subject");
        Encoding.UTF8.GetString(record.Payload.Span).ShouldBe("payload data");
    }

    // Go: loadMsgs — reading a non-existent sequence returns nil
    [Fact]
    public void Read_NonexistentSequence_ReturnsNull()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
        block.Write("a", ReadOnlyMemory<byte>.Empty, "data"u8.ToArray());

        var record = block.Read(999);
        record.ShouldBeNull();
    }

    // Go: filestore.go rbytes check — block seals when size exceeds maxBytes
    [Fact]
    public void IsSealed_ReturnsTrueWhenFull()
    {
        // Use a very small maxBytes so the block seals quickly.
        // A single record with subject "a", empty headers, and 32-byte payload is ~61 bytes.
        // Set maxBytes to 50 so one write seals the block.
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 50);

        var payload = new byte[32];
        block.Write("a", ReadOnlyMemory<byte>.Empty, payload);
        block.IsSealed.ShouldBeTrue();
    }

    // Go: filestore.go errNoRoom — cannot write to sealed block
    [Fact]
    public void Write_ThrowsWhenSealed()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 50);
        block.Write("a", ReadOnlyMemory<byte>.Empty, new byte[32]);
        block.IsSealed.ShouldBeTrue();

        Should.Throw<InvalidOperationException>(() =>
            block.Write("b", ReadOnlyMemory<byte>.Empty, "more"u8.ToArray()));
    }

    // Go: dmap soft-delete — deleted message reads back as null
    [Fact]
    public void Delete_MarksSequenceAsDeleted()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
        block.Write("a", ReadOnlyMemory<byte>.Empty, "data"u8.ToArray());

        block.Delete(1).ShouldBeTrue();
        block.Read(1).ShouldBeNull();
    }

    // Go: dmap — MessageCount reflects only non-deleted messages
    [Fact]
    public void Delete_DecreasesMessageCount()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);
        block.Write("a", ReadOnlyMemory<byte>.Empty, "one"u8.ToArray());
        block.Write("b", ReadOnlyMemory<byte>.Empty, "two"u8.ToArray());
        block.Write("c", ReadOnlyMemory<byte>.Empty, "three"u8.ToArray());

        block.MessageCount.ShouldBe(3UL);
        block.DeletedCount.ShouldBe(0UL);

        block.Delete(2).ShouldBeTrue();

        block.MessageCount.ShouldBe(2UL);
        block.DeletedCount.ShouldBe(1UL);

        // Double delete returns false
        block.Delete(2).ShouldBeFalse();
    }

    // Go: recovery path — rebuild index from existing block file
    [Fact]
    public void Recover_RebuildsIndexFromFile()
    {
        // Write messages and dispose
        using (var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024))
        {
            block.Write("a.b", ReadOnlyMemory<byte>.Empty, "first"u8.ToArray());
            block.Write("c.d", ReadOnlyMemory<byte>.Empty, "second"u8.ToArray());
            block.Write("e.f", ReadOnlyMemory<byte>.Empty, "third"u8.ToArray());
            block.Flush();
        }

        // Recover and verify all messages readable
        using var recovered = MsgBlock.Recover(0, _testDir);
        recovered.MessageCount.ShouldBe(3UL);
        recovered.FirstSequence.ShouldBe(1UL);
        recovered.LastSequence.ShouldBe(3UL);

        var r1 = recovered.Read(1);
        r1.ShouldNotBeNull();
        r1.Subject.ShouldBe("a.b");
        Encoding.UTF8.GetString(r1.Payload.Span).ShouldBe("first");

        var r2 = recovered.Read(2);
        r2.ShouldNotBeNull();
        r2.Subject.ShouldBe("c.d");

        var r3 = recovered.Read(3);
        r3.ShouldNotBeNull();
        r3.Subject.ShouldBe("e.f");
    }

    // Go: recovery with dmap — deleted records still show as null after recovery
    [Fact]
    public void Recover_PreservesDeletedState()
    {
        // Write messages, delete one, flush and dispose
        using (var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024))
        {
            block.Write("a", ReadOnlyMemory<byte>.Empty, "one"u8.ToArray());
            block.Write("b", ReadOnlyMemory<byte>.Empty, "two"u8.ToArray());
            block.Write("c", ReadOnlyMemory<byte>.Empty, "three"u8.ToArray());
            block.Delete(2);
            block.Flush();
        }

        // Recover — seq 2 should still be deleted
        using var recovered = MsgBlock.Recover(0, _testDir);
        recovered.MessageCount.ShouldBe(2UL);
        recovered.DeletedCount.ShouldBe(1UL);

        recovered.Read(1).ShouldNotBeNull();
        recovered.Read(2).ShouldBeNull();
        recovered.Read(3).ShouldNotBeNull();
    }

    // Go: sync.RWMutex on msgBlock — concurrent reads during writes should not crash
    [Fact]
    public async Task ConcurrentReads_DuringWrite()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024);

        // Pre-populate some messages
        for (var i = 0; i < 10; i++)
            block.Write($"subj.{i}", ReadOnlyMemory<byte>.Empty, Encoding.UTF8.GetBytes($"msg-{i}"));

        // Run concurrent reads and writes
        var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
        var exceptions = new List<Exception>();

        var writerTask = Task.Run(() =>
        {
            try
            {
                while (!cts.Token.IsCancellationRequested)
                {
                    try
                    {
                        block.Write("concurrent", ReadOnlyMemory<byte>.Empty, "data"u8.ToArray());
                    }
                    catch (InvalidOperationException)
                    {
                        // Block sealed — expected eventually
                        break;
                    }
                }
            }
            catch (Exception ex) { lock (exceptions) { exceptions.Add(ex); } }
        });

        var readerTasks = Enumerable.Range(0, 4).Select(t => Task.Run(() =>
        {
            try
            {
                while (!cts.Token.IsCancellationRequested)
                {
                    for (ulong seq = 1; seq <= 10; seq++)
                        _ = block.Read(seq);
                }
            }
            catch (Exception ex) { lock (exceptions) { exceptions.Add(ex); } }
        })).ToArray();

        await Task.WhenAll([writerTask, .. readerTasks]).WaitAsync(TimeSpan.FromSeconds(5));

        exceptions.ShouldBeEmpty();
    }

    // Go: msgBlock first/last — custom firstSequence offsets sequence numbering
    [Fact]
    public void Write_WithCustomFirstSequence()
    {
        using var block = MsgBlock.Create(0, _testDir, maxBytes: 1024 * 1024, firstSequence: 100);
        var s1 = block.Write("x", ReadOnlyMemory<byte>.Empty, "a"u8.ToArray());
        var s2 = block.Write("y", ReadOnlyMemory<byte>.Empty, "b"u8.ToArray());

        s1.ShouldBe(100UL);
        s2.ShouldBe(101UL);
        block.FirstSequence.ShouldBe(100UL);
        block.LastSequence.ShouldBe(101UL);

        var r = block.Read(100);
        r.ShouldNotBeNull();
        r.Subject.ShouldBe("x");
    }
}