dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add FileStore tombstone, TTL & consumer state persistence (Task 2)

Browse Source 
Port Go filestore tombstone/deletion tests, consumer state encode/decode,
consumer file store persistence, and message TTL enforcement. Adds
ConsumerStateCodec and ConsumerFileStore implementations.

17 new tests ported from filestore_test.go.
This commit is contained in:
Joseph Doherty
2026-02-24 20:17:35 -05:00
parent a9967d3077
commit 7eb06c8ac5
6 changed files with 1557 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/NATS.Server/JetStream/Storage/FileStore.cs
View File

@@ -665,6 +665,20 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
DisposeAllBlocks();
}
/// <summary>
/// Stops the store and deletes all persisted data (blocks, index files).
/// Reference: golang/nats-server/server/filestore.go — fileStore.Delete.
/// </summary>
public void Delete()
{
DisposeAllBlocks();
if (Directory.Exists(_options.Directory))
{
try { Directory.Delete(_options.Directory, recursive: true); }
catch { /* best effort */ }
}
}
// -------------------------------------------------------------------------
// Block management
@@ -831,12 +845,24 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
PruneExpired(DateTime.UtcNow);
// After recovery, sync _last watermark from block metadata only when
// no messages were recovered (e.g., after a full purge). This ensures
// FirstSeq/LastSeq watermarks survive a restart after purge.
// We do NOT override _last if messages were found — truncation may have
// reduced _last below the block's raw LastSequence.
// After recovery, sync _last from skip-sequence high-water marks.
// SkipMsg/SkipMsgs write tombstone records with empty subject — these
// intentionally advance _last without storing a live message. We must
// include them in the high-water mark so the next StoreMsg gets the
// correct sequence number.
// We do NOT use block.LastSequence blindly because that includes
// soft-deleted real messages at the tail (e.g., after Truncate or
// RemoveMsg of the last message), which must not inflate _last.
// Go: filestore.go — recovery sets state.LastSeq from lmb.last.seq.
foreach (var blk in _blocks)
{
var maxSkip = blk.MaxSkipSequence;
if (maxSkip > _last)
_last = maxSkip;
}
// If no messages and no skips were found, fall back to block.LastSequence
// to preserve watermarks from purge or full-delete scenarios.
if (_last == 0)
{
foreach (var blk in _blocks)
@@ -1320,8 +1346,9 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
var removed = _messages.Remove(seq);
if (removed)
{
if (seq == _last)
_last = _messages.Count == 0 ? _last : _messages.Keys.Max();
// Go: filestore.go — LastSeq (lmb.last.seq) is a high-water mark and is
// never decremented on removal. Only FirstSeq advances when the first
// live message is removed.
if (_messages.Count == 0)
_first = _last + 1; // All gone — next first would be after last
else
@@ -1577,6 +1604,25 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
}
// -------------------------------------------------------------------------
// ConsumerStore factory
// Reference: golang/nats-server/server/filestore.go — fileStore.ConsumerStore
// -------------------------------------------------------------------------
/// <summary>
/// Creates or opens a per-consumer state store backed by a binary file.
/// The state file is located at <c>{Directory}/obs/{name}/o.dat</c>,
/// matching the Go server's consumer directory layout.
/// Reference: golang/nats-server/server/filestore.go — newConsumerFileStore.
/// </summary>
public IConsumerStore ConsumerStore(string name, DateTime created, ConsumerConfig cfg)
{
var consumerDir = Path.Combine(_options.Directory, "obs", name);
Directory.CreateDirectory(consumerDir);
var stateFile = Path.Combine(consumerDir, "o.dat");
return new ConsumerFileStore(stateFile, cfg);
}
private sealed class FileRecord
{
public ulong Sequence { get; init; }