feat(storage): port FileStore Go tests and add sync methods (Go parity)
Add 67 Go-parity tests from filestore_test.go covering: - SkipMsg/SkipMsgs sequence reservation - RemoveMsg/EraseMsg soft-delete - LoadMsg/LoadLastMsg/LoadNextMsg message retrieval - AllLastSeqs/MultiLastSeqs per-subject last sequences - SubjectForSeq reverse lookup - NumPending with filters and last-per-subject mode - Recovery watermark preservation after purge - FastState NumDeleted/LastTime correctness - PurgeEx with empty subject + keep parameter - Compact _first watermark tracking - Multi-block operations and state verification Implements missing IStreamStore sync methods on FileStore: RemoveMsg, EraseMsg, SkipMsg, SkipMsgs, LoadMsg, LoadLastMsg, LoadNextMsg, AllLastSeqs, MultiLastSeqs, SubjectForSeq, NumPending. Adds MsgBlock.WriteSkip() for tombstone sequence reservation. Adds IDisposable to FileStore for synchronous test disposal.
This commit is contained in:
Joseph Doherty
@@ -19,7 +19,7 @@ namespace NATS.Server.JetStream.Storage;
|
||||
/// Reference: golang/nats-server/server/filestore.go — block manager, block rotation,
|
||||
/// recovery via scanning .blk files, soft-delete via dmap.
|
||||
/// </summary>
|
||||
public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
|
||||
{
|
||||
private readonly FileStoreOptions _options;
|
||||
|
||||
@@ -33,6 +33,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
private int _nextBlockId;
|
||||
|
||||
private ulong _last;
|
||||
private ulong _first; // Go: first.seq — watermark for the first live or expected-first sequence
|
||||
|
||||
// Resolved at construction time: which format family to use.
|
||||
private readonly bool _useS2; // true -> S2Codec (FSV2 compression path)
|
||||
@@ -332,7 +333,10 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
/// </summary>
|
||||
public ulong PurgeEx(string subject, ulong seq, ulong keep)
|
||||
{
|
||||
if (string.IsNullOrEmpty(subject))
|
||||
// Go parity: empty subject with keep=0 and seq=0 is a full purge.
|
||||
// If keep > 0 or seq > 0, fall through to the candidate-based path
|
||||
// treating all messages as candidates.
|
||||
if (string.IsNullOrEmpty(subject) && keep == 0 && seq == 0)
|
||||
return Purge();
|
||||
|
||||
// Collect all messages matching the subject (with wildcard support) at or below seq, ordered by sequence.
|
||||
@@ -389,9 +393,18 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
}
|
||||
|
||||
if (_messages.Count == 0)
|
||||
_last = 0;
|
||||
else if (!_messages.ContainsKey(_last))
|
||||
_last = _messages.Keys.Max();
|
||||
{
|
||||
// Go: preserve _last (monotonically increasing), advance _first to seq.
|
||||
// Compact(seq) removes everything < seq; the new first is seq.
|
||||
_first = seq;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!_messages.ContainsKey(_last))
|
||||
_last = _messages.Keys.Max();
|
||||
// Update _first to reflect the real first message.
|
||||
_first = _messages.Keys.Min();
|
||||
}
|
||||
|
||||
return (ulong)toRemove.Length;
|
||||
}
|
||||
@@ -580,15 +593,39 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
|
||||
if (_messages.Count == 0)
|
||||
{
|
||||
state.FirstSeq = 0;
|
||||
// Go: when all messages are removed/expired, first.seq tracks the watermark.
|
||||
// If _first > 0 use it (set by Compact / SkipMsg); otherwise 0.
|
||||
state.FirstSeq = _first > 0 ? _first : 0;
|
||||
state.FirstTime = default;
|
||||
state.NumDeleted = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
var firstSeq = _messages.Keys.Min();
|
||||
state.FirstSeq = firstSeq;
|
||||
state.FirstTime = _messages[firstSeq].TimestampUtc;
|
||||
state.LastTime = _messages[_last].TimestampUtc;
|
||||
|
||||
// Go parity: LastTime from the actual last stored message (not _last,
|
||||
// which may be a skip/tombstone sequence with no corresponding message).
|
||||
if (_messages.TryGetValue(_last, out var lastMsg))
|
||||
state.LastTime = lastMsg.TimestampUtc;
|
||||
else
|
||||
{
|
||||
// _last is a skip — use the highest actual message time.
|
||||
var actualLast = _messages.Keys.Max();
|
||||
state.LastTime = _messages[actualLast].TimestampUtc;
|
||||
}
|
||||
|
||||
// Go parity: NumDeleted = gaps between firstSeq and lastSeq not in _messages.
|
||||
// Reference: filestore.go — FastState sets state.NumDeleted.
|
||||
if (_last >= firstSeq)
|
||||
{
|
||||
var span = _last - firstSeq + 1;
|
||||
var liveCount = (ulong)_messages.Count;
|
||||
state.NumDeleted = span > liveCount ? (int)(span - liveCount) : 0;
|
||||
}
|
||||
else
|
||||
state.NumDeleted = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -619,6 +656,16 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Synchronous dispose — releases all block file handles.
|
||||
/// Allows the store to be used in synchronous test contexts with <c>using</c> blocks.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
DisposeAllBlocks();
|
||||
}
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Block management
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -783,6 +830,28 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
}
|
||||
|
||||
PruneExpired(DateTime.UtcNow);
|
||||
|
||||
// After recovery, sync _last watermark from block metadata only when
|
||||
// no messages were recovered (e.g., after a full purge). This ensures
|
||||
// FirstSeq/LastSeq watermarks survive a restart after purge.
|
||||
// We do NOT override _last if messages were found — truncation may have
|
||||
// reduced _last below the block's raw LastSequence.
|
||||
// Go: filestore.go — recovery sets state.LastSeq from lmb.last.seq.
|
||||
if (_last == 0)
|
||||
{
|
||||
foreach (var blk in _blocks)
|
||||
{
|
||||
var blkLast = blk.LastSequence;
|
||||
if (blkLast > _last)
|
||||
_last = blkLast;
|
||||
}
|
||||
}
|
||||
|
||||
// Sync _first from _messages; if empty, set to _last+1 (watermark).
|
||||
if (_messages.Count > 0)
|
||||
_first = _messages.Keys.Min();
|
||||
else if (_last > 0)
|
||||
_first = _last + 1;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1235,6 +1304,279 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
|
||||
private const int EnvelopeHeaderSize = 17; // 4 magic + 1 flags + 4 keyHash + 8 payloadHash
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Go-parity sync methods not yet in the interface default implementations
|
||||
// Reference: golang/nats-server/server/filestore.go
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Soft-deletes a message by sequence number.
|
||||
/// Returns <c>true</c> if the sequence existed and was removed.
|
||||
/// Reference: golang/nats-server/server/filestore.go — RemoveMsg.
|
||||
/// </summary>
|
||||
public bool RemoveMsg(ulong seq)
|
||||
{
|
||||
var removed = _messages.Remove(seq);
|
||||
if (removed)
|
||||
{
|
||||
if (seq == _last)
|
||||
_last = _messages.Count == 0 ? _last : _messages.Keys.Max();
|
||||
if (_messages.Count == 0)
|
||||
_first = _last + 1; // All gone — next first would be after last
|
||||
else
|
||||
_first = _messages.Keys.Min();
|
||||
DeleteInBlock(seq);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Overwrites a message with zeros and then soft-deletes it.
|
||||
/// Returns <c>true</c> if the sequence existed and was erased.
|
||||
/// Reference: golang/nats-server/server/filestore.go — EraseMsg.
|
||||
/// </summary>
|
||||
public bool EraseMsg(ulong seq)
|
||||
{
|
||||
// In .NET we don't do physical overwrite — just remove from the in-memory
|
||||
// cache and soft-delete in the block layer (same semantics as RemoveMsg).
|
||||
return RemoveMsg(seq);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reserves a sequence without storing a message. Advances <see cref="_last"/>
|
||||
/// to <paramref name="seq"/> (or _last+1 when seq is 0), recording the gap in
|
||||
/// the block as a tombstone-style skip.
|
||||
/// Returns the skipped sequence number.
|
||||
/// Reference: golang/nats-server/server/filestore.go — SkipMsg.
|
||||
/// </summary>
|
||||
public ulong SkipMsg(ulong seq)
|
||||
{
|
||||
// When seq is 0, auto-assign next sequence.
|
||||
var skipSeq = seq == 0 ? _last + 1 : seq;
|
||||
_last = skipSeq;
|
||||
// Do NOT add to _messages — it is a skip (tombstone).
|
||||
// We still need to write a record to the block so recovery
|
||||
// can reconstruct the sequence gap. Use an empty subject sentinel.
|
||||
EnsureActiveBlock();
|
||||
try
|
||||
{
|
||||
_activeBlock!.WriteSkip(skipSeq);
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
RotateBlock();
|
||||
_activeBlock!.WriteSkip(skipSeq);
|
||||
}
|
||||
|
||||
if (_activeBlock!.IsSealed)
|
||||
RotateBlock();
|
||||
|
||||
// After a skip, if there are no real messages, the next real first
|
||||
// would be skipSeq+1. Track this so FastState reports correctly.
|
||||
if (_messages.Count == 0)
|
||||
_first = skipSeq + 1;
|
||||
|
||||
return skipSeq;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reserves a contiguous range of sequences starting at <paramref name="seq"/>
|
||||
/// for <paramref name="num"/> slots.
|
||||
/// Reference: golang/nats-server/server/filestore.go — SkipMsgs.
|
||||
/// Go parity: when seq is non-zero it must match the expected next sequence
|
||||
/// (_last + 1); otherwise an <see cref="InvalidOperationException"/> is thrown
|
||||
/// (Go: ErrSequenceMismatch).
|
||||
/// </summary>
|
||||
public void SkipMsgs(ulong seq, ulong num)
|
||||
{
|
||||
if (seq != 0)
|
||||
{
|
||||
var expectedNext = _last + 1;
|
||||
if (seq != expectedNext)
|
||||
throw new InvalidOperationException($"Sequence mismatch: expected {expectedNext}, got {seq}.");
|
||||
}
|
||||
else
|
||||
{
|
||||
seq = _last + 1;
|
||||
}
|
||||
|
||||
for (var i = 0UL; i < num; i++)
|
||||
SkipMsg(seq + i);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads a message by exact sequence number into the optional reusable container
|
||||
/// <paramref name="sm"/>. Throws <see cref="KeyNotFoundException"/> if not found.
|
||||
/// Reference: golang/nats-server/server/filestore.go — LoadMsg.
|
||||
/// </summary>
|
||||
public StoreMsg LoadMsg(ulong seq, StoreMsg? sm)
|
||||
{
|
||||
if (!_messages.TryGetValue(seq, out var stored))
|
||||
throw new KeyNotFoundException($"Message sequence {seq} not found.");
|
||||
|
||||
sm ??= new StoreMsg();
|
||||
sm.Clear();
|
||||
sm.Subject = stored.Subject;
|
||||
sm.Data = stored.Payload.Length > 0 ? stored.Payload.ToArray() : null;
|
||||
sm.Sequence = stored.Sequence;
|
||||
sm.Timestamp = new DateTimeOffset(stored.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
|
||||
return sm;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the most recent message on <paramref name="subject"/> into the optional
|
||||
/// reusable container <paramref name="sm"/>.
|
||||
/// Throws <see cref="KeyNotFoundException"/> if no message exists on the subject.
|
||||
/// Reference: golang/nats-server/server/filestore.go — LoadLastMsg.
|
||||
/// </summary>
|
||||
public StoreMsg LoadLastMsg(string subject, StoreMsg? sm)
|
||||
{
|
||||
var match = _messages.Values
|
||||
.Where(m => string.IsNullOrEmpty(subject)
|
||||
|| SubjectMatchesFilter(m.Subject, subject))
|
||||
.MaxBy(m => m.Sequence);
|
||||
|
||||
if (match is null)
|
||||
throw new KeyNotFoundException($"No message found for subject '{subject}'.");
|
||||
|
||||
sm ??= new StoreMsg();
|
||||
sm.Clear();
|
||||
sm.Subject = match.Subject;
|
||||
sm.Data = match.Payload.Length > 0 ? match.Payload.ToArray() : null;
|
||||
sm.Sequence = match.Sequence;
|
||||
sm.Timestamp = new DateTimeOffset(match.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
|
||||
return sm;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the next message at or after <paramref name="start"/> whose subject
|
||||
/// matches <paramref name="filter"/>. Returns the message and the number of
|
||||
/// sequences skipped to reach it.
|
||||
/// Reference: golang/nats-server/server/filestore.go — LoadNextMsg.
|
||||
/// </summary>
|
||||
public (StoreMsg Msg, ulong Skip) LoadNextMsg(string filter, bool wc, ulong start, StoreMsg? sm)
|
||||
{
|
||||
var match = _messages
|
||||
.Where(kv => kv.Key >= start)
|
||||
.Where(kv => string.IsNullOrEmpty(filter) || SubjectMatchesFilter(kv.Value.Subject, filter))
|
||||
.OrderBy(kv => kv.Key)
|
||||
.Cast<KeyValuePair<ulong, StoredMessage>?>()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (match is null)
|
||||
throw new KeyNotFoundException($"No message found at or after seq {start} matching filter '{filter}'.");
|
||||
|
||||
var found = match.Value;
|
||||
var skip = found.Key > start ? found.Key - start : 0UL;
|
||||
|
||||
sm ??= new StoreMsg();
|
||||
sm.Clear();
|
||||
sm.Subject = found.Value.Subject;
|
||||
sm.Data = found.Value.Payload.Length > 0 ? found.Value.Payload.ToArray() : null;
|
||||
sm.Sequence = found.Key;
|
||||
sm.Timestamp = new DateTimeOffset(found.Value.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
|
||||
return (sm, skip);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the last sequence for every distinct subject in the stream,
|
||||
/// sorted ascending.
|
||||
/// Reference: golang/nats-server/server/filestore.go — AllLastSeqs.
|
||||
/// </summary>
|
||||
public ulong[] AllLastSeqs()
|
||||
{
|
||||
var lastPerSubject = new Dictionary<string, ulong>(StringComparer.Ordinal);
|
||||
foreach (var kv in _messages)
|
||||
{
|
||||
var subj = kv.Value.Subject;
|
||||
if (!lastPerSubject.TryGetValue(subj, out var existing) || kv.Key > existing)
|
||||
lastPerSubject[subj] = kv.Key;
|
||||
}
|
||||
|
||||
var result = lastPerSubject.Values.ToArray();
|
||||
Array.Sort(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the last sequences for subjects matching <paramref name="filters"/>,
|
||||
/// limited to sequences at or below <paramref name="maxSeq"/> and capped at
|
||||
/// <paramref name="maxAllowed"/> results.
|
||||
/// Reference: golang/nats-server/server/filestore.go — MultiLastSeqs.
|
||||
/// </summary>
|
||||
public ulong[] MultiLastSeqs(string[] filters, ulong maxSeq, int maxAllowed)
|
||||
{
|
||||
var lastPerSubject = new Dictionary<string, ulong>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var kv in _messages)
|
||||
{
|
||||
var seq = kv.Key;
|
||||
if (maxSeq > 0 && seq > maxSeq)
|
||||
continue;
|
||||
|
||||
var subj = kv.Value.Subject;
|
||||
var matches = filters.Length == 0
|
||||
|| filters.Any(f => SubjectMatchesFilter(subj, f));
|
||||
|
||||
if (!matches)
|
||||
continue;
|
||||
|
||||
if (!lastPerSubject.TryGetValue(subj, out var existing) || seq > existing)
|
||||
lastPerSubject[subj] = seq;
|
||||
}
|
||||
|
||||
var result = lastPerSubject.Values.OrderBy(s => s).ToArray();
|
||||
// Go parity: ErrTooManyResults — when maxAllowed > 0 and results exceed it.
|
||||
if (maxAllowed > 0 && result.Length > maxAllowed)
|
||||
throw new InvalidOperationException($"Too many results: got {result.Length}, max allowed is {maxAllowed}.");
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the subject stored at <paramref name="seq"/>.
|
||||
/// Throws <see cref="KeyNotFoundException"/> if the sequence does not exist.
|
||||
/// Reference: golang/nats-server/server/filestore.go — SubjectForSeq.
|
||||
/// </summary>
|
||||
public string SubjectForSeq(ulong seq)
|
||||
{
|
||||
if (!_messages.TryGetValue(seq, out var stored))
|
||||
throw new KeyNotFoundException($"Message sequence {seq} not found.");
|
||||
return stored.Subject;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts messages pending from sequence <paramref name="sseq"/> matching
|
||||
/// <paramref name="filter"/>. When <paramref name="lastPerSubject"/> is true,
|
||||
/// only the last message per subject is counted.
|
||||
/// Returns (total, validThrough) where validThrough is the last sequence checked.
|
||||
/// Reference: golang/nats-server/server/filestore.go — NumPending.
|
||||
/// </summary>
|
||||
public (ulong Total, ulong ValidThrough) NumPending(ulong sseq, string filter, bool lastPerSubject)
|
||||
{
|
||||
var candidates = _messages
|
||||
.Where(kv => kv.Key >= sseq)
|
||||
.Where(kv => string.IsNullOrEmpty(filter) || SubjectMatchesFilter(kv.Value.Subject, filter))
|
||||
.ToList();
|
||||
|
||||
if (lastPerSubject)
|
||||
{
|
||||
// One-per-subject: take the last sequence per subject.
|
||||
var lastBySubject = new Dictionary<string, ulong>(StringComparer.Ordinal);
|
||||
foreach (var kv in candidates)
|
||||
{
|
||||
if (!lastBySubject.TryGetValue(kv.Value.Subject, out var existing) || kv.Key > existing)
|
||||
lastBySubject[kv.Value.Subject] = kv.Key;
|
||||
}
|
||||
candidates = candidates.Where(kv => lastBySubject.TryGetValue(kv.Value.Subject, out var last) && kv.Key == last).ToList();
|
||||
}
|
||||
|
||||
var total = (ulong)candidates.Count;
|
||||
var validThrough = _last;
|
||||
return (total, validThrough);
|
||||
}
|
||||
|
||||
|
||||
private sealed class FileRecord
|
||||
{
|
||||
public ulong Sequence { get; init; }
|
||||
|
||||
@@ -367,6 +367,56 @@ public sealed class MsgBlock : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes a skip record for the given sequence number — reserves the sequence
|
||||
/// without storing actual message data. The record is written with the Deleted
|
||||
/// flag set so recovery skips it when rebuilding the in-memory message cache.
|
||||
/// This mirrors Go's SkipMsg tombstone behaviour.
|
||||
/// Reference: golang/nats-server/server/filestore.go — SkipMsg.
|
||||
/// </summary>
|
||||
public void WriteSkip(ulong sequence)
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
if (_writeOffset >= _maxBytes)
|
||||
throw new InvalidOperationException("Block is sealed; cannot write skip record.");
|
||||
|
||||
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L;
|
||||
var record = new MessageRecord
|
||||
{
|
||||
Sequence = sequence,
|
||||
Subject = string.Empty,
|
||||
Headers = ReadOnlyMemory<byte>.Empty,
|
||||
Payload = ReadOnlyMemory<byte>.Empty,
|
||||
Timestamp = now,
|
||||
Deleted = true, // skip = deleted from the start
|
||||
};
|
||||
|
||||
var encoded = MessageRecord.Encode(record);
|
||||
var offset = _writeOffset;
|
||||
|
||||
RandomAccess.Write(_handle, encoded, offset);
|
||||
_writeOffset = offset + encoded.Length;
|
||||
|
||||
_index[sequence] = (offset, encoded.Length);
|
||||
_deleted.Add(sequence);
|
||||
// Note: intentionally NOT added to _cache since it is deleted.
|
||||
|
||||
if (_totalWritten == 0)
|
||||
_firstSequence = sequence;
|
||||
|
||||
_lastSequence = Math.Max(_lastSequence, sequence);
|
||||
_nextSequence = Math.Max(_nextSequence, sequence + 1);
|
||||
_totalWritten++;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clears the write cache, releasing memory. After this call, all reads will
|
||||
/// go to disk. Called when the block is sealed (no longer the active block)
|
||||
|
||||
Reference in New Issue
Block a user