feat(filestore): implement StoreRawMsg, LoadPrevMsg, Type, Stop on FileStore

Complete IStreamStore Batch 1 — all core operations now have FileStore
implementations instead of throwing NotSupportedException:
- StoreRawMsg: caller-specified seq/ts for replication/mirroring
- LoadPrevMsg: backward scan for message before given sequence
- Type: returns StorageType.File
- Stop: flush + dispose blocks, reject further writes

14 new tests in FileStoreStreamStoreTests.
This commit is contained in:
Joseph Doherty
2026-02-25 01:20:03 -05:00
parent f402fd364f
commit be432c3224
2 changed files with 432 additions and 0 deletions

View File

@@ -35,6 +35,9 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
private ulong _last;
private ulong _first; // Go: first.seq — watermark for the first live or expected-first sequence
// Set to true after Stop() is called. Prevents further writes.
private bool _stopped;
// Resolved at construction time: which format family to use.
private readonly bool _useS2; // true -> S2Codec (FSV2 compression path)
private readonly bool _useAead; // true -> AeadEncryptor (FSV2 encryption path)
@@ -66,6 +69,9 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
public async ValueTask<ulong> AppendAsync(string subject, ReadOnlyMemory<byte> payload, CancellationToken ct)
{
if (_stopped)
throw new ObjectDisposedException(nameof(FileStore), "Store has been stopped.");
// Go: check and remove expired messages before each append.
// Reference: golang/nats-server/server/filestore.go — storeMsg, expire check.
ExpireFromWheel();
@@ -255,6 +261,9 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
/// </summary>
public (ulong Seq, long Ts) StoreMsg(string subject, byte[]? hdr, byte[] msg, long ttl)
{
if (_stopped)
throw new ObjectDisposedException(nameof(FileStore), "Store has been stopped.");
// Go: expire check before each store (same as AppendAsync).
// Reference: golang/nats-server/server/filestore.go:6793 (expireMsgs call).
ExpireFromWheel();
@@ -1614,6 +1623,135 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
}
// -------------------------------------------------------------------------
// Go-parity IStreamStore methods: StoreRawMsg, LoadPrevMsg, Type, Stop
// Reference: golang/nats-server/server/filestore.go
// -------------------------------------------------------------------------
/// <summary>
/// Stores a message at a caller-specified sequence number and timestamp.
/// Used for replication and mirroring — the caller (NRG, mirror source) controls
/// the sequence/timestamp rather than the store auto-incrementing them.
/// <para>Unlike <see cref="StoreMsg"/>, this does NOT call <c>ExpireFromWheel</c>
/// or auto-increment <c>_last</c>. It updates <c>_last</c> via
/// <c>Math.Max(_last, seq)</c> so the watermark reflects the highest stored
/// sequence.</para>
/// Reference: golang/nats-server/server/filestore.go:6756 (storeRawMsg).
/// </summary>
public void StoreRawMsg(string subject, byte[]? hdr, byte[] msg, ulong seq, long ts, long ttl, bool discardNewCheck)
{
if (_stopped)
throw new ObjectDisposedException(nameof(FileStore), "Store has been stopped.");
// Combine headers and payload, same as StoreMsg.
byte[] combined;
if (hdr is { Length: > 0 })
{
combined = new byte[hdr.Length + msg.Length];
hdr.CopyTo(combined, 0);
msg.CopyTo(combined, hdr.Length);
}
else
{
combined = msg;
}
var persistedPayload = TransformForPersist(combined.AsSpan());
// Recover UTC DateTime from caller-supplied Unix nanosecond timestamp.
var storedUtc = DateTimeOffset.FromUnixTimeMilliseconds(ts / 1_000_000L).UtcDateTime;
var stored = new StoredMessage
{
Sequence = seq,
Subject = subject,
Payload = combined,
TimestampUtc = storedUtc,
};
_messages[seq] = stored;
// Go: update _last to the high-water mark — do not decrement.
_last = Math.Max(_last, seq);
// Register TTL using the caller-supplied timestamp and TTL.
var effectiveTtlNs = ttl > 0 ? ttl : (_options.MaxAgeMs > 0 ? (long)_options.MaxAgeMs * 1_000_000L : 0L);
RegisterTtl(seq, ts, effectiveTtlNs);
EnsureActiveBlock();
try
{
_activeBlock!.WriteAt(seq, subject, ReadOnlyMemory<byte>.Empty, persistedPayload, ts);
}
catch (InvalidOperationException)
{
RotateBlock();
_activeBlock!.WriteAt(seq, subject, ReadOnlyMemory<byte>.Empty, persistedPayload, ts);
}
if (_activeBlock!.IsSealed)
RotateBlock();
}
/// <summary>
/// Loads the message immediately before <paramref name="start"/> by walking
/// backward from <c>start - 1</c> to <c>_first</c>.
/// Throws <see cref="KeyNotFoundException"/> if no such message exists.
/// Reference: golang/nats-server/server/filestore.go — LoadPrevMsg.
/// </summary>
public StoreMsg LoadPrevMsg(ulong start, StoreMsg? sm)
{
if (start == 0)
throw new KeyNotFoundException("No message found before seq 0.");
var first = _messages.Count > 0 ? _messages.Keys.Min() : 1UL;
for (var seq = start - 1; seq >= first && seq <= _last; seq--)
{
if (_messages.TryGetValue(seq, out var stored))
{
sm ??= new StoreMsg();
sm.Clear();
sm.Subject = stored.Subject;
sm.Data = stored.Payload.Length > 0 ? stored.Payload.ToArray() : null;
sm.Sequence = stored.Sequence;
sm.Timestamp = new DateTimeOffset(stored.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
return sm;
}
// Prevent underflow on ulong subtraction.
if (seq == 0)
break;
}
throw new KeyNotFoundException($"No message found before seq {start}.");
}
/// <summary>
/// Returns the storage backend type for this store instance.
/// Reference: golang/nats-server/server/filestore.go — fileStore.Type.
/// </summary>
public StorageType Type() => StorageType.File;
/// <summary>
/// Flushes the active block to disk and marks the store as stopped.
/// After <c>Stop()</c> returns, calls to <see cref="StoreMsg"/> or
/// <see cref="AppendAsync"/> will throw <see cref="ObjectDisposedException"/>.
/// Blocks are NOT deleted — use <see cref="Delete"/> if data removal is needed.
/// Reference: golang/nats-server/server/filestore.go — fileStore.Stop.
/// </summary>
public void Stop()
{
if (_stopped)
return;
_stopped = true;
// Flush the active block to ensure all buffered writes reach disk.
_activeBlock?.Flush();
// Dispose all blocks to release OS file handles. The files remain on disk.
DisposeAllBlocks();
}
// -------------------------------------------------------------------------
// ConsumerStore factory
// Reference: golang/nats-server/server/filestore.go — fileStore.ConsumerStore