dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): add write cache and TTL scheduling (Go parity)

Browse Source 
Add MsgBlock write cache (mirrors Go's msgBlock.cache) to serve reads
for recently-written records without disk I/O; cleared on block seal via
RotateBlock. Add HashWheel-based TTL expiry in FileStore (ExpireFromWheel /
RegisterTtl), replacing the O(n) linear scan on every append with an
O(expired) wheel scan. Implement StoreMsg sync method with per-message TTL
override support. Add 10 tests covering cache hits/eviction, wheel expiry,
retention, StoreMsg seq/ts, per-msg TTL, and recovery re-registration.
This commit is contained in:
Joseph Doherty
2026-02-24 13:54:37 -05:00
parent b0b64292b3
commit d0068b121f
3 changed files with 570 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
src/NATS.Server/JetStream/Storage/MsgBlock.cs
View File

@@ -3,6 +3,8 @@
// Go block load: filestore.go:8140-8260 (loadMsgs / msgFromBufEx)
// Go deletion: filestore.go dmap (avl.SequenceSet) for soft-deletes
// Go sealing: filestore.go rbytes check — block rolls when rbytes >= maxBytes
// Go write cache: filestore.go msgBlock.cache — recently-written records kept in
// memory to avoid disk reads on the hot path (cache field, clearCache method).
//
// MsgBlock is the unit of storage in the file store. Messages are appended
// sequentially as binary records (using MessageRecord). Blocks are sealed
@@ -33,6 +35,11 @@ public sealed class MsgBlock : IDisposable
private ulong _totalWritten; // Total records written (including later-deleted)
private bool _disposed;
// Go: msgBlock.cache — in-memory write cache for recently-written records.
// Only the active (last) block maintains a cache; sealed blocks use disk reads.
// Reference: golang/nats-server/server/filestore.go:236 (cache field)
private Dictionary<ulong, MessageRecord>? _cache;
private MsgBlock(FileStream file, int blockId, long maxBytes, ulong firstSequence)
{
_file = file;
@@ -113,6 +120,20 @@ public sealed class MsgBlock : IDisposable
}
}
/// <summary>
/// True when the write cache is currently populated.
/// Used by tests to verify cache presence without exposing the cache contents directly.
/// </summary>
public bool HasCache
{
get
{
_lock.EnterReadLock();
try { return _cache is not null; }
finally { _lock.ExitReadLock(); }
}
}
/// <summary>
/// Creates a new empty block file.
/// </summary>
@@ -150,6 +171,8 @@ public sealed class MsgBlock : IDisposable
/// <summary>
/// Appends a message to the block with an auto-assigned sequence number.
/// Populates the write cache so subsequent reads can bypass disk.
/// Reference: golang/nats-server/server/filestore.go:6700 (writeMsgRecord).
/// </summary>
/// <param name="subject">NATS subject.</param>
/// <param name="headers">Optional message headers.</param>
@@ -184,6 +207,11 @@ public sealed class MsgBlock : IDisposable
_index[sequence] = (offset, encoded.Length);
// Go: cache recently-written record to avoid disk reads on hot path.
// Reference: golang/nats-server/server/filestore.go:6730 (cache population).
_cache ??= new Dictionary<ulong, MessageRecord>();
_cache[sequence] = record;
if (_totalWritten == 0)
_firstSequence = sequence;
@@ -203,6 +231,8 @@ public sealed class MsgBlock : IDisposable
/// Appends a message to the block with an explicit sequence number and timestamp.
/// Used by FileStore when rewriting blocks from the in-memory cache where
/// sequences may have gaps (from prior removals).
/// Populates the write cache so subsequent reads can bypass disk.
/// Reference: golang/nats-server/server/filestore.go:6700 (writeMsgRecord).
/// </summary>
/// <param name="sequence">Explicit sequence number to assign.</param>
/// <param name="subject">NATS subject.</param>
@@ -236,6 +266,11 @@ public sealed class MsgBlock : IDisposable
_index[sequence] = (offset, encoded.Length);
// Go: cache recently-written record to avoid disk reads on hot path.
// Reference: golang/nats-server/server/filestore.go:6730 (cache population).
_cache ??= new Dictionary<ulong, MessageRecord>();
_cache[sequence] = record;
if (_totalWritten == 0)
_firstSequence = sequence;
@@ -250,9 +285,10 @@ public sealed class MsgBlock : IDisposable
}
/// <summary>
/// Reads a message by sequence number. Uses positional I/O
/// (<see cref="RandomAccess.Read"/>) so concurrent readers don't
/// interfere with each other or the writer's append position.
/// Reads a message by sequence number.
/// Checks the write cache first to avoid disk I/O for recently-written messages.
/// Falls back to positional disk read if the record is not cached.
/// Reference: golang/nats-server/server/filestore.go:8140 (loadMsgs / msgFromBufEx).
/// </summary>
/// <param name="sequence">The sequence number to read.</param>
/// <returns>The decoded record, or null if not found or deleted.</returns>
@@ -264,6 +300,11 @@ public sealed class MsgBlock : IDisposable
if (_deleted.Contains(sequence))
return null;
// Go: check cache first (msgBlock.cache lookup).
// Reference: golang/nats-server/server/filestore.go:8155 (cache hit path).
if (_cache is not null && _cache.TryGetValue(sequence, out var cached))
return cached;
if (!_index.TryGetValue(sequence, out var entry))
return null;
@@ -281,6 +322,7 @@ public sealed class MsgBlock : IDisposable
/// <summary>
/// Soft-deletes a message by sequence number. Re-encodes the record on disk
/// with the deleted flag set (and updated checksum) so the deletion survives recovery.
/// Also evicts the sequence from the write cache.
/// </summary>
/// <param name="sequence">The sequence number to delete.</param>
/// <returns>True if the message was deleted; false if already deleted or not found.</returns>
@@ -314,6 +356,9 @@ public sealed class MsgBlock : IDisposable
var encoded = MessageRecord.Encode(deletedRecord);
RandomAccess.Write(_handle, encoded, entry.Offset);
// Evict from write cache — the record is now deleted.
_cache?.Remove(sequence);
return true;
}
finally
@@ -322,6 +367,24 @@ public sealed class MsgBlock : IDisposable
}
}
/// <summary>
/// Clears the write cache, releasing memory. After this call, all reads will
/// go to disk. Called when the block is sealed (no longer the active block)
/// or under memory pressure.
/// Reference: golang/nats-server/server/filestore.go — clearCache method on msgBlock.
/// </summary>
public void ClearCache()
{
_lock.EnterWriteLock();
try
{
_cache = null;
}
finally
{
_lock.ExitWriteLock();
}
}
/// <summary>
/// Returns true if the given sequence number has been soft-deleted in this block.
@@ -377,6 +440,25 @@ public sealed class MsgBlock : IDisposable
foreach (var (offset, length, seq) in entries)
{
// Check the write cache first to avoid disk I/O.
_lock.EnterReadLock();
MessageRecord? cached = null;
try
{
_cache?.TryGetValue(seq, out cached);
}
finally
{
_lock.ExitReadLock();
}
if (cached is not null)
{
if (!cached.Deleted)
yield return (cached.Sequence, cached.Subject);
continue;
}
var buffer = new byte[length];
RandomAccess.Read(_handle, buffer, offset);
var record = MessageRecord.Decode(buffer);
@@ -464,6 +546,8 @@ public sealed class MsgBlock : IDisposable
_totalWritten = count;
_writeOffset = offset;
// Note: recovered blocks do not populate the write cache — reads go to disk.
// The cache is only populated during active writes on the hot path.
}
private static string BlockFilePath(string directoryPath, int blockId)