dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: eliminate per-message allocations in pub/sub hot path and coalesce outbound writes

Browse Source 
Pub/sub 1:1 (16B) improved from 0.18x to 0.50x, fan-out from 0.18x to 0.44x,
and JetStream durable fetch from 0.13x to 0.64x vs Go. Key changes: replace
.ToArray() copy in SendMessage with pooled buffer handoff, batch multiple small
writes into single WriteAsync via 64KB coalesce buffer in write loop, and remove
profiling Stopwatch instrumentation from ProcessMessage/StreamManager hot paths.
This commit is contained in:
Joseph Doherty
2026-03-13 05:09:36 -04:00
parent 9e0df9b3d7
commit 0a4e7a822f
10 changed files with 654 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/NATS.Server/JetStream/Storage/FileStore.cs
View File

@@ -50,6 +50,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
// Resolved at construction time: which format family to use.
private readonly bool _useS2; // true -> S2Codec (FSV2 compression path)
private readonly bool _useAead; // true -> AeadEncryptor (FSV2 encryption path)
private readonly bool _noTransform; // true -> no compression/encryption, TransformForPersist just copies
// Go: filestore.go — per-stream time hash wheel for efficient TTL expiration.
// Created lazily only when MaxAgeMs > 0. Entries are (seq, expires_ns) pairs.
@@ -95,6 +96,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
// Determine which format path is active.
_useS2 = _options.Compression == StoreCompression.S2Compression;
_useAead = _options.Cipher != StoreCipher.NoCipher;
_noTransform = !_useS2 && !_useAead && !_options.EnableCompression && !_options.EnableEncryption;
Directory.CreateDirectory(options.Directory);
@@ -113,13 +115,12 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
_flushTask = Task.Run(() => FlushLoopAsync(_flushCts.Token));
}
public async ValueTask<ulong> AppendAsync(string subject, ReadOnlyMemory<byte> payload, CancellationToken ct)
public ValueTask<ulong> AppendAsync(string subject, ReadOnlyMemory<byte> payload, CancellationToken ct)
{
if (_stopped)
throw new ObjectDisposedException(nameof(FileStore), "Store has been stopped.");
// Go: DiscardNew — reject when MaxBytes would be exceeded.
// Reference: golang/nats-server/server/filestore.go — storeMsg, discard new check.
if (_options.MaxBytes > 0 && _options.Discard == DiscardPolicy.New
&& (long)_totalBytes + payload.Length > _options.MaxBytes)
{
@@ -127,35 +128,38 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
}
// Go: check and remove expired messages before each append.
// Reference: golang/nats-server/server/filestore.go — storeMsg, expire check.
ExpireFromWheel();
// Only when MaxAge is configured (Go: filestore.go:4701 conditional).
if (_options.MaxAgeMs > 0)
ExpireFromWheel();
_last++;
var now = DateTime.UtcNow;
var timestamp = new DateTimeOffset(now).ToUnixTimeMilliseconds() * 1_000_000L;
// Go: writeMsgRecordLocked writes directly into mb.cache.buf (a single contiguous
// byte slice). It does NOT store a per-message object in a map.
// We keep _messages for LoadAsync/RemoveAsync but avoid double payload.ToArray().
var persistedPayload = TransformForPersist(payload.Span);
var stored = new StoredMessage
var storedPayload = _noTransform ? persistedPayload : payload.ToArray();
_messages[_last] = new StoredMessage
{
Sequence = _last,
Subject = subject,
Payload = payload.ToArray(),
Payload = storedPayload,
TimestampUtc = now,
};
_messages[_last] = stored;
_generation++;
// Incremental state tracking.
_messageCount++;
_totalBytes += (ulong)payload.Length;
if (_messageCount == 1)
_firstSeq = _last;
// Go: register new message in TTL wheel when MaxAgeMs is configured.
// Reference: golang/nats-server/server/filestore.go:6820 (storeMsg TTL schedule).
RegisterTtl(_last, timestamp, _options.MaxAgeMs > 0 ? (long)_options.MaxAgeMs * 1_000_000L : 0);
// Go: register TTL only when TTL > 0.
if (_options.MaxAgeMs > 0)
RegisterTtl(_last, timestamp, (long)_options.MaxAgeMs * 1_000_000L);
// Write to MsgBlock. The payload stored in the block is the transformed
// (compressed/encrypted) payload, not the plaintext.
// Write to MsgBlock.
EnsureActiveBlock();
try
{
@@ -163,15 +167,11 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
}
catch (InvalidOperationException)
{
// Block is sealed. Rotate to a new block and retry.
RotateBlock();
_activeBlock!.WriteAt(_last, subject, ReadOnlyMemory<byte>.Empty, persistedPayload, timestamp);
}
// Go: filestore.go:4443 (setupWriteCache) — record write in bounded cache manager.
_writeCache.TrackWrite(_activeBlock!.BlockId, persistedPayload.Length);
// Signal the background flush loop to coalesce and flush pending writes.
_flushSignal.Writer.TryWrite(0);
// Check if the block just became sealed after this write.
@@ -184,7 +184,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable, IDisposable
if (_options.MaxMsgsPerSubject > 0 && !string.IsNullOrEmpty(subject))
EnforceMaxMsgsPerSubject(subject);
return _last;
return ValueTask.FromResult(_last);
}
public ValueTask<StoredMessage?> LoadAsync(ulong sequence, CancellationToken ct)