dohertj2/natsnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port session 18 — JetStream File Store

Browse Source 
- FileStoreTypes: FileStoreConfig, FileStreamInfo, FileConsumerInfo, Psi,
  Cache, MsgId, CompressionInfo, ErrBadMsg, FileStoreDefaults constants
- FileStore: JetStreamFileStore implementing IStreamStore (26 methods stubbed)
  with State/Type/Stop/Register* properly implemented
- MessageBlock: MessageBlock type with all 40+ fields, ConsumerFileStore stub
- 312 features complete (IDs 951-1262)
This commit is contained in:
Joseph Doherty
2026-02-26 16:06:50 -05:00
parent 5a2c8a3250
commit 3cffa5b156
6 changed files with 1269 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

428
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/FileStore.cs Normal file
View File

@@ -0,0 +1,428 @@
// Copyright 2019-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Adapted from server/filestore.go (fileStore struct and methods)
using System.Threading.Channels;
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
namespace ZB.MOM.NatsNet.Server;
/// <summary>
/// File-backed implementation of <see cref="IStreamStore"/>.
/// Stores JetStream messages in per-block files on disk with optional
/// encryption and compression.
/// Mirrors the <c>fileStore</c> struct in filestore.go.
/// </summary>
public sealed class JetStreamFileStore : IStreamStore, IDisposable
{
// -----------------------------------------------------------------------
// Fields — mirrors fileStore struct fields
// -----------------------------------------------------------------------
private readonly ReaderWriterLockSlim _mu = new(LockRecursionPolicy.NoRecursion);
// State
private StreamState _state = new();
private List<ulong>? _tombs;
private LostStreamData? _ld;
// Callbacks
private StorageUpdateHandler? _scb;
private StorageRemoveMsgHandler? _rmcb;
private ProcessJetStreamMsgHandler? _pmsgcb;
// Age-check timer
private Timer? _ageChk;
private bool _ageChkRun;
private long _ageChkTime;
// Background sync timer
private Timer? _syncTmr;
// Configuration
private FileStreamInfo _cfg;
private FileStoreConfig _fcfg;
// Message block list and index
private MessageBlock? _lmb; // last (active write) block
private List<MessageBlock> _blks = [];
private Dictionary<uint, MessageBlock> _bim = [];
// Per-subject index map
private SubjectTree<Psi>? _psim;
// Total subject-list length (sum of subject-string lengths)
private int _tsl;
// writeFullState concurrency guard
private readonly object _wfsmu = new();
private long _wfsrun; // Interlocked: is writeFullState running?
private int _wfsadml; // Average dmap length (protected by _wfsmu)
// Quit / load-done channels (Channel<byte> mimics chan struct{})
private Channel<byte>? _qch;
private Channel<byte>? _fsld;
// Consumer list
private readonly ReaderWriterLockSlim _cmu = new(LockRecursionPolicy.NoRecursion);
private List<IConsumerStore> _cfs = [];
// Snapshot-in-progress count
private int _sips;
// Dirty-write counter (incremented when writes are pending flush)
private int _dirty;
// Lifecycle flags
private bool _closing;
private volatile bool _closed;
// Flush-in-progress flag
private bool _fip;
// Whether the store has ever received a message
private bool _receivedAny;
// Whether the first sequence has been moved forward
private bool _firstMoved;
// Last PurgeEx call time (for throttle logic)
private DateTime _lpex;
// -----------------------------------------------------------------------
// Constructor
// -----------------------------------------------------------------------
/// <summary>
/// Initialises a file-backed stream store using the supplied file-store
/// configuration and stream information.
/// </summary>
/// <param name="fcfg">File-store configuration (block size, cipher, paths, etc.).</param>
/// <param name="cfg">Stream metadata (created time and stream config).</param>
/// <exception cref="ArgumentNullException">
/// Thrown when <paramref name="fcfg"/> or <paramref name="cfg"/> is null.
/// </exception>
public JetStreamFileStore(FileStoreConfig fcfg, FileStreamInfo cfg)
{
ArgumentNullException.ThrowIfNull(fcfg);
ArgumentNullException.ThrowIfNull(cfg);
_fcfg = fcfg;
_cfg = cfg;
// Apply defaults (mirrors newFileStoreWithCreated in filestore.go).
if (_fcfg.BlockSize == 0)
_fcfg.BlockSize = FileStoreDefaults.DefaultLargeBlockSize;
if (_fcfg.CacheExpire == TimeSpan.Zero)
_fcfg.CacheExpire = FileStoreDefaults.DefaultCacheBufferExpiration;
if (_fcfg.SubjectStateExpire == TimeSpan.Zero)
_fcfg.SubjectStateExpire = FileStoreDefaults.DefaultFssExpiration;
if (_fcfg.SyncInterval == TimeSpan.Zero)
_fcfg.SyncInterval = FileStoreDefaults.DefaultSyncInterval;
_psim = new SubjectTree<Psi>();
_bim = new Dictionary<uint, MessageBlock>();
_qch = Channel.CreateUnbounded<byte>();
_fsld = Channel.CreateUnbounded<byte>();
}
// -----------------------------------------------------------------------
// IStreamStore — type / state
// -----------------------------------------------------------------------
/// <inheritdoc/>
public StorageType Type() => StorageType.FileStorage;
/// <inheritdoc/>
public StreamState State()
{
_mu.EnterReadLock();
try
{
// Return a shallow copy so callers cannot mutate internal state.
return new StreamState
{
Msgs = _state.Msgs,
Bytes = _state.Bytes,
FirstSeq = _state.FirstSeq,
FirstTime = _state.FirstTime,
LastSeq = _state.LastSeq,
LastTime = _state.LastTime,
NumSubjects = _state.NumSubjects,
NumDeleted = _state.NumDeleted,
Deleted = _state.Deleted,
Lost = _state.Lost,
Consumers = _state.Consumers,
};
}
finally
{
_mu.ExitReadLock();
}
}
/// <inheritdoc/>
public void FastState(StreamState state)
{
_mu.EnterReadLock();
try
{
state.Msgs = _state.Msgs;
state.Bytes = _state.Bytes;
state.FirstSeq = _state.FirstSeq;
state.FirstTime = _state.FirstTime;
state.LastSeq = _state.LastSeq;
state.LastTime = _state.LastTime;
state.NumDeleted = _state.NumDeleted;
state.Consumers = _state.Consumers;
}
finally
{
_mu.ExitReadLock();
}
}
// -----------------------------------------------------------------------
// IStreamStore — callback registration
// -----------------------------------------------------------------------
/// <inheritdoc/>
public void RegisterStorageUpdates(StorageUpdateHandler cb)
{
_mu.EnterWriteLock();
try { _scb = cb; }
finally { _mu.ExitWriteLock(); }
}
/// <inheritdoc/>
public void RegisterStorageRemoveMsg(StorageRemoveMsgHandler cb)
{
_mu.EnterWriteLock();
try { _rmcb = cb; }
finally { _mu.ExitWriteLock(); }
}
/// <inheritdoc/>
public void RegisterProcessJetStreamMsg(ProcessJetStreamMsgHandler cb)
{
_mu.EnterWriteLock();
try { _pmsgcb = cb; }
finally { _mu.ExitWriteLock(); }
}
// -----------------------------------------------------------------------
// IStreamStore — lifecycle
// -----------------------------------------------------------------------
/// <inheritdoc/>
public void Stop()
{
_mu.EnterWriteLock();
try
{
if (_closing) return;
_closing = true;
}
finally
{
_mu.ExitWriteLock();
}
_ageChk?.Dispose();
_ageChk = null;
_syncTmr?.Dispose();
_syncTmr = null;
_closed = true;
}
/// <inheritdoc/>
public void Dispose() => Stop();
// -----------------------------------------------------------------------
// IStreamStore — store / load (all stubs)
// -----------------------------------------------------------------------
/// <inheritdoc/>
public (ulong Seq, long Ts) StoreMsg(string subject, byte[]? hdr, byte[]? msg, long ttl)
=> throw new NotImplementedException("TODO: session 18 — filestore StoreMsg");
/// <inheritdoc/>
public void StoreRawMsg(string subject, byte[]? hdr, byte[]? msg, ulong seq, long ts, long ttl, bool discardNewCheck)
=> throw new NotImplementedException("TODO: session 18 — filestore StoreRawMsg");
/// <inheritdoc/>
public (ulong Seq, Exception? Error) SkipMsg(ulong seq)
=> throw new NotImplementedException("TODO: session 18 — filestore SkipMsg");
/// <inheritdoc/>
public void SkipMsgs(ulong seq, ulong num)
=> throw new NotImplementedException("TODO: session 18 — filestore SkipMsgs");
/// <inheritdoc/>
public void FlushAllPending()
=> throw new NotImplementedException("TODO: session 18 — filestore FlushAllPending");
/// <inheritdoc/>
public StoreMsg? LoadMsg(ulong seq, StoreMsg? sm)
=> throw new NotImplementedException("TODO: session 18 — filestore LoadMsg");
/// <inheritdoc/>
public (StoreMsg? Sm, ulong Skip) LoadNextMsg(string filter, bool wc, ulong start, StoreMsg? smp)
=> throw new NotImplementedException("TODO: session 18 — filestore LoadNextMsg");
/// <inheritdoc/>
public (StoreMsg? Sm, ulong Skip) LoadNextMsgMulti(object? sl, ulong start, StoreMsg? smp)
=> throw new NotImplementedException("TODO: session 18 — filestore LoadNextMsgMulti");
/// <inheritdoc/>
public StoreMsg? LoadLastMsg(string subject, StoreMsg? sm)
=> throw new NotImplementedException("TODO: session 18 — filestore LoadLastMsg");
/// <inheritdoc/>
public (StoreMsg? Sm, Exception? Error) LoadPrevMsg(ulong start, StoreMsg? smp)
=> throw new NotImplementedException("TODO: session 18 — filestore LoadPrevMsg");
/// <inheritdoc/>
public (StoreMsg? Sm, ulong Skip, Exception? Error) LoadPrevMsgMulti(object? sl, ulong start, StoreMsg? smp)
=> throw new NotImplementedException("TODO: session 18 — filestore LoadPrevMsgMulti");
/// <inheritdoc/>
public (bool Removed, Exception? Error) RemoveMsg(ulong seq)
=> throw new NotImplementedException("TODO: session 18 — filestore RemoveMsg");
/// <inheritdoc/>
public (bool Removed, Exception? Error) EraseMsg(ulong seq)
=> throw new NotImplementedException("TODO: session 18 — filestore EraseMsg");
/// <inheritdoc/>
public (ulong Purged, Exception? Error) Purge()
=> throw new NotImplementedException("TODO: session 18 — filestore Purge");
/// <inheritdoc/>
public (ulong Purged, Exception? Error) PurgeEx(string subject, ulong seq, ulong keep)
=> throw new NotImplementedException("TODO: session 18 — filestore PurgeEx");
/// <inheritdoc/>
public (ulong Purged, Exception? Error) Compact(ulong seq)
=> throw new NotImplementedException("TODO: session 18 — filestore Compact");
/// <inheritdoc/>
public void Truncate(ulong seq)
=> throw new NotImplementedException("TODO: session 18 — filestore Truncate");
// -----------------------------------------------------------------------
// IStreamStore — query methods (all stubs)
// -----------------------------------------------------------------------
/// <inheritdoc/>
public ulong GetSeqFromTime(DateTime t)
=> throw new NotImplementedException("TODO: session 18 — filestore GetSeqFromTime");
/// <inheritdoc/>
public SimpleState FilteredState(ulong seq, string subject)
=> throw new NotImplementedException("TODO: session 18 — filestore FilteredState");
/// <inheritdoc/>
public Dictionary<string, SimpleState> SubjectsState(string filterSubject)
=> throw new NotImplementedException("TODO: session 18 — filestore SubjectsState");
/// <inheritdoc/>
public Dictionary<string, ulong> SubjectsTotals(string filterSubject)
=> throw new NotImplementedException("TODO: session 18 — filestore SubjectsTotals");
/// <inheritdoc/>
public (ulong[] Seqs, Exception? Error) AllLastSeqs()
=> throw new NotImplementedException("TODO: session 18 — filestore AllLastSeqs");
/// <inheritdoc/>
public (ulong[] Seqs, Exception? Error) MultiLastSeqs(string[] filters, ulong maxSeq, int maxAllowed)
=> throw new NotImplementedException("TODO: session 18 — filestore MultiLastSeqs");
/// <inheritdoc/>
public (string Subject, Exception? Error) SubjectForSeq(ulong seq)
=> throw new NotImplementedException("TODO: session 18 — filestore SubjectForSeq");
/// <inheritdoc/>
public (ulong Total, ulong ValidThrough, Exception? Error) NumPending(ulong sseq, string filter, bool lastPerSubject)
=> throw new NotImplementedException("TODO: session 18 — filestore NumPending");
/// <inheritdoc/>
public (ulong Total, ulong ValidThrough, Exception? Error) NumPendingMulti(ulong sseq, object? sl, bool lastPerSubject)
=> throw new NotImplementedException("TODO: session 18 — filestore NumPendingMulti");
// -----------------------------------------------------------------------
// IStreamStore — stream state encoding (stubs)
// -----------------------------------------------------------------------
/// <inheritdoc/>
public (byte[] Enc, Exception? Error) EncodedStreamState(ulong failed)
=> throw new NotImplementedException("TODO: session 18 — filestore EncodedStreamState");
/// <inheritdoc/>
public void SyncDeleted(DeleteBlocks dbs)
=> throw new NotImplementedException("TODO: session 18 — filestore SyncDeleted");
// -----------------------------------------------------------------------
// IStreamStore — config / admin (stubs)
// -----------------------------------------------------------------------
/// <inheritdoc/>
public void UpdateConfig(StreamConfig cfg)
=> throw new NotImplementedException("TODO: session 18 — filestore UpdateConfig");
/// <inheritdoc/>
public void Delete(bool inline)
=> throw new NotImplementedException("TODO: session 18 — filestore Delete");
/// <inheritdoc/>
public void ResetState()
=> throw new NotImplementedException("TODO: session 18 — filestore ResetState");
// -----------------------------------------------------------------------
// IStreamStore — consumer management (stubs)
// -----------------------------------------------------------------------
/// <inheritdoc/>
public IConsumerStore ConsumerStore(string name, DateTime created, ConsumerConfig cfg)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerStore");
/// <inheritdoc/>
public void AddConsumer(IConsumerStore o)
{
_cmu.EnterWriteLock();
try { _cfs.Add(o); }
finally { _cmu.ExitWriteLock(); }
}
/// <inheritdoc/>
public void RemoveConsumer(IConsumerStore o)
{
_cmu.EnterWriteLock();
try { _cfs.Remove(o); }
finally { _cmu.ExitWriteLock(); }
}
// -----------------------------------------------------------------------
// IStreamStore — snapshot / utilization (stubs)
// -----------------------------------------------------------------------
/// <inheritdoc/>
public (SnapshotResult? Result, Exception? Error) Snapshot(TimeSpan deadline, bool includeConsumers, bool checkMsgs)
=> throw new NotImplementedException("TODO: session 18 — filestore Snapshot");
/// <inheritdoc/>
public (ulong Total, ulong Reported, Exception? Error) Utilization()
=> throw new NotImplementedException("TODO: session 18 — filestore Utilization");
}

416
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/FileStoreTypes.cs Normal file
View File

@@ -0,0 +1,416 @@
// Copyright 2019-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Adapted from server/filestore.go
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
namespace ZB.MOM.NatsNet.Server;
// ---------------------------------------------------------------------------
// FileStoreConfig
// ---------------------------------------------------------------------------
/// <summary>
/// Configuration for a file-backed JetStream stream store.
/// Mirrors <c>FileStoreConfig</c> in filestore.go.
/// </summary>
public sealed class FileStoreConfig
{
/// <summary>Parent directory for all storage.</summary>
public string StoreDir { get; set; } = string.Empty;
/// <summary>
/// File block size. Also represents the maximum per-block overhead.
/// Defaults to <see cref="FileStoreDefaults.DefaultBlockSize"/>.
/// </summary>
public ulong BlockSize { get; set; }
/// <summary>How long with no activity until the in-memory cache is expired.</summary>
public TimeSpan CacheExpire { get; set; }
/// <summary>How long with no activity until a message block's subject state is expired.</summary>
public TimeSpan SubjectStateExpire { get; set; }
/// <summary>How often the store syncs data to disk in the background.</summary>
public TimeSpan SyncInterval { get; set; }
/// <summary>When true, every write is immediately synced to disk.</summary>
public bool SyncAlways { get; set; }
/// <summary>When true, write operations may be batched and flushed asynchronously.</summary>
public bool AsyncFlush { get; set; }
/// <summary>Encryption cipher used when encrypting blocks.</summary>
public StoreCipher Cipher { get; set; }
/// <summary>Compression algorithm applied to stored blocks.</summary>
public StoreCompression Compression { get; set; }
// Internal reference to the owning server — not serialised.
// Equivalent to srv *Server in Go; kept as object to avoid circular project deps.
internal object? Server { get; set; }
}
// ---------------------------------------------------------------------------
// FileStreamInfo
// ---------------------------------------------------------------------------
/// <summary>
/// Remembers the creation time alongside the stream configuration.
/// Mirrors <c>FileStreamInfo</c> in filestore.go.
/// </summary>
public sealed class FileStreamInfo
{
/// <summary>UTC time at which the stream was created.</summary>
public DateTime Created { get; set; }
/// <summary>Stream configuration.</summary>
public StreamConfig Config { get; set; } = new();
}
// ---------------------------------------------------------------------------
// FileConsumerInfo
// ---------------------------------------------------------------------------
/// <summary>
/// Used for creating and restoring consumer stores from disk.
/// Mirrors <c>FileConsumerInfo</c> in filestore.go.
/// </summary>
public sealed class FileConsumerInfo
{
/// <summary>UTC time at which the consumer was created.</summary>
public DateTime Created { get; set; }
/// <summary>Durable consumer name.</summary>
public string Name { get; set; } = string.Empty;
/// <summary>Consumer configuration.</summary>
public ConsumerConfig Config { get; set; } = new();
}
// ---------------------------------------------------------------------------
// Psi — per-subject index entry (internal)
// ---------------------------------------------------------------------------
/// <summary>
/// Per-subject index entry stored in the subject tree.
/// Mirrors the <c>psi</c> struct in filestore.go.
/// </summary>
internal sealed class Psi
{
/// <summary>Total messages for this subject across all blocks.</summary>
public ulong Total { get; set; }
/// <summary>Index of the first block that holds messages for this subject.</summary>
public uint Fblk { get; set; }
/// <summary>Index of the last block that holds messages for this subject.</summary>
public uint Lblk { get; set; }
}
// ---------------------------------------------------------------------------
// Cache — write-through and load cache (internal)
// ---------------------------------------------------------------------------
/// <summary>
/// Write-through caching layer also used when loading messages from disk.
/// Mirrors the <c>cache</c> struct in filestore.go.
/// </summary>
internal sealed class Cache
{
/// <summary>Raw message data buffer.</summary>
public byte[] Buf { get; set; } = Array.Empty<byte>();
/// <summary>Write position into <see cref="Buf"/>.</summary>
public int Wp { get; set; }
/// <summary>Per-sequence byte offsets into <see cref="Buf"/>.</summary>
public uint[] Idx { get; set; } = Array.Empty<uint>();
/// <summary>First sequence number this cache covers.</summary>
public ulong Fseq { get; set; }
/// <summary>No-random-access flag: when true sequential access is assumed.</summary>
public bool Nra { get; set; }
}
// ---------------------------------------------------------------------------
// MsgId — sequence + timestamp pair (internal)
// ---------------------------------------------------------------------------
/// <summary>
/// Pairs a message sequence number with its nanosecond timestamp.
/// Mirrors the <c>msgId</c> struct in filestore.go.
/// </summary>
internal struct MsgId
{
/// <summary>Sequence number.</summary>
public ulong Seq;
/// <summary>Nanosecond Unix timestamp.</summary>
public long Ts;
}
// ---------------------------------------------------------------------------
// CompressionInfo — compression metadata
// ---------------------------------------------------------------------------
/// <summary>
/// Compression metadata attached to a message block.
/// Mirrors <c>CompressionInfo</c> in filestore.go.
/// </summary>
public sealed class CompressionInfo
{
/// <summary>Compression algorithm in use.</summary>
public StoreCompression Type { get; set; }
/// <summary>Original (uncompressed) size in bytes.</summary>
public ulong Original { get; set; }
/// <summary>Compressed size in bytes.</summary>
public ulong Compressed { get; set; }
/// <summary>
/// Serialises compression metadata as a compact binary prefix.
/// Format: 'c' 'm' 'p' &lt;algorithmByte&gt; &lt;uvarint originalSize&gt;
/// </summary>
public byte[] MarshalMetadata()
{
// TODO: session 18 — implement varint encoding
throw new NotImplementedException("TODO: session 18 — filestore CompressionInfo.MarshalMetadata");
}
/// <summary>
/// Deserialises compression metadata from a binary buffer.
/// Returns the number of bytes consumed, or 0 if the buffer does not start with the expected prefix.
/// </summary>
public int UnmarshalMetadata(byte[] b)
{
// TODO: session 18 — implement varint decoding
throw new NotImplementedException("TODO: session 18 — filestore CompressionInfo.UnmarshalMetadata");
}
}
// ---------------------------------------------------------------------------
// ErrBadMsg — corrupt/malformed message error (internal)
// ---------------------------------------------------------------------------
/// <summary>
/// Indicates a malformed or corrupt message was detected in a block file.
/// Mirrors the <c>errBadMsg</c> type in filestore.go.
/// </summary>
internal sealed class ErrBadMsg : Exception
{
/// <summary>Path to the block file that contained the bad message.</summary>
public string FileName { get; }
/// <summary>Optional additional detail about the corruption.</summary>
public string Detail { get; }
public ErrBadMsg(string fileName, string detail = "")
: base(BuildMessage(fileName, detail))
{
FileName = fileName;
Detail = detail;
}
private static string BuildMessage(string fileName, string detail)
{
var baseName = Path.GetFileName(fileName);
return string.IsNullOrEmpty(detail)
? $"malformed or corrupt message in {baseName}"
: $"malformed or corrupt message in {baseName}: {detail}";
}
}
// ---------------------------------------------------------------------------
// FileStoreDefaults — well-known constants
// ---------------------------------------------------------------------------
/// <summary>
/// Well-known constants from filestore.go, exposed for cross-assembly use.
/// </summary>
public static class FileStoreDefaults
{
// Magic / version markers written into block files.
/// <summary>Magic byte used to identify file-store block files.</summary>
public const byte FileStoreMagic = 22;
/// <summary>Current block file version.</summary>
public const byte FileStoreVersion = 1;
/// <summary>New-format index version.</summary>
internal const byte NewVersion = 2;
/// <summary>Header length in bytes for block records.</summary>
internal const int HdrLen = 2;
// Directory names
/// <summary>Top-level directory that holds per-stream subdirectories.</summary>
public const string StreamsDir = "streams";
/// <summary>Directory that holds in-flight batch data for a stream.</summary>
public const string BatchesDir = "batches";
/// <summary>Directory that holds message block files.</summary>
public const string MsgDir = "msgs";
/// <summary>Temporary directory name used during a full purge.</summary>
public const string PurgeDir = "__msgs__";
/// <summary>Temporary directory name for the new message block during purge.</summary>
public const string NewMsgDir = "__new_msgs__";
/// <summary>Directory name that holds per-consumer state.</summary>
public const string ConsumerDir = "obs";
// File name patterns
/// <summary>Format string for block file names (<c>{index}.blk</c>).</summary>
public const string BlkScan = "{0}.blk";
/// <summary>Suffix for active block files.</summary>
public const string BlkSuffix = ".blk";
/// <summary>Format string for compacted-block staging files (<c>{index}.new</c>).</summary>
public const string NewScan = "{0}.new";
/// <summary>Format string for index files (<c>{index}.idx</c>).</summary>
public const string IndexScan = "{0}.idx";
/// <summary>Format string for per-block encryption-key files (<c>{index}.key</c>).</summary>
public const string KeyScan = "{0}.key";
/// <summary>Glob pattern used to find orphaned key files.</summary>
public const string KeyScanAll = "*.key";
/// <summary>Suffix for temporary rewrite/compression staging files.</summary>
public const string BlkTmpSuffix = ".tmp";
// Meta files
/// <summary>Stream / consumer metadata file name.</summary>
public const string JetStreamMetaFile = "meta.inf";
/// <summary>Checksum file for the metadata file.</summary>
public const string JetStreamMetaFileSum = "meta.sum";
/// <summary>Encrypted metadata key file name.</summary>
public const string JetStreamMetaFileKey = "meta.key";
/// <summary>Full stream-state snapshot file name.</summary>
public const string StreamStateFile = "index.db";
/// <summary>Encoded TTL hash-wheel persistence file name.</summary>
public const string TtlStreamStateFile = "thw.db";
/// <summary>Encoded message-scheduling persistence file name.</summary>
public const string MsgSchedulingStreamStateFile = "sched.db";
/// <summary>Consumer state file name inside a consumer directory.</summary>
public const string ConsumerState = "o.dat";
// Block size defaults (bytes)
/// <summary>Default block size for large (limits-based) streams: 8 MB.</summary>
public const ulong DefaultLargeBlockSize = 8 * 1024 * 1024;
/// <summary>Default block size for work-queue / interest streams: 4 MB.</summary>
public const ulong DefaultMediumBlockSize = 4 * 1024 * 1024;
/// <summary>Default block size used by mirrors/sources: 1 MB.</summary>
public const ulong DefaultSmallBlockSize = 1 * 1024 * 1024;
/// <summary>Tiny pool block size (256 KB) — avoids large allocations at low write rates.</summary>
public const ulong DefaultTinyBlockSize = 256 * 1024;
/// <summary>Maximum encrypted-head block size: 2 MB.</summary>
public const ulong MaximumEncryptedBlockSize = 2 * 1024 * 1024;
/// <summary>Default block size for KV-based streams (same as medium).</summary>
public const ulong DefaultKvBlockSize = DefaultMediumBlockSize;
/// <summary>Hard upper limit on block size.</summary>
public const ulong MaxBlockSize = DefaultLargeBlockSize;
/// <summary>Minimum allowed block size: 32 KiB.</summary>
public const ulong FileStoreMinBlkSize = 32 * 1000;
/// <summary>Maximum allowed block size (same as <see cref="MaxBlockSize"/>).</summary>
public const ulong FileStoreMaxBlkSize = MaxBlockSize;
/// <summary>
/// Default block size exposed publicly; resolves to <see cref="DefaultSmallBlockSize"/> (1 MB)
/// to match the spec note in the porting plan.
/// </summary>
public const ulong DefaultBlockSize = DefaultSmallBlockSize;
// Timing defaults
/// <summary>Default duration before an idle cache buffer is expired: 10 seconds.</summary>
public static readonly TimeSpan DefaultCacheBufferExpiration = TimeSpan.FromSeconds(10);
/// <summary>Default interval for background disk sync: 2 minutes.</summary>
public static readonly TimeSpan DefaultSyncInterval = TimeSpan.FromMinutes(2);
/// <summary>Default idle timeout before file descriptors are closed: 30 seconds.</summary>
public static readonly TimeSpan CloseFdsIdle = TimeSpan.FromSeconds(30);
/// <summary>Default expiration time for idle per-block subject state: 2 minutes.</summary>
public static readonly TimeSpan DefaultFssExpiration = TimeSpan.FromMinutes(2);
// Thresholds
/// <summary>Minimum coalesce size for write batching: 16 KiB.</summary>
public const int CoalesceMinimum = 16 * 1024;
/// <summary>Maximum wait time when gathering messages to flush: 8 ms.</summary>
public static readonly TimeSpan MaxFlushWait = TimeSpan.FromMilliseconds(8);
/// <summary>Minimum block size before compaction is attempted: 2 MB.</summary>
public const int CompactMinimum = 2 * 1024 * 1024;
/// <summary>Threshold above which a record length is considered corrupt: 32 MB.</summary>
public const int RlBadThresh = 32 * 1024 * 1024;
/// <summary>Size of the per-record hash checksum in bytes.</summary>
public const int RecordHashSize = 8;
// Encryption key size minimums
/// <summary>Minimum size of a metadata encryption key: 64 bytes.</summary>
internal const int MinMetaKeySize = 64;
/// <summary>Minimum size of a block encryption key: 64 bytes.</summary>
internal const int MinBlkKeySize = 64;
// Cache-index bit flags
/// <summary>Bit set in a cache index slot to mark that the checksum has been validated.</summary>
internal const uint Cbit = 1u << 31;
/// <summary>Bit set in a cache index slot to mark the message as deleted.</summary>
internal const uint Dbit = 1u << 30;
/// <summary>Bit set in a record length field to indicate the record has headers.</summary>
internal const uint Hbit = 1u << 31;
/// <summary>Bit set in a sequence number to mark an erased message.</summary>
internal const ulong Ebit = 1UL << 63;
/// <summary>Bit set in a sequence number to mark a tombstone.</summary>
internal const ulong Tbit = 1UL << 62;
}

382
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/MessageBlock.cs Normal file
View File

@@ -0,0 +1,382 @@
// Copyright 2019-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Adapted from server/filestore.go (msgBlock struct and consumerFileStore struct)
using System.Threading.Channels;
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
namespace ZB.MOM.NatsNet.Server;
// ---------------------------------------------------------------------------
// MessageBlock
// ---------------------------------------------------------------------------
/// <summary>
/// Represents a single on-disk message block file together with its
/// in-memory cache and index state.
/// Mirrors the <c>msgBlock</c> struct in filestore.go.
/// </summary>
internal sealed class MessageBlock
{
// ------------------------------------------------------------------
// Identity fields — first/last use volatile-style access in Go via
// atomic.LoadUint64 on the embedded msgId structs.
// We replicate those as plain fields; callers must acquire _mu before
// reading/writing unless using the Interlocked helpers on Seq/Ts.
// ------------------------------------------------------------------
/// <summary>First message in this block (sequence + nanosecond timestamp).</summary>
public MsgId First;
/// <summary>Last message in this block (sequence + nanosecond timestamp).</summary>
public MsgId Last;
// ------------------------------------------------------------------
// Lock
// ------------------------------------------------------------------
/// <summary>Guards all mutable fields in this block.</summary>
public readonly ReaderWriterLockSlim Mu = new(LockRecursionPolicy.NoRecursion);
// ------------------------------------------------------------------
// Back-reference
// ------------------------------------------------------------------
/// <summary>Owning file store.</summary>
public JetStreamFileStore? Fs { get; set; }
// ------------------------------------------------------------------
// File I/O
// ------------------------------------------------------------------
/// <summary>Path to the <c>.blk</c> message data file.</summary>
public string Mfn { get; set; } = string.Empty;
/// <summary>Open file stream for the block data file (null when closed/idle).</summary>
public FileStream? Mfd { get; set; }
/// <summary>Path to the per-block encryption key file.</summary>
public string Kfn { get; set; } = string.Empty;
// ------------------------------------------------------------------
// Compression
// ------------------------------------------------------------------
/// <summary>Effective compression algorithm when the block was last loaded.</summary>
public StoreCompression Cmp { get; set; }
/// <summary>
/// Last index write size in bytes; used to detect whether re-compressing
/// the block would save meaningful space.
/// </summary>
public long Liwsz { get; set; }
// ------------------------------------------------------------------
// Block identity
// ------------------------------------------------------------------
/// <summary>Monotonically increasing block index number (used in file names).</summary>
public uint Index { get; set; }
// ------------------------------------------------------------------
// Counters
// ------------------------------------------------------------------
/// <summary>User-visible byte count (excludes deleted-message bytes).</summary>
public ulong Bytes { get; set; }
/// <summary>
/// Total raw byte count including deleted messages.
/// Used to decide when to roll to a new block.
/// </summary>
public ulong RBytes { get; set; }
/// <summary>Byte count captured at the last compaction (0 if never compacted).</summary>
public ulong CBytes { get; set; }
/// <summary>User-visible message count (excludes deleted messages).</summary>
public ulong Msgs { get; set; }
// ------------------------------------------------------------------
// Per-subject state
// ------------------------------------------------------------------
/// <summary>
/// Optional per-subject state tree for this block.
/// Lazily populated and expired when idle.
/// Mirrors <c>mb.fss</c> in filestore.go.
/// </summary>
public SubjectTree<SimpleState>? Fss { get; set; }
// ------------------------------------------------------------------
// Deleted-sequence tracking
// ------------------------------------------------------------------
/// <summary>
/// Set of deleted sequence numbers within this block.
/// Uses the AVL-backed <see cref="SequenceSet"/> to match Go's <c>avl.SequenceSet</c>.
/// </summary>
public SequenceSet Dmap { get; set; } = new();
// ------------------------------------------------------------------
// Timestamps (nanosecond Unix times, matches Go int64)
// ------------------------------------------------------------------
/// <summary>Nanosecond timestamp of the last write to this block.</summary>
public long Lwts { get; set; }
/// <summary>Nanosecond timestamp of the last load (cache fill) of this block.</summary>
public long Llts { get; set; }
/// <summary>Nanosecond timestamp of the last read from this block.</summary>
public long Lrts { get; set; }
/// <summary>Nanosecond timestamp of the last subject-state (fss) access.</summary>
public long Lsts { get; set; }
/// <summary>Last sequence that was looked up; used to detect linear scans.</summary>
public ulong Llseq { get; set; }
// ------------------------------------------------------------------
// Cache
// ------------------------------------------------------------------
/// <summary>
/// Active in-memory cache. May be null when evicted.
/// Mirrors <c>mb.cache</c>; the elastic-pointer field (<c>mb.ecache</c>) is
/// not ported — a plain nullable field is sufficient here.
/// </summary>
public Cache? CacheData { get; set; }
/// <summary>Number of times the cache has been (re)loaded from disk.</summary>
public ulong Cloads { get; set; }
/// <summary>Cache buffer expiration duration for this block.</summary>
public TimeSpan Cexp { get; set; }
/// <summary>Per-block subject-state (fss) expiration duration.</summary>
public TimeSpan Fexp { get; set; }
/// <summary>Timer used to expire the cache buffer when idle.</summary>
public Timer? Ctmr { get; set; }
// ------------------------------------------------------------------
// State flags
// ------------------------------------------------------------------
/// <summary>Whether the in-memory cache is currently populated.</summary>
public bool HaveCache => CacheData != null;
/// <summary>Whether this block has been closed and must not be written to.</summary>
public bool Closed { get; set; }
/// <summary>Whether this block has unflushed data that must be synced to disk.</summary>
public bool NeedSync { get; set; }
/// <summary>When true every write is immediately synced (SyncAlways mode).</summary>
public bool SyncAlways { get; set; }
/// <summary>When true compaction is suppressed for this block.</summary>
public bool NoCompact { get; set; }
/// <summary>
/// When true the block's messages are not tracked in the per-subject index.
/// Used for blocks that only contain tombstone or deleted markers.
/// </summary>
public bool NoTrack { get; set; }
/// <summary>Whether a background flusher goroutine equivalent is running.</summary>
public bool Flusher { get; set; }
/// <summary>Whether a cache-load is currently in progress.</summary>
public bool Loading { get; set; }
// ------------------------------------------------------------------
// Write error
// ------------------------------------------------------------------
/// <summary>Captured write error; non-null means the block is in a bad state.</summary>
public Exception? Werr { get; set; }
// ------------------------------------------------------------------
// TTL / scheduling counters
// ------------------------------------------------------------------
/// <summary>Number of messages in this block that have a TTL set.</summary>
public ulong Ttls { get; set; }
/// <summary>Number of messages in this block that have a schedule set.</summary>
public ulong Schedules { get; set; }
// ------------------------------------------------------------------
// Channels (equivalent to Go chan struct{})
// ------------------------------------------------------------------
/// <summary>Flush-request channel: signals the flusher to run.</summary>
public Channel<byte>? Fch { get; set; }
/// <summary>Quit channel: closing signals background goroutine equivalents to stop.</summary>
public Channel<byte>? Qch { get; set; }
// ------------------------------------------------------------------
// Checksum
// ------------------------------------------------------------------
/// <summary>
/// Last-check hash: 8-byte rolling checksum of the last validated record.
/// Mirrors <c>mb.lchk [8]byte</c>.
/// </summary>
public byte[] Lchk { get; set; } = new byte[8];
// ------------------------------------------------------------------
// Test hook
// ------------------------------------------------------------------
/// <summary>When true, simulates a write failure. Used by unit tests only.</summary>
public bool MockWriteErr { get; set; }
}
// ---------------------------------------------------------------------------
// ConsumerFileStore
// ---------------------------------------------------------------------------
/// <summary>
/// File-backed implementation of <see cref="IConsumerStore"/>.
/// Persists consumer delivery and ack state to a directory under the stream's
/// <c>obs/</c> subdirectory.
/// Mirrors the <c>consumerFileStore</c> struct in filestore.go.
/// </summary>
public sealed class ConsumerFileStore : IConsumerStore
{
// ------------------------------------------------------------------
// Fields — mirrors consumerFileStore struct
// ------------------------------------------------------------------
private readonly object _mu = new();
/// <summary>Back-reference to the owning file store.</summary>
private readonly JetStreamFileStore _fs;
/// <summary>Consumer metadata (name, created time, config).</summary>
private FileConsumerInfo _cfg;
/// <summary>Durable consumer name.</summary>
private readonly string _name;
/// <summary>Path to the consumer's state directory.</summary>
private readonly string _odir;
/// <summary>Path to the consumer state file (<c>o.dat</c>).</summary>
private readonly string _ifn;
/// <summary>Consumer delivery/ack state.</summary>
private ConsumerState _state = new();
/// <summary>Flush-request channel.</summary>
private Channel<byte>? _fch;
/// <summary>Quit channel.</summary>
private Channel<byte>? _qch;
/// <summary>Whether a background flusher is running.</summary>
private bool _flusher;
/// <summary>Whether a write is currently in progress.</summary>
private bool _writing;
/// <summary>Whether the state is dirty (pending flush).</summary>
private bool _dirty;
/// <summary>Whether this consumer store is closed.</summary>
private bool _closed;
// ------------------------------------------------------------------
// Constructor
// ------------------------------------------------------------------
/// <summary>
/// Creates a new file-backed consumer store.
/// </summary>
public ConsumerFileStore(JetStreamFileStore fs, FileConsumerInfo cfg, string name, string odir)
{
_fs = fs;
_cfg = cfg;
_name = name;
_odir = odir;
_ifn = Path.Combine(odir, FileStoreDefaults.ConsumerState);
}
// ------------------------------------------------------------------
// IConsumerStore — all methods stubbed
// ------------------------------------------------------------------
/// <inheritdoc/>
public void SetStarting(ulong sseq)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.SetStarting");
/// <inheritdoc/>
public void UpdateStarting(ulong sseq)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateStarting");
/// <inheritdoc/>
public void Reset(ulong sseq)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Reset");
/// <inheritdoc/>
public bool HasState()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.HasState");
/// <inheritdoc/>
public void UpdateDelivered(ulong dseq, ulong sseq, ulong dc, long ts)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateDelivered");
/// <inheritdoc/>
public void UpdateAcks(ulong dseq, ulong sseq)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateAcks");
/// <inheritdoc/>
public void UpdateConfig(ConsumerConfig cfg)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateConfig");
/// <inheritdoc/>
public void Update(ConsumerState state)
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Update");
/// <inheritdoc/>
public (ConsumerState? State, Exception? Error) State()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.State");
/// <inheritdoc/>
public (ConsumerState? State, Exception? Error) BorrowState()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.BorrowState");
/// <inheritdoc/>
public byte[] EncodedState()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.EncodedState");
/// <inheritdoc/>
public StorageType Type() => StorageType.FileStorage;
/// <inheritdoc/>
public void Stop()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Stop");
/// <inheritdoc/>
public void Delete()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Delete");
/// <inheritdoc/>
public void StreamDelete()
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.StreamDelete");
}

BIN
porting.db
View File

Binary file not shown.

8
reports/current.md
View File

@@ -1,6 +1,6 @@
# NATS .NET Porting Status Report # NATS .NET Porting Status Report
Generated: 2026-02-26 21:02:04 UTC Generated: 2026-02-26 21:06:51 UTC
## Modules (12 total) ## Modules (12 total)
@@ -13,9 +13,9 @@ Generated: 2026-02-26 21:02:04 UTC
| Status | Count | | Status | Count |
|--------|-------| |--------|-------|
| complete | 1736 | | complete | 2048 |
| n_a | 77 | | n_a | 77 |
| not_started | 1767 | | not_started | 1455 |
| stub | 93 | | stub | 93 |
## Unit Tests (3257 total) ## Unit Tests (3257 total)
@@ -36,4 +36,4 @@ Generated: 2026-02-26 21:02:04 UTC
## Overall Progress ## Overall Progress
**2324/6942 items complete (33.5%)** **2636/6942 items complete (38.0%)**

39
reports/report_5a2c8a3.md Normal file
View File

@@ -0,0 +1,39 @@
# NATS .NET Porting Status Report
Generated: 2026-02-26 21:06:51 UTC
## Modules (12 total)
| Status | Count |
|--------|-------|
| complete | 11 |
| not_started | 1 |
## Features (3673 total)
| Status | Count |
|--------|-------|
| complete | 2048 |
| n_a | 77 |
| not_started | 1455 |
| stub | 93 |
## Unit Tests (3257 total)
| Status | Count |
|--------|-------|
| complete | 319 |
| n_a | 181 |
| not_started | 2533 |
| stub | 224 |
## Library Mappings (36 total)
| Status | Count |
|--------|-------|
| mapped | 36 |
## Overall Progress
**2636/6942 items complete (38.0%)**