feat: port session 18 — JetStream File Store
- FileStoreTypes: FileStoreConfig, FileStreamInfo, FileConsumerInfo, Psi, Cache, MsgId, CompressionInfo, ErrBadMsg, FileStoreDefaults constants - FileStore: JetStreamFileStore implementing IStreamStore (26 methods stubbed) with State/Type/Stop/Register* properly implemented - MessageBlock: MessageBlock type with all 40+ fields, ConsumerFileStore stub - 312 features complete (IDs 951-1262)
This commit is contained in:
Joseph Doherty
428
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/FileStore.cs
Normal file
428
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/FileStore.cs
Normal file
@@ -0,0 +1,428 @@
|
||||
// Copyright 2019-2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Adapted from server/filestore.go (fileStore struct and methods)
|
||||
|
||||
using System.Threading.Channels;
|
||||
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
|
||||
|
||||
namespace ZB.MOM.NatsNet.Server;
|
||||
|
||||
/// <summary>
|
||||
/// File-backed implementation of <see cref="IStreamStore"/>.
|
||||
/// Stores JetStream messages in per-block files on disk with optional
|
||||
/// encryption and compression.
|
||||
/// Mirrors the <c>fileStore</c> struct in filestore.go.
|
||||
/// </summary>
|
||||
public sealed class JetStreamFileStore : IStreamStore, IDisposable
|
||||
{
|
||||
// -----------------------------------------------------------------------
|
||||
// Fields — mirrors fileStore struct fields
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private readonly ReaderWriterLockSlim _mu = new(LockRecursionPolicy.NoRecursion);
|
||||
|
||||
// State
|
||||
private StreamState _state = new();
|
||||
private List<ulong>? _tombs;
|
||||
private LostStreamData? _ld;
|
||||
|
||||
// Callbacks
|
||||
private StorageUpdateHandler? _scb;
|
||||
private StorageRemoveMsgHandler? _rmcb;
|
||||
private ProcessJetStreamMsgHandler? _pmsgcb;
|
||||
|
||||
// Age-check timer
|
||||
private Timer? _ageChk;
|
||||
private bool _ageChkRun;
|
||||
private long _ageChkTime;
|
||||
|
||||
// Background sync timer
|
||||
private Timer? _syncTmr;
|
||||
|
||||
// Configuration
|
||||
private FileStreamInfo _cfg;
|
||||
private FileStoreConfig _fcfg;
|
||||
|
||||
// Message block list and index
|
||||
private MessageBlock? _lmb; // last (active write) block
|
||||
private List<MessageBlock> _blks = [];
|
||||
private Dictionary<uint, MessageBlock> _bim = [];
|
||||
|
||||
// Per-subject index map
|
||||
private SubjectTree<Psi>? _psim;
|
||||
|
||||
// Total subject-list length (sum of subject-string lengths)
|
||||
private int _tsl;
|
||||
|
||||
// writeFullState concurrency guard
|
||||
private readonly object _wfsmu = new();
|
||||
private long _wfsrun; // Interlocked: is writeFullState running?
|
||||
private int _wfsadml; // Average dmap length (protected by _wfsmu)
|
||||
|
||||
// Quit / load-done channels (Channel<byte> mimics chan struct{})
|
||||
private Channel<byte>? _qch;
|
||||
private Channel<byte>? _fsld;
|
||||
|
||||
// Consumer list
|
||||
private readonly ReaderWriterLockSlim _cmu = new(LockRecursionPolicy.NoRecursion);
|
||||
private List<IConsumerStore> _cfs = [];
|
||||
|
||||
// Snapshot-in-progress count
|
||||
private int _sips;
|
||||
|
||||
// Dirty-write counter (incremented when writes are pending flush)
|
||||
private int _dirty;
|
||||
|
||||
// Lifecycle flags
|
||||
private bool _closing;
|
||||
private volatile bool _closed;
|
||||
|
||||
// Flush-in-progress flag
|
||||
private bool _fip;
|
||||
|
||||
// Whether the store has ever received a message
|
||||
private bool _receivedAny;
|
||||
|
||||
// Whether the first sequence has been moved forward
|
||||
private bool _firstMoved;
|
||||
|
||||
// Last PurgeEx call time (for throttle logic)
|
||||
private DateTime _lpex;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Constructor
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Initialises a file-backed stream store using the supplied file-store
|
||||
/// configuration and stream information.
|
||||
/// </summary>
|
||||
/// <param name="fcfg">File-store configuration (block size, cipher, paths, etc.).</param>
|
||||
/// <param name="cfg">Stream metadata (created time and stream config).</param>
|
||||
/// <exception cref="ArgumentNullException">
|
||||
/// Thrown when <paramref name="fcfg"/> or <paramref name="cfg"/> is null.
|
||||
/// </exception>
|
||||
public JetStreamFileStore(FileStoreConfig fcfg, FileStreamInfo cfg)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fcfg);
|
||||
ArgumentNullException.ThrowIfNull(cfg);
|
||||
|
||||
_fcfg = fcfg;
|
||||
_cfg = cfg;
|
||||
|
||||
// Apply defaults (mirrors newFileStoreWithCreated in filestore.go).
|
||||
if (_fcfg.BlockSize == 0)
|
||||
_fcfg.BlockSize = FileStoreDefaults.DefaultLargeBlockSize;
|
||||
if (_fcfg.CacheExpire == TimeSpan.Zero)
|
||||
_fcfg.CacheExpire = FileStoreDefaults.DefaultCacheBufferExpiration;
|
||||
if (_fcfg.SubjectStateExpire == TimeSpan.Zero)
|
||||
_fcfg.SubjectStateExpire = FileStoreDefaults.DefaultFssExpiration;
|
||||
if (_fcfg.SyncInterval == TimeSpan.Zero)
|
||||
_fcfg.SyncInterval = FileStoreDefaults.DefaultSyncInterval;
|
||||
|
||||
_psim = new SubjectTree<Psi>();
|
||||
_bim = new Dictionary<uint, MessageBlock>();
|
||||
_qch = Channel.CreateUnbounded<byte>();
|
||||
_fsld = Channel.CreateUnbounded<byte>();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — type / state
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public StorageType Type() => StorageType.FileStorage;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public StreamState State()
|
||||
{
|
||||
_mu.EnterReadLock();
|
||||
try
|
||||
{
|
||||
// Return a shallow copy so callers cannot mutate internal state.
|
||||
return new StreamState
|
||||
{
|
||||
Msgs = _state.Msgs,
|
||||
Bytes = _state.Bytes,
|
||||
FirstSeq = _state.FirstSeq,
|
||||
FirstTime = _state.FirstTime,
|
||||
LastSeq = _state.LastSeq,
|
||||
LastTime = _state.LastTime,
|
||||
NumSubjects = _state.NumSubjects,
|
||||
NumDeleted = _state.NumDeleted,
|
||||
Deleted = _state.Deleted,
|
||||
Lost = _state.Lost,
|
||||
Consumers = _state.Consumers,
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
_mu.ExitReadLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void FastState(StreamState state)
|
||||
{
|
||||
_mu.EnterReadLock();
|
||||
try
|
||||
{
|
||||
state.Msgs = _state.Msgs;
|
||||
state.Bytes = _state.Bytes;
|
||||
state.FirstSeq = _state.FirstSeq;
|
||||
state.FirstTime = _state.FirstTime;
|
||||
state.LastSeq = _state.LastSeq;
|
||||
state.LastTime = _state.LastTime;
|
||||
state.NumDeleted = _state.NumDeleted;
|
||||
state.Consumers = _state.Consumers;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_mu.ExitReadLock();
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — callback registration
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void RegisterStorageUpdates(StorageUpdateHandler cb)
|
||||
{
|
||||
_mu.EnterWriteLock();
|
||||
try { _scb = cb; }
|
||||
finally { _mu.ExitWriteLock(); }
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void RegisterStorageRemoveMsg(StorageRemoveMsgHandler cb)
|
||||
{
|
||||
_mu.EnterWriteLock();
|
||||
try { _rmcb = cb; }
|
||||
finally { _mu.ExitWriteLock(); }
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void RegisterProcessJetStreamMsg(ProcessJetStreamMsgHandler cb)
|
||||
{
|
||||
_mu.EnterWriteLock();
|
||||
try { _pmsgcb = cb; }
|
||||
finally { _mu.ExitWriteLock(); }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — lifecycle
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Stop()
|
||||
{
|
||||
_mu.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
if (_closing) return;
|
||||
_closing = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_mu.ExitWriteLock();
|
||||
}
|
||||
|
||||
_ageChk?.Dispose();
|
||||
_ageChk = null;
|
||||
_syncTmr?.Dispose();
|
||||
_syncTmr = null;
|
||||
|
||||
_closed = true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose() => Stop();
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — store / load (all stubs)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Seq, long Ts) StoreMsg(string subject, byte[]? hdr, byte[]? msg, long ttl)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore StoreMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void StoreRawMsg(string subject, byte[]? hdr, byte[]? msg, ulong seq, long ts, long ttl, bool discardNewCheck)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore StoreRawMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Seq, Exception? Error) SkipMsg(ulong seq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore SkipMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SkipMsgs(ulong seq, ulong num)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore SkipMsgs");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void FlushAllPending()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore FlushAllPending");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public StoreMsg? LoadMsg(ulong seq, StoreMsg? sm)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore LoadMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (StoreMsg? Sm, ulong Skip) LoadNextMsg(string filter, bool wc, ulong start, StoreMsg? smp)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore LoadNextMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (StoreMsg? Sm, ulong Skip) LoadNextMsgMulti(object? sl, ulong start, StoreMsg? smp)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore LoadNextMsgMulti");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public StoreMsg? LoadLastMsg(string subject, StoreMsg? sm)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore LoadLastMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (StoreMsg? Sm, Exception? Error) LoadPrevMsg(ulong start, StoreMsg? smp)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore LoadPrevMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (StoreMsg? Sm, ulong Skip, Exception? Error) LoadPrevMsgMulti(object? sl, ulong start, StoreMsg? smp)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore LoadPrevMsgMulti");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (bool Removed, Exception? Error) RemoveMsg(ulong seq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore RemoveMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (bool Removed, Exception? Error) EraseMsg(ulong seq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore EraseMsg");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Purged, Exception? Error) Purge()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore Purge");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Purged, Exception? Error) PurgeEx(string subject, ulong seq, ulong keep)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore PurgeEx");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Purged, Exception? Error) Compact(ulong seq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore Compact");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Truncate(ulong seq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore Truncate");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — query methods (all stubs)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ulong GetSeqFromTime(DateTime t)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore GetSeqFromTime");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public SimpleState FilteredState(ulong seq, string subject)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore FilteredState");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Dictionary<string, SimpleState> SubjectsState(string filterSubject)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore SubjectsState");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Dictionary<string, ulong> SubjectsTotals(string filterSubject)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore SubjectsTotals");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong[] Seqs, Exception? Error) AllLastSeqs()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore AllLastSeqs");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong[] Seqs, Exception? Error) MultiLastSeqs(string[] filters, ulong maxSeq, int maxAllowed)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore MultiLastSeqs");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (string Subject, Exception? Error) SubjectForSeq(ulong seq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore SubjectForSeq");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Total, ulong ValidThrough, Exception? Error) NumPending(ulong sseq, string filter, bool lastPerSubject)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore NumPending");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Total, ulong ValidThrough, Exception? Error) NumPendingMulti(ulong sseq, object? sl, bool lastPerSubject)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore NumPendingMulti");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — stream state encoding (stubs)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (byte[] Enc, Exception? Error) EncodedStreamState(ulong failed)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore EncodedStreamState");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SyncDeleted(DeleteBlocks dbs)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore SyncDeleted");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — config / admin (stubs)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void UpdateConfig(StreamConfig cfg)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore UpdateConfig");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Delete(bool inline)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore Delete");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void ResetState()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ResetState");
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — consumer management (stubs)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IConsumerStore ConsumerStore(string name, DateTime created, ConsumerConfig cfg)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerStore");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void AddConsumer(IConsumerStore o)
|
||||
{
|
||||
_cmu.EnterWriteLock();
|
||||
try { _cfs.Add(o); }
|
||||
finally { _cmu.ExitWriteLock(); }
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void RemoveConsumer(IConsumerStore o)
|
||||
{
|
||||
_cmu.EnterWriteLock();
|
||||
try { _cfs.Remove(o); }
|
||||
finally { _cmu.ExitWriteLock(); }
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IStreamStore — snapshot / utilization (stubs)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (SnapshotResult? Result, Exception? Error) Snapshot(TimeSpan deadline, bool includeConsumers, bool checkMsgs)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore Snapshot");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ulong Total, ulong Reported, Exception? Error) Utilization()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore Utilization");
|
||||
}
|
||||
416
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/FileStoreTypes.cs
Normal file
416
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/FileStoreTypes.cs
Normal file
@@ -0,0 +1,416 @@
|
||||
// Copyright 2019-2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Adapted from server/filestore.go
|
||||
|
||||
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
|
||||
|
||||
namespace ZB.MOM.NatsNet.Server;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileStoreConfig
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a file-backed JetStream stream store.
|
||||
/// Mirrors <c>FileStoreConfig</c> in filestore.go.
|
||||
/// </summary>
|
||||
public sealed class FileStoreConfig
|
||||
{
|
||||
/// <summary>Parent directory for all storage.</summary>
|
||||
public string StoreDir { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// File block size. Also represents the maximum per-block overhead.
|
||||
/// Defaults to <see cref="FileStoreDefaults.DefaultBlockSize"/>.
|
||||
/// </summary>
|
||||
public ulong BlockSize { get; set; }
|
||||
|
||||
/// <summary>How long with no activity until the in-memory cache is expired.</summary>
|
||||
public TimeSpan CacheExpire { get; set; }
|
||||
|
||||
/// <summary>How long with no activity until a message block's subject state is expired.</summary>
|
||||
public TimeSpan SubjectStateExpire { get; set; }
|
||||
|
||||
/// <summary>How often the store syncs data to disk in the background.</summary>
|
||||
public TimeSpan SyncInterval { get; set; }
|
||||
|
||||
/// <summary>When true, every write is immediately synced to disk.</summary>
|
||||
public bool SyncAlways { get; set; }
|
||||
|
||||
/// <summary>When true, write operations may be batched and flushed asynchronously.</summary>
|
||||
public bool AsyncFlush { get; set; }
|
||||
|
||||
/// <summary>Encryption cipher used when encrypting blocks.</summary>
|
||||
public StoreCipher Cipher { get; set; }
|
||||
|
||||
/// <summary>Compression algorithm applied to stored blocks.</summary>
|
||||
public StoreCompression Compression { get; set; }
|
||||
|
||||
// Internal reference to the owning server — not serialised.
|
||||
// Equivalent to srv *Server in Go; kept as object to avoid circular project deps.
|
||||
internal object? Server { get; set; }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileStreamInfo
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Remembers the creation time alongside the stream configuration.
|
||||
/// Mirrors <c>FileStreamInfo</c> in filestore.go.
|
||||
/// </summary>
|
||||
public sealed class FileStreamInfo
|
||||
{
|
||||
/// <summary>UTC time at which the stream was created.</summary>
|
||||
public DateTime Created { get; set; }
|
||||
|
||||
/// <summary>Stream configuration.</summary>
|
||||
public StreamConfig Config { get; set; } = new();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileConsumerInfo
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Used for creating and restoring consumer stores from disk.
|
||||
/// Mirrors <c>FileConsumerInfo</c> in filestore.go.
|
||||
/// </summary>
|
||||
public sealed class FileConsumerInfo
|
||||
{
|
||||
/// <summary>UTC time at which the consumer was created.</summary>
|
||||
public DateTime Created { get; set; }
|
||||
|
||||
/// <summary>Durable consumer name.</summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Consumer configuration.</summary>
|
||||
public ConsumerConfig Config { get; set; } = new();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Psi — per-subject index entry (internal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Per-subject index entry stored in the subject tree.
|
||||
/// Mirrors the <c>psi</c> struct in filestore.go.
|
||||
/// </summary>
|
||||
internal sealed class Psi
|
||||
{
|
||||
/// <summary>Total messages for this subject across all blocks.</summary>
|
||||
public ulong Total { get; set; }
|
||||
|
||||
/// <summary>Index of the first block that holds messages for this subject.</summary>
|
||||
public uint Fblk { get; set; }
|
||||
|
||||
/// <summary>Index of the last block that holds messages for this subject.</summary>
|
||||
public uint Lblk { get; set; }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache — write-through and load cache (internal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Write-through caching layer also used when loading messages from disk.
|
||||
/// Mirrors the <c>cache</c> struct in filestore.go.
|
||||
/// </summary>
|
||||
internal sealed class Cache
|
||||
{
|
||||
/// <summary>Raw message data buffer.</summary>
|
||||
public byte[] Buf { get; set; } = Array.Empty<byte>();
|
||||
|
||||
/// <summary>Write position into <see cref="Buf"/>.</summary>
|
||||
public int Wp { get; set; }
|
||||
|
||||
/// <summary>Per-sequence byte offsets into <see cref="Buf"/>.</summary>
|
||||
public uint[] Idx { get; set; } = Array.Empty<uint>();
|
||||
|
||||
/// <summary>First sequence number this cache covers.</summary>
|
||||
public ulong Fseq { get; set; }
|
||||
|
||||
/// <summary>No-random-access flag: when true sequential access is assumed.</summary>
|
||||
public bool Nra { get; set; }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MsgId — sequence + timestamp pair (internal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Pairs a message sequence number with its nanosecond timestamp.
|
||||
/// Mirrors the <c>msgId</c> struct in filestore.go.
|
||||
/// </summary>
|
||||
internal struct MsgId
|
||||
{
|
||||
/// <summary>Sequence number.</summary>
|
||||
public ulong Seq;
|
||||
|
||||
/// <summary>Nanosecond Unix timestamp.</summary>
|
||||
public long Ts;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CompressionInfo — compression metadata
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Compression metadata attached to a message block.
|
||||
/// Mirrors <c>CompressionInfo</c> in filestore.go.
|
||||
/// </summary>
|
||||
public sealed class CompressionInfo
|
||||
{
|
||||
/// <summary>Compression algorithm in use.</summary>
|
||||
public StoreCompression Type { get; set; }
|
||||
|
||||
/// <summary>Original (uncompressed) size in bytes.</summary>
|
||||
public ulong Original { get; set; }
|
||||
|
||||
/// <summary>Compressed size in bytes.</summary>
|
||||
public ulong Compressed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Serialises compression metadata as a compact binary prefix.
|
||||
/// Format: 'c' 'm' 'p' <algorithmByte> <uvarint originalSize>
|
||||
/// </summary>
|
||||
public byte[] MarshalMetadata()
|
||||
{
|
||||
// TODO: session 18 — implement varint encoding
|
||||
throw new NotImplementedException("TODO: session 18 — filestore CompressionInfo.MarshalMetadata");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deserialises compression metadata from a binary buffer.
|
||||
/// Returns the number of bytes consumed, or 0 if the buffer does not start with the expected prefix.
|
||||
/// </summary>
|
||||
public int UnmarshalMetadata(byte[] b)
|
||||
{
|
||||
// TODO: session 18 — implement varint decoding
|
||||
throw new NotImplementedException("TODO: session 18 — filestore CompressionInfo.UnmarshalMetadata");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ErrBadMsg — corrupt/malformed message error (internal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Indicates a malformed or corrupt message was detected in a block file.
|
||||
/// Mirrors the <c>errBadMsg</c> type in filestore.go.
|
||||
/// </summary>
|
||||
internal sealed class ErrBadMsg : Exception
|
||||
{
|
||||
/// <summary>Path to the block file that contained the bad message.</summary>
|
||||
public string FileName { get; }
|
||||
|
||||
/// <summary>Optional additional detail about the corruption.</summary>
|
||||
public string Detail { get; }
|
||||
|
||||
public ErrBadMsg(string fileName, string detail = "")
|
||||
: base(BuildMessage(fileName, detail))
|
||||
{
|
||||
FileName = fileName;
|
||||
Detail = detail;
|
||||
}
|
||||
|
||||
private static string BuildMessage(string fileName, string detail)
|
||||
{
|
||||
var baseName = Path.GetFileName(fileName);
|
||||
return string.IsNullOrEmpty(detail)
|
||||
? $"malformed or corrupt message in {baseName}"
|
||||
: $"malformed or corrupt message in {baseName}: {detail}";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FileStoreDefaults — well-known constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Well-known constants from filestore.go, exposed for cross-assembly use.
|
||||
/// </summary>
|
||||
public static class FileStoreDefaults
|
||||
{
|
||||
// Magic / version markers written into block files.
|
||||
|
||||
/// <summary>Magic byte used to identify file-store block files.</summary>
|
||||
public const byte FileStoreMagic = 22;
|
||||
|
||||
/// <summary>Current block file version.</summary>
|
||||
public const byte FileStoreVersion = 1;
|
||||
|
||||
/// <summary>New-format index version.</summary>
|
||||
internal const byte NewVersion = 2;
|
||||
|
||||
/// <summary>Header length in bytes for block records.</summary>
|
||||
internal const int HdrLen = 2;
|
||||
|
||||
// Directory names
|
||||
|
||||
/// <summary>Top-level directory that holds per-stream subdirectories.</summary>
|
||||
public const string StreamsDir = "streams";
|
||||
|
||||
/// <summary>Directory that holds in-flight batch data for a stream.</summary>
|
||||
public const string BatchesDir = "batches";
|
||||
|
||||
/// <summary>Directory that holds message block files.</summary>
|
||||
public const string MsgDir = "msgs";
|
||||
|
||||
/// <summary>Temporary directory name used during a full purge.</summary>
|
||||
public const string PurgeDir = "__msgs__";
|
||||
|
||||
/// <summary>Temporary directory name for the new message block during purge.</summary>
|
||||
public const string NewMsgDir = "__new_msgs__";
|
||||
|
||||
/// <summary>Directory name that holds per-consumer state.</summary>
|
||||
public const string ConsumerDir = "obs";
|
||||
|
||||
// File name patterns
|
||||
|
||||
/// <summary>Format string for block file names (<c>{index}.blk</c>).</summary>
|
||||
public const string BlkScan = "{0}.blk";
|
||||
|
||||
/// <summary>Suffix for active block files.</summary>
|
||||
public const string BlkSuffix = ".blk";
|
||||
|
||||
/// <summary>Format string for compacted-block staging files (<c>{index}.new</c>).</summary>
|
||||
public const string NewScan = "{0}.new";
|
||||
|
||||
/// <summary>Format string for index files (<c>{index}.idx</c>).</summary>
|
||||
public const string IndexScan = "{0}.idx";
|
||||
|
||||
/// <summary>Format string for per-block encryption-key files (<c>{index}.key</c>).</summary>
|
||||
public const string KeyScan = "{0}.key";
|
||||
|
||||
/// <summary>Glob pattern used to find orphaned key files.</summary>
|
||||
public const string KeyScanAll = "*.key";
|
||||
|
||||
/// <summary>Suffix for temporary rewrite/compression staging files.</summary>
|
||||
public const string BlkTmpSuffix = ".tmp";
|
||||
|
||||
// Meta files
|
||||
|
||||
/// <summary>Stream / consumer metadata file name.</summary>
|
||||
public const string JetStreamMetaFile = "meta.inf";
|
||||
|
||||
/// <summary>Checksum file for the metadata file.</summary>
|
||||
public const string JetStreamMetaFileSum = "meta.sum";
|
||||
|
||||
/// <summary>Encrypted metadata key file name.</summary>
|
||||
public const string JetStreamMetaFileKey = "meta.key";
|
||||
|
||||
/// <summary>Full stream-state snapshot file name.</summary>
|
||||
public const string StreamStateFile = "index.db";
|
||||
|
||||
/// <summary>Encoded TTL hash-wheel persistence file name.</summary>
|
||||
public const string TtlStreamStateFile = "thw.db";
|
||||
|
||||
/// <summary>Encoded message-scheduling persistence file name.</summary>
|
||||
public const string MsgSchedulingStreamStateFile = "sched.db";
|
||||
|
||||
/// <summary>Consumer state file name inside a consumer directory.</summary>
|
||||
public const string ConsumerState = "o.dat";
|
||||
|
||||
// Block size defaults (bytes)
|
||||
|
||||
/// <summary>Default block size for large (limits-based) streams: 8 MB.</summary>
|
||||
public const ulong DefaultLargeBlockSize = 8 * 1024 * 1024;
|
||||
|
||||
/// <summary>Default block size for work-queue / interest streams: 4 MB.</summary>
|
||||
public const ulong DefaultMediumBlockSize = 4 * 1024 * 1024;
|
||||
|
||||
/// <summary>Default block size used by mirrors/sources: 1 MB.</summary>
|
||||
public const ulong DefaultSmallBlockSize = 1 * 1024 * 1024;
|
||||
|
||||
/// <summary>Tiny pool block size (256 KB) — avoids large allocations at low write rates.</summary>
|
||||
public const ulong DefaultTinyBlockSize = 256 * 1024;
|
||||
|
||||
/// <summary>Maximum encrypted-head block size: 2 MB.</summary>
|
||||
public const ulong MaximumEncryptedBlockSize = 2 * 1024 * 1024;
|
||||
|
||||
/// <summary>Default block size for KV-based streams (same as medium).</summary>
|
||||
public const ulong DefaultKvBlockSize = DefaultMediumBlockSize;
|
||||
|
||||
/// <summary>Hard upper limit on block size.</summary>
|
||||
public const ulong MaxBlockSize = DefaultLargeBlockSize;
|
||||
|
||||
/// <summary>Minimum allowed block size: 32 KiB.</summary>
|
||||
public const ulong FileStoreMinBlkSize = 32 * 1000;
|
||||
|
||||
/// <summary>Maximum allowed block size (same as <see cref="MaxBlockSize"/>).</summary>
|
||||
public const ulong FileStoreMaxBlkSize = MaxBlockSize;
|
||||
|
||||
/// <summary>
|
||||
/// Default block size exposed publicly; resolves to <see cref="DefaultSmallBlockSize"/> (1 MB)
|
||||
/// to match the spec note in the porting plan.
|
||||
/// </summary>
|
||||
public const ulong DefaultBlockSize = DefaultSmallBlockSize;
|
||||
|
||||
// Timing defaults
|
||||
|
||||
/// <summary>Default duration before an idle cache buffer is expired: 10 seconds.</summary>
|
||||
public static readonly TimeSpan DefaultCacheBufferExpiration = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>Default interval for background disk sync: 2 minutes.</summary>
|
||||
public static readonly TimeSpan DefaultSyncInterval = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>Default idle timeout before file descriptors are closed: 30 seconds.</summary>
|
||||
public static readonly TimeSpan CloseFdsIdle = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Default expiration time for idle per-block subject state: 2 minutes.</summary>
|
||||
public static readonly TimeSpan DefaultFssExpiration = TimeSpan.FromMinutes(2);
|
||||
|
||||
// Thresholds
|
||||
|
||||
/// <summary>Minimum coalesce size for write batching: 16 KiB.</summary>
|
||||
public const int CoalesceMinimum = 16 * 1024;
|
||||
|
||||
/// <summary>Maximum wait time when gathering messages to flush: 8 ms.</summary>
|
||||
public static readonly TimeSpan MaxFlushWait = TimeSpan.FromMilliseconds(8);
|
||||
|
||||
/// <summary>Minimum block size before compaction is attempted: 2 MB.</summary>
|
||||
public const int CompactMinimum = 2 * 1024 * 1024;
|
||||
|
||||
/// <summary>Threshold above which a record length is considered corrupt: 32 MB.</summary>
|
||||
public const int RlBadThresh = 32 * 1024 * 1024;
|
||||
|
||||
/// <summary>Size of the per-record hash checksum in bytes.</summary>
|
||||
public const int RecordHashSize = 8;
|
||||
|
||||
// Encryption key size minimums
|
||||
|
||||
/// <summary>Minimum size of a metadata encryption key: 64 bytes.</summary>
|
||||
internal const int MinMetaKeySize = 64;
|
||||
|
||||
/// <summary>Minimum size of a block encryption key: 64 bytes.</summary>
|
||||
internal const int MinBlkKeySize = 64;
|
||||
|
||||
// Cache-index bit flags
|
||||
|
||||
/// <summary>Bit set in a cache index slot to mark that the checksum has been validated.</summary>
|
||||
internal const uint Cbit = 1u << 31;
|
||||
|
||||
/// <summary>Bit set in a cache index slot to mark the message as deleted.</summary>
|
||||
internal const uint Dbit = 1u << 30;
|
||||
|
||||
/// <summary>Bit set in a record length field to indicate the record has headers.</summary>
|
||||
internal const uint Hbit = 1u << 31;
|
||||
|
||||
/// <summary>Bit set in a sequence number to mark an erased message.</summary>
|
||||
internal const ulong Ebit = 1UL << 63;
|
||||
|
||||
/// <summary>Bit set in a sequence number to mark a tombstone.</summary>
|
||||
internal const ulong Tbit = 1UL << 62;
|
||||
}
|
||||
382
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/MessageBlock.cs
Normal file
382
dotnet/src/ZB.MOM.NatsNet.Server/JetStream/MessageBlock.cs
Normal file
@@ -0,0 +1,382 @@
|
||||
// Copyright 2019-2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Adapted from server/filestore.go (msgBlock struct and consumerFileStore struct)
|
||||
|
||||
using System.Threading.Channels;
|
||||
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
|
||||
|
||||
namespace ZB.MOM.NatsNet.Server;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MessageBlock
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Represents a single on-disk message block file together with its
|
||||
/// in-memory cache and index state.
|
||||
/// Mirrors the <c>msgBlock</c> struct in filestore.go.
|
||||
/// </summary>
|
||||
internal sealed class MessageBlock
|
||||
{
|
||||
// ------------------------------------------------------------------
|
||||
// Identity fields — first/last use volatile-style access in Go via
|
||||
// atomic.LoadUint64 on the embedded msgId structs.
|
||||
// We replicate those as plain fields; callers must acquire _mu before
|
||||
// reading/writing unless using the Interlocked helpers on Seq/Ts.
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>First message in this block (sequence + nanosecond timestamp).</summary>
|
||||
public MsgId First;
|
||||
|
||||
/// <summary>Last message in this block (sequence + nanosecond timestamp).</summary>
|
||||
public MsgId Last;
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Lock
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Guards all mutable fields in this block.</summary>
|
||||
public readonly ReaderWriterLockSlim Mu = new(LockRecursionPolicy.NoRecursion);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Back-reference
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Owning file store.</summary>
|
||||
public JetStreamFileStore? Fs { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// File I/O
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Path to the <c>.blk</c> message data file.</summary>
|
||||
public string Mfn { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Open file stream for the block data file (null when closed/idle).</summary>
|
||||
public FileStream? Mfd { get; set; }
|
||||
|
||||
/// <summary>Path to the per-block encryption key file.</summary>
|
||||
public string Kfn { get; set; } = string.Empty;
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Compression
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Effective compression algorithm when the block was last loaded.</summary>
|
||||
public StoreCompression Cmp { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Last index write size in bytes; used to detect whether re-compressing
|
||||
/// the block would save meaningful space.
|
||||
/// </summary>
|
||||
public long Liwsz { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Block identity
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Monotonically increasing block index number (used in file names).</summary>
|
||||
public uint Index { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Counters
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>User-visible byte count (excludes deleted-message bytes).</summary>
|
||||
public ulong Bytes { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Total raw byte count including deleted messages.
|
||||
/// Used to decide when to roll to a new block.
|
||||
/// </summary>
|
||||
public ulong RBytes { get; set; }
|
||||
|
||||
/// <summary>Byte count captured at the last compaction (0 if never compacted).</summary>
|
||||
public ulong CBytes { get; set; }
|
||||
|
||||
/// <summary>User-visible message count (excludes deleted messages).</summary>
|
||||
public ulong Msgs { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Per-subject state
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Optional per-subject state tree for this block.
|
||||
/// Lazily populated and expired when idle.
|
||||
/// Mirrors <c>mb.fss</c> in filestore.go.
|
||||
/// </summary>
|
||||
public SubjectTree<SimpleState>? Fss { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Deleted-sequence tracking
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Set of deleted sequence numbers within this block.
|
||||
/// Uses the AVL-backed <see cref="SequenceSet"/> to match Go's <c>avl.SequenceSet</c>.
|
||||
/// </summary>
|
||||
public SequenceSet Dmap { get; set; } = new();
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Timestamps (nanosecond Unix times, matches Go int64)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Nanosecond timestamp of the last write to this block.</summary>
|
||||
public long Lwts { get; set; }
|
||||
|
||||
/// <summary>Nanosecond timestamp of the last load (cache fill) of this block.</summary>
|
||||
public long Llts { get; set; }
|
||||
|
||||
/// <summary>Nanosecond timestamp of the last read from this block.</summary>
|
||||
public long Lrts { get; set; }
|
||||
|
||||
/// <summary>Nanosecond timestamp of the last subject-state (fss) access.</summary>
|
||||
public long Lsts { get; set; }
|
||||
|
||||
/// <summary>Last sequence that was looked up; used to detect linear scans.</summary>
|
||||
public ulong Llseq { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Cache
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Active in-memory cache. May be null when evicted.
|
||||
/// Mirrors <c>mb.cache</c>; the elastic-pointer field (<c>mb.ecache</c>) is
|
||||
/// not ported — a plain nullable field is sufficient here.
|
||||
/// </summary>
|
||||
public Cache? CacheData { get; set; }
|
||||
|
||||
/// <summary>Number of times the cache has been (re)loaded from disk.</summary>
|
||||
public ulong Cloads { get; set; }
|
||||
|
||||
/// <summary>Cache buffer expiration duration for this block.</summary>
|
||||
public TimeSpan Cexp { get; set; }
|
||||
|
||||
/// <summary>Per-block subject-state (fss) expiration duration.</summary>
|
||||
public TimeSpan Fexp { get; set; }
|
||||
|
||||
/// <summary>Timer used to expire the cache buffer when idle.</summary>
|
||||
public Timer? Ctmr { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// State flags
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Whether the in-memory cache is currently populated.</summary>
|
||||
public bool HaveCache => CacheData != null;
|
||||
|
||||
/// <summary>Whether this block has been closed and must not be written to.</summary>
|
||||
public bool Closed { get; set; }
|
||||
|
||||
/// <summary>Whether this block has unflushed data that must be synced to disk.</summary>
|
||||
public bool NeedSync { get; set; }
|
||||
|
||||
/// <summary>When true every write is immediately synced (SyncAlways mode).</summary>
|
||||
public bool SyncAlways { get; set; }
|
||||
|
||||
/// <summary>When true compaction is suppressed for this block.</summary>
|
||||
public bool NoCompact { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When true the block's messages are not tracked in the per-subject index.
|
||||
/// Used for blocks that only contain tombstone or deleted markers.
|
||||
/// </summary>
|
||||
public bool NoTrack { get; set; }
|
||||
|
||||
/// <summary>Whether a background flusher goroutine equivalent is running.</summary>
|
||||
public bool Flusher { get; set; }
|
||||
|
||||
/// <summary>Whether a cache-load is currently in progress.</summary>
|
||||
public bool Loading { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Write error
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Captured write error; non-null means the block is in a bad state.</summary>
|
||||
public Exception? Werr { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// TTL / scheduling counters
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Number of messages in this block that have a TTL set.</summary>
|
||||
public ulong Ttls { get; set; }
|
||||
|
||||
/// <summary>Number of messages in this block that have a schedule set.</summary>
|
||||
public ulong Schedules { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Channels (equivalent to Go chan struct{})
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>Flush-request channel: signals the flusher to run.</summary>
|
||||
public Channel<byte>? Fch { get; set; }
|
||||
|
||||
/// <summary>Quit channel: closing signals background goroutine equivalents to stop.</summary>
|
||||
public Channel<byte>? Qch { get; set; }
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Checksum
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Last-check hash: 8-byte rolling checksum of the last validated record.
|
||||
/// Mirrors <c>mb.lchk [8]byte</c>.
|
||||
/// </summary>
|
||||
public byte[] Lchk { get; set; } = new byte[8];
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Test hook
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>When true, simulates a write failure. Used by unit tests only.</summary>
|
||||
public bool MockWriteErr { get; set; }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ConsumerFileStore
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// File-backed implementation of <see cref="IConsumerStore"/>.
|
||||
/// Persists consumer delivery and ack state to a directory under the stream's
|
||||
/// <c>obs/</c> subdirectory.
|
||||
/// Mirrors the <c>consumerFileStore</c> struct in filestore.go.
|
||||
/// </summary>
|
||||
public sealed class ConsumerFileStore : IConsumerStore
|
||||
{
|
||||
// ------------------------------------------------------------------
|
||||
// Fields — mirrors consumerFileStore struct
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private readonly object _mu = new();
|
||||
|
||||
/// <summary>Back-reference to the owning file store.</summary>
|
||||
private readonly JetStreamFileStore _fs;
|
||||
|
||||
/// <summary>Consumer metadata (name, created time, config).</summary>
|
||||
private FileConsumerInfo _cfg;
|
||||
|
||||
/// <summary>Durable consumer name.</summary>
|
||||
private readonly string _name;
|
||||
|
||||
/// <summary>Path to the consumer's state directory.</summary>
|
||||
private readonly string _odir;
|
||||
|
||||
/// <summary>Path to the consumer state file (<c>o.dat</c>).</summary>
|
||||
private readonly string _ifn;
|
||||
|
||||
/// <summary>Consumer delivery/ack state.</summary>
|
||||
private ConsumerState _state = new();
|
||||
|
||||
/// <summary>Flush-request channel.</summary>
|
||||
private Channel<byte>? _fch;
|
||||
|
||||
/// <summary>Quit channel.</summary>
|
||||
private Channel<byte>? _qch;
|
||||
|
||||
/// <summary>Whether a background flusher is running.</summary>
|
||||
private bool _flusher;
|
||||
|
||||
/// <summary>Whether a write is currently in progress.</summary>
|
||||
private bool _writing;
|
||||
|
||||
/// <summary>Whether the state is dirty (pending flush).</summary>
|
||||
private bool _dirty;
|
||||
|
||||
/// <summary>Whether this consumer store is closed.</summary>
|
||||
private bool _closed;
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Constructor
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new file-backed consumer store.
|
||||
/// </summary>
|
||||
public ConsumerFileStore(JetStreamFileStore fs, FileConsumerInfo cfg, string name, string odir)
|
||||
{
|
||||
_fs = fs;
|
||||
_cfg = cfg;
|
||||
_name = name;
|
||||
_odir = odir;
|
||||
_ifn = Path.Combine(odir, FileStoreDefaults.ConsumerState);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// IConsumerStore — all methods stubbed
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SetStarting(ulong sseq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.SetStarting");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void UpdateStarting(ulong sseq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateStarting");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Reset(ulong sseq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Reset");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool HasState()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.HasState");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void UpdateDelivered(ulong dseq, ulong sseq, ulong dc, long ts)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateDelivered");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void UpdateAcks(ulong dseq, ulong sseq)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateAcks");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void UpdateConfig(ConsumerConfig cfg)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.UpdateConfig");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Update(ConsumerState state)
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Update");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ConsumerState? State, Exception? Error) State()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.State");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public (ConsumerState? State, Exception? Error) BorrowState()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.BorrowState");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public byte[] EncodedState()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.EncodedState");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public StorageType Type() => StorageType.FileStorage;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Stop()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Stop");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Delete()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.Delete");
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void StreamDelete()
|
||||
=> throw new NotImplementedException("TODO: session 18 — filestore ConsumerFileStore.StreamDelete");
|
||||
}
|
||||
Reference in New Issue
Block a user