78b4bc2486
Move 225 JetStream-related test files from NATS.Server.Tests into a dedicated NATS.Server.JetStream.Tests project. This includes root-level JetStream*.cs files, storage test files (FileStore, MemStore, StreamStoreContract), and the full JetStream/ subfolder tree (Api, Cluster, Consumers, MirrorSource, Snapshots, Storage, Streams). Updated all namespaces, added InternalsVisibleTo, registered in the solution file, and added the JETSTREAM_INTEGRATION_MATRIX define.
1312 lines
52 KiB
C#
1312 lines
52 KiB
C#
// Reference: golang/nats-server/server/filestore_test.go
|
|
// Tests ported from (Go parity — T1 FileStore Block Recovery & Compaction):
|
|
// TestFileStoreInvalidIndexesRebuilt → InvalidIndexes_RebuildAndLoadCorrectSeq
|
|
// TestFileStoreMsgBlockHolesAndIndexing → MsgBlockHoles_FetchMsgWithGaps
|
|
// TestFileStoreMsgBlockCompactionAndHoles → MsgBlockCompaction_UtilizationAfterCompact
|
|
// TestFileStoreCompactAndPSIMWhenDeletingBlocks → Compact_PSIM_MultiBlock
|
|
// TestFileStoreReloadAndLoseLastSequence → Reload_SkipMsg_LoseLastSeq
|
|
// TestFileStoreRestoreIndexWithMatchButLeftOverBlocks → RestoreIndex_LeftoverBlocks_StatePreserved
|
|
// TestFileStoreLargeFullStatePSIM → LargeFullState_PSIM_StopsWithoutError
|
|
// TestFileStorePartialIndexes → PartialIndexes_StoreAndLoadAfterCacheExpiry (skipped in Go)
|
|
// TestFileStoreCheckSkipFirstBlockBug → CheckSkip_FirstBlock_LoadNextMsgWorks
|
|
// TestFileStoreSyncCompressOnlyIfDirty → SyncCompress_OnlyIfDirty_CompactFlagBehavior
|
|
// TestFileStoreErrPartialLoad → ErrPartialLoad_ConcurrentWriteAndLoad
|
|
// TestFileStoreStreamFailToRollBug → StreamFailToRoll_StoreAndLoadMessages
|
|
// TestFileStoreBadFirstAndFailedExpireAfterRestart → BadFirst_ExpireAfterRestart_StateCorrect
|
|
// TestFileStoreSyncIntervals → SyncIntervals_StoreThenFlushOnRestart
|
|
// TestFileStoreAsyncFlushOnSkipMsgs → AsyncFlush_SkipMsgs_StateAfterSkip
|
|
// TestFileStoreLeftoverSkipMsgInDmap → LeftoverSkipMsg_NotInDeleted_AfterRestart
|
|
// TestFileStoreCacheLookupOnEmptyBlock → CacheLookup_EmptyBlock_ReturnsNotFound
|
|
// TestFileStoreWriteFullStateHighSubjectCardinality → WriteFullState_HighSubjectCardinality (skipped in Go)
|
|
// TestFileStoreWriteFullStateDetectCorruptState → WriteFullState_DetectCorruptState
|
|
// TestFileStoreRecoverFullStateDetectCorruptState → RecoverFullState_DetectCorruptState
|
|
// TestFileStoreFullStateTestUserRemoveWAL → FullState_UserRemoveWAL_StatePreserved
|
|
// TestFileStoreFullStateTestSysRemovals → FullState_SysRemovals_StatePreserved
|
|
// TestFileStoreTrackSubjLenForPSIM → TrackSubjLen_PSIM_TotalLength
|
|
// TestFileStoreMsgBlockFirstAndLastSeqCorrupt → MsgBlock_FirstAndLastSeqCorrupt_RebuildOk
|
|
// TestFileStoreCorruptPSIMOnDisk → CorruptPSIM_OnDisk_RecoveryLoadsCorrectly
|
|
// TestFileStoreCorruptionSetsHbitWithoutHeaders → Corruption_HbitSanityCheck (skipped)
|
|
// TestFileStoreWriteFullStateThrowsPermissionErrorIfFSModeReadOnly → WriteFullState_ReadOnlyDir_ThrowsPermissionError
|
|
// TestFileStoreStoreRawMessageThrowsPermissionErrorIfFSModeReadOnly → StoreMsg_ReadOnlyDir_ThrowsPermissionError
|
|
// TestFileStoreWriteFailures → WriteFailures_FullDisk_SkipEnvironmentSpecific
|
|
// TestFileStoreStreamDeleteDirNotEmpty → StreamDelete_DirNotEmpty_DeleteSucceeds
|
|
// TestFileStoreFSSCloseAndKeepOnExpireOnRecoverBug → FSS_CloseAndKeepOnExpireOnRecover_NoSubjects
|
|
// TestFileStoreExpireOnRecoverSubjectAccounting → ExpireOnRecover_SubjectAccounting_CorrectNumSubjects
|
|
// TestFileStoreFSSExpireNumPendingBug → FSS_ExpireNumPending_FilteredStateCorrect
|
|
// TestFileStoreSelectMsgBlockBinarySearch → SelectMsgBlock_BinarySearch_CorrectBlockSelected (partially ported)
|
|
// TestFileStoreRecoverFullState → RecoverFullState_StoreAndReopen (indirectly covered by existing tests)
|
|
// TestFileStoreLargeFullState → LargeFullState_StoreReopen_StatePreserved
|
|
|
|
using NATS.Server.JetStream.Storage;
|
|
|
|
namespace NATS.Server.JetStream.Tests.JetStream.Storage;
|
|
|
|
/// <summary>
|
|
/// Go FileStore parity tests — T1: Block Recovery and Compaction.
|
|
/// Each test mirrors a specific Go test from filestore_test.go.
|
|
/// </summary>
|
|
public sealed class FileStoreRecovery2Tests : IDisposable
|
|
{
|
|
private readonly string _root;
|
|
|
|
public FileStoreRecovery2Tests()
|
|
{
|
|
_root = Path.Combine(Path.GetTempPath(), $"nats-js-recovery2-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(_root);
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
if (Directory.Exists(_root))
|
|
{
|
|
try { Directory.Delete(_root, recursive: true); }
|
|
catch { /* best-effort cleanup */ }
|
|
}
|
|
}
|
|
|
|
private FileStore CreateStore(string subDir, FileStoreOptions? opts = null)
|
|
{
|
|
var dir = Path.Combine(_root, subDir);
|
|
Directory.CreateDirectory(dir);
|
|
var o = opts ?? new FileStoreOptions();
|
|
o.Directory = dir;
|
|
return new FileStore(o);
|
|
}
|
|
|
|
private static string UniqueDir() => Guid.NewGuid().ToString("N");
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Index rebuild / invalid index tests
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreInvalidIndexesRebuilt (filestore_test.go:1755)
|
|
// Verifies that after the in-memory index is mangled (simulated by deleting then
|
|
// re-storing), the store can still load messages correctly after a restart.
|
|
// The Go test directly corrupts the in-memory cache bit; here we test the
|
|
// observable behaviour: store → delete block → re-create → all messages loadable.
|
|
[Fact]
|
|
public void InvalidIndexes_RebuildAndLoadCorrectSeq()
|
|
{
|
|
var dir = UniqueDir();
|
|
|
|
using (var store = CreateStore(dir))
|
|
{
|
|
for (var i = 0; i < 5; i++)
|
|
store.StoreMsg("foo", null, "ok-1"u8.ToArray(), 0);
|
|
}
|
|
|
|
// Reopen: the recovered store must serve all 5 messages correctly.
|
|
using (var store = CreateStore(dir))
|
|
{
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(5UL);
|
|
state.FirstSeq.ShouldBe(1UL);
|
|
state.LastSeq.ShouldBe(5UL);
|
|
|
|
for (ulong seq = 1; seq <= 5; seq++)
|
|
{
|
|
var sm = store.LoadMsg(seq, null);
|
|
sm.Sequence.ShouldBe(seq);
|
|
sm.Subject.ShouldBe("foo");
|
|
}
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// MsgBlock holes and indexing
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreMsgBlockHolesAndIndexing (filestore_test.go:5965)
|
|
// Verifies that messages with gaps in sequence numbers (holes) can be stored and
|
|
// loaded correctly, and that missing sequences are reported as deleted.
|
|
[Fact]
|
|
public void MsgBlockHoles_FetchMsgWithGaps()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir);
|
|
|
|
// Store 3 messages, then remove 2 to create holes.
|
|
store.StoreMsg("A", null, "A"u8.ToArray(), 0);
|
|
store.StoreMsg("B", null, "B"u8.ToArray(), 0);
|
|
store.StoreMsg("C", null, "C"u8.ToArray(), 0);
|
|
store.StoreMsg("D", null, "D"u8.ToArray(), 0);
|
|
store.StoreMsg("E", null, "E"u8.ToArray(), 0);
|
|
|
|
// Remove 2 and 4 to create holes.
|
|
store.RemoveMsg(2).ShouldBeTrue();
|
|
store.RemoveMsg(4).ShouldBeTrue();
|
|
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(3UL);
|
|
|
|
// Remaining messages should be loadable.
|
|
store.LoadMsg(1, null).Subject.ShouldBe("A");
|
|
store.LoadMsg(3, null).Subject.ShouldBe("C");
|
|
store.LoadMsg(5, null).Subject.ShouldBe("E");
|
|
|
|
// Deleted messages should not be loadable.
|
|
Should.Throw<KeyNotFoundException>(() => store.LoadMsg(2, null));
|
|
Should.Throw<KeyNotFoundException>(() => store.LoadMsg(4, null));
|
|
|
|
// State should include the deleted sequences.
|
|
state.NumDeleted.ShouldBe(2);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Compaction and holes
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreMsgBlockCompactionAndHoles (filestore_test.go:6027)
|
|
// After storing 10 messages and deleting most of them, a compact should
|
|
// leave only the bytes used by live messages.
|
|
[Fact]
|
|
public void MsgBlockCompaction_UtilizationAfterCompact()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 4096 });
|
|
|
|
var msg = new byte[1024];
|
|
Array.Fill(msg, (byte)'Z');
|
|
|
|
foreach (var subj in new[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" })
|
|
store.StoreMsg(subj, null, msg, 0);
|
|
|
|
// Leave first one but delete the rest (seq 2 through 9).
|
|
for (ulong seq = 2; seq < 10; seq++)
|
|
store.RemoveMsg(seq).ShouldBeTrue();
|
|
|
|
var state = store.State();
|
|
// Only seq 1 and seq 10 remain.
|
|
state.Msgs.ShouldBe(2UL);
|
|
state.FirstSeq.ShouldBe(1UL);
|
|
state.LastSeq.ShouldBe(10UL);
|
|
|
|
// After restart, state is preserved.
|
|
{
|
|
using var store2 = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 4096 });
|
|
var state2 = store2.State();
|
|
state2.Msgs.ShouldBe(2UL);
|
|
store2.LoadMsg(1, null).Subject.ShouldBe("A");
|
|
store2.LoadMsg(10, null).Subject.ShouldBe("J");
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Compact + PSIM across multiple blocks
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreCompactAndPSIMWhenDeletingBlocks (filestore_test.go:6259)
|
|
// Compact(10) should remove 9 messages and leave 1, reducing to a single block.
|
|
[Fact]
|
|
public void Compact_PSIM_MultiBlock()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 512 });
|
|
|
|
var msg = new byte[99];
|
|
Array.Fill(msg, (byte)'A');
|
|
|
|
// Add 10 messages on subject "A".
|
|
for (var i = 0; i < 10; i++)
|
|
store.StoreMsg("A", null, msg, 0);
|
|
|
|
// The store may have split into multiple blocks; compact down to seq 10 (removes < 10).
|
|
var removed = store.Compact(10);
|
|
removed.ShouldBe(9UL);
|
|
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(1UL);
|
|
state.FirstSeq.ShouldBe(10UL);
|
|
state.LastSeq.ShouldBe(10UL);
|
|
|
|
// Subject "A" should have exactly 1 message, and it should be loadable.
|
|
var sm = store.LoadMsg(10, null);
|
|
sm.Subject.ShouldBe("A");
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Reload and lose last sequence
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreReloadAndLoseLastSequence (filestore_test.go:7242)
|
|
// After issuing 22 SkipMsgs, restarts should always yield FirstSeq=23, LastSeq=22.
|
|
[Fact]
|
|
public void Reload_SkipMsg_LoseLastSeq()
|
|
{
|
|
var dir = UniqueDir();
|
|
|
|
{
|
|
using var store = CreateStore(dir);
|
|
for (var i = 0; i < 22; i++)
|
|
store.SkipMsg(0);
|
|
}
|
|
|
|
// Restart 5 times and verify state is stable.
|
|
for (var i = 0; i < 5; i++)
|
|
{
|
|
using var store = CreateStore(dir);
|
|
var state = store.State();
|
|
state.FirstSeq.ShouldBe(23UL);
|
|
state.LastSeq.ShouldBe(22UL);
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Restore index with match but leftover blocks
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreRestoreIndexWithMatchButLeftOverBlocks (filestore_test.go:7931)
|
|
// After filling 2 blocks, stopping, adding more messages, then restoring the old
|
|
// index file, state should be rebuilt correctly from the block files.
|
|
[Fact]
|
|
public void RestoreIndex_LeftoverBlocks_StatePreserved()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
// Fill 12 messages across 2 blocks (small block size: each holds ~6 msgs).
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 256 };
|
|
using var store = new FileStore(o);
|
|
for (var i = 1; i <= 12; i++)
|
|
store.StoreMsg($"foo.{i}", null, "hello"u8.ToArray(), 0);
|
|
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(12UL);
|
|
state.LastSeq.ShouldBe(12UL);
|
|
}
|
|
|
|
// Reopen and add 6 more messages.
|
|
StreamState beforeStop;
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 256 };
|
|
using var store = new FileStore(o);
|
|
for (var i = 13; i <= 18; i++)
|
|
store.StoreMsg($"foo.{i}", null, "hello"u8.ToArray(), 0);
|
|
beforeStop = store.State();
|
|
}
|
|
|
|
// Reopen again — state should be preserved after recovery.
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 256 };
|
|
using var store = new FileStore(o);
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(beforeStop.Msgs);
|
|
state.LastSeq.ShouldBe(beforeStop.LastSeq);
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Large full state PSIM
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreLargeFullStatePSIM (filestore_test.go:6368)
|
|
// Stores 100,000 messages with random varying-length subjects and stops.
|
|
// Verifies the store can shut down without errors.
|
|
[Fact]
|
|
public void LargeFullState_PSIM_StopsWithoutError()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir);
|
|
|
|
var rng = new Random(42);
|
|
var hex = "0123456789abcdef";
|
|
for (var i = 0; i < 1000; i++) // reduced from 100k for test speed
|
|
{
|
|
var numTokens = rng.Next(1, 7);
|
|
var tokens = new string[numTokens];
|
|
for (var t = 0; t < numTokens; t++)
|
|
{
|
|
var tLen = rng.Next(2, 9);
|
|
var buf = new char[tLen * 2];
|
|
for (var b = 0; b < tLen; b++)
|
|
{
|
|
var v = rng.Next(256);
|
|
buf[b * 2] = hex[v >> 4];
|
|
buf[b * 2 + 1] = hex[v & 0xf];
|
|
}
|
|
tokens[t] = new string(buf);
|
|
}
|
|
var subj = string.Join(".", tokens);
|
|
store.StoreMsg(subj, null, Array.Empty<byte>(), 0);
|
|
}
|
|
|
|
// Should complete without exception.
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(1000UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// CheckSkip first block bug
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreCheckSkipFirstBlockBug (filestore_test.go:7648)
|
|
// After storing messages across blocks and removing two from a block,
|
|
// LoadNextMsg should still work correctly.
|
|
[Fact]
|
|
public void CheckSkip_FirstBlock_LoadNextMsgWorks()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 128 });
|
|
|
|
var msg = "hello"u8.ToArray();
|
|
|
|
store.StoreMsg("foo.BB.bar", null, msg, 0); // seq 1
|
|
store.StoreMsg("foo.BB.bar", null, msg, 0); // seq 2
|
|
store.StoreMsg("foo.AA.bar", null, msg, 0); // seq 3
|
|
for (var i = 0; i < 5; i++)
|
|
store.StoreMsg("foo.BB.bar", null, msg, 0); // seq 4-8
|
|
store.StoreMsg("foo.AA.bar", null, msg, 0); // seq 9
|
|
store.StoreMsg("foo.AA.bar", null, msg, 0); // seq 10
|
|
|
|
// Remove sequences 3 and 4.
|
|
store.RemoveMsg(3).ShouldBeTrue();
|
|
store.RemoveMsg(4).ShouldBeTrue();
|
|
|
|
// LoadNextMsg for "foo.AA.bar" from seq 4 should find seq 9.
|
|
var (sm, _) = store.LoadNextMsg("foo.AA.bar", false, 4, null);
|
|
sm.ShouldNotBeNull();
|
|
sm.Subject.ShouldBe("foo.AA.bar");
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Sync compress only if dirty
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreSyncCompressOnlyIfDirty (filestore_test.go:7813)
|
|
// After deleting messages to create holes and then adding new messages,
|
|
// the state must still be correct (compaction via sync does not lose data).
|
|
[Fact]
|
|
public void SyncCompress_OnlyIfDirty_CompactFlagBehavior()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 256 });
|
|
|
|
var msg = "hello"u8.ToArray();
|
|
|
|
// Fill 2 blocks (6 per block at blockSize=256).
|
|
for (var i = 0; i < 12; i++)
|
|
store.StoreMsg("foo.BB", null, msg, 0);
|
|
|
|
// Add one more to start a third block.
|
|
store.StoreMsg("foo.BB", null, msg, 0); // seq 13
|
|
|
|
// Delete a bunch to create holes in blocks 1 and 2.
|
|
foreach (var seq in new ulong[] { 2, 3, 4, 5, 8, 9, 10, 11 })
|
|
store.RemoveMsg(seq).ShouldBeTrue();
|
|
|
|
// Add more to create a 4th/5th block.
|
|
for (var i = 0; i < 6; i++)
|
|
store.StoreMsg("foo.BB", null, msg, 0);
|
|
|
|
// Total live: 13 + 6 = 19 - 8 deleted = 11.
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(11UL);
|
|
|
|
// After restart, state should be preserved.
|
|
using var store2 = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 256 });
|
|
var state2 = store2.State();
|
|
state2.Msgs.ShouldBe(11UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Err partial load (concurrent write and load)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreErrPartialLoad (filestore_test.go:5271)
|
|
// Under a series of stores, cache-clears, and random loads, no load error should occur.
|
|
// The Go test uses concurrent goroutines; the .NET FileStore is single-threaded,
|
|
// so we test the sequential equivalent: rapid store + clear + load cycles.
|
|
[Fact]
|
|
public void ErrPartialLoad_ConcurrentWriteAndLoad()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir);
|
|
|
|
// Prime with 100 messages.
|
|
for (var i = 0; i < 100; i++)
|
|
store.StoreMsg("Z", null, "ZZZZZZZZZZZZZ"u8.ToArray(), 0);
|
|
|
|
var rng = new Random(1);
|
|
|
|
// Simulate cache-expiry + load cycles sequentially (Go uses goroutines for this).
|
|
for (var i = 0; i < 500; i++)
|
|
{
|
|
// Store 5 more messages.
|
|
for (var j = 0; j < 5; j++)
|
|
store.StoreMsg("Z", null, "ZZZZZZZZZZZZZ"u8.ToArray(), 0);
|
|
|
|
var state = store.State();
|
|
if (state.FirstSeq > 0 && state.LastSeq >= state.FirstSeq)
|
|
{
|
|
var range = (int)(state.LastSeq - state.FirstSeq);
|
|
var seq = state.FirstSeq + (range > 0 ? (ulong)rng.Next(range) : 0);
|
|
// Load should not throw (except KeyNotFound for removed messages).
|
|
try { store.LoadMsg(seq, null); }
|
|
catch (KeyNotFoundException) { /* ok: sequence may be out of range */ }
|
|
}
|
|
}
|
|
|
|
// Verify final state is consistent.
|
|
var finalState = store.State();
|
|
finalState.Msgs.ShouldBeGreaterThan(0UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Stream fail to roll bug
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreStreamFailToRollBug (filestore_test.go:2965)
|
|
// Store and load messages on streams that span multiple blocks.
|
|
[Fact]
|
|
public void StreamFailToRoll_StoreAndLoadMessages()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 256 });
|
|
|
|
var msg = "hello"u8.ToArray();
|
|
for (var i = 0; i < 30; i++)
|
|
store.StoreMsg("foo", null, msg, 0);
|
|
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(30UL);
|
|
|
|
// All messages should be loadable.
|
|
for (ulong seq = 1; seq <= 30; seq++)
|
|
{
|
|
var sm = store.LoadMsg(seq, null);
|
|
sm.Subject.ShouldBe("foo");
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Bad first and failed expire after restart
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreBadFirstAndFailedExpireAfterRestart (filestore_test.go:4604)
|
|
// After storing 7 messages with TTL, waiting for expiry, removing one from the
|
|
// second block, and restarting, state should still reflect the correct first/last seq.
|
|
[Fact]
|
|
public async Task BadFirst_ExpireAfterRestart_StateCorrect()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
const int maxAgeMs = 400;
|
|
|
|
// Store 7 messages that will expire.
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 256, MaxAgeMs = maxAgeMs };
|
|
using var store = new FileStore(o);
|
|
for (var i = 0; i < 7; i++)
|
|
store.StoreMsg("foo", null, "ZZ"u8.ToArray(), 0);
|
|
|
|
// Add 2 more with enough delay that they won't expire.
|
|
store.StoreMsg("foo", null, "ZZ"u8.ToArray(), 0); // seq 8
|
|
store.StoreMsg("foo", null, "ZZ"u8.ToArray(), 0); // seq 9
|
|
|
|
// Remove seq 8 so its block has a hole.
|
|
store.RemoveMsg(8).ShouldBeTrue();
|
|
}
|
|
|
|
// Wait for the first 7 to expire.
|
|
await Task.Delay(maxAgeMs * 2);
|
|
|
|
// Reopen with same TTL — old messages should be expired, seq 9 should remain.
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 256, MaxAgeMs = maxAgeMs };
|
|
using var store = new FileStore(o);
|
|
|
|
// Trigger expire by storing a new message.
|
|
store.StoreMsg("foo", null, "ZZ"u8.ToArray(), 0);
|
|
|
|
var state = store.State();
|
|
// seq 9 should still be present (it was stored late enough), plus the new one.
|
|
state.Msgs.ShouldBeGreaterThanOrEqualTo(1UL);
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Sync intervals
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreSyncIntervals (filestore_test.go:5366)
|
|
// Storing a message marks the block dirty; after enough time it should be flushed.
|
|
[Fact]
|
|
public async Task SyncIntervals_StoreThenFlushOnRestart()
|
|
{
|
|
var dir = UniqueDir();
|
|
|
|
{
|
|
using var store = CreateStore(dir);
|
|
store.StoreMsg("Z", null, "hello"u8.ToArray(), 0);
|
|
}
|
|
|
|
// Give any async flush a moment.
|
|
await Task.Delay(50);
|
|
|
|
// Reopening recovers the stored message — confirms flush/sync completed.
|
|
using var store2 = CreateStore(dir);
|
|
var state = store2.State();
|
|
state.Msgs.ShouldBe(1UL);
|
|
var sm = store2.LoadMsg(1, null);
|
|
sm.Subject.ShouldBe("Z");
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Async flush on skip msgs
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreAsyncFlushOnSkipMsgs (filestore_test.go:10054)
|
|
// After a SkipMsgs call that spans two blocks, state should reflect the skip.
|
|
[Fact]
|
|
public void AsyncFlush_SkipMsgs_StateAfterSkip()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 8192 });
|
|
|
|
// Store one message.
|
|
store.StoreMsg("foo", null, Array.Empty<byte>(), 0); // seq 1
|
|
|
|
// Skip 100,000 messages starting at seq 2.
|
|
store.SkipMsgs(2, 100_000);
|
|
|
|
var state = store.State();
|
|
// FirstSeq is 1 (the one real message), LastSeq is the last skip.
|
|
state.FirstSeq.ShouldBe(1UL);
|
|
state.LastSeq.ShouldBe(100_001UL);
|
|
state.Msgs.ShouldBe(1UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Leftover skip msg in dmap
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreLeftoverSkipMsgInDmap (filestore_test.go:9170)
|
|
// A single SkipMsg(0) should leave FirstSeq=2, LastSeq=1, NumDeleted=0.
|
|
// After restart (recovery from blk file), state should be identical.
|
|
[Fact]
|
|
public void LeftoverSkipMsg_NotInDeleted_AfterRestart()
|
|
{
|
|
var dir = UniqueDir();
|
|
|
|
{
|
|
using var store = CreateStore(dir);
|
|
store.SkipMsg(0);
|
|
|
|
var state = store.State();
|
|
state.FirstSeq.ShouldBe(2UL);
|
|
state.LastSeq.ShouldBe(1UL);
|
|
state.NumDeleted.ShouldBe(0);
|
|
}
|
|
|
|
// After restart, state should still match.
|
|
using var store2 = CreateStore(dir);
|
|
var state2 = store2.State();
|
|
state2.FirstSeq.ShouldBe(2UL);
|
|
state2.LastSeq.ShouldBe(1UL);
|
|
state2.NumDeleted.ShouldBe(0);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Cache lookup on empty block
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreCacheLookupOnEmptyBlock (filestore_test.go:10754)
|
|
// Loading from an empty store should return not-found, not a cache error.
|
|
[Fact]
|
|
public void CacheLookup_EmptyBlock_ReturnsNotFound()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir);
|
|
|
|
// Empty store — LoadMsg should throw KeyNotFoundException (not found), not a cache error.
|
|
Should.Throw<KeyNotFoundException>(() => store.LoadMsg(1, null));
|
|
|
|
// State should show an empty store.
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(0UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// WriteFullState detect corrupt state (observable behavior)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreWriteFullStateDetectCorruptState (filestore_test.go:8542)
|
|
// After storing 10 messages, the full state should reflect all messages.
|
|
// The Go test directly tweaks internal msg block state (mb.msgs--) and expects
|
|
// writeFullState to detect and correct this. Here we verify the observable outcome:
|
|
// the state is consistent after a restart.
|
|
[Fact]
|
|
public void WriteFullState_DetectCorruptState()
|
|
{
|
|
var dir = UniqueDir();
|
|
|
|
{
|
|
using var store = CreateStore(dir);
|
|
var msg = "abc"u8.ToArray();
|
|
for (var i = 1; i <= 10; i++)
|
|
store.StoreMsg($"foo.{i}", null, msg, 0);
|
|
|
|
var state = store.State();
|
|
state.FirstSeq.ShouldBe(1UL);
|
|
state.LastSeq.ShouldBe(10UL);
|
|
state.Msgs.ShouldBe(10UL);
|
|
}
|
|
|
|
// After restart, the state must still be 10 messages.
|
|
using var store2 = CreateStore(dir);
|
|
var state2 = store2.State();
|
|
state2.FirstSeq.ShouldBe(1UL);
|
|
state2.LastSeq.ShouldBe(10UL);
|
|
state2.Msgs.ShouldBe(10UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// RecoverFullState detect corrupt state
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreRecoverFullStateDetectCorruptState (filestore_test.go:8577)
|
|
// If the state file is corrupted (wrong message count), recovery should rebuild
|
|
// from block files and produce the correct state.
|
|
[Fact]
|
|
public void RecoverFullState_DetectCorruptState()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath };
|
|
using var store = new FileStore(o);
|
|
var msg = "abc"u8.ToArray();
|
|
for (var i = 1; i <= 10; i++)
|
|
store.StoreMsg($"foo.{i}", null, msg, 0);
|
|
}
|
|
|
|
// Corrupt the manifest/index file by writing garbage bytes, then check
|
|
// that recovery falls back to scanning block files.
|
|
var indexFiles = Directory.GetFiles(dirPath, "*.manifest.json");
|
|
foreach (var f in indexFiles)
|
|
File.WriteAllBytes(f, [0xFF, 0xFE, 0xFD]);
|
|
|
|
// Recovery should still work (falling back to block file scan).
|
|
using var store2 = new FileStore(new FileStoreOptions { Directory = dirPath });
|
|
var state = store2.State();
|
|
// The store should still have all 10 messages (or at least not crash).
|
|
state.Msgs.ShouldBe(10UL);
|
|
state.FirstSeq.ShouldBe(1UL);
|
|
state.LastSeq.ShouldBe(10UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// FullState test user remove WAL
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreFullStateTestUserRemoveWAL (filestore_test.go:5746)
|
|
// After storing 2 messages, removing 1, stopping, restarting, the state must match.
|
|
// The Go test specifically verifies tombstone WAL behavior; here we verify the
|
|
// observable outcome: state is consistent across restarts.
|
|
[Fact]
|
|
public void FullState_UserRemoveWAL_StatePreserved()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
var msgA = new byte[19];
|
|
var msgZ = new byte[19];
|
|
Array.Fill(msgA, (byte)'A');
|
|
Array.Fill(msgZ, (byte)'Z');
|
|
|
|
StreamState firstState;
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 132 };
|
|
using var store = new FileStore(o);
|
|
store.StoreMsg("A", null, msgA, 0); // seq 1
|
|
store.StoreMsg("Z", null, msgZ, 0); // seq 2
|
|
store.RemoveMsg(1).ShouldBeTrue();
|
|
|
|
// Seq 2 must be loadable.
|
|
var sm = store.LoadMsg(2, null);
|
|
sm.Subject.ShouldBe("Z");
|
|
|
|
firstState = store.State();
|
|
firstState.Msgs.ShouldBe(1UL);
|
|
firstState.FirstSeq.ShouldBe(2UL);
|
|
firstState.LastSeq.ShouldBe(2UL);
|
|
}
|
|
|
|
// Restart — state should be preserved.
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 132 };
|
|
using var store = new FileStore(o);
|
|
|
|
// Seq 2 must still be loadable.
|
|
var sm = store.LoadMsg(2, null);
|
|
sm.Subject.ShouldBe("Z");
|
|
|
|
// Seq 1 must be gone.
|
|
Should.Throw<KeyNotFoundException>(() => store.LoadMsg(1, null));
|
|
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(firstState.Msgs);
|
|
state.FirstSeq.ShouldBe(firstState.FirstSeq);
|
|
state.LastSeq.ShouldBe(firstState.LastSeq);
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// FullState sys removals
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreFullStateTestSysRemovals (filestore_test.go:5841)
|
|
// When MaxMsgs is exceeded, the oldest messages are removed automatically.
|
|
// After restart, state should match what was set before stop.
|
|
// Note: Go uses MaxMsgsPer=1 (not yet in .NET FileStoreOptions), so we simulate
|
|
// the behavior by manually removing older messages and verifying state persistence.
|
|
[Fact]
|
|
public void FullState_SysRemovals_StatePreserved()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
var msg = new byte[19];
|
|
Array.Fill(msg, (byte)'A');
|
|
|
|
StreamState firstState;
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 100 };
|
|
using var store = new FileStore(o);
|
|
|
|
// Store 4 messages on subjects A and B.
|
|
store.StoreMsg("A", null, msg, 0); // seq 1
|
|
store.StoreMsg("B", null, msg, 0); // seq 2
|
|
store.StoreMsg("A", null, msg, 0); // seq 3
|
|
store.StoreMsg("B", null, msg, 0); // seq 4
|
|
|
|
// Simulate system removal of seq 1 and 2 (old messages replaced by newer ones).
|
|
store.RemoveMsg(1).ShouldBeTrue();
|
|
store.RemoveMsg(2).ShouldBeTrue();
|
|
|
|
firstState = store.State();
|
|
firstState.Msgs.ShouldBe(2UL);
|
|
firstState.FirstSeq.ShouldBe(3UL);
|
|
firstState.LastSeq.ShouldBe(4UL);
|
|
}
|
|
|
|
// Restart — state should match.
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 100 };
|
|
using var store = new FileStore(o);
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(firstState.Msgs);
|
|
state.FirstSeq.ShouldBe(firstState.FirstSeq);
|
|
state.LastSeq.ShouldBe(firstState.LastSeq);
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Track subject length for PSIM
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreTrackSubjLenForPSIM (filestore_test.go:6289)
|
|
// Verifies that total subject length tracking is consistent with the set of
|
|
// live subjects — after stores, removes, and purge.
|
|
[Fact]
|
|
public void TrackSubjLen_PSIM_TotalLength()
|
|
{
|
|
var dir = UniqueDir();
|
|
var rng = new Random(17);
|
|
using var store = CreateStore(dir);
|
|
|
|
var subjects = new List<string>();
|
|
for (var i = 0; i < 100; i++) // reduced from 1000 for speed
|
|
{
|
|
var numTokens = rng.Next(1, 7);
|
|
var hex = "0123456789abcdef";
|
|
var tokens = new string[numTokens];
|
|
for (var t = 0; t < numTokens; t++)
|
|
{
|
|
var tLen = rng.Next(2, 5);
|
|
var buf = new char[tLen * 2];
|
|
for (var b = 0; b < tLen; b++)
|
|
{
|
|
var v = rng.Next(256);
|
|
buf[b * 2] = hex[v >> 4];
|
|
buf[b * 2 + 1] = hex[v & 0xf];
|
|
}
|
|
tokens[t] = new string(buf);
|
|
}
|
|
var subj = string.Join(".", tokens);
|
|
if (!subjects.Contains(subj)) // avoid dupes
|
|
{
|
|
subjects.Add(subj);
|
|
store.StoreMsg(subj, null, Array.Empty<byte>(), 0);
|
|
}
|
|
}
|
|
|
|
// Verify all messages are present.
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe((ulong)subjects.Count);
|
|
|
|
// Remove ~half.
|
|
var removed = subjects.Count / 2;
|
|
for (var i = 0; i < removed; i++)
|
|
store.RemoveMsg((ulong)(i + 1)).ShouldBeTrue();
|
|
|
|
var stateAfterRemove = store.State();
|
|
stateAfterRemove.Msgs.ShouldBe((ulong)(subjects.Count - removed));
|
|
|
|
// Restart and verify.
|
|
using var store2 = CreateStore(dir);
|
|
store2.State().Msgs.ShouldBe(stateAfterRemove.Msgs);
|
|
|
|
// Purge.
|
|
store2.Purge();
|
|
store2.State().Msgs.ShouldBe(0UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// MsgBlock first and last seq corrupt
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreMsgBlockFirstAndLastSeqCorrupt (filestore_test.go:7023)
|
|
// After a purge, accessing a message block whose first/last seq is incorrect
|
|
// should rebuild correctly. Observable behavior: store 10, purge, state is empty.
|
|
[Fact]
|
|
public void MsgBlock_FirstAndLastSeqCorrupt_RebuildOk()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir);
|
|
|
|
var msg = "abc"u8.ToArray();
|
|
for (var i = 1; i <= 10; i++)
|
|
store.StoreMsg($"foo.{i}", null, msg, 0);
|
|
|
|
store.Purge();
|
|
|
|
var state = store.State();
|
|
state.Msgs.ShouldBe(0UL);
|
|
|
|
// After restart, should still be empty.
|
|
using var store2 = CreateStore(dir);
|
|
var state2 = store2.State();
|
|
state2.Msgs.ShouldBe(0UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Corrupt PSIM on disk (recovery)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreCorruptPSIMOnDisk (filestore_test.go:6560)
|
|
// If the subject index on disk has a corrupted entry, recovery should rebuild
|
|
// from block files and allow LoadLastMsg to work.
|
|
[Fact]
|
|
public void CorruptPSIM_OnDisk_RecoveryLoadsCorrectly()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
{
|
|
var o = new FileStoreOptions { Directory = dirPath };
|
|
using var store = new FileStore(o);
|
|
store.StoreMsg("foo.bar", null, "ABC"u8.ToArray(), 0);
|
|
store.StoreMsg("foo.baz", null, "XYZ"u8.ToArray(), 0);
|
|
}
|
|
|
|
// Corrupt any manifest files on disk.
|
|
foreach (var f in Directory.GetFiles(dirPath, "*.manifest.json"))
|
|
File.WriteAllBytes(f, [0x00, 0x00]);
|
|
|
|
// After restart, both messages should be recoverable.
|
|
using var store2 = new FileStore(new FileStoreOptions { Directory = dirPath });
|
|
|
|
var sm1 = store2.LoadLastMsg("foo.bar", null);
|
|
sm1.ShouldNotBeNull();
|
|
sm1.Subject.ShouldBe("foo.bar");
|
|
|
|
var sm2 = store2.LoadLastMsg("foo.baz", null);
|
|
sm2.ShouldNotBeNull();
|
|
sm2.Subject.ShouldBe("foo.baz");
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Stream delete dir not empty
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreStreamDeleteDirNotEmpty (filestore_test.go:2841)
|
|
// Even if another goroutine is writing a file to the store directory concurrently,
|
|
// Purge/Dispose should succeed without error when extra files are present.
|
|
// Note: Go's fs.Delete(true) removes the directory entirely; here we test that
|
|
// Purge + Dispose succeeds even with a spurious file in the store directory.
|
|
[Fact]
|
|
public void StreamDelete_DirNotEmpty_DeleteSucceeds()
|
|
{
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 64 * 1024 };
|
|
using var store = new FileStore(o);
|
|
|
|
for (ulong i = 1; i <= 10; i++)
|
|
store.StoreMsg("foo", null, System.Text.Encoding.UTF8.GetBytes($"[{i:D8}] Hello World!"), 0);
|
|
|
|
// Write a spurious file to the directory (simulating concurrent file creation).
|
|
var spuriousFile = Path.Combine(dirPath, "g");
|
|
File.WriteAllText(spuriousFile, "OK");
|
|
|
|
// Purge should succeed even with the extra file present.
|
|
Should.NotThrow(() => store.Purge());
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// FSS close and keep on expire on recover bug
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreFSSCloseAndKeepOnExpireOnRecoverBug (filestore_test.go:4369)
|
|
// After storing a message with MaxAge, stopping, waiting for expiry, and restarting,
|
|
// the expired message must not appear in the subject count.
|
|
[Fact]
|
|
public async Task FSS_CloseAndKeepOnExpireOnRecover_NoSubjects()
|
|
{
|
|
var dir = UniqueDir();
|
|
const int ttlMs = 200;
|
|
|
|
{
|
|
using var store = CreateStore(dir, new FileStoreOptions { MaxAgeMs = ttlMs });
|
|
store.StoreMsg("foo", null, Array.Empty<byte>(), 0);
|
|
}
|
|
|
|
// Wait for the message to expire.
|
|
await Task.Delay(ttlMs * 2 + 50);
|
|
|
|
// Reopen with same TTL — expired message should not be present.
|
|
using var store2 = CreateStore(dir, new FileStoreOptions { MaxAgeMs = ttlMs });
|
|
|
|
// Trigger expiry by storing something.
|
|
store2.StoreMsg("bar", null, Array.Empty<byte>(), 0);
|
|
|
|
var state = store2.State();
|
|
// Subject "foo" should be gone; only "bar" remains.
|
|
state.NumSubjects.ShouldBe(1);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Expire on recover subject accounting
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreExpireOnRecoverSubjectAccounting (filestore_test.go:4393)
|
|
// When a whole block expires during recovery, NumSubjects in the state should
|
|
// only count subjects that still have live messages.
|
|
[Fact]
|
|
public async Task ExpireOnRecover_SubjectAccounting_CorrectNumSubjects()
|
|
{
|
|
var dir = UniqueDir();
|
|
const int ttlMs = 300;
|
|
|
|
{
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 100, MaxAgeMs = ttlMs });
|
|
|
|
// These messages will expire.
|
|
store.StoreMsg("A", null, new byte[19], 0);
|
|
store.StoreMsg("B", null, new byte[19], 0);
|
|
|
|
// Wait half TTL.
|
|
await Task.Delay(ttlMs / 2);
|
|
|
|
// This message will survive.
|
|
store.StoreMsg("C", null, new byte[19], 0);
|
|
}
|
|
|
|
// Wait for A and B to expire, but C should survive.
|
|
await Task.Delay(ttlMs / 2 + 50);
|
|
|
|
using var store2 = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 100, MaxAgeMs = ttlMs });
|
|
|
|
// Trigger expiry.
|
|
store2.StoreMsg("D", null, new byte[19], 0);
|
|
|
|
var state = store2.State();
|
|
// A and B should be expired; C and D should remain (2 subjects).
|
|
state.NumSubjects.ShouldBe(2);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// FSS expire num pending bug
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreFSSExpireNumPendingBug (filestore_test.go:4426)
|
|
// After an FSS meta expiry period, storing a message and then calling
|
|
// FilteredState should still return the correct count.
|
|
[Fact]
|
|
public async Task FSS_ExpireNumPending_FilteredStateCorrect()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir);
|
|
|
|
// Let any initial cache expire.
|
|
await Task.Delay(50);
|
|
|
|
store.StoreMsg("KV.X", null, "Y"u8.ToArray(), 0);
|
|
|
|
var filtered = store.FilteredState(1, "KV.X");
|
|
filtered.Msgs.ShouldBe(1UL);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Select MsgBlock binary search
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreSelectMsgBlockBinarySearch (filestore_test.go:12810)
|
|
// Verifies that LoadMsg correctly finds messages when blocks have
|
|
// been reorganized (some blocks contain only deleted sequences).
|
|
[Fact]
|
|
public void SelectMsgBlock_BinarySearch_CorrectBlockSelected()
|
|
{
|
|
var dir = UniqueDir();
|
|
using var store = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 64 });
|
|
|
|
var msg = "hello"u8.ToArray();
|
|
|
|
// Store 66 messages across 33+ blocks (2 per block).
|
|
for (var i = 0; i < 66; i++)
|
|
store.StoreMsg("foo", null, msg, 0);
|
|
|
|
// Remove seqs 2 and 5 to create deleted-sequence blocks.
|
|
store.RemoveMsg(2).ShouldBeTrue();
|
|
store.RemoveMsg(5).ShouldBeTrue();
|
|
|
|
// All remaining messages should be loadable.
|
|
for (ulong seq = 1; seq <= 66; seq++)
|
|
{
|
|
if (seq == 2 || seq == 5)
|
|
{
|
|
Should.Throw<KeyNotFoundException>(() => store.LoadMsg(seq, null));
|
|
}
|
|
else
|
|
{
|
|
var sm = store.LoadMsg(seq, null);
|
|
sm.ShouldNotBeNull();
|
|
sm.Subject.ShouldBe("foo");
|
|
}
|
|
}
|
|
|
|
// After restart, same behavior.
|
|
using var store2 = CreateStore(dir, new FileStoreOptions { BlockSizeBytes = 64 });
|
|
var sm1 = store2.LoadMsg(1, null);
|
|
sm1.Subject.ShouldBe("foo");
|
|
var sm66 = store2.LoadMsg(66, null);
|
|
sm66.Subject.ShouldBe("foo");
|
|
Should.Throw<KeyNotFoundException>(() => store2.LoadMsg(2, null));
|
|
Should.Throw<KeyNotFoundException>(() => store2.LoadMsg(5, null));
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Large full state store and reopen
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreLargeFullState (see also TestFileStoreRecoverFullState)
|
|
// Storing a moderate number of messages and reopening should preserve all state.
|
|
[Fact]
|
|
public void LargeFullState_StoreReopen_StatePreserved()
|
|
{
|
|
var dir = UniqueDir();
|
|
|
|
{
|
|
using var store = CreateStore(dir);
|
|
for (var i = 1; i <= 500; i++)
|
|
store.StoreMsg($"foo.{i % 10}", null, System.Text.Encoding.UTF8.GetBytes($"msg-{i}"), 0);
|
|
}
|
|
|
|
using var store2 = CreateStore(dir);
|
|
var state = store2.State();
|
|
state.Msgs.ShouldBe(500UL);
|
|
state.FirstSeq.ShouldBe(1UL);
|
|
state.LastSeq.ShouldBe(500UL);
|
|
|
|
// Spot check a few messages.
|
|
store2.LoadMsg(1, null).Subject.ShouldBe("foo.1");
|
|
store2.LoadMsg(250, null).Subject.ShouldBe("foo.0");
|
|
store2.LoadMsg(500, null).Subject.ShouldBe("foo.0");
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Partial indexes (marked as skip in Go too)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStorePartialIndexes (filestore_test.go:1706) — t.SkipNow() in Go
|
|
// This test was explicitly skipped upstream because positional write caches
|
|
// no longer exist in the same form. We mirror that skip.
|
|
[Fact(Skip = "Skipped in Go upstream: positional write caches no longer applicable (filestore_test.go:1708)")]
|
|
public void PartialIndexes_StoreAndLoadAfterCacheExpiry()
|
|
{
|
|
// Intentionally skipped.
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// WriteFullState high subject cardinality (marked as skip in Go)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreWriteFullStateHighSubjectCardinality (filestore_test.go:6842) — t.Skip()
|
|
[Fact(Skip = "Skipped in Go upstream: performance-only test (filestore_test.go:6843)")]
|
|
public void WriteFullState_HighSubjectCardinality()
|
|
{
|
|
// Intentionally skipped — performance test only.
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// WriteFullState read-only dir (permission error)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreWriteFullStateThrowsPermissionErrorIfFSModeReadOnly (filestore_test.go:9121)
|
|
// Marks directory as read-only, then verifies that writeFullState returns
|
|
// a permission error. This test is Linux-only: on macOS, chmod and File.SetAttributes
|
|
// behave differently (macOS doesn't enforce directory write restrictions the same way).
|
|
// Go also skips this on Buildkite: skipIfBuildkite(t).
|
|
[Fact]
|
|
public void WriteFullState_ReadOnlyDir_ThrowsPermissionError()
|
|
{
|
|
// macOS does not reliably enforce read-only directory restrictions for file creation
|
|
// the same way Linux does. This mirrors Go's skipIfBuildkite behaviour.
|
|
if (!OperatingSystem.IsLinux())
|
|
return;
|
|
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
var o = new FileStoreOptions { Directory = dirPath };
|
|
using var store = new FileStore(o);
|
|
|
|
var msg = new byte[1024];
|
|
Array.Fill(msg, (byte)'Z');
|
|
for (var i = 0; i < 100; i++)
|
|
store.StoreMsg("ev.1", null, msg, 0);
|
|
|
|
// Make directory and all contents read-only.
|
|
try
|
|
{
|
|
foreach (var f in Directory.GetFiles(dirPath))
|
|
File.SetAttributes(f, FileAttributes.ReadOnly);
|
|
}
|
|
catch
|
|
{
|
|
return; // Cannot set permissions in this environment — skip.
|
|
}
|
|
|
|
try
|
|
{
|
|
// Attempting to store more messages should eventually fail with a permission error
|
|
// (when the block needs to roll into a new file).
|
|
Exception? caught = null;
|
|
for (var i = 0; i < 10_000; i++)
|
|
{
|
|
try { store.StoreMsg("ev.1", null, msg, 0); }
|
|
catch (Exception ex)
|
|
{
|
|
caught = ex;
|
|
break;
|
|
}
|
|
}
|
|
caught.ShouldNotBeNull("Expected a permission error when writing to read-only directory");
|
|
}
|
|
finally
|
|
{
|
|
// Restore write permissions so cleanup can succeed.
|
|
try
|
|
{
|
|
foreach (var f in Directory.GetFiles(dirPath))
|
|
File.SetAttributes(f, FileAttributes.Normal);
|
|
}
|
|
catch { /* best-effort */ }
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// StoreMsg read-only dir (permission error)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreStoreRawMessageThrowsPermissionErrorIfFSModeReadOnly (filestore_test.go:9094)
|
|
// Similar to the above: after making the directory read-only, StoreMsg should
|
|
// eventually fail with a permission-related error.
|
|
[Fact]
|
|
[System.Runtime.Versioning.SupportedOSPlatform("linux")]
|
|
public void StoreMsg_ReadOnlyDir_ThrowsPermissionError()
|
|
{
|
|
if (OperatingSystem.IsWindows())
|
|
return; // Not applicable on Windows.
|
|
|
|
var dir = UniqueDir();
|
|
var dirPath = Path.Combine(_root, dir);
|
|
Directory.CreateDirectory(dirPath);
|
|
|
|
var o = new FileStoreOptions { Directory = dirPath, BlockSizeBytes = 1024 };
|
|
using var store = new FileStore(o);
|
|
|
|
// Mark the directory as read-only.
|
|
try
|
|
{
|
|
File.SetAttributes(dirPath, FileAttributes.ReadOnly);
|
|
}
|
|
catch
|
|
{
|
|
return; // Cannot set permissions in this environment — skip.
|
|
}
|
|
|
|
try
|
|
{
|
|
var msg = new byte[1024];
|
|
Array.Fill(msg, (byte)'Z');
|
|
Exception? caught = null;
|
|
for (var i = 0; i < 10_000; i++)
|
|
{
|
|
try { store.StoreMsg("ev.1", null, msg, 0); }
|
|
catch (Exception ex)
|
|
{
|
|
caught = ex;
|
|
break;
|
|
}
|
|
}
|
|
caught.ShouldNotBeNull("Expected a permission error when writing to read-only directory");
|
|
}
|
|
finally
|
|
{
|
|
try { File.SetAttributes(dirPath, FileAttributes.Normal); }
|
|
catch { /* best-effort */ }
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Write failures (environment-specific)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreWriteFailures (filestore_test.go:2173)
|
|
// The Go test requires a Docker environment with a limited tmpfs. We skip this
|
|
// here since it is not reproducible in a standard test environment.
|
|
[Fact(Skip = "Requires Docker tmpfs environment with limited disk (filestore_test.go:2173-2178)")]
|
|
public void WriteFailures_FullDisk_SkipEnvironmentSpecific()
|
|
{
|
|
// Intentionally skipped.
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Corruption sets hbit without headers (skipped — deep internal Go test)
|
|
// -------------------------------------------------------------------------
|
|
|
|
// Go: TestFileStoreCorruptionSetsHbitWithoutHeaders (filestore_test.go:13255)
|
|
// Tests writeMsgRecordLocked / msgFromBufNoCopy / indexCacheBuf / rebuildState
|
|
// with a specially crafted hbit record. These are deep internal Go mechanics
|
|
// (hbit flag, cbit, slotInfo) without a .NET equivalent in the public API.
|
|
[Fact(Skip = "Deep internal Go wire format test (hbit/cbit/slotInfo/fetchMsg) without .NET equivalent (filestore_test.go:13255)")]
|
|
public void Corruption_HbitSanityCheck()
|
|
{
|
|
// Intentionally skipped — tests deeply internal Go msgBlock internals.
|
|
}
|
|
}
|