using System.Security.Cryptography; using System.Text; using System.Text.Json; using Shouldly; using ZB.MOM.NatsNet.Server; namespace ZB.MOM.NatsNet.Server.Tests.ImplBacklog; public sealed partial class JetStreamFileStoreTests { [Fact] // T:351 public void FileStoreBasics_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("foo", null, "m1"u8.ToArray(), 0).Seq.ShouldBe(1UL); fs.StoreMsg("bar", null, "m2"u8.ToArray(), 0).Seq.ShouldBe(2UL); var sm = fs.LoadMsg(2, null); sm.ShouldNotBeNull(); sm!.Subject.ShouldBe("bar"); fs.State().Msgs.ShouldBe(2UL); }); } [Fact] // T:352 public void FileStoreMsgHeaders_ShouldSucceed() { WithStore((fs, _) => { var hdr = new byte[] { 1, 2, 3, 4 }; fs.StoreMsg("hdr", hdr, "body"u8.ToArray(), 0); var sm = fs.LoadMsg(1, null); sm.ShouldNotBeNull(); sm!.Hdr.ShouldNotBeNull(); sm.Hdr.ShouldBe(hdr); }); } [Fact] // T:353 public void FileStoreBasicWriteMsgsAndRestore_ShouldSucceed() { var root = NewRoot(); Directory.CreateDirectory(root); try { var cfg = DefaultStreamConfig(); var fs1 = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, cfg); fs1.StoreMsg("foo", null, "one"u8.ToArray(), 0); fs1.Stop(); var fs2 = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, cfg); fs2.StoreMsg("bar", null, "two"u8.ToArray(), 0).Seq.ShouldBe(1UL); File.Exists(Path.Combine(root, FileStoreDefaults.JetStreamMetaFile)).ShouldBeTrue(); File.Exists(Path.Combine(root, FileStoreDefaults.JetStreamMetaFileSum)).ShouldBeTrue(); fs2.Stop(); } finally { Directory.Delete(root, recursive: true); } } [Fact] // T:355 public void FileStoreSkipMsg_ShouldSucceed() { WithStore((fs, _) => { var (seq, err) = fs.SkipMsg(0); err.ShouldBeNull(); seq.ShouldBeGreaterThan(0UL); fs.StoreMsg("foo", null, "payload"u8.ToArray(), 0).Seq.ShouldBe(seq + 1); }); } [Fact] // T:357 public void FileStoreMsgLimit_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("s", null, "1"u8.ToArray(), 0); fs.StoreMsg("s", null, "2"u8.ToArray(), 0); fs.StoreMsg("s", null, "3"u8.ToArray(), 0); fs.State().Msgs.ShouldBeLessThanOrEqualTo(2UL); }, cfg: DefaultStreamConfig(maxMsgs: 2)); } [Fact] // T:358 public void FileStoreMsgLimitBug_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("s", null, "1"u8.ToArray(), 0); fs.StoreMsg("s", null, "2"u8.ToArray(), 0); var state = fs.State(); state.Msgs.ShouldBeLessThanOrEqualTo(1UL); state.FirstSeq.ShouldBeGreaterThan(0UL); }, cfg: DefaultStreamConfig(maxMsgs: 1)); } [Fact] // T:359 public void FileStoreBytesLimit_ShouldSucceed() { var subj = "foo"; var msg = new byte[64]; var storedMsgSize = JetStreamMemStore.MemStoreMsgSize(subj, null, msg); const ulong toStore = 8; var maxBytes = (long)(storedMsgSize * toStore); WithStore((fs, _) => { for (ulong i = 0; i < toStore; i++) { fs.StoreMsg(subj, null, msg, 0).Seq.ShouldBe(i + 1); } var state = fs.State(); state.Msgs.ShouldBe(toStore); state.Bytes.ShouldBe(storedMsgSize * toStore); for (var i = 0; i < 3; i++) { fs.StoreMsg(subj, null, msg, 0).Seq.ShouldBeGreaterThan(0UL); } state = fs.State(); state.Msgs.ShouldBe(toStore); state.Bytes.ShouldBe(storedMsgSize * toStore); state.FirstSeq.ShouldBe(4UL); state.LastSeq.ShouldBe(toStore + 3); }, cfg: DefaultStreamConfig(maxBytes: maxBytes)); } [Fact] // T:360 public void FileStoreBytesLimitWithDiscardNew_ShouldSucceed() { var subj = "tiny"; var msg = new byte[7]; var storedMsgSize = JetStreamMemStore.MemStoreMsgSize(subj, null, msg); const ulong toStore = 2; var maxBytes = (long)(storedMsgSize * toStore); WithStore((fs, _) => { for (var i = 0; i < 10; i++) { var (seq, _) = fs.StoreMsg(subj, null, msg, 0); if (i < (int)toStore) seq.ShouldBeGreaterThan(0UL); else seq.ShouldBe(0UL); } var state = fs.State(); state.Msgs.ShouldBe(toStore); state.Bytes.ShouldBe(storedMsgSize * toStore); }, cfg: DefaultStreamConfig(maxBytes: maxBytes, discard: DiscardPolicy.DiscardNew)); } [Fact] // T:361 public void FileStoreAgeLimit_ShouldSucceed() { WithStore((fs, _) => { var (_, ts) = fs.StoreMsg("ttl", null, "v"u8.ToArray(), 0); ts.ShouldBeGreaterThan(0L); fs.State().Msgs.ShouldBe(1UL); }, cfg: DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(20))); } [Fact] // T:362 public void FileStoreTimeStamps_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("ts", null, "one"u8.ToArray(), 0); var cutoff = DateTime.UtcNow; Thread.Sleep(2); fs.StoreMsg("ts", null, "two"u8.ToArray(), 0); fs.GetSeqFromTime(cutoff).ShouldBeGreaterThanOrEqualTo(2UL); }); } [Fact] // T:369 public void FileStoreRemovePartialRecovery_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("s", null, "1"u8.ToArray(), 0); fs.StoreMsg("s", null, "2"u8.ToArray(), 0); fs.StoreMsg("s", null, "3"u8.ToArray(), 0); fs.RemoveMsg(2).Removed.ShouldBeTrue(); fs.State().Msgs.ShouldBe(2UL); }); } [Fact] // T:370 public void FileStoreRemoveOutOfOrderRecovery_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("b", null, "2"u8.ToArray(), 0); fs.StoreMsg("c", null, "3"u8.ToArray(), 0); fs.RemoveMsg(2).Removed.ShouldBeTrue(); fs.RemoveMsg(1).Removed.ShouldBeTrue(); fs.LoadMsg(3, null)!.Subject.ShouldBe("c"); }); } [Fact] // T:371 public void FileStoreAgeLimitRecovery_ShouldSucceed() { WithStore((fs, root) => { fs.StoreMsg("age", null, "one"u8.ToArray(), 0); fs.Stop(); var recovered = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(20))); recovered.StoreMsg("age", null, "two"u8.ToArray(), 0).Seq.ShouldBe(1UL); recovered.Stop(); }, cfg: DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(20))); } [Fact] // T:374 public void FileStoreEraseAndNoIndexRecovery_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("erase", null, "x"u8.ToArray(), 0); fs.EraseMsg(1).Removed.ShouldBeTrue(); Should.Throw(() => fs.LoadMsg(1, null)); }); } [Fact] // T:375 public void FileStoreMeta_ShouldSucceed() { WithStore((_, root) => { var meta = Path.Combine(root, FileStoreDefaults.JetStreamMetaFile); var sum = Path.Combine(root, FileStoreDefaults.JetStreamMetaFileSum); File.Exists(meta).ShouldBeTrue(); File.Exists(sum).ShouldBeTrue(); new FileInfo(meta).Length.ShouldBeGreaterThan(0); new FileInfo(sum).Length.ShouldBeGreaterThan(0); }); } [Fact] // T:376 public void FileStoreWriteAndReadSameBlock_ShouldSucceed() { WithStore((fs, root) => { var blk = CreateBlock(root, 1, Encoding.ASCII.GetBytes("abcdefgh")); var mb = fs.RecoverMsgBlock(1); mb.Index.ShouldBe(1u); mb.RBytes.ShouldBe((ulong)new FileInfo(blk).Length); }); } [Fact] // T:377 public void FileStoreAndRetrieveMultiBlock_ShouldSucceed() { WithStore((fs, root) => { CreateBlock(root, 1, Encoding.ASCII.GetBytes("12345678")); CreateBlock(root, 2, Encoding.ASCII.GetBytes("ABCDEFGH")); fs.RecoverMsgBlock(1).Index.ShouldBe(1u); fs.RecoverMsgBlock(2).Index.ShouldBe(2u); }); } [Fact] // T:380 public void FileStorePartialCacheExpiration_ShouldSucceed() { var buf = JetStreamFileStore.GetMsgBlockBuf(512); buf.Length.ShouldBeGreaterThanOrEqualTo((int)FileStoreDefaults.DefaultTinyBlockSize); JetStreamFileStore.RecycleMsgBlockBuf(buf); } [Fact] // T:381 public void FileStorePartialIndexes_ShouldSucceed() { WithStore((fs, root) => { var blk = Encoding.ASCII.GetBytes("abcdefgh"); CreateBlock(root, 1, blk); WriteIndex(root, 1, blk[^8..], matchingChecksum: true); var mb = fs.RecoverMsgBlock(1); mb.Msgs.ShouldBe(1UL); mb.First.Seq.ShouldBe(1UL); }); } [Fact] // T:388 public void FileStoreWriteFailures_ShouldSucceed() { WithStore((fs, _) => { var mb = fs.InitMsgBlock(7); mb.MockWriteErr = true; mb.MockWriteErr.ShouldBeTrue(); }); } [Fact] // T:397 public void FileStoreStreamStateDeleted_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("s", null, "1"u8.ToArray(), 0); fs.Purge().Purged.ShouldBe(1UL); fs.State().Msgs.ShouldBe(0UL); }); } [Fact] // T:398 public void FileStoreStreamDeleteDirNotEmpty_ShouldSucceed() { WithStore((fs, root) => { var extra = Path.Combine(root, "extra.txt"); File.WriteAllText(extra, "leftover"); fs.Delete(false); File.Exists(extra).ShouldBeTrue(); }); } [Fact] // T:400 public void FileStoreStreamDeleteCacheBug_ShouldSucceed() { WithStore((fs, _) => { var mb = fs.InitMsgBlock(1); mb.CacheData = new Cache { Buf = JetStreamFileStore.GetMsgBlockBuf(16) }; mb.TryForceExpireCacheLocked(); mb.HaveCache.ShouldBeFalse(); }); } [Fact] // T:401 public void FileStoreStreamFailToRollBug_ShouldSucceed() { WithStore((fs, _) => { var mb1 = fs.InitMsgBlock(1); var mb2 = fs.InitMsgBlock(2); mb1.Mfn.ShouldNotBe(mb2.Mfn); mb1.Index.ShouldBeLessThan(mb2.Index); }); } [Fact] // T:421 public void FileStoreEncrypted_ShouldSucceed() { var root = NewRoot(); Directory.CreateDirectory(root); try { var fs = JetStreamFileStore.NewFileStoreWithCreated( new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes }, DefaultStreamConfig(), DateTime.UtcNow, DeterministicKeyGen, null); var keyFile = Path.Combine(root, FileStoreDefaults.JetStreamMetaFileKey); var metaFile = Path.Combine(root, FileStoreDefaults.JetStreamMetaFile); File.Exists(keyFile).ShouldBeTrue(); new FileInfo(keyFile).Length.ShouldBeGreaterThan(0); File.ReadAllBytes(metaFile)[0].ShouldNotBe((byte)'{'); fs.Stop(); } finally { Directory.Delete(root, recursive: true); } } [Fact] // T:422 public void FileStoreNoFSSWhenNoSubjects_ShouldSucceed() { WithStore((fs, _) => fs.NoTrackSubjects().ShouldBeTrue(), cfg: DefaultStreamConfig(subjects: [])); } [Fact] // T:423 public void FileStoreNoFSSBugAfterRemoveFirst_ShouldSucceed() { WithStore((fs, _) => { fs.NoTrackSubjects().ShouldBeFalse(); fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("a", null, "2"u8.ToArray(), 0); fs.RemoveMsg(1).Removed.ShouldBeTrue(); fs.State().FirstSeq.ShouldBe(2UL); }); } [Fact] // T:424 public void FileStoreNoFSSAfterRecover_ShouldSucceed() { WithStore((fs, root) => { CreateBlock(root, 1, Encoding.ASCII.GetBytes("abcdefgh")); var mb = fs.RecoverMsgBlock(1); mb.Fss.ShouldBeNull(); }, cfg: DefaultStreamConfig(subjects: ["foo"])); } [Fact] // T:425 public void FileStoreFSSCloseAndKeepOnExpireOnRecoverBug_ShouldSucceed() { WithStore((fs, _) => { var mb = fs.InitMsgBlock(1); mb.CacheData = new Cache { Buf = JetStreamFileStore.GetMsgBlockBuf(32) }; mb.Fss = new ZB.MOM.NatsNet.Server.Internal.DataStructures.SubjectTree(); mb.TryForceExpireCacheLocked(); mb.Fss.ShouldBeNull(); }); } [Fact] // T:426 public void FileStoreExpireOnRecoverSubjectAccounting_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("b", null, "2"u8.ToArray(), 0); fs.SubjectsTotals(">").Count.ShouldBe(2); }); } [Fact] // T:427 public void FileStoreFSSExpireNumPendingBug_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("b", null, "2"u8.ToArray(), 0); var (total, validThrough, err) = fs.NumPending(1, ">", false); err.ShouldBeNull(); total.ShouldBeGreaterThanOrEqualTo(2UL); validThrough.ShouldBeGreaterThanOrEqualTo(2UL); }); } [Fact] // T:429 public void FileStoreOutOfSpaceRebuildState_ShouldSucceed() { WithStore((fs, root) => { var blk = Encoding.ASCII.GetBytes("abcdefgh"); CreateBlock(root, 1, blk); WriteIndex(root, 1, new byte[8], matchingChecksum: false); var mb = fs.RecoverMsgBlock(1); mb.Lchk.Length.ShouldBe(8); }); } [Fact] // T:430 public void FileStoreRebuildStateProperlyWithMaxMsgsPerSubject_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("a", null, "2"u8.ToArray(), 0); fs.SubjectsTotals("a")["a"].ShouldBeLessThanOrEqualTo(1UL); }, cfg: DefaultStreamConfig(maxMsgsPer: 1)); } [Fact] // T:431 public void FileStoreUpdateMaxMsgsPerSubject_ShouldSucceed() { WithStore((fs, _) => { fs.UpdateConfig(DefaultStreamConfig(maxMsgsPer: 1)); fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("a", null, "2"u8.ToArray(), 0); fs.SubjectsTotals("a")["a"].ShouldBeLessThanOrEqualTo(1UL); }); } [Fact] // T:432 public void FileStoreBadFirstAndFailedExpireAfterRestart_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0).Seq.ShouldBe(100UL); }, cfg: DefaultStreamConfig(firstSeq: 100, maxAge: TimeSpan.FromMilliseconds(10))); } [Fact] // T:433 public void FileStoreCompactAllWithDanglingLMB_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.Compact(10).Error.ShouldBeNull(); }); } [Fact] // T:434 public void FileStoreStateWithBlkFirstDeleted_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "1"u8.ToArray(), 0); fs.StoreMsg("a", null, "2"u8.ToArray(), 0); fs.RemoveMsg(1).Removed.ShouldBeTrue(); fs.State().FirstSeq.ShouldBe(2UL); }); } [Fact] // T:439 public void FileStoreSubjectsTotals_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("foo", null, "1"u8.ToArray(), 0); fs.StoreMsg("foo", null, "2"u8.ToArray(), 0); fs.StoreMsg("bar", null, "3"u8.ToArray(), 0); var totals = fs.SubjectsTotals(">"); totals["foo"].ShouldBe(2UL); totals["bar"].ShouldBe(1UL); }); } [Fact] // T:443 public void FileStoreRestoreEncryptedWithNoKeyFuncFails_ShouldSucceed() { var root = NewRoot(); Directory.CreateDirectory(root); try { var cfg = DefaultStreamConfig(); var encrypted = JetStreamFileStore.NewFileStoreWithCreated( new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes }, cfg, DateTime.UtcNow, DeterministicKeyGen, null); encrypted.Stop(); Should.Throw(() => JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes }, cfg)); } finally { Directory.Delete(root, recursive: true); } } [Fact] // T:444 public void FileStoreInitialFirstSeq_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("a", null, "payload"u8.ToArray(), 0).Seq.ShouldBe(42UL); }, cfg: DefaultStreamConfig(firstSeq: 42)); } [Fact] // T:532 public void FileStoreRecoverOnlyBlkFiles_ShouldSucceed() { WithStore((fs, root) => { CreateBlock(root, 1, Encoding.ASCII.GetBytes("abcdefgh")); var mb = fs.RecoverMsgBlock(1); mb.Index.ShouldBe(1u); mb.Msgs.ShouldBe(0UL); }); } [Fact] // T:575 public void JetStreamFileStoreSubjectsRemovedAfterSecureErase_ShouldSucceed() { WithStore((fs, _) => { fs.StoreMsg("test.1", null, "msg1"u8.ToArray(), 0).Seq.ShouldBe(1UL); fs.StoreMsg("test.2", null, "msg2"u8.ToArray(), 0).Seq.ShouldBe(2UL); fs.StoreMsg("test.3", null, "msg3"u8.ToArray(), 0).Seq.ShouldBe(3UL); var before = fs.SubjectsTotals(">"); before.Count.ShouldBe(3); before.ShouldContainKey("test.1"); before.ShouldContainKey("test.2"); before.ShouldContainKey("test.3"); var (removed, err) = fs.EraseMsg(1); removed.ShouldBeTrue(); err.ShouldBeNull(); var after = fs.SubjectsTotals(">"); after.Count.ShouldBe(2); after.ContainsKey("test.1").ShouldBeFalse(); after["test.2"].ShouldBe(1UL); after["test.3"].ShouldBe(1UL); }); } private static void WithStore( Action action, StreamConfig? cfg = null, FileStoreConfig? fcfg = null, KeyGen? prf = null, KeyGen? oldPrf = null) { var root = NewRoot(); Directory.CreateDirectory(root); JetStreamFileStore? fs = null; try { var streamCfg = cfg ?? DefaultStreamConfig(); var storeCfg = fcfg ?? new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes }; storeCfg.StoreDir = root; fs = prf == null && oldPrf == null ? JetStreamFileStore.NewFileStore(storeCfg, streamCfg) : JetStreamFileStore.NewFileStoreWithCreated(storeCfg, streamCfg, DateTime.UtcNow, prf, oldPrf); action(fs, root); } finally { fs?.Stop(); if (Directory.Exists(root)) Directory.Delete(root, recursive: true); } } private static StreamConfig DefaultStreamConfig( long maxMsgs = -1, long maxBytes = -1, TimeSpan? maxAge = null, long maxMsgsPer = -1, ulong firstSeq = 0, DiscardPolicy discard = DiscardPolicy.DiscardOld, string[]? subjects = null) { return new StreamConfig { Name = "TEST", Storage = StorageType.FileStorage, Subjects = subjects ?? ["test.>"], MaxMsgs = maxMsgs, MaxBytes = maxBytes, MaxAge = maxAge ?? TimeSpan.Zero, MaxMsgsPer = maxMsgsPer, FirstSeq = firstSeq, Discard = discard, Retention = RetentionPolicy.LimitsPolicy, }; } private static string CreateBlock(string root, uint index, byte[] payload) { var mdir = Path.Combine(root, FileStoreDefaults.MsgDir); Directory.CreateDirectory(mdir); var blockPath = Path.Combine(mdir, string.Format(FileStoreDefaults.BlkScan, index)); File.WriteAllBytes(blockPath, payload); return blockPath; } private static void WriteIndex(string root, uint index, byte[] checksum, bool matchingChecksum) { var mdir = Path.Combine(root, FileStoreDefaults.MsgDir); Directory.CreateDirectory(mdir); var lastChecksum = matchingChecksum ? checksum : new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; var info = new { Msgs = 1UL, Bytes = 8UL, RawBytes = 8UL, FirstSeq = 1UL, FirstTs = 1L, LastSeq = 1UL, LastTs = 1L, LastChecksum = lastChecksum, NoTrack = false, }; var indexPath = Path.Combine(mdir, string.Format(FileStoreDefaults.IndexScan, index)); File.WriteAllText(indexPath, JsonSerializer.Serialize(info)); } private static byte[] DeterministicKeyGen(byte[] context) { using var sha = SHA256.Create(); return sha.ComputeHash(context); } private static string NewRoot() => Path.Combine(Path.GetTempPath(), $"impl-fs-{Guid.NewGuid():N}"); }