821 lines
25 KiB
C#
821 lines
25 KiB
C#
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using Shouldly;
|
|
using ZB.MOM.NatsNet.Server;
|
|
|
|
namespace ZB.MOM.NatsNet.Server.Tests.ImplBacklog;
|
|
|
|
public sealed partial class JetStreamFileStoreTests
|
|
{
|
|
[Fact] // T:351
|
|
public void FileStoreBasics_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("foo", null, "m1"u8.ToArray(), 0).Seq.ShouldBe(1UL);
|
|
fs.StoreMsg("bar", null, "m2"u8.ToArray(), 0).Seq.ShouldBe(2UL);
|
|
|
|
var sm = fs.LoadMsg(2, null);
|
|
sm.ShouldNotBeNull();
|
|
sm!.Subject.ShouldBe("bar");
|
|
fs.State().Msgs.ShouldBe(2UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:352
|
|
public void FileStoreMsgHeaders_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var hdr = new byte[] { 1, 2, 3, 4 };
|
|
fs.StoreMsg("hdr", hdr, "body"u8.ToArray(), 0);
|
|
|
|
var sm = fs.LoadMsg(1, null);
|
|
sm.ShouldNotBeNull();
|
|
sm!.Hdr.ShouldNotBeNull();
|
|
sm.Hdr.ShouldBe(hdr);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:353
|
|
public void FileStoreBasicWriteMsgsAndRestore_ShouldSucceed()
|
|
{
|
|
var root = NewRoot();
|
|
Directory.CreateDirectory(root);
|
|
|
|
try
|
|
{
|
|
var cfg = DefaultStreamConfig();
|
|
var fs1 = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, cfg);
|
|
fs1.StoreMsg("foo", null, "one"u8.ToArray(), 0);
|
|
fs1.Stop();
|
|
|
|
var fs2 = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, cfg);
|
|
fs2.StoreMsg("bar", null, "two"u8.ToArray(), 0).Seq.ShouldBe(1UL);
|
|
File.Exists(Path.Combine(root, FileStoreDefaults.JetStreamMetaFile)).ShouldBeTrue();
|
|
File.Exists(Path.Combine(root, FileStoreDefaults.JetStreamMetaFileSum)).ShouldBeTrue();
|
|
fs2.Stop();
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact] // T:355
|
|
public void FileStoreSkipMsg_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var (seq, err) = fs.SkipMsg(0);
|
|
err.ShouldBeNull();
|
|
seq.ShouldBeGreaterThan(0UL);
|
|
fs.StoreMsg("foo", null, "payload"u8.ToArray(), 0).Seq.ShouldBe(seq + 1);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:357
|
|
public void FileStoreMsgLimit_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("s", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("s", null, "2"u8.ToArray(), 0);
|
|
fs.StoreMsg("s", null, "3"u8.ToArray(), 0);
|
|
|
|
fs.State().Msgs.ShouldBeLessThanOrEqualTo(2UL);
|
|
}, cfg: DefaultStreamConfig(maxMsgs: 2));
|
|
}
|
|
|
|
[Fact] // T:358
|
|
public void FileStoreMsgLimitBug_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("s", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("s", null, "2"u8.ToArray(), 0);
|
|
|
|
var state = fs.State();
|
|
state.Msgs.ShouldBeLessThanOrEqualTo(1UL);
|
|
state.FirstSeq.ShouldBeGreaterThan(0UL);
|
|
}, cfg: DefaultStreamConfig(maxMsgs: 1));
|
|
}
|
|
|
|
[Fact] // T:359
|
|
public void FileStoreBytesLimit_ShouldSucceed()
|
|
{
|
|
var subj = "foo";
|
|
var msg = new byte[64];
|
|
var storedMsgSize = JetStreamMemStore.MemStoreMsgSize(subj, null, msg);
|
|
const ulong toStore = 8;
|
|
var maxBytes = (long)(storedMsgSize * toStore);
|
|
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (ulong i = 0; i < toStore; i++)
|
|
{
|
|
fs.StoreMsg(subj, null, msg, 0).Seq.ShouldBe(i + 1);
|
|
}
|
|
|
|
var state = fs.State();
|
|
state.Msgs.ShouldBe(toStore);
|
|
state.Bytes.ShouldBe(storedMsgSize * toStore);
|
|
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
fs.StoreMsg(subj, null, msg, 0).Seq.ShouldBeGreaterThan(0UL);
|
|
}
|
|
|
|
state = fs.State();
|
|
state.Msgs.ShouldBe(toStore);
|
|
state.Bytes.ShouldBe(storedMsgSize * toStore);
|
|
state.FirstSeq.ShouldBe(4UL);
|
|
state.LastSeq.ShouldBe(toStore + 3);
|
|
}, cfg: DefaultStreamConfig(maxBytes: maxBytes));
|
|
}
|
|
|
|
[Fact] // T:360
|
|
public void FileStoreBytesLimitWithDiscardNew_ShouldSucceed()
|
|
{
|
|
var subj = "tiny";
|
|
var msg = new byte[7];
|
|
var storedMsgSize = JetStreamMemStore.MemStoreMsgSize(subj, null, msg);
|
|
const ulong toStore = 2;
|
|
var maxBytes = (long)(storedMsgSize * toStore);
|
|
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (var i = 0; i < 10; i++)
|
|
{
|
|
var (seq, _) = fs.StoreMsg(subj, null, msg, 0);
|
|
if (i < (int)toStore)
|
|
seq.ShouldBeGreaterThan(0UL);
|
|
else
|
|
seq.ShouldBe(0UL);
|
|
}
|
|
|
|
var state = fs.State();
|
|
state.Msgs.ShouldBe(toStore);
|
|
state.Bytes.ShouldBe(storedMsgSize * toStore);
|
|
}, cfg: DefaultStreamConfig(maxBytes: maxBytes, discard: DiscardPolicy.DiscardNew));
|
|
}
|
|
|
|
[Fact] // T:361
|
|
public void FileStoreAgeLimit_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var (_, ts) = fs.StoreMsg("ttl", null, "v"u8.ToArray(), 0);
|
|
ts.ShouldBeGreaterThan(0L);
|
|
fs.State().Msgs.ShouldBe(1UL);
|
|
}, cfg: DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(20)));
|
|
}
|
|
|
|
[Fact] // T:362
|
|
public void FileStoreTimeStamps_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var (_, firstTs) = fs.StoreMsg("ts", null, "one"u8.ToArray(), 0);
|
|
var cutoff = DateTime.UnixEpoch.AddTicks((firstTs / 100) + 1);
|
|
fs.StoreMsg("ts", null, "two"u8.ToArray(), 0);
|
|
|
|
fs.GetSeqFromTime(cutoff).ShouldBeGreaterThanOrEqualTo(2UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:369
|
|
public void FileStoreRemovePartialRecovery_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("s", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("s", null, "2"u8.ToArray(), 0);
|
|
fs.StoreMsg("s", null, "3"u8.ToArray(), 0);
|
|
|
|
fs.RemoveMsg(2).Removed.ShouldBeTrue();
|
|
fs.State().Msgs.ShouldBe(2UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:370
|
|
public void FileStoreRemoveOutOfOrderRecovery_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("b", null, "2"u8.ToArray(), 0);
|
|
fs.StoreMsg("c", null, "3"u8.ToArray(), 0);
|
|
|
|
fs.RemoveMsg(2).Removed.ShouldBeTrue();
|
|
fs.RemoveMsg(1).Removed.ShouldBeTrue();
|
|
|
|
fs.LoadMsg(3, null)!.Subject.ShouldBe("c");
|
|
});
|
|
}
|
|
|
|
[Fact] // T:371
|
|
public void FileStoreAgeLimitRecovery_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
fs.StoreMsg("age", null, "one"u8.ToArray(), 0);
|
|
fs.Stop();
|
|
|
|
var recovered = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(20)));
|
|
recovered.StoreMsg("age", null, "two"u8.ToArray(), 0).Seq.ShouldBe(1UL);
|
|
recovered.Stop();
|
|
}, cfg: DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(20)));
|
|
}
|
|
|
|
[Fact] // T:374
|
|
public void FileStoreEraseAndNoIndexRecovery_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("erase", null, "x"u8.ToArray(), 0);
|
|
fs.EraseMsg(1).Removed.ShouldBeTrue();
|
|
Should.Throw<KeyNotFoundException>(() => fs.LoadMsg(1, null));
|
|
});
|
|
}
|
|
|
|
[Fact] // T:375
|
|
public void FileStoreMeta_ShouldSucceed()
|
|
{
|
|
WithStore((_, root) =>
|
|
{
|
|
var meta = Path.Combine(root, FileStoreDefaults.JetStreamMetaFile);
|
|
var sum = Path.Combine(root, FileStoreDefaults.JetStreamMetaFileSum);
|
|
File.Exists(meta).ShouldBeTrue();
|
|
File.Exists(sum).ShouldBeTrue();
|
|
new FileInfo(meta).Length.ShouldBeGreaterThan(0);
|
|
new FileInfo(sum).Length.ShouldBeGreaterThan(0);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:376
|
|
public void FileStoreWriteAndReadSameBlock_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
var blk = CreateBlock(root, 1, Encoding.ASCII.GetBytes("abcdefgh"));
|
|
var mb = fs.RecoverMsgBlock(1);
|
|
mb.Index.ShouldBe(1u);
|
|
mb.RBytes.ShouldBe((ulong)new FileInfo(blk).Length);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:377
|
|
public void FileStoreAndRetrieveMultiBlock_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
CreateBlock(root, 1, Encoding.ASCII.GetBytes("12345678"));
|
|
CreateBlock(root, 2, Encoding.ASCII.GetBytes("ABCDEFGH"));
|
|
|
|
fs.RecoverMsgBlock(1).Index.ShouldBe(1u);
|
|
fs.RecoverMsgBlock(2).Index.ShouldBe(2u);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:380
|
|
public void FileStorePartialCacheExpiration_ShouldSucceed()
|
|
{
|
|
var buf = JetStreamFileStore.GetMsgBlockBuf(512);
|
|
buf.Length.ShouldBeGreaterThanOrEqualTo((int)FileStoreDefaults.DefaultTinyBlockSize);
|
|
JetStreamFileStore.RecycleMsgBlockBuf(buf);
|
|
}
|
|
|
|
[Fact] // T:381
|
|
public void FileStorePartialIndexes_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
var blk = Encoding.ASCII.GetBytes("abcdefgh");
|
|
CreateBlock(root, 1, blk);
|
|
WriteIndex(root, 1, blk[^8..], matchingChecksum: true);
|
|
|
|
var mb = fs.RecoverMsgBlock(1);
|
|
mb.Msgs.ShouldBe(1UL);
|
|
mb.First.Seq.ShouldBe(1UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:388
|
|
public void FileStoreWriteFailures_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var mb = fs.InitMsgBlock(7);
|
|
mb.MockWriteErr = true;
|
|
mb.MockWriteErr.ShouldBeTrue();
|
|
});
|
|
}
|
|
|
|
[Fact] // T:397
|
|
public void FileStoreStreamStateDeleted_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("s", null, "1"u8.ToArray(), 0);
|
|
fs.Purge().Purged.ShouldBe(1UL);
|
|
fs.State().Msgs.ShouldBe(0UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:398
|
|
public void FileStoreStreamDeleteDirNotEmpty_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
var extra = Path.Combine(root, "extra.txt");
|
|
File.WriteAllText(extra, "leftover");
|
|
fs.Delete(false);
|
|
File.Exists(extra).ShouldBeTrue();
|
|
});
|
|
}
|
|
|
|
[Fact] // T:400
|
|
public void FileStoreStreamDeleteCacheBug_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var mb = fs.InitMsgBlock(1);
|
|
mb.CacheData = new Cache { Buf = JetStreamFileStore.GetMsgBlockBuf(16) };
|
|
mb.TryForceExpireCacheLocked();
|
|
mb.HaveCache.ShouldBeFalse();
|
|
});
|
|
}
|
|
|
|
[Fact] // T:401
|
|
public void FileStoreStreamFailToRollBug_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var mb1 = fs.InitMsgBlock(1);
|
|
var mb2 = fs.InitMsgBlock(2);
|
|
mb1.Mfn.ShouldNotBe(mb2.Mfn);
|
|
mb1.Index.ShouldBeLessThan(mb2.Index);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:421
|
|
public void FileStoreEncrypted_ShouldSucceed()
|
|
{
|
|
var root = NewRoot();
|
|
Directory.CreateDirectory(root);
|
|
|
|
try
|
|
{
|
|
var fs = JetStreamFileStore.NewFileStoreWithCreated(
|
|
new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes },
|
|
DefaultStreamConfig(),
|
|
DateTime.UtcNow,
|
|
DeterministicKeyGen,
|
|
null);
|
|
|
|
var keyFile = Path.Combine(root, FileStoreDefaults.JetStreamMetaFileKey);
|
|
var metaFile = Path.Combine(root, FileStoreDefaults.JetStreamMetaFile);
|
|
File.Exists(keyFile).ShouldBeTrue();
|
|
new FileInfo(keyFile).Length.ShouldBeGreaterThan(0);
|
|
File.ReadAllBytes(metaFile)[0].ShouldNotBe((byte)'{');
|
|
fs.Stop();
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact] // T:422
|
|
public void FileStoreNoFSSWhenNoSubjects_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) => fs.NoTrackSubjects().ShouldBeTrue(), cfg: DefaultStreamConfig(subjects: []));
|
|
}
|
|
|
|
[Fact] // T:423
|
|
public void FileStoreNoFSSBugAfterRemoveFirst_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.NoTrackSubjects().ShouldBeFalse();
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("a", null, "2"u8.ToArray(), 0);
|
|
fs.RemoveMsg(1).Removed.ShouldBeTrue();
|
|
fs.State().FirstSeq.ShouldBe(2UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:424
|
|
public void FileStoreNoFSSAfterRecover_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
CreateBlock(root, 1, Encoding.ASCII.GetBytes("abcdefgh"));
|
|
var mb = fs.RecoverMsgBlock(1);
|
|
mb.Fss.ShouldBeNull();
|
|
}, cfg: DefaultStreamConfig(subjects: ["foo"]));
|
|
}
|
|
|
|
[Fact] // T:425
|
|
public void FileStoreFSSCloseAndKeepOnExpireOnRecoverBug_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
var mb = fs.InitMsgBlock(1);
|
|
mb.CacheData = new Cache { Buf = JetStreamFileStore.GetMsgBlockBuf(32) };
|
|
mb.Fss = new ZB.MOM.NatsNet.Server.Internal.DataStructures.SubjectTree<SimpleState>();
|
|
mb.TryForceExpireCacheLocked();
|
|
mb.Fss.ShouldBeNull();
|
|
});
|
|
}
|
|
|
|
[Fact] // T:426
|
|
public void FileStoreExpireOnRecoverSubjectAccounting_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("b", null, "2"u8.ToArray(), 0);
|
|
fs.SubjectsTotals(">").Count.ShouldBe(2);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:427
|
|
public void FileStoreFSSExpireNumPendingBug_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("b", null, "2"u8.ToArray(), 0);
|
|
var (total, validThrough, err) = fs.NumPending(1, ">", false);
|
|
err.ShouldBeNull();
|
|
total.ShouldBeGreaterThanOrEqualTo(2UL);
|
|
validThrough.ShouldBeGreaterThanOrEqualTo(2UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:429
|
|
public void FileStoreOutOfSpaceRebuildState_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
var blk = Encoding.ASCII.GetBytes("abcdefgh");
|
|
CreateBlock(root, 1, blk);
|
|
WriteIndex(root, 1, new byte[8], matchingChecksum: false);
|
|
|
|
var mb = fs.RecoverMsgBlock(1);
|
|
mb.Lchk.Length.ShouldBe(8);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:430
|
|
public void FileStoreRebuildStateProperlyWithMaxMsgsPerSubject_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("a", null, "2"u8.ToArray(), 0);
|
|
fs.SubjectsTotals("a")["a"].ShouldBeLessThanOrEqualTo(1UL);
|
|
}, cfg: DefaultStreamConfig(maxMsgsPer: 1));
|
|
}
|
|
|
|
[Fact] // T:431
|
|
public void FileStoreUpdateMaxMsgsPerSubject_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.UpdateConfig(DefaultStreamConfig(maxMsgsPer: 1));
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("a", null, "2"u8.ToArray(), 0);
|
|
fs.SubjectsTotals("a")["a"].ShouldBeLessThanOrEqualTo(1UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:432
|
|
public void FileStoreBadFirstAndFailedExpireAfterRestart_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0).Seq.ShouldBe(100UL);
|
|
}, cfg: DefaultStreamConfig(firstSeq: 100, maxAge: TimeSpan.FromMilliseconds(10)));
|
|
}
|
|
|
|
[Fact] // T:433
|
|
public void FileStoreCompactAllWithDanglingLMB_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.Compact(10).Error.ShouldBeNull();
|
|
});
|
|
}
|
|
|
|
[Fact] // T:434
|
|
public void FileStoreStateWithBlkFirstDeleted_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("a", null, "2"u8.ToArray(), 0);
|
|
fs.RemoveMsg(1).Removed.ShouldBeTrue();
|
|
fs.State().FirstSeq.ShouldBe(2UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:439
|
|
public void FileStoreSubjectsTotals_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("foo", null, "1"u8.ToArray(), 0);
|
|
fs.StoreMsg("foo", null, "2"u8.ToArray(), 0);
|
|
fs.StoreMsg("bar", null, "3"u8.ToArray(), 0);
|
|
|
|
var totals = fs.SubjectsTotals(">");
|
|
totals["foo"].ShouldBe(2UL);
|
|
totals["bar"].ShouldBe(1UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:443
|
|
public void FileStoreRestoreEncryptedWithNoKeyFuncFails_ShouldSucceed()
|
|
{
|
|
var root = NewRoot();
|
|
Directory.CreateDirectory(root);
|
|
|
|
try
|
|
{
|
|
var cfg = DefaultStreamConfig();
|
|
var encrypted = JetStreamFileStore.NewFileStoreWithCreated(
|
|
new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes },
|
|
cfg,
|
|
DateTime.UtcNow,
|
|
DeterministicKeyGen,
|
|
null);
|
|
encrypted.Stop();
|
|
|
|
Should.Throw<InvalidOperationException>(() =>
|
|
JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes }, cfg));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact] // T:444
|
|
public void FileStoreInitialFirstSeq_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("a", null, "payload"u8.ToArray(), 0).Seq.ShouldBe(42UL);
|
|
}, cfg: DefaultStreamConfig(firstSeq: 42));
|
|
}
|
|
|
|
[Fact] // T:532
|
|
public void FileStoreRecoverOnlyBlkFiles_ShouldSucceed()
|
|
{
|
|
WithStore((fs, root) =>
|
|
{
|
|
CreateBlock(root, 1, Encoding.ASCII.GetBytes("abcdefgh"));
|
|
var mb = fs.RecoverMsgBlock(1);
|
|
mb.Index.ShouldBe(1u);
|
|
mb.Msgs.ShouldBe(0UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:575
|
|
public void JetStreamFileStoreSubjectsRemovedAfterSecureErase_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("test.1", null, "msg1"u8.ToArray(), 0).Seq.ShouldBe(1UL);
|
|
fs.StoreMsg("test.2", null, "msg2"u8.ToArray(), 0).Seq.ShouldBe(2UL);
|
|
fs.StoreMsg("test.3", null, "msg3"u8.ToArray(), 0).Seq.ShouldBe(3UL);
|
|
|
|
var before = fs.SubjectsTotals(">");
|
|
before.Count.ShouldBe(3);
|
|
before.ShouldContainKey("test.1");
|
|
before.ShouldContainKey("test.2");
|
|
before.ShouldContainKey("test.3");
|
|
|
|
var (removed, err) = fs.EraseMsg(1);
|
|
removed.ShouldBeTrue();
|
|
err.ShouldBeNull();
|
|
|
|
var after = fs.SubjectsTotals(">");
|
|
after.Count.ShouldBe(2);
|
|
after.ContainsKey("test.1").ShouldBeFalse();
|
|
after["test.2"].ShouldBe(1UL);
|
|
after["test.3"].ShouldBe(1UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:356
|
|
public void FileStoreWriteExpireWrite_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("expire", null, "first"u8.ToArray(), 0).Seq.ShouldBe(1UL);
|
|
fs.StoreMsg("expire", null, "second"u8.ToArray(), 0).Seq.ShouldBeGreaterThan(0UL);
|
|
|
|
var state = fs.State();
|
|
state.Msgs.ShouldBeLessThanOrEqualTo(2UL);
|
|
state.LastSeq.ShouldBe(2UL);
|
|
}, cfg: DefaultStreamConfig(maxAge: TimeSpan.FromMilliseconds(10)));
|
|
}
|
|
|
|
[Fact] // T:379
|
|
public void FileStoreReadCache_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
fs.StoreMsg("cache", null, "payload"u8.ToArray(), 0).Seq.ShouldBe(1UL);
|
|
|
|
var first = fs.LoadMsg(1, null);
|
|
var second = fs.LoadMsg(1, null);
|
|
first.ShouldNotBeNull();
|
|
second.ShouldNotBeNull();
|
|
second!.Msg.ShouldBe(first!.Msg);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:389
|
|
public void FileStorePerf_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (var i = 0; i < 250; i++)
|
|
{
|
|
fs.StoreMsg("perf", null, "x"u8.ToArray(), 0).Seq.ShouldBeGreaterThan(0UL);
|
|
}
|
|
|
|
var state = fs.State();
|
|
state.Msgs.ShouldBe(250UL);
|
|
state.LastSeq.ShouldBe(250UL);
|
|
});
|
|
}
|
|
|
|
[Fact] // T:390
|
|
public void FileStoreReadBackMsgPerf_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
fs.StoreMsg("readback", null, "m"u8.ToArray(), 0);
|
|
|
|
for (ulong seq = 100; seq >= 90; seq--)
|
|
{
|
|
var msg = fs.LoadMsg(seq, null);
|
|
msg.ShouldNotBeNull();
|
|
msg!.Subject.ShouldBe("readback");
|
|
}
|
|
});
|
|
}
|
|
|
|
[Fact] // T:391
|
|
public void FileStoreStoreLimitRemovePerf_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (var i = 0; i < 120; i++)
|
|
fs.StoreMsg("limit", null, "x"u8.ToArray(), 0);
|
|
|
|
var state = fs.State();
|
|
state.Msgs.ShouldBeLessThanOrEqualTo(50UL);
|
|
state.FirstSeq.ShouldBeGreaterThan(1UL);
|
|
}, cfg: DefaultStreamConfig(maxMsgs: 50));
|
|
}
|
|
|
|
[Fact] // T:392
|
|
public void FileStorePubPerfWithSmallBlkSize_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (var i = 0; i < 40; i++)
|
|
{
|
|
fs.StoreMsg("blk", null, "payload"u8.ToArray(), 0).Seq.ShouldBeGreaterThan(0UL);
|
|
}
|
|
|
|
fs.State().Msgs.ShouldBe(40UL);
|
|
}, fcfg: new FileStoreConfig
|
|
{
|
|
BlockSize = FileStoreDefaults.DefaultTinyBlockSize,
|
|
Cipher = StoreCipher.Aes,
|
|
});
|
|
}
|
|
|
|
[Fact] // T:463
|
|
public void FileStoreCompactingBlocksOnSync_ShouldSucceed()
|
|
{
|
|
WithStore((fs, _) =>
|
|
{
|
|
for (var i = 0; i < 60; i++)
|
|
fs.StoreMsg("compact", null, "x"u8.ToArray(), 0);
|
|
|
|
for (ulong seq = 1; seq <= 30; seq++)
|
|
fs.RemoveMsg(seq).Removed.ShouldBeTrue();
|
|
|
|
fs.Compact(35).Error.ShouldBeNull();
|
|
fs.State().Msgs.ShouldBeInRange(1UL, 30UL);
|
|
});
|
|
}
|
|
|
|
private static void WithStore(
|
|
Action<JetStreamFileStore, string> action,
|
|
StreamConfig? cfg = null,
|
|
FileStoreConfig? fcfg = null,
|
|
KeyGen? prf = null,
|
|
KeyGen? oldPrf = null)
|
|
{
|
|
var root = NewRoot();
|
|
Directory.CreateDirectory(root);
|
|
JetStreamFileStore? fs = null;
|
|
|
|
try
|
|
{
|
|
var streamCfg = cfg ?? DefaultStreamConfig();
|
|
var storeCfg = fcfg ?? new FileStoreConfig { StoreDir = root, Cipher = StoreCipher.Aes };
|
|
storeCfg.StoreDir = root;
|
|
|
|
fs = prf == null && oldPrf == null
|
|
? JetStreamFileStore.NewFileStore(storeCfg, streamCfg)
|
|
: JetStreamFileStore.NewFileStoreWithCreated(storeCfg, streamCfg, DateTime.UtcNow, prf, oldPrf);
|
|
|
|
action(fs, root);
|
|
}
|
|
finally
|
|
{
|
|
fs?.Stop();
|
|
if (Directory.Exists(root))
|
|
Directory.Delete(root, recursive: true);
|
|
}
|
|
}
|
|
|
|
private static StreamConfig DefaultStreamConfig(
|
|
long maxMsgs = -1,
|
|
long maxBytes = -1,
|
|
TimeSpan? maxAge = null,
|
|
long maxMsgsPer = -1,
|
|
ulong firstSeq = 0,
|
|
DiscardPolicy discard = DiscardPolicy.DiscardOld,
|
|
string[]? subjects = null)
|
|
{
|
|
return new StreamConfig
|
|
{
|
|
Name = "TEST",
|
|
Storage = StorageType.FileStorage,
|
|
Subjects = subjects ?? ["test.>"],
|
|
MaxMsgs = maxMsgs,
|
|
MaxBytes = maxBytes,
|
|
MaxAge = maxAge ?? TimeSpan.Zero,
|
|
MaxMsgsPer = maxMsgsPer,
|
|
FirstSeq = firstSeq,
|
|
Discard = discard,
|
|
Retention = RetentionPolicy.LimitsPolicy,
|
|
};
|
|
}
|
|
|
|
private static string CreateBlock(string root, uint index, byte[] payload)
|
|
{
|
|
var mdir = Path.Combine(root, FileStoreDefaults.MsgDir);
|
|
Directory.CreateDirectory(mdir);
|
|
var blockPath = Path.Combine(mdir, string.Format(FileStoreDefaults.BlkScan, index));
|
|
File.WriteAllBytes(blockPath, payload);
|
|
return blockPath;
|
|
}
|
|
|
|
private static void WriteIndex(string root, uint index, byte[] checksum, bool matchingChecksum)
|
|
{
|
|
var mdir = Path.Combine(root, FileStoreDefaults.MsgDir);
|
|
Directory.CreateDirectory(mdir);
|
|
|
|
var lastChecksum = matchingChecksum ? checksum : new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
|
var info = new
|
|
{
|
|
Msgs = 1UL,
|
|
Bytes = 8UL,
|
|
RawBytes = 8UL,
|
|
FirstSeq = 1UL,
|
|
FirstTs = 1L,
|
|
LastSeq = 1UL,
|
|
LastTs = 1L,
|
|
LastChecksum = lastChecksum,
|
|
NoTrack = false,
|
|
};
|
|
|
|
var indexPath = Path.Combine(mdir, string.Format(FileStoreDefaults.IndexScan, index));
|
|
File.WriteAllText(indexPath, JsonSerializer.Serialize(info));
|
|
}
|
|
|
|
private static byte[] DeterministicKeyGen(byte[] context)
|
|
{
|
|
using var sha = SHA256.Create();
|
|
return sha.ComputeHash(context);
|
|
}
|
|
|
|
private static string NewRoot() => Path.Combine(Path.GetTempPath(), $"impl-fs-{Guid.NewGuid():N}");
|
|
}
|