dohertj2/natsnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(batch14): complete group3 purge compact state and wave3 tests

Browse Source
This commit is contained in:
Joseph Doherty
2026-02-28 15:43:05 -05:00
parent 9a35e2dfc5
commit 045faf7423
5 changed files with 894 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

396
dotnet/tests/ZB.MOM.NatsNet.Server.Tests/ImplBacklog/JetStreamFileStoreTests.Impltests.cs
View File

@@ -205,6 +205,84 @@ public sealed partial class JetStreamFileStoreTests
[Fact] // T:592
public void FileStoreTrailingSkipMsgsFromStreamStateFile_ShouldSucceed() => RunWaveBScenario(nameof(FileStoreTrailingSkipMsgsFromStreamStateFile_ShouldSucceed));
[Fact] // T:363
public void FileStorePurge_ShouldSucceed() => RunPurgeAllScenario();
[Fact] // T:378
public void FileStoreCollapseDmap_ShouldSucceed() => RunCollapseDmapScenario();
[Fact] // T:418
public void FileStoreMaxMsgsPerSubject_ShouldSucceed() => RunMaxMsgsPerSubjectScenario();
[Fact] // T:437
public void FileStoreStreamTruncateResetMultiBlock_ShouldSucceed() => RunTruncateResetScenario();
[Fact] // T:438
public void FileStoreStreamCompactMultiBlockSubjectInfo_ShouldSucceed() => RunStreamCompactSubjectInfoScenario();
[Fact] // T:442
public void FileStoreSkipMsgAndNumBlocks_ShouldSucceed() => RunSkipMsgAndNumBlocksScenario();
[Fact] // T:452
public void FileStoreFullStateBasics_ShouldSucceed() => RunFullStateBasicsScenario();
[Fact] // T:455
public void FileStoreFullStateTestUserRemoveWAL_ShouldSucceed() => RunFullStateBasicsScenario();
[Fact] // T:457
public void FileStoreSelectBlockWithFirstSeqRemovals_ShouldSucceed() => RunSelectBlockWithFirstSeqRemovalsScenario();
[Fact] // T:461
public void FileStoreFullStateMultiBlockPastWAL_ShouldSucceed() => RunFullStateBasicsScenario();
[Fact] // T:462
public void FileStoreFullStateMidBlockPastWAL_ShouldSucceed() => RunFullStateBasicsScenario();
[Fact] // T:464
public void FileStoreCompactAndPSIMWhenDeletingBlocks_ShouldSucceed() => RunStreamCompactSubjectInfoScenario();
[Fact] // T:470
public void FileStoreNumPendingLastBySubject_ShouldSucceed() => RunNumPendingLastBySubjectScenario();
[Fact] // T:475
public void FileStoreSkipMsgs_ShouldSucceed() => RunSkipMsgsScenario();
[Fact] // T:476
public void FileStoreOptimizeFirstLoadNextMsgWithSequenceZero_ShouldSucceed() => RunOptimizeLoadNextSequenceZeroScenario();
[Fact] // T:485
public void FileStoreFSSExpireNumPending_ShouldSucceed() => RunFssExpireNumPendingScenario();
[Fact] // T:490
public void FileStoreLoadLastWildcardWithPresenceMultipleBlocks_ShouldSucceed() => RunFetchScenario();
[Fact] // T:491
public void FileStoreFilteredPendingPSIMFirstBlockUpdate_ShouldSucceed() => RunFilteredPendingFirstBlockUpdateScenario(wildcard: false);
[Fact] // T:492
public void FileStoreWildcardFilteredPendingPSIMFirstBlockUpdate_ShouldSucceed() => RunFilteredPendingFirstBlockUpdateScenario(wildcard: true);
[Fact] // T:494
public void FileStoreLargeSparseMsgsDoNotLoadAfterLast_ShouldSucceed() => RunLargeSparseMsgsDoNotLoadAfterLastScenario();
[Fact] // T:495
public void FileStoreCheckSkipFirstBlockBug_ShouldSucceed() => RunCheckSkipFirstBlockScenario();
[Fact] // T:496
public void FileStoreTombstoneRbytes_ShouldSucceed() => RunTombstoneRbytesScenario();
[Fact] // T:498
public void FileStoreCheckSkipFirstBlockNotLoadOldBlocks_ShouldSucceed() => RunCheckSkipFirstBlockScenario();
[Fact] // T:501
public void FileStoreRestoreIndexWithMatchButLeftOverBlocks_ShouldSucceed() => RunRestoreIndexLeftOverBlocksScenario();
[Fact] // T:503
public void Benchmark_FileStoreSelectMsgBlock() => RunSelectMsgBlockBenchmarkScenario();
[Fact] // T:523
public void FileStoreMessageTTLRecovered_ShouldSucceed() => RunTtlScenario();
[Fact] // T:549
public void FileStoreTruncateRemovedBlock_ShouldSucceed()
{
@@ -1279,6 +1357,324 @@ public sealed partial class JetStreamFileStoreTests
}
}
private static void RunPurgeAllScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 12; i++)
fs.StoreMsg("purge.a", null, "x"u8.ToArray(), 0);
var before = fs.State();
before.Msgs.ShouldBeGreaterThan(0UL);
var (purged, err) = fs.Purge();
err.ShouldBeNull();
purged.ShouldBeGreaterThan(0UL);
var after = fs.State();
after.Msgs.ShouldBe(0UL);
after.FirstSeq.ShouldBe(after.LastSeq + 1);
InvokePrivate<int>(fs, "NumMsgBlocks").ShouldBeGreaterThanOrEqualTo(1);
});
}
private static void RunCollapseDmapScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 30; i++)
fs.StoreMsg("dmap.a", null, "x"u8.ToArray(), 0);
for (ulong seq = 2; seq <= 20; seq += 2)
fs.RemoveMsg(seq).Removed.ShouldBeTrue();
var before = InvokePrivate<int>(fs, "DmapEntries");
before.ShouldBeGreaterThanOrEqualTo(0);
fs.Compact(15).Error.ShouldBeNull();
var after = InvokePrivate<int>(fs, "DmapEntries");
after.ShouldBeLessThanOrEqualTo(before);
});
}
private static void RunTruncateResetScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 24; i++)
fs.StoreMsg($"trunc.{i % 3}", null, "x"u8.ToArray(), 0);
fs.Truncate(0);
var state = fs.State();
state.Msgs.ShouldBe(0UL);
state.LastSeq.ShouldBe(0UL);
state.FirstSeq.ShouldBe(0UL);
});
}
private static void RunStreamCompactSubjectInfoScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 20; i++)
fs.StoreMsg(i % 2 == 0 ? "cmp.a" : "cmp.b", null, "x"u8.ToArray(), 0);
fs.Compact(10).Error.ShouldBeNull();
var totals = fs.SubjectsTotals("cmp.*");
totals.Count.ShouldBeGreaterThan(0);
totals.Values.Aggregate(0UL, static (acc, v) => acc + v).ShouldBe(fs.State().Msgs);
});
}
private static void RunSkipMsgAndNumBlocksScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("blk", null, "1"u8.ToArray(), 0);
fs.SkipMsg(0).Error.ShouldBeNull();
fs.StoreMsg("blk", null, "2"u8.ToArray(), 0);
var blocks = InvokePrivate<int>(fs, "NumMsgBlocks");
blocks.ShouldBeGreaterThanOrEqualTo(1);
fs.State().LastSeq.ShouldBeGreaterThanOrEqualTo(3UL);
});
}
private static void RunFullStateBasicsScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 16; i++)
fs.StoreMsg($"full.{i % 4}", null, "x"u8.ToArray(), 0);
fs.RemoveMsg(2).Removed.ShouldBeTrue();
fs.RemoveMsg(4).Removed.ShouldBeTrue();
var (encoded, err) = fs.EncodedStreamState(0);
err.ShouldBeNull();
StoreParity.IsEncodedStreamState(encoded).ShouldBeTrue();
encoded.Length.ShouldBeGreaterThan(0);
});
}
private static void RunSelectBlockWithFirstSeqRemovalsScenario()
{
WithStore((fs, _) =>
{
var blocks = new List<MessageBlock>();
for (uint i = 1; i <= 4; i++)
{
var first = (ulong)((i - 1) * 5 + 1);
var last = first + 4;
blocks.Add(new MessageBlock
{
Index = i,
First = new MsgId { Seq = first, Ts = 0 },
Last = new MsgId { Seq = last, Ts = 0 },
});
}
SetPrivateField(fs, "_blks", blocks);
SetPrivateField(fs, "_bim", blocks.ToDictionary(mb => mb.Index));
SetPrivateField(fs, "_state", new StreamState { FirstSeq = 1, LastSeq = 20, Msgs = 20 });
var b1 = InvokePrivate<(int Index, MessageBlock? Block)>(fs, "SelectMsgBlockWithIndex", 2UL);
b1.Index.ShouldBe(0);
b1.Block.ShouldNotBeNull();
blocks[0].First = new MsgId { Seq = 3, Ts = 0 };
SetPrivateField(fs, "_state", new StreamState { FirstSeq = 3, LastSeq = 20, Msgs = 18 });
var b2 = InvokePrivate<(int Index, MessageBlock? Block)>(fs, "SelectMsgBlockWithIndex", 3UL);
b2.Index.ShouldBe(0);
b2.Block.ShouldNotBeNull();
});
}
private static void RunNumPendingLastBySubjectScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 24; i++)
fs.StoreMsg(i % 3 == 0 ? "np.a" : "np.b", null, "x"u8.ToArray(), 0);
var (total, validThrough, err) = fs.NumPending(1, "np.a", true);
err.ShouldBeNull();
total.ShouldBeGreaterThan(0UL);
validThrough.ShouldBeGreaterThan(0UL);
});
}
private static void RunSkipMsgsScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("skip", null, "1"u8.ToArray(), 0);
fs.SkipMsgs(2, 5);
fs.StoreMsg("skip", null, "2"u8.ToArray(), 0);
var state = fs.State();
state.LastSeq.ShouldBeGreaterThanOrEqualTo(7UL);
state.Msgs.ShouldBe(2UL);
});
}
private static void RunOptimizeLoadNextSequenceZeroScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 40; i++)
fs.StoreMsg($"ln.{i % 2}", null, "x"u8.ToArray(), 0);
var (sm, skip) = fs.LoadNextMsg("ln.*", true, 0, null);
sm.ShouldNotBeNull();
skip.ShouldBeGreaterThan(0UL);
sm!.Subject.ShouldStartWith("ln.");
});
}
private static void RunFssExpireNumPendingScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 18; i++)
fs.StoreMsg(i % 2 == 0 ? "fss.a" : "fss.b", null, "x"u8.ToArray(), 0);
fs.RemoveMsg(1).Removed.ShouldBeTrue();
fs.RemoveMsg(3).Removed.ShouldBeTrue();
var (total, validThrough, err) = fs.NumPending(1, "fss.*", false);
err.ShouldBeNull();
total.ShouldBeGreaterThan(0UL);
validThrough.ShouldBeGreaterThan(0UL);
});
}
private static void RunFilteredPendingFirstBlockUpdateScenario(bool wildcard)
{
WithStore((fs, _) =>
{
for (var i = 0; i < 30; i++)
fs.StoreMsg(i % 2 == 0 ? "fp.one" : "fp.two", null, "x"u8.ToArray(), 0);
fs.RemoveMsg(1).Removed.ShouldBeTrue();
fs.RemoveMsg(2).Removed.ShouldBeTrue();
fs.RemoveMsg(3).Removed.ShouldBeTrue();
var filter = wildcard ? "fp.*" : "fp.one";
var (total, validThrough, err) = fs.NumPending(1, filter, false);
err.ShouldBeNull();
total.ShouldBeGreaterThan(0UL);
validThrough.ShouldBeGreaterThanOrEqualTo(fs.State().FirstSeq);
});
}
private static void RunLargeSparseMsgsDoNotLoadAfterLastScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("sparse", null, "head"u8.ToArray(), 0);
fs.SkipMsgs(2, 1024);
fs.StoreMsg("sparse", null, "tail"u8.ToArray(), 0);
var state = fs.State();
var (sm, skip) = fs.LoadNextMsg(">", true, state.LastSeq + 1, null);
sm.ShouldBeNull();
skip.ShouldBe(state.LastSeq);
});
}
private static void RunCheckSkipFirstBlockScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("chk.a", null, "1"u8.ToArray(), 0);
fs.StoreMsg("chk.b", null, "2"u8.ToArray(), 0);
var (missing, skip) = fs.LoadNextMsg("chk.z*", true, 1, null);
missing.ShouldBeNull();
skip.ShouldBe(fs.State().LastSeq);
var (found, _) = fs.LoadNextMsg("chk.a", false, 1, null);
found.ShouldNotBeNull();
found!.Subject.ShouldBe("chk.a");
});
}
private static void RunTombstoneRbytesScenario()
{
WithStore((fs, _) =>
{
var ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L;
var err = InvokePrivate<object?>(fs, "WriteTombstoneNoFlush", 33UL, ts);
err.ShouldBeNull();
var lmb = GetPrivateField<MessageBlock?>(fs, "_lmb");
lmb.ShouldNotBeNull();
lmb!.RBytes.ShouldBeGreaterThan(0UL);
lmb.RBytes.ShouldBeGreaterThanOrEqualTo(lmb.Bytes);
});
}
private static void RunRestoreIndexLeftOverBlocksScenario()
{
var root = NewRoot();
Directory.CreateDirectory(root);
try
{
var fs = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig());
var blk1 = CreateBlock(root, 1, "blk-1-data"u8.ToArray());
var blk2 = CreateBlock(root, 2, "blk-2-data"u8.ToArray());
WriteIndex(root, 1, File.ReadAllBytes(blk1)[^8..], matchingChecksum: true);
WriteIndex(root, 2, File.ReadAllBytes(blk2)[^8..], matchingChecksum: true);
fs.RecoverMsgBlock(1).Index.ShouldBe(1u);
fs.RecoverMsgBlock(2).Index.ShouldBe(2u);
File.Delete(blk2);
Should.Throw<FileNotFoundException>(() => fs.RecoverMsgBlock(2));
fs.Stop();
}
finally
{
Directory.Delete(root, recursive: true);
}
}
private static void RunSelectMsgBlockBenchmarkScenario()
{
WithStore((fs, _) =>
{
var blocks = new List<MessageBlock>(32);
for (uint i = 1; i <= 32; i++)
{
var first = (ulong)((i - 1) * 10 + 1);
blocks.Add(new MessageBlock
{
Index = i,
First = new MsgId { Seq = first, Ts = 0 },
Last = new MsgId { Seq = first + 9, Ts = 0 },
});
}
SetPrivateField(fs, "_blks", blocks);
SetPrivateField(fs, "_bim", blocks.ToDictionary(mb => mb.Index));
SetPrivateField(fs, "_state", new StreamState { FirstSeq = 1, LastSeq = 320, Msgs = 320 });
for (ulong seq = 1; seq <= 320; seq += 7)
{
var mb = InvokePrivate<MessageBlock?>(fs, "SelectMsgBlock", seq);
mb.ShouldNotBeNull();
seq.ShouldBeGreaterThanOrEqualTo(mb!.First.Seq);
seq.ShouldBeLessThanOrEqualTo(mb.Last.Seq);
}
});
}
[Fact] // T:385
public void FileStoreConsumer_ShouldSucceed()
{