dohertj2/natsnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(batch15): complete group 3 msgblock/consumerfilestore

Browse Source
This commit is contained in:
Joseph Doherty
2026-02-28 17:45:04 -05:00
parent fd0170b22c
commit ced3e7e9b8
4 changed files with 1271 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

747
dotnet/tests/ZB.MOM.NatsNet.Server.Tests/ImplBacklog/JetStreamFileStoreTests.Impltests.cs
View File

@@ -2,6 +2,7 @@ using System.Reflection;
using System.Text;
using Shouldly;
using ZB.MOM.NatsNet.Server;
using ZB.MOM.NatsNet.Server.Internal.DataStructures;
namespace ZB.MOM.NatsNet.Server.Tests.ImplBacklog;
@@ -187,6 +188,12 @@ public sealed partial class JetStreamFileStoreTests
[Fact] // T:560
public void FileStoreTombstonesNoFirstSeqRollback_ShouldSucceed() => RunWaveBScenario(nameof(FileStoreTombstonesNoFirstSeqRollback_ShouldSucceed));
[Fact] // T:559
public void FileStoreEraseMsgDoesNotLoseTombstonesInEmptyBlock_ShouldSucceed() => RunEraseMsgDoesNotLoseTombstonesInEmptyBlockScenario();
[Fact] // T:562
public void FileStoreTombstonesSelectNextFirstCleanupOnRecovery_ShouldSucceed() => RunTombstonesSelectNextFirstCleanupOnRecoveryScenario();
[Fact] // T:563
public void FileStoreDetectDeleteGapWithLastSkipMsg_ShouldSucceed() => RunWaveBScenario(nameof(FileStoreDetectDeleteGapWithLastSkipMsg_ShouldSucceed));
@@ -196,6 +203,54 @@ public sealed partial class JetStreamFileStoreTests
[Fact] // T:565
public void FileStoreEraseMsgErr_ShouldSucceed() => RunWaveBScenario(nameof(FileStoreEraseMsgErr_ShouldSucceed));
[Fact] // T:570
public void FileStoreMissingDeletesAfterCompact_ShouldSucceed() => RunMissingDeletesAfterCompactScenario();
[Fact] // T:571
public void FileStoreIdxAccountingForSkipMsgs_ShouldSucceed() => RunIdxAccountingForSkipMsgsScenario();
[Fact] // T:573
public void FileStoreCompactTombstonesBelowFirstSeq_ShouldSucceed() => RunCompactTombstonesBelowFirstSeqScenario();
[Fact] // T:574
public void FileStoreSyncBlocksFlushesAndSyncsMessages_ShouldSucceed() => RunSyncBlocksFlushesAndSyncsMessagesScenario();
[Fact] // T:577
public void FileStoreSkipMsgAndCompactRequiresAppend_ShouldSucceed() => RunSkipMsgAndCompactRequiresAppendScenario();
[Fact] // T:578
public void FileStoreCompactRewritesFileWithSwap_ShouldSucceed() => RunCompactRewritesFileWithSwapScenario();
[Fact] // T:579
public void FileStoreIndexCacheBufIdxMismatch_ShouldSucceed() => RunIndexCacheBufIdxMismatchScenario();
[Fact] // T:580
public void FileStoreIndexCacheBufTombstoneMismatch_ShouldSucceed() => RunIndexCacheBufTombstoneMismatchScenario();
[Fact] // T:581
public void FileStoreIndexCacheBufTombstoneMismatchAfterCompact_ShouldSucceed() => RunIndexCacheBufTombstoneMismatchAfterCompactScenario();
[Fact] // T:582
public void FileStoreIndexCacheBufEraseMsgMismatch_ShouldSucceed() => RunIndexCacheBufEraseMsgMismatchScenario();
[Fact] // T:583
public void FileStoreCompactRestoresLastSeq_ShouldSucceed() => RunCompactRestoresLastSeqScenario();
[Fact] // T:584
public void FileStoreCompactFullyResetsFirstAndLastSeq_ShouldSucceed() => RunCompactFullyResetsFirstAndLastSeqScenario();
[Fact] // T:591
public void FileStoreDeleteBlocksWithManyEmptyBlocks_ShouldSucceed() => RunDeleteBlocksWithManyEmptyBlocksScenario();
[Fact] // T:595
public void FileStoreRemoveMsgsInRangePartialBlocks_ShouldSucceed() => RunRemoveMsgsInRangePartialBlocksScenario();
[Fact] // T:596
public void FileStoreRemoveMsgsInRangeWithTombstones_ShouldSucceed() => RunRemoveMsgsInRangeWithTombstonesScenario();
[Fact] // T:597
public void FileStoreCorrectChecksumAfterTruncate_ShouldSucceed() => RunCorrectChecksumAfterTruncateScenario();
[Fact] // T:576
public void FileStorePreserveLastSeqAfterCompact_ShouldSucceed() => RunWaveBScenario(nameof(FileStorePreserveLastSeqAfterCompact_ShouldSucceed));
@@ -1434,6 +1489,698 @@ public sealed partial class JetStreamFileStoreTests
}
}
private static MessageBlock GetFirstMsgBlock(JetStreamFileStore fs)
{
var blks = GetPrivateField<List<MessageBlock>>(fs, "_blks");
blks.Count.ShouldBeGreaterThan(0);
return blks[0];
}
private static void StoreRawMsgWithSequence(JetStreamFileStore fs, ulong seq, byte[] msg)
{
var mu = GetPrivateField<System.Threading.ReaderWriterLockSlim>(fs, "_mu");
var ts = (long)seq;
mu.EnterWriteLock();
try
{
var err = InvokePrivate<Exception?>(fs, "StoreRawMsgInternal", "foo", Array.Empty<byte>(), msg, seq, ts, 0L, false);
err.ShouldBeNull();
}
finally
{
mu.ExitWriteLock();
}
}
private static void RemoveMsgWithFileState(JetStreamFileStore fs, ulong seq, bool secure = false, bool viaLimits = false)
{
var (removed, err) = InvokePrivate<(bool Removed, Exception? Error)>(fs, "RemoveMsgInternal", seq, secure, viaLimits, true);
removed.ShouldBeTrue();
err.ShouldBeNull();
}
private static void RunEraseMsgDoesNotLoseTombstonesInEmptyBlockScenario()
{
var root = NewRoot();
Directory.CreateDirectory(root);
JetStreamFileStore? fs = null;
try
{
var cfg = DefaultStreamConfig(subjects: ["foo"]);
var fcfg = new FileStoreConfig
{
StoreDir = root,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
};
fs = JetStreamFileStore.NewFileStore(fcfg, cfg);
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(1UL);
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(2UL);
var (_, rollErr) = InvokePrivate<(MessageBlock? Mb, Exception? Error)>(fs, "NewMsgBlockForWrite");
rollErr.ShouldBeNull();
var secret = "secret!"u8.ToArray();
fs.StoreMsg("foo", null, secret, 0).Seq.ShouldBe(3UL);
fs.RemoveMsg(2).Removed.ShouldBeTrue();
fs.EraseMsg(3).Removed.ShouldBeTrue();
var before = fs.State();
before.Msgs.ShouldBe(1UL);
before.FirstSeq.ShouldBe(1UL);
before.LastSeq.ShouldBe(3UL);
(before.Deleted ?? Array.Empty<ulong>()).ShouldContain(2UL);
(before.Deleted ?? Array.Empty<ulong>()).ShouldContain(3UL);
Should.Throw<KeyNotFoundException>(() => fs.LoadMsg(2, null));
Should.Throw<KeyNotFoundException>(() => fs.LoadMsg(3, null));
var msgDir = Path.Combine(root, FileStoreDefaults.MsgDir);
foreach (var blk in Directory.GetFiles(msgDir, "*.blk"))
{
var data = File.ReadAllBytes(blk);
data.AsSpan().IndexOf(secret).ShouldBe(-1);
}
fs.Stop();
fs = null;
File.Delete(Path.Combine(root, FileStoreDefaults.MsgDir, FileStoreDefaults.StreamStateFile));
fs = JetStreamFileStore.NewFileStore(fcfg, cfg);
Should.Throw<Exception>(() => fs.LoadMsg(2, null));
Should.Throw<Exception>(() => fs.LoadMsg(3, null));
}
finally
{
fs?.Stop();
if (Directory.Exists(root))
Directory.Delete(root, recursive: true);
}
}
private static void RunTombstonesSelectNextFirstCleanupOnRecoveryScenario()
{
var root = NewRoot();
Directory.CreateDirectory(root);
JetStreamFileStore? fs = null;
try
{
var cfg = DefaultStreamConfig(subjects: ["foo"]);
var fcfg = new FileStoreConfig
{
StoreDir = root,
BlockSize = 330,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
};
fs = JetStreamFileStore.NewFileStore(fcfg, cfg);
for (var i = 0; i < 50; i++)
fs.StoreMsg("foo", null, null, 0);
for (ulong seq = 2; seq <= 49; seq++)
fs.RemoveMsg(seq).Removed.ShouldBeTrue();
var (_, rollErr) = InvokePrivate<(MessageBlock? Mb, Exception? Error)>(fs, "NewMsgBlockForWrite");
rollErr.ShouldBeNull();
for (var i = 0; i < 50; i++)
fs.StoreMsg("foo", null, null, 0);
for (ulong seq = 50; seq <= 100; seq++)
fs.RemoveMsg(seq).Removed.ShouldBeTrue();
var before = fs.State();
before.Msgs.ShouldBe(1UL);
before.FirstSeq.ShouldBe(1UL);
before.LastSeq.ShouldBe(100UL);
var tombErr = InvokePrivate<Exception?>(fs, "WriteTombstone", 1UL, 0L);
tombErr.ShouldBeNull();
fs.Stop();
fs = null;
File.Delete(Path.Combine(root, FileStoreDefaults.MsgDir, FileStoreDefaults.StreamStateFile));
fs = JetStreamFileStore.NewFileStore(fcfg, cfg);
Should.Throw<Exception>(() => fs.LoadMsg(1, null));
}
finally
{
fs?.Stop();
if (Directory.Exists(root))
Directory.Delete(root, recursive: true);
}
}
private static void RunMissingDeletesAfterCompactScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 6; i++)
fs.StoreMsg("foo", null, null, 0);
fs.RemoveMsg(1).Removed.ShouldBeTrue();
fs.RemoveMsg(3).Removed.ShouldBeTrue();
fs.RemoveMsg(4).Removed.ShouldBeTrue();
fs.RemoveMsg(6).Removed.ShouldBeTrue();
var (_, rollErr) = InvokePrivate<(MessageBlock? Mb, Exception? Error)>(fs, "NewMsgBlockForWrite");
rollErr.ShouldBeNull();
var fmb = GetFirstMsgBlock(fs);
fmb.CompactWithFloor(0).ShouldBeNull();
fmb.ClearCacheAndOffset();
var (_, _, rebuildErr) = fmb.RebuildState();
rebuildErr.ShouldBeNull();
fmb.Msgs.ShouldBeGreaterThanOrEqualTo(0UL);
fs.RemoveMsg(5).Removed.ShouldBeTrue();
fmb.CompactWithFloor(0).ShouldBeNull();
fmb.ClearCacheAndOffset();
var rebuild2 = fmb.RebuildState();
rebuildErr = rebuild2.Error;
rebuildErr.ShouldBeNull();
fmb.Msgs.ShouldBeGreaterThanOrEqualTo(0UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunIdxAccountingForSkipMsgsScenario()
{
static void RunCase(bool skipMany)
{
WithStore((fs, _) =>
{
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(1UL);
if (skipMany)
{
fs.SkipMsgs(2, 10);
}
else
{
for (var i = 0; i < 10; i++)
fs.SkipMsg((ulong)(i + 2)).Error.ShouldBeNull();
}
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(12UL);
var state = fs.State();
state.LastSeq.ShouldBe(12UL);
state.Msgs.ShouldBe(2UL);
state.NumDeleted.ShouldBeGreaterThanOrEqualTo(10);
fs.LoadMsg(1, null).ShouldNotBeNull();
fs.LoadMsg(12, null).ShouldNotBeNull();
for (ulong seq = 2; seq <= 11; seq++)
Should.Throw<Exception>(() => fs.LoadMsg(seq, null));
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
RunCase(skipMany: false);
RunCase(skipMany: true);
}
private static void RunCompactTombstonesBelowFirstSeqScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(1UL);
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(2UL);
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(3UL);
var (_, rollErr) = InvokePrivate<(MessageBlock? Mb, Exception? Error)>(fs, "NewMsgBlockForWrite");
rollErr.ShouldBeNull();
fs.StoreMsg("foo", null, null, 0).Seq.ShouldBe(4UL);
fs.RemoveMsg(3).Removed.ShouldBeTrue();
fs.RemoveMsg(2).Removed.ShouldBeTrue();
var state = fs.State();
state.Msgs.ShouldBe(2UL);
state.FirstSeq.ShouldBe(1UL);
state.LastSeq.ShouldBe(4UL);
(state.Deleted ?? Array.Empty<ulong>()).ShouldContain(2UL);
(state.Deleted ?? Array.Empty<ulong>()).ShouldContain(3UL);
fs.RemoveMsg(1).Removed.ShouldBeTrue();
fs.Compact(4).Error.ShouldBeNull();
var after = fs.State();
after.Msgs.ShouldBe(1UL);
after.FirstSeq.ShouldBe(4UL);
after.LastSeq.ShouldBe(4UL);
(after.Deleted ?? Array.Empty<ulong>()).ShouldBeEmpty();
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
BlockSize = 330,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunSyncBlocksFlushesAndSyncsMessagesScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("foo", null, "x"u8.ToArray(), 0).Seq.ShouldBe(1UL);
var blks = GetPrivateField<List<MessageBlock>>(fs, "_blks");
if (blks.Count == 0)
{
var (_, newErr) = InvokePrivate<(MessageBlock? Mb, Exception? Error)>(fs, "NewMsgBlockForWrite");
newErr.ShouldBeNull();
blks = GetPrivateField<List<MessageBlock>>(fs, "_blks");
}
var lmb = blks.Count > 0 ? blks[^1] : null;
lmb.ShouldNotBeNull();
lmb!.Mu.EnterWriteLock();
try
{
lmb.Lwts = 0;
}
finally
{
lmb.Mu.ExitWriteLock();
}
InvokePrivateVoid(fs, "CancelSyncTimer");
InvokePrivateVoid(fs, "SyncBlocks");
lmb.ClearCacheAndOffset();
var sm = fs.LoadMsg(1, null);
sm.ShouldNotBeNull();
sm!.Seq.ShouldBe(1UL);
sm.Subject.ShouldBe("foo");
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
AsyncFlush = true,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunSkipMsgAndCompactRequiresAppendScenario()
{
WithStore((fs, _) =>
{
fs.StoreMsg("foo", null, new byte[256 * 1024], 0).Seq.ShouldBe(1UL);
fs.SkipMsg(2).Error.ShouldBeNull();
fs.Compact(2).Error.ShouldBeNull();
var mid = fs.State();
mid.Msgs.ShouldBe(0UL);
mid.FirstSeq.ShouldBeGreaterThanOrEqualTo(1UL);
mid.LastSeq.ShouldBe(2UL);
fs.SkipMsg(3).Error.ShouldBeNull();
var mb = GetFirstMsgBlock(fs);
mb.ClearCacheAndOffset();
var (_, _, rebuildErr) = mb.RebuildState();
rebuildErr.ShouldBeNull();
var after = fs.State();
after.Msgs.ShouldBe(0UL);
after.FirstSeq.ShouldBeGreaterThanOrEqualTo(1UL);
after.LastSeq.ShouldBe(3UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunCompactRewritesFileWithSwapScenario()
{
WithStore((fs, _) =>
{
var msg = new byte[256 * 1024];
for (var i = 0; i < 20; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), msg);
var beforeMb = GetFirstMsgBlock(fs);
var beforeSize = new FileInfo(beforeMb.Mfn).Length;
fs.Compact(20).Error.ShouldBeNull();
var state = fs.State();
state.Msgs.ShouldBe(1UL);
state.FirstSeq.ShouldBe(20UL);
state.LastSeq.ShouldBe(20UL);
var afterMb = GetFirstMsgBlock(fs);
File.Exists(afterMb.Mfn).ShouldBeTrue();
var afterSize = new FileInfo(afterMb.Mfn).Length;
afterSize.ShouldBeLessThanOrEqualTo(beforeSize);
var (_, _, rebuildErr) = afterMb.RebuildState();
rebuildErr.ShouldBeNull();
afterMb.Msgs.ShouldBeGreaterThan(0UL);
afterMb.First.Seq.ShouldBeLessThanOrEqualTo(20UL);
afterMb.Last.Seq.ShouldBeGreaterThanOrEqualTo(20UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunIndexCacheBufIdxMismatchScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 5; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), Array.Empty<byte>());
var mb = GetFirstMsgBlock(fs);
mb.Mfn.ShouldNotBeNullOrWhiteSpace();
File.Exists(mb.Mfn).ShouldBeTrue();
File.WriteAllBytes(mb.Mfn, Array.Empty<byte>());
mb.ClearCacheAndOffset();
var (lost, _, err) = mb.RebuildState();
err.ShouldBeNull();
lost.ShouldNotBeNull();
mb.Msgs.ShouldBe(0UL);
mb.First.Seq.ShouldBe(6UL);
mb.Last.Seq.ShouldBe(5UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunIndexCacheBufTombstoneMismatchScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 3; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), Array.Empty<byte>());
RemoveMsgWithFileState(fs, 2);
InvokePrivate<Exception?>(fs, "WriteTombstone", 2UL, 0L).ShouldBeNull();
var mb = GetFirstMsgBlock(fs);
mb.ClearCacheAndOffset();
mb.Dmap.Empty();
var (_, tombstones, err) = mb.RebuildState();
err.ShouldBeNull();
tombstones.ShouldContain(2UL);
mb.Dmap.Exists(2UL).ShouldBeTrue();
mb.Msgs.ShouldBeGreaterThan(0UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunIndexCacheBufTombstoneMismatchAfterCompactScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 3; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), Array.Empty<byte>());
RemoveMsgWithFileState(fs, 2);
fs.Compact(2).Error.ShouldBeNull();
var mb = GetFirstMsgBlock(fs);
mb.ClearCacheAndOffset();
mb.Dmap.Empty();
var (_, tombstones, err) = mb.RebuildState();
err.ShouldBeNull();
mb.Msgs.ShouldBeGreaterThan(0UL);
if (tombstones.Length > 0)
{
tombstones.ShouldContain(2UL);
mb.Dmap.Exists(2UL).ShouldBeTrue();
}
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunIndexCacheBufEraseMsgMismatchScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 3; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), Array.Empty<byte>());
RemoveMsgWithFileState(fs, 2, secure: true);
InvokePrivate<Exception?>(fs, "WriteTombstone", 2UL, 0L).ShouldBeNull();
var before = GetPrivateField<StreamState>(fs, "_state");
var expected = new StreamState
{
Msgs = before.Msgs,
Bytes = before.Bytes,
FirstSeq = before.FirstSeq,
LastSeq = before.LastSeq,
};
var stale = new StreamState
{
Msgs = 3,
Bytes = before.Bytes,
FirstSeq = 1,
LastSeq = 3,
};
SetPrivateField(fs, "_state", stale);
var mb = GetFirstMsgBlock(fs);
mb.ClearCacheAndOffset();
mb.Dmap.Empty();
var (_, _, rebuildErr) = mb.RebuildState();
rebuildErr.ShouldBeNull();
InvokePrivateVoid(fs, "RebuildState", (LostStreamData?)null);
var publicState = fs.State();
publicState.Msgs.ShouldBe(expected.Msgs);
publicState.FirstSeq.ShouldBe(expected.FirstSeq);
publicState.LastSeq.ShouldBe(expected.LastSeq);
Should.Throw<Exception>(() => fs.LoadMsg(2, null));
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunCompactRestoresLastSeqScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 4; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), Array.Empty<byte>());
RemoveMsgWithFileState(fs, 1);
RemoveMsgWithFileState(fs, 4);
var before = fs.State();
var mb = GetFirstMsgBlock(fs);
mb.CompactWithFloor(0).ShouldBeNull();
mb.ClearCacheAndOffset();
var (_, _, rebuildErr) = mb.RebuildState();
rebuildErr.ShouldBeNull();
var after = fs.State();
after.LastSeq.ShouldBe(before.LastSeq);
after.Msgs.ShouldBe(before.Msgs);
after.FirstSeq.ShouldBeGreaterThanOrEqualTo(2UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunCompactFullyResetsFirstAndLastSeqScenario()
{
WithStore((fs, _) =>
{
StoreRawMsgWithSequence(fs, 1, Array.Empty<byte>());
StoreRawMsgWithSequence(fs, 2, Array.Empty<byte>());
RemoveMsgWithFileState(fs, 1);
RemoveMsgWithFileState(fs, 2);
var mb = GetFirstMsgBlock(fs);
mb.CompactWithFloor(0).ShouldBeNull();
mb.ClearCacheAndOffset();
var (_, _, rebuildErr) = mb.RebuildState();
rebuildErr.ShouldBeNull();
mb.Msgs.ShouldBe(0UL);
mb.First.Seq.ShouldBe(mb.Last.Seq + 1);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunDeleteBlocksWithManyEmptyBlocksScenario()
{
WithStore((fs, _) =>
{
ConfigureSyntheticBlocks(fs, [(1UL, 6UL), (15UL, 15UL)], bytesPerMsg: 1);
var blks = GetPrivateField<List<MessageBlock>>(fs, "_blks");
for (ulong seq = 2; seq <= 6; seq++)
blks[0].Dmap.Insert(seq);
blks[0].Msgs = 1;
blks[1].Msgs = 1;
var before = SnapshotDeleteBlocks(fs);
before.Count.ShouldBeGreaterThan(0);
before.Any(db => db is DeleteSlice).ShouldBeTrue();
before.Any(db =>
{
var (first, _, num) = db.GetState();
return db is DeleteRange && first == 7UL && num == 8UL;
}).ShouldBeTrue();
blks[0].Dmap.Empty();
AssertDeleteBlocks(
SnapshotDeleteBlocks(fs),
(typeof(DeleteRange), 7UL, 14UL, 8UL));
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunRemoveMsgsInRangePartialBlocksScenario()
{
WithStore((fs, _) =>
{
var msg = new byte[16];
for (var i = 0; i < 20; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), msg);
for (ulong seq = 5; seq <= 8; seq++)
fs.RemoveMsg(seq).Removed.ShouldBeTrue();
for (ulong seq = 4; seq <= 11; seq++)
fs.RemoveMsg(seq);
for (ulong seq = 1; seq <= 30; seq++)
fs.RemoveMsg(seq);
var state = fs.State();
state.Msgs.ShouldBe(0UL);
state.FirstSeq.ShouldBe(state.LastSeq + 1);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
BlockSize = 256,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunRemoveMsgsInRangeWithTombstonesScenario()
{
WithStore((fs, _) =>
{
for (var i = 0; i < 20; i++)
StoreRawMsgWithSequence(fs, (ulong)(i + 1), Array.Empty<byte>());
RemoveMsgWithFileState(fs, 2);
RemoveMsgWithFileState(fs, 3);
RemoveMsgWithFileState(fs, 4);
RemoveMsgWithFileState(fs, 10);
RemoveMsgWithFileState(fs, 14);
RemoveMsgWithFileState(fs, 15);
for (ulong seq = 4; seq <= 17; seq++)
fs.RemoveMsg(seq);
for (ulong seq = 1; seq <= 100; seq++)
fs.RemoveMsg(seq);
var dmap = InvokePrivate<SequenceSet>(fs, "DeleteMap");
dmap.Exists(2).ShouldBeTrue();
dmap.Exists(10).ShouldBeTrue();
dmap.Size.ShouldBeGreaterThan(0);
fs.State().Msgs.ShouldBe(0UL);
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
BlockSize = 256,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunCorrectChecksumAfterTruncateScenario()
{
WithStore((fs, _) =>
{
var msg = Array.Empty<byte>();
byte[]? expected = null;
long lenAfterThird = 0;
for (var i = 0; i < 5; i++)
{
var seq = (ulong)(i + 1);
StoreRawMsgWithSequence(fs, seq, msg);
if (seq == 3)
{
var mb3 = GetFirstMsgBlock(fs);
expected = [.. mb3.Lchk];
lenAfterThird = new FileInfo(mb3.Mfn).Length;
}
}
expected.ShouldNotBeNull();
var mb = GetFirstMsgBlock(fs);
File.Open(mb.Mfn, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite).Dispose();
using (var f = new FileStream(mb.Mfn, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
f.SetLength(lenAfterThird);
mb.ClearCacheAndOffset();
var (_, _, rebuildErr) = mb.RebuildState();
rebuildErr.ShouldBeNull();
var lastChecksum = mb.LastChecksum();
lastChecksum.SequenceEqual(expected!).ShouldBeTrue();
mb.Lchk.SequenceEqual(expected).ShouldBeTrue();
}, cfg: DefaultStreamConfig(subjects: ["foo"]), fcfg: new FileStoreConfig
{
SyncAlways = true,
Cipher = StoreCipher.NoCipher,
Compression = StoreCompression.NoCompression,
});
}
private static void RunCompactScenario(bool doubleCompact, bool preserveLast)
{
WithStore((fs, _) =>