perf: add FileStore buffered writes, O(1) state tracking, and eliminate redundant per-publish work
Implement Go-parity background flush loop (coalesce 16KB/8ms) in MsgBlock/FileStore, replace O(n) GetStateAsync with incremental counters, skip PruneExpired/LoadAsync/ PrunePerSubject when not needed, and bypass RAFT for single-replica streams. Fix counter tracking bugs in RemoveMsg/EraseMsg/TTL expiry and ObjectDisposedException races in flush loop disposal. FileStore optimizations verified with 3112/3112 JetStream tests passing; async publish benchmark remains at ~174 msg/s due to E2E protocol path bottleneck.
This commit is contained in:
Joseph Doherty
@@ -114,52 +114,181 @@ public sealed class FileStoreSubjectTests : IDisposable
|
||||
}
|
||||
|
||||
// Go: TestFileStoreSubjectStateCacheExpiration server/filestore_test.go:4143
|
||||
[Fact(Skip = "SubjectsState not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task Subject_state_cache_expiration()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("subj-state-cache");
|
||||
|
||||
await store.AppendAsync("foo.1", "a"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.2", "b"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar.1", "c"u8.ToArray(), default);
|
||||
|
||||
// Initial state: 3 subjects, each with 1 message.
|
||||
var initial = store.SubjectsState(">");
|
||||
initial.Count.ShouldBe(3);
|
||||
initial["foo.1"].Msgs.ShouldBe((ulong)1);
|
||||
initial["foo.2"].Msgs.ShouldBe((ulong)1);
|
||||
initial["bar.1"].Msgs.ShouldBe((ulong)1);
|
||||
|
||||
// Add a second message to "foo.1" — cache must be invalidated.
|
||||
await store.AppendAsync("foo.1", "d"u8.ToArray(), default);
|
||||
|
||||
var updated = store.SubjectsState(">");
|
||||
updated.Count.ShouldBe(3);
|
||||
updated["foo.1"].Msgs.ShouldBe((ulong)2);
|
||||
updated["foo.1"].First.ShouldBe((ulong)1);
|
||||
updated["foo.1"].Last.ShouldBe((ulong)4);
|
||||
|
||||
// Remove one "foo.1" message — cache must be invalidated again.
|
||||
(await store.RemoveAsync(1, default)).ShouldBeTrue();
|
||||
|
||||
var afterRemove = store.SubjectsState(">");
|
||||
afterRemove.Count.ShouldBe(3);
|
||||
afterRemove["foo.1"].Msgs.ShouldBe((ulong)1);
|
||||
afterRemove["foo.1"].First.ShouldBe((ulong)4);
|
||||
afterRemove["foo.1"].Last.ShouldBe((ulong)4);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreSubjectsTotals server/filestore_test.go:4948
|
||||
[Fact(Skip = "SubjectsTotals not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task Subjects_totals_with_wildcards()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("subj-totals");
|
||||
|
||||
await store.AppendAsync("foo.a", "1"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.b", "2"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.a", "3"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar.c", "4"u8.ToArray(), default);
|
||||
|
||||
// Filter to foo.> — should only see foo subjects.
|
||||
var fooTotals = store.SubjectsTotals("foo.>");
|
||||
fooTotals.Count.ShouldBe(2);
|
||||
fooTotals["foo.a"].ShouldBe((ulong)2);
|
||||
fooTotals["foo.b"].ShouldBe((ulong)1);
|
||||
fooTotals.ContainsKey("bar.c").ShouldBeFalse();
|
||||
|
||||
// Filter to > — should see all subjects.
|
||||
var allTotals = store.SubjectsTotals(">");
|
||||
allTotals.Count.ShouldBe(3);
|
||||
allTotals["foo.a"].ShouldBe((ulong)2);
|
||||
allTotals["foo.b"].ShouldBe((ulong)1);
|
||||
allTotals["bar.c"].ShouldBe((ulong)1);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreSubjectCorruption server/filestore_test.go:6466
|
||||
[Fact(Skip = "SubjectForSeq not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task Subject_corruption_detection()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("subj-corruption");
|
||||
|
||||
await store.AppendAsync("foo", "a"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar", "b"u8.ToArray(), default);
|
||||
await store.AppendAsync("baz", "c"u8.ToArray(), default);
|
||||
|
||||
// Each sequence should map to the correct subject.
|
||||
store.SubjectForSeq(1).ShouldBe("foo");
|
||||
store.SubjectForSeq(2).ShouldBe("bar");
|
||||
store.SubjectForSeq(3).ShouldBe("baz");
|
||||
|
||||
// Remove seq 2 — SubjectForSeq should throw for the removed sequence.
|
||||
(await store.RemoveAsync(2, default)).ShouldBeTrue();
|
||||
|
||||
Should.Throw<KeyNotFoundException>(() => store.SubjectForSeq(2));
|
||||
|
||||
// Non-existent sequence should also throw.
|
||||
Should.Throw<KeyNotFoundException>(() => store.SubjectForSeq(999));
|
||||
|
||||
// Remaining sequences still resolve correctly.
|
||||
store.SubjectForSeq(1).ShouldBe("foo");
|
||||
store.SubjectForSeq(3).ShouldBe("baz");
|
||||
}
|
||||
|
||||
// Go: TestFileStoreFilteredPendingBug server/filestore_test.go:3414
|
||||
[Fact(Skip = "FilteredState not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task Filtered_pending_no_match_returns_zero()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("filtered-pending-nomatch");
|
||||
|
||||
await store.AppendAsync("foo", "a"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo", "b"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo", "c"u8.ToArray(), default);
|
||||
|
||||
// Filter "bar" matches no messages — Msgs should be 0.
|
||||
var state = store.FilteredState(1, "bar");
|
||||
state.Msgs.ShouldBe((ulong)0);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreFilteredFirstMatchingBug server/filestore_test.go:4448
|
||||
[Fact(Skip = "LoadNextMsg not yet implemented in .NET FileStore")]
|
||||
// The bug was that LoadNextMsg with a filter could return a message whose subject
|
||||
// did not match the filter when fss (per-subject state) was regenerated from only
|
||||
// part of the block. The fix: when no matching message exists at or after start,
|
||||
// throw KeyNotFoundException rather than returning a wrong-subject message.
|
||||
[Fact]
|
||||
public async Task Filtered_first_matching_finds_correct_sequence()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("filtered-first-match");
|
||||
|
||||
// seqs 1-3: "foo.foo", seq 4: "foo.bar" (no more "foo.foo" after seq 3)
|
||||
await store.AppendAsync("foo.foo", "A"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.foo", "B"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.foo", "C"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.bar", "X"u8.ToArray(), default);
|
||||
|
||||
// Starting at seq 4, filter "foo.foo" — seq 4 is "foo.bar", and there are no
|
||||
// further "foo.foo" messages, so LoadNextMsg must throw rather than return a
|
||||
// message with the wrong subject.
|
||||
Should.Throw<KeyNotFoundException>(() =>
|
||||
store.LoadNextMsg("foo.foo", false, 4, null));
|
||||
|
||||
// Sanity: starting at seq 1 should find "foo.foo" at seq 1 with no skip.
|
||||
var (msg, skip) = store.LoadNextMsg("foo.foo", false, 1, null);
|
||||
msg.Subject.ShouldBe("foo.foo");
|
||||
msg.Sequence.ShouldBe((ulong)1);
|
||||
skip.ShouldBe((ulong)0);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreExpireSubjectMeta server/filestore_test.go:4014
|
||||
[Fact(Skip = "SubjectsState not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task Expired_subject_metadata_cleans_up()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("expire-subj-meta");
|
||||
|
||||
await store.AppendAsync("foo.1", "a"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.1", "b"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo.2", "c"u8.ToArray(), default);
|
||||
|
||||
// Remove ALL messages on "foo.1".
|
||||
(await store.RemoveAsync(1, default)).ShouldBeTrue();
|
||||
(await store.RemoveAsync(2, default)).ShouldBeTrue();
|
||||
|
||||
// "foo.1" should have been cleaned up — not present in SubjectsState.
|
||||
var state = store.SubjectsState(">");
|
||||
state.ContainsKey("foo.1").ShouldBeFalse();
|
||||
|
||||
// "foo.2" is still alive.
|
||||
state.ContainsKey("foo.2").ShouldBeTrue();
|
||||
state["foo.2"].Msgs.ShouldBe((ulong)1);
|
||||
}
|
||||
|
||||
// Go: TestFileStoreAllFilteredStateWithDeleted server/filestore_test.go:4827
|
||||
[Fact(Skip = "FilteredState not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task Filtered_state_with_deleted_messages()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("filtered-state-deleted");
|
||||
|
||||
// Store 5 messages on "foo" — seqs 1..5.
|
||||
for (var i = 0; i < 5; i++)
|
||||
await store.AppendAsync("foo", "x"u8.ToArray(), default);
|
||||
|
||||
// Remove seqs 2 and 4 — seqs 1, 3, 5 remain.
|
||||
(await store.RemoveAsync(2, default)).ShouldBeTrue();
|
||||
(await store.RemoveAsync(4, default)).ShouldBeTrue();
|
||||
|
||||
// FilteredState from seq 1 on "foo" should report 3 remaining messages.
|
||||
var state = store.FilteredState(1, "foo");
|
||||
state.Msgs.ShouldBe((ulong)3);
|
||||
state.First.ShouldBe((ulong)1);
|
||||
state.Last.ShouldBe((ulong)5);
|
||||
}
|
||||
|
||||
// Test LoadLastBySubject with multiple subjects and removes.
|
||||
@@ -277,10 +406,30 @@ public sealed class FileStoreSubjectTests : IDisposable
|
||||
}
|
||||
|
||||
// Go: TestFileStoreNumPendingLastBySubject server/filestore_test.go:6501
|
||||
[Fact(Skip = "NumPending not yet implemented in .NET FileStore")]
|
||||
[Fact]
|
||||
public async Task NumPending_last_per_subject()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
await using var store = CreateStore("num-pending-lps");
|
||||
|
||||
// "foo" x3, "bar" x2 — 2 distinct subjects.
|
||||
await store.AppendAsync("foo", "1"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo", "2"u8.ToArray(), default);
|
||||
await store.AppendAsync("foo", "3"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar", "4"u8.ToArray(), default);
|
||||
await store.AppendAsync("bar", "5"u8.ToArray(), default);
|
||||
|
||||
// lastPerSubject=true: count only the last message per distinct subject.
|
||||
// 2 distinct subjects → Total == 2.
|
||||
var (total, _) = store.NumPending(1, ">", true);
|
||||
total.ShouldBe((ulong)2);
|
||||
|
||||
// lastPerSubject=false: count all messages at or after sseq 1.
|
||||
var (totalAll, _) = store.NumPending(1, ">", false);
|
||||
totalAll.ShouldBe((ulong)5);
|
||||
|
||||
// Filter to just "foo" with lastPerSubject=true → 1.
|
||||
var (fooLps, _) = store.NumPending(1, "foo", true);
|
||||
fooLps.ShouldBe((ulong)1);
|
||||
}
|
||||
|
||||
// Test many distinct subjects.
|
||||
|
||||
Reference in New Issue
Block a user