dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: add FileStore buffered writes, O(1) state tracking, and eliminate redundant per-publish work

Browse Source 
Implement Go-parity background flush loop (coalesce 16KB/8ms) in MsgBlock/FileStore,
replace O(n) GetStateAsync with incremental counters, skip PruneExpired/LoadAsync/
PrunePerSubject when not needed, and bypass RAFT for single-replica streams. Fix counter
tracking bugs in RemoveMsg/EraseMsg/TTL expiry and ObjectDisposedException races in
flush loop disposal. FileStore optimizations verified with 3112/3112 JetStream tests
passing; async publish benchmark remains at ~174 msg/s due to E2E protocol path bottleneck.
This commit is contained in:
Joseph Doherty
2026-03-13 03:11:11 -04:00
parent 37575dc41c
commit 4de691c9c5
30 changed files with 1514 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/NATS.Server.JetStream.Tests/JetStream/Cluster/JetStreamClusterGoParityTests.cs
View File

@@ -533,7 +533,7 @@ public class JetStreamClusterGoParityTests
// Go reference: TestJetStreamClusterMetaSyncOrphanCleanup — meta state clean after stream delete
// Skip: delete API handler doesn't yet propagate to meta group
[Fact(Skip = "Stream delete API handler does not yet call ProposeDeleteStreamAsync on meta group")]
[Fact]
public async Task Meta_state_does_not_track_deleted_streams()
{
await using var cluster = await JetStreamClusterFixture.StartAsync(3);

10
tests/NATS.Server.JetStream.Tests/JetStream/Cluster/JsCluster1GoParityTests.cs
View File

@@ -398,7 +398,7 @@ public class JsCluster1GoParityTests
// Go: TestJetStreamClusterMetaSnapshotsAndCatchup — streams delete and meta state is updated
// Skip: StreamManager.Delete does not call ProposeDeleteStreamAsync on meta group,
// so meta state still contains deleted streams (same limitation as Meta_state_does_not_track_deleted_streams)
[Fact(Skip = "StreamManager.Delete does not yet call ProposeDeleteStreamAsync on meta group")]
[Fact]
public async Task Deleted_streams_not_in_meta_state()
{
// Go: TestJetStreamClusterMetaSnapshotsAndCatchup (jetstream_cluster_1_test.go:833)
@@ -428,7 +428,7 @@ public class JsCluster1GoParityTests
// Go: TestJetStreamClusterMetaSnapshotsMultiChange — adding and deleting streams/consumers changes meta state
// Skip: StreamManager.Delete does not call ProposeDeleteStreamAsync on meta group so meta
// state still contains deleted streams — stream create/add/delete meta parity not yet complete.
[Fact(Skip = "StreamManager.Delete does not yet call ProposeDeleteStreamAsync on meta group")]
[Fact]
public async Task Meta_state_reflects_multi_stream_and_consumer_changes()
{
// Go: TestJetStreamClusterMetaSnapshotsMultiChange (jetstream_cluster_1_test.go:881)
@@ -467,7 +467,7 @@ public class JsCluster1GoParityTests
// Go: TestJetStreamClusterStreamOverlapSubjects — overlapping subjects rejected
// Skip: subject overlap validation not yet enforced by StreamManager.CreateOrUpdate
[Fact(Skip = "Subject overlap validation not yet enforced by .NET StreamManager.CreateOrUpdate")]
[Fact]
public async Task Creating_stream_with_overlapping_subjects_returns_error()
{
// Go: TestJetStreamClusterStreamOverlapSubjects (jetstream_cluster_1_test.go:1248)
@@ -482,7 +482,7 @@ public class JsCluster1GoParityTests
// Go: TestJetStreamClusterStreamOverlapSubjects — only one stream in list after overlap attempt
// Skip: subject overlap validation not yet enforced by StreamManager.CreateOrUpdate
[Fact(Skip = "Subject overlap validation not yet enforced by .NET StreamManager.CreateOrUpdate")]
[Fact]
public async Task Stream_list_contains_only_non_overlapping_stream()
{
// Go: TestJetStreamClusterStreamOverlapSubjects (jetstream_cluster_1_test.go:1248)
@@ -606,7 +606,7 @@ public class JsCluster1GoParityTests
// Go: TestJetStreamClusterStreamUpdate — update with wrong stream name fails
// Skip: StreamManager.CreateOrUpdate upserts rather than rejecting unknown stream names
[Fact(Skip = "StreamManager.CreateOrUpdate upserts rather than rejecting unknown stream names")]
[Fact]
public async Task Stream_update_with_mismatched_name_returns_error()
{
// Go: TestJetStreamClusterStreamUpdate (jetstream_cluster_1_test.go:1433)

10
tests/NATS.Server.JetStream.Tests/JetStream/Cluster/JsSuperClusterTests.cs
View File

@@ -407,9 +407,11 @@ public class JsSuperClusterTests
// Stream info returns 3 alternates, sorted by proximity.
await using var cluster = await JetStreamClusterFixture.StartAsync(9);
// In Go, mirrors live in separate clusters (separate jsAccounts) so subjects can overlap.
// Our fixture uses a single StreamManager, so we use distinct subjects per stream.
await cluster.CreateStreamAsync("SOURCE", ["foo", "bar", "baz"], replicas: 3);
await cluster.CreateStreamAsync("MIRROR-1", ["foo", "bar", "baz"], replicas: 1);
await cluster.CreateStreamAsync("MIRROR-2", ["foo", "bar", "baz"], replicas: 2);
await cluster.CreateStreamAsync("MIRROR-1", ["m1foo", "m1bar", "m1baz"], replicas: 1);
await cluster.CreateStreamAsync("MIRROR-2", ["m2foo", "m2bar", "m2baz"], replicas: 2);
// All three streams should exist and be accessible.
var src = await cluster.GetStreamInfoAsync("SOURCE");
@@ -715,7 +717,9 @@ public class JsSuperClusterTests
});
source.Error.ShouldBeNull();
var mirror = await cluster.CreateStreamAsync("MIRROR_AD", ["src.>"], replicas: 1);
// In Go, mirror lives in a separate cluster so subjects can overlap.
// Our fixture uses a single StreamManager, so we use distinct subjects.
var mirror = await cluster.CreateStreamAsync("MIRROR_AD", ["msrc.>"], replicas: 1);
mirror.Error.ShouldBeNull();
// Both source and mirror exist and are accessible.