// Go parity: golang/nats-server/server/jetstream_cluster_1_test.go // Covers: meta group leadership, API routing through meta leader, // stream/consumer placement decisions, asset distribution, // R1/R3 placement, preferred tags, cluster-wide operations. using System.Collections.Concurrent; using System.Reflection; using System.Text; using NATS.Server.JetStream; using NATS.Server.JetStream.Api; using NATS.Server.JetStream.Cluster; using NATS.Server.JetStream.Consumers; using NATS.Server.JetStream.Models; using NATS.Server.JetStream.Publish; namespace NATS.Server.Tests.JetStream.Cluster; /// /// Tests covering JetStream meta controller leadership, API routing through /// the meta leader, stream/consumer placement decisions, asset distribution, /// R1/R3 placement, and cluster-wide operations. /// Ported from Go jetstream_cluster_1_test.go and jetstream_cluster_2_test.go. /// public class JetStreamMetaControllerTests { // --------------------------------------------------------------- // Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73 // --------------------------------------------------------------- [Fact] public void Meta_group_initial_leader_is_meta_1() { var meta = new JetStreamMetaGroup(3); var state = meta.GetState(); state.LeaderId.ShouldBe("meta-1"); state.ClusterSize.ShouldBe(3); state.LeadershipVersion.ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73 // --------------------------------------------------------------- [Fact] public void Meta_group_stepdown_advances_leader_id() { var meta = new JetStreamMetaGroup(3); meta.GetState().LeaderId.ShouldBe("meta-1"); meta.StepDown(); meta.GetState().LeaderId.ShouldBe("meta-2"); meta.StepDown(); meta.GetState().LeaderId.ShouldBe("meta-3"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73 // --------------------------------------------------------------- [Fact] public void Meta_group_stepdown_wraps_around_to_first_node() { var meta = new JetStreamMetaGroup(3); meta.StepDown(); // meta-2 meta.StepDown(); // meta-3 meta.StepDown(); // meta-1 (wrap) meta.GetState().LeaderId.ShouldBe("meta-1"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73 // --------------------------------------------------------------- [Fact] public void Meta_group_leadership_version_increments_on_each_stepdown() { var meta = new JetStreamMetaGroup(3); for (var i = 1; i <= 5; i++) { meta.GetState().LeadershipVersion.ShouldBe(i); meta.StepDown(); } meta.GetState().LeadershipVersion.ShouldBe(6); } // --------------------------------------------------------------- // Go: TestJetStreamClusterConfig server/jetstream_cluster_1_test.go:43 // --------------------------------------------------------------- [Fact] public async Task Meta_group_propose_creates_stream_record() { var meta = new JetStreamMetaGroup(3); await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "TEST" }, default); var state = meta.GetState(); state.Streams.Count.ShouldBe(1); state.Streams.ShouldContain("TEST"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299 // --------------------------------------------------------------- [Fact] public async Task Meta_group_tracks_multiple_stream_proposals() { var meta = new JetStreamMetaGroup(5); for (var i = 0; i < 10; i++) await meta.ProposeCreateStreamAsync(new StreamConfig { Name = $"S{i}" }, default); var state = meta.GetState(); state.Streams.Count.ShouldBe(10); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299 // --------------------------------------------------------------- [Fact] public async Task Meta_group_streams_are_sorted_alphabetically() { var meta = new JetStreamMetaGroup(3); await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "ZULU" }, default); await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "ALPHA" }, default); await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "MIKE" }, default); var state = meta.GetState(); state.Streams[0].ShouldBe("ALPHA"); state.Streams[1].ShouldBe("MIKE"); state.Streams[2].ShouldBe("ZULU"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterConfig server/jetstream_cluster_1_test.go:43 // --------------------------------------------------------------- [Fact] public async Task Meta_group_duplicate_stream_proposal_is_idempotent() { var meta = new JetStreamMetaGroup(3); await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "DUP" }, default); await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "DUP" }, default); meta.GetState().Streams.Count.ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterExpandCluster server/jetstream_cluster_1_test.go:86 // --------------------------------------------------------------- [Fact] public void Meta_group_single_node_cluster_has_leader() { var meta = new JetStreamMetaGroup(1); var state = meta.GetState(); state.ClusterSize.ShouldBe(1); state.LeaderId.ShouldBe("meta-1"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterExpandCluster server/jetstream_cluster_1_test.go:86 // --------------------------------------------------------------- [Fact] public void Meta_group_single_node_stepdown_returns_to_same_leader() { var meta = new JetStreamMetaGroup(1); meta.StepDown(); meta.GetState().LeaderId.ShouldBe("meta-1"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterLeaderStepdown server/jetstream_cluster_1_test.go:5464 // --------------------------------------------------------------- [Fact] public async Task Api_meta_leader_stepdown_changes_leader_and_preserves_streams() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("KEEPME", ["keep.>"], replicas: 3); var before = fx.GetMetaState(); var leaderBefore = before.LeaderId; var resp = await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}"); resp.Success.ShouldBeTrue(); var after = fx.GetMetaState(); after.LeaderId.ShouldNotBe(leaderBefore); after.Streams.ShouldContain("KEEPME"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterAccountInfo server/jetstream_cluster_1_test.go:94 // --------------------------------------------------------------- [Fact] public async Task Api_routing_through_meta_leader_returns_account_info() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("A", ["a.>"], replicas: 3); await fx.CreateStreamAsync("B", ["b.>"], replicas: 3); await fx.CreateConsumerAsync("A", "c1"); var resp = await fx.RequestAsync(JetStreamApiSubjects.Info, "{}"); resp.AccountInfo.ShouldNotBeNull(); resp.AccountInfo!.Streams.ShouldBe(2); resp.AccountInfo.Consumers.ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamLimitWithAccountDefaults server/jetstream_cluster_1_test.go:124 // --------------------------------------------------------------- [Fact] public async Task Placement_planner_r1_creates_single_node_placement() { var planner = new AssetPlacementPlanner(nodes: 5); var placement = planner.PlanReplicas(replicas: 1); placement.Count.ShouldBe(1); placement[0].ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299 // --------------------------------------------------------------- [Fact] public void Placement_planner_r3_creates_three_node_placement() { var planner = new AssetPlacementPlanner(nodes: 5); var placement = planner.PlanReplicas(replicas: 3); placement.Count.ShouldBe(3); placement[0].ShouldBe(1); placement[1].ShouldBe(2); placement[2].ShouldBe(3); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299 // --------------------------------------------------------------- [Fact] public void Placement_planner_caps_replicas_at_cluster_size() { var planner = new AssetPlacementPlanner(nodes: 3); var placement = planner.PlanReplicas(replicas: 7); placement.Count.ShouldBe(3); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299 // --------------------------------------------------------------- [Fact] public void Placement_planner_negative_replicas_returns_one() { var planner = new AssetPlacementPlanner(nodes: 5); var placement = planner.PlanReplicas(replicas: -1); placement.Count.ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterConfig server/jetstream_cluster_1_test.go:43 // --------------------------------------------------------------- [Fact] public void Placement_planner_zero_nodes_returns_one() { var planner = new AssetPlacementPlanner(nodes: 0); var placement = planner.PlanReplicas(replicas: 3); placement.Count.ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160 // --------------------------------------------------------------- [Fact] public async Task Stream_create_via_meta_leader_sets_replica_group() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 5); var resp = await fx.CreateStreamAsync("REPGRP", ["rg.>"], replicas: 3); resp.Error.ShouldBeNull(); // The stream manager creates a replica group internally var meta = fx.GetMetaState(); meta.Streams.ShouldContain("REPGRP"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMaxStreamsReached server/jetstream_cluster_1_test.go:3177 // --------------------------------------------------------------- [Fact] public async Task Multiple_stream_creates_all_tracked_in_meta_group() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); for (var i = 0; i < 20; i++) await fx.CreateStreamAsync($"MS{i}", [$"ms{i}.>"], replicas: 3); var meta = fx.GetMetaState(); meta.Streams.Count.ShouldBe(20); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamNames server/jetstream_cluster_1_test.go:1284 // --------------------------------------------------------------- [Fact] public async Task Stream_names_api_returns_all_streams_through_meta_leader() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("S1", ["s1.>"], replicas: 3); await fx.CreateStreamAsync("S2", ["s2.>"], replicas: 1); await fx.CreateStreamAsync("S3", ["s3.>"], replicas: 3); var resp = await fx.RequestAsync(JetStreamApiSubjects.StreamNames, "{}"); resp.StreamNames.ShouldNotBeNull(); resp.StreamNames!.Count.ShouldBe(3); } // --------------------------------------------------------------- // Go: TestJetStreamClusterDelete server/jetstream_cluster_1_test.go:472 // --------------------------------------------------------------- [Fact] public async Task Stream_delete_removes_from_active_names() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("DEL1", ["d1.>"], replicas: 3); await fx.CreateStreamAsync("DEL2", ["d2.>"], replicas: 3); var del = await fx.RequestAsync($"{JetStreamApiSubjects.StreamDelete}DEL1", "{}"); del.Success.ShouldBeTrue(); var names = await fx.RequestAsync(JetStreamApiSubjects.StreamNames, "{}"); names.StreamNames!.Count.ShouldBe(1); names.StreamNames.ShouldContain("DEL2"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterDoubleAdd server/jetstream_cluster_1_test.go:1551 // --------------------------------------------------------------- [Fact] public async Task Stream_create_idempotent_with_same_config() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); var first = await fx.CreateStreamAsync("IDEM", ["idem.>"], replicas: 3); first.Error.ShouldBeNull(); var second = await fx.CreateStreamAsync("IDEM", ["idem.>"], replicas: 3); second.Error.ShouldBeNull(); var meta = fx.GetMetaState(); meta.Streams.Count.ShouldBe(1); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamInfoList server/jetstream_cluster_1_test.go:1284 // --------------------------------------------------------------- [Fact] public async Task Consumer_create_tracked_in_cluster() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("CC", ["cc.>"], replicas: 3); await fx.CreateConsumerAsync("CC", "d1"); await fx.CreateConsumerAsync("CC", "d2"); var names = await fx.RequestAsync($"{JetStreamApiSubjects.ConsumerNames}CC", "{}"); names.ConsumerNames.ShouldNotBeNull(); names.ConsumerNames!.Count.ShouldBe(2); } // --------------------------------------------------------------- // Go: TestJetStreamClusterPeerRemovalAPI server/jetstream_cluster_1_test.go:3469 // --------------------------------------------------------------- [Fact] public async Task Peer_removal_api_routed_through_meta() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("PR", ["pr.>"], replicas: 3); var resp = await fx.RequestAsync($"{JetStreamApiSubjects.StreamPeerRemove}PR", """{"peer":"n2"}"""); resp.Success.ShouldBeTrue(); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMetaSnapshotsAndCatchup server/jetstream_cluster_1_test.go:833 // --------------------------------------------------------------- [Fact] public async Task Meta_state_preserved_across_multiple_stepdowns() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("M1", ["m1.>"], replicas: 3); await fx.CreateStreamAsync("M2", ["m2.>"], replicas: 3); (await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); (await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); (await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); var state = fx.GetMetaState(); state.Streams.ShouldContain("M1"); state.Streams.ShouldContain("M2"); state.LeadershipVersion.ShouldBe(4); } // --------------------------------------------------------------- // Go: TestJetStreamClusterMetaSnapshotsMultiChange server/jetstream_cluster_1_test.go:881 // --------------------------------------------------------------- [Fact] public async Task Create_and_delete_across_stepdowns_reflected_in_names() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("A", ["a.>"], replicas: 3); (await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); await fx.CreateStreamAsync("B", ["b.>"], replicas: 3); (await fx.RequestAsync($"{JetStreamApiSubjects.StreamDelete}A", "{}")).Success.ShouldBeTrue(); (await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); var names = await fx.RequestAsync(JetStreamApiSubjects.StreamNames, "{}"); names.StreamNames!.Count.ShouldBe(1); names.StreamNames.ShouldContain("B"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160 // --------------------------------------------------------------- [Fact] public async Task Stream_info_for_nonexistent_stream_returns_404() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); var resp = await fx.RequestAsync($"{JetStreamApiSubjects.StreamInfo}MISSING", "{}"); resp.Error.ShouldNotBeNull(); resp.Error!.Code.ShouldBe(404); } // --------------------------------------------------------------- // Go: TestJetStreamClusterConsumerCreate server/jetstream_cluster_1_test.go:700 // --------------------------------------------------------------- [Fact] public async Task Consumer_info_for_nonexistent_consumer_returns_404() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("NOCON", ["nc.>"], replicas: 3); var resp = await fx.RequestAsync($"{JetStreamApiSubjects.ConsumerInfo}NOCON.MISSING", "{}"); resp.Error.ShouldNotBeNull(); resp.Error!.Code.ShouldBe(404); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160 // --------------------------------------------------------------- [Fact] public void Stream_create_without_name_returns_error() { var streamManager = new StreamManager(); var resp = streamManager.CreateOrUpdate(new StreamConfig { Name = "" }); resp.Error.ShouldNotBeNull(); resp.Error!.Description.ShouldContain("name"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160 // --------------------------------------------------------------- [Fact] public async Task Unknown_api_subject_returns_404() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); var resp = await fx.RequestAsync("$JS.API.UNKNOWN.SUBJECT", "{}"); resp.Error.ShouldNotBeNull(); resp.Error!.Code.ShouldBe(404); } // --------------------------------------------------------------- // Go: TestJetStreamClusterAccountPurge server/jetstream_cluster_1_test.go:3891 // --------------------------------------------------------------- [Fact] public async Task Account_purge_via_meta_returns_success() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); await fx.CreateStreamAsync("P", ["p.>"], replicas: 3); var resp = await fx.RequestAsync($"{JetStreamApiSubjects.AccountPurge}GLOBAL", "{}"); resp.Success.ShouldBeTrue(); } // --------------------------------------------------------------- // Go: TestJetStreamClusterServerRemove server/jetstream_cluster_1_test.go:3620 // --------------------------------------------------------------- [Fact] public async Task Server_remove_via_meta_returns_success() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); var resp = await fx.RequestAsync(JetStreamApiSubjects.ServerRemove, "{}"); resp.Success.ShouldBeTrue(); } // --------------------------------------------------------------- // Go: TestJetStreamClusterAccountStreamMove server/jetstream_cluster_1_test.go:3750 // --------------------------------------------------------------- [Fact] public async Task Account_stream_move_via_meta_returns_success() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); var resp = await fx.RequestAsync($"{JetStreamApiSubjects.AccountStreamMove}TEST", "{}"); resp.Success.ShouldBeTrue(); } // --------------------------------------------------------------- // Go: TestJetStreamClusterAccountStreamMoveCancel server/jetstream_cluster_1_test.go:3780 // --------------------------------------------------------------- [Fact] public async Task Account_stream_move_cancel_via_meta_returns_success() { await using var fx = await MetaControllerFixture.StartAsync(nodes: 3); var resp = await fx.RequestAsync($"{JetStreamApiSubjects.AccountStreamMoveCancel}TEST", "{}"); resp.Success.ShouldBeTrue(); } } /// /// Self-contained fixture for JetStream meta controller tests. /// internal sealed class MetaControllerFixture : IAsyncDisposable { private readonly JetStreamMetaGroup _metaGroup; private readonly StreamManager _streamManager; private readonly ConsumerManager _consumerManager; private readonly JetStreamApiRouter _router; private readonly JetStreamPublisher _publisher; private MetaControllerFixture( JetStreamMetaGroup metaGroup, StreamManager streamManager, ConsumerManager consumerManager, JetStreamApiRouter router, JetStreamPublisher publisher) { _metaGroup = metaGroup; _streamManager = streamManager; _consumerManager = consumerManager; _router = router; _publisher = publisher; } public static Task StartAsync(int nodes) { var meta = new JetStreamMetaGroup(nodes); var consumerManager = new ConsumerManager(meta); var streamManager = new StreamManager(meta, consumerManager: consumerManager); var router = new JetStreamApiRouter(streamManager, consumerManager, meta); var publisher = new JetStreamPublisher(streamManager); return Task.FromResult(new MetaControllerFixture(meta, streamManager, consumerManager, router, publisher)); } public Task CreateStreamAsync(string name, string[] subjects, int replicas) { var response = _streamManager.CreateOrUpdate(new StreamConfig { Name = name, Subjects = [.. subjects], Replicas = replicas, }); return Task.FromResult(response); } public Task CreateConsumerAsync(string stream, string durableName) { return Task.FromResult(_consumerManager.CreateOrUpdate(stream, new ConsumerConfig { DurableName = durableName, })); } public MetaGroupState GetMetaState() => _metaGroup.GetState(); public Task RequestAsync(string subject, string payload) { var response = _router.Route(subject, Encoding.UTF8.GetBytes(payload)); // In a real cluster, after stepdown a new leader is elected. // Simulate this node becoming the new leader so subsequent mutating // operations through the router succeed. if (subject.Equals(JetStreamApiSubjects.MetaLeaderStepdown, StringComparison.Ordinal) && response.Success) _metaGroup.BecomeLeader(); return Task.FromResult(response); } public ValueTask DisposeAsync() => ValueTask.CompletedTask; }