// Go parity: golang/nats-server/server/jetstream_cluster_1_test.go // Covers: meta-leader election (3-node and 5-node clusters), stream leader // selection (R1 and R3), consumer leader selection, leader ID non-empty checks, // meta stepdown producing new leader, stream stepdown producing new leader, // multiple stepdowns cycling through different leaders, leader ID consistency, // meta state reflecting correct cluster size and leadership version increments, // and meta state tracking all created streams. // // Go reference functions: // TestJetStreamClusterLeader (line 73) // TestJetStreamClusterStreamLeaderStepDown (line 4925) // TestJetStreamClusterLeaderStepdown (line 5464) // TestJetStreamClusterMultiReplicaStreams (line 299) // waitOnStreamLeader, waitOnConsumerLeader, c.leader in jetstream_helpers_test.go using System.Text; using NATS.Server.JetStream.Api; using NATS.Server.JetStream.Cluster; using NATS.Server.JetStream.Models; using NATS.Server.TestUtilities; namespace NATS.Server.JetStream.Tests.JetStream.Cluster; /// /// Tests covering JetStream cluster leader election for the meta-cluster, /// streams, and consumers. Uses the unified JetStreamClusterFixture. /// Ported from Go jetstream_cluster_1_test.go. /// public class JsClusterLeaderElectionTests { // --------------------------------------------------------------- // Go: TestJetStreamClusterLeader line 73 — meta leader election // --------------------------------------------------------------- // Go ref: c.leader() in jetstream_helpers_test.go [Fact] public async Task Three_node_cluster_elects_nonempty_meta_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var leader = cluster.GetMetaLeaderId(); leader.ShouldNotBeNullOrWhiteSpace(); } // Go ref: c.leader() in jetstream_helpers_test.go [Fact] public async Task Five_node_cluster_elects_nonempty_meta_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(5); var leader = cluster.GetMetaLeaderId(); leader.ShouldNotBeNullOrWhiteSpace(); } // Go ref: checkClusterFormed — meta cluster size is equal to node count [Fact] public async Task Three_node_cluster_meta_state_reports_correct_size() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var state = cluster.GetMetaState(); state.ShouldNotBeNull(); state!.ClusterSize.ShouldBe(3); } // Go ref: checkClusterFormed — meta cluster size is equal to node count [Fact] public async Task Five_node_cluster_meta_state_reports_correct_size() { await using var cluster = await JetStreamClusterFixture.StartAsync(5); var state = cluster.GetMetaState(); state.ShouldNotBeNull(); state!.ClusterSize.ShouldBe(5); } // Go ref: TestJetStreamClusterLeader — initial leadership version is 1 [Fact] public async Task Three_node_cluster_initial_leadership_version_is_one() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var state = cluster.GetMetaState(); state!.LeadershipVersion.ShouldBe(1); } // --------------------------------------------------------------- // Stream leader selection — R1 // --------------------------------------------------------------- // Go ref: streamLeader in jetstream_helpers_test.go [Fact] public async Task R1_stream_has_nonempty_leader_after_creation() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("R1ELECT", ["r1e.>"], replicas: 1); var leader = cluster.GetStreamLeaderId("R1ELECT"); leader.ShouldNotBeNullOrWhiteSpace(); } // Go ref: streamLeader in jetstream_helpers_test.go [Fact] public async Task R3_stream_has_nonempty_leader_after_creation_in_3_node_cluster() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("R3ELECT", ["r3e.>"], replicas: 3); var leader = cluster.GetStreamLeaderId("R3ELECT"); leader.ShouldNotBeNullOrWhiteSpace(); } // Go ref: streamLeader in jetstream_helpers_test.go [Fact] public async Task R3_stream_has_nonempty_leader_after_creation_in_5_node_cluster() { await using var cluster = await JetStreamClusterFixture.StartAsync(5); await cluster.CreateStreamAsync("R3E5", ["r3e5.>"], replicas: 3); var leader = cluster.GetStreamLeaderId("R3E5"); leader.ShouldNotBeNullOrWhiteSpace(); } // --------------------------------------------------------------- // Go: waitOnStreamLeader in jetstream_helpers_test.go // --------------------------------------------------------------- [Fact] public async Task WaitOnStreamLeader_completes_immediately_when_stream_already_has_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("WAITLDR", ["wl.>"], replicas: 3); await cluster.WaitOnStreamLeaderAsync("WAITLDR", timeoutMs: 2000); cluster.GetStreamLeaderId("WAITLDR").ShouldNotBeNullOrWhiteSpace(); } [Fact] public async Task WaitOnStreamLeader_throws_timeout_for_nonexistent_stream() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var ex = await Should.ThrowAsync( () => cluster.WaitOnStreamLeaderAsync("GHOST", timeoutMs: 100)); ex.Message.ShouldContain("GHOST"); } // --------------------------------------------------------------- // Consumer leader selection // --------------------------------------------------------------- // Go ref: consumerLeader in jetstream_helpers_test.go [Fact] public async Task Durable_consumer_on_R3_stream_has_nonempty_leader_id() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("CLELECT", ["cle.>"], replicas: 3); await cluster.CreateConsumerAsync("CLELECT", "dlc"); var leader = cluster.GetConsumerLeaderId("CLELECT", "dlc"); leader.ShouldNotBeNullOrWhiteSpace(); } // Go ref: consumerLeader in jetstream_helpers_test.go [Fact] public async Task Durable_consumer_on_R1_stream_has_nonempty_leader_id() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("CLELECTR1", ["cler1.>"], replicas: 1); await cluster.CreateConsumerAsync("CLELECTR1", "consumer1"); var leader = cluster.GetConsumerLeaderId("CLELECTR1", "consumer1"); leader.ShouldNotBeNullOrWhiteSpace(); } // Go ref: waitOnConsumerLeader in jetstream_helpers_test.go [Fact] public async Task WaitOnConsumerLeader_completes_when_consumer_exists() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("WCLE", ["wcle.>"], replicas: 3); await cluster.CreateConsumerAsync("WCLE", "dur1"); await cluster.WaitOnConsumerLeaderAsync("WCLE", "dur1", timeoutMs: 2000); cluster.GetConsumerLeaderId("WCLE", "dur1").ShouldNotBeNullOrWhiteSpace(); } [Fact] public async Task WaitOnConsumerLeader_throws_timeout_when_consumer_missing() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("WCLETOUT", ["wclet.>"], replicas: 3); var ex = await Should.ThrowAsync( () => cluster.WaitOnConsumerLeaderAsync("WCLETOUT", "ghost-consumer", timeoutMs: 100)); ex.Message.ShouldContain("ghost-consumer"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterLeaderStepdown line 5464 — meta leader stepdown // --------------------------------------------------------------- // Go ref: c.leader().Shutdown() + waitOnLeader in jetstream_helpers_test.go [Fact] public async Task Meta_leader_stepdown_produces_different_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var before = cluster.GetMetaLeaderId(); cluster.StepDownMetaLeader(); var after = cluster.GetMetaLeaderId(); after.ShouldNotBe(before); after.ShouldNotBeNullOrWhiteSpace(); } // Go ref: meta stepdown via API subject $JS.API.META.LEADER.STEPDOWN [Fact] public async Task Meta_leader_stepdown_via_api_returns_success() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var resp = await cluster.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}"); resp.Success.ShouldBeTrue(); } // Go ref: meta step-down increments leadership version [Fact] public async Task Meta_leader_stepdown_increments_leadership_version() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var versionBefore = cluster.GetMetaState()!.LeadershipVersion; cluster.StepDownMetaLeader(); var versionAfter = cluster.GetMetaState()!.LeadershipVersion; versionAfter.ShouldBe(versionBefore + 1); } // Go ref: multiple meta step-downs each increment the version [Fact] public async Task Multiple_meta_stepdowns_increment_leadership_version_sequentially() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); cluster.StepDownMetaLeader(); cluster.StepDownMetaLeader(); cluster.StepDownMetaLeader(); cluster.GetMetaState()!.LeadershipVersion.ShouldBe(4); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamLeaderStepDown line 4925 — stream leader stepdown // --------------------------------------------------------------- // Go ref: JSApiStreamLeaderStepDownT in jetstream_helpers_test.go [Fact] public async Task Stream_leader_stepdown_produces_different_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("SLEADSD", ["sls.>"], replicas: 3); var before = cluster.GetStreamLeaderId("SLEADSD"); var resp = await cluster.StepDownStreamLeaderAsync("SLEADSD"); resp.Success.ShouldBeTrue(); var after = cluster.GetStreamLeaderId("SLEADSD"); after.ShouldNotBe(before); after.ShouldNotBeNullOrWhiteSpace(); } // Go ref: TestJetStreamClusterStreamLeaderStepDown — new leader still accepts writes [Fact] public async Task Stream_leader_stepdown_new_leader_accepts_writes() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("SDWRITE", ["sdw.>"], replicas: 3); await cluster.PublishAsync("sdw.pre", "before"); await cluster.StepDownStreamLeaderAsync("SDWRITE"); var ack = await cluster.PublishAsync("sdw.post", "after"); ack.Stream.ShouldBe("SDWRITE"); ack.ErrorCode.ShouldBeNull(); } // --------------------------------------------------------------- // Multiple stepdowns cycle through different leaders // --------------------------------------------------------------- // Go ref: TestJetStreamClusterLeader line 73 — consecutive elections [Fact] public async Task Two_consecutive_stream_stepdowns_cycle_through_different_leaders() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("CYCLE2", ["cy2.>"], replicas: 3); var l0 = cluster.GetStreamLeaderId("CYCLE2"); (await cluster.StepDownStreamLeaderAsync("CYCLE2")).Success.ShouldBeTrue(); var l1 = cluster.GetStreamLeaderId("CYCLE2"); (await cluster.StepDownStreamLeaderAsync("CYCLE2")).Success.ShouldBeTrue(); var l2 = cluster.GetStreamLeaderId("CYCLE2"); l1.ShouldNotBe(l0); l2.ShouldNotBe(l1); } // Go ref: multiple stepdowns in sequence — each produces a distinct leader [Fact] public async Task Three_consecutive_meta_stepdowns_cycle_through_distinct_leaders() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var observed = new HashSet(); observed.Add(cluster.GetMetaLeaderId()); cluster.StepDownMetaLeader(); observed.Add(cluster.GetMetaLeaderId()); cluster.StepDownMetaLeader(); observed.Add(cluster.GetMetaLeaderId()); cluster.StepDownMetaLeader(); // With 3 nodes cycling round-robin we see at least 2 unique leaders observed.Count.ShouldBeGreaterThanOrEqualTo(2); } // Go ref: TestJetStreamClusterLeader — wraps around after exhausting peers [Fact] public async Task Meta_stepdowns_wrap_around_producing_only_node_count_unique_leaders() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var observed = new HashSet(); for (var i = 0; i < 9; i++) { observed.Add(cluster.GetMetaLeaderId()); cluster.StepDownMetaLeader(); } // 3-node cluster cycles through exactly 3 unique leader IDs observed.Count.ShouldBe(3); } // --------------------------------------------------------------- // Leader ID consistency // --------------------------------------------------------------- // Go ref: streamLeader queried multiple times returns same stable ID [Fact] public async Task Stream_leader_id_is_stable_across_repeated_queries_without_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("STABLE", ["stb.>"], replicas: 3); var ids = Enumerable.Range(0, 5) .Select(_ => cluster.GetStreamLeaderId("STABLE")) .ToList(); ids.Distinct().Count().ShouldBe(1); ids[0].ShouldNotBeNullOrWhiteSpace(); } // Go ref: meta leader queried multiple times is stable between stepdowns [Fact] public async Task Meta_leader_id_is_stable_between_stepdowns() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var a = cluster.GetMetaLeaderId(); var b = cluster.GetMetaLeaderId(); a.ShouldBe(b); cluster.StepDownMetaLeader(); var c = cluster.GetMetaLeaderId(); var d = cluster.GetMetaLeaderId(); c.ShouldBe(d); c.ShouldNotBe(a); } // --------------------------------------------------------------- // Meta state reflecting all created streams // --------------------------------------------------------------- // Go ref: getMetaState in tests — streams tracked in meta state [Fact] public async Task Meta_state_tracks_single_created_stream() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("MTRACK1", ["mt1.>"], replicas: 3); var state = cluster.GetMetaState(); state.ShouldNotBeNull(); state!.Streams.ShouldContain("MTRACK1"); } // Go ref: getMetaState tracks multiple streams [Fact] public async Task Meta_state_tracks_all_created_streams() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("MTRK_A", ["mta.>"], replicas: 3); await cluster.CreateStreamAsync("MTRK_B", ["mtb.>"], replicas: 3); await cluster.CreateStreamAsync("MTRK_C", ["mtc.>"], replicas: 1); var state = cluster.GetMetaState(); state!.Streams.ShouldContain("MTRK_A"); state.Streams.ShouldContain("MTRK_B"); state.Streams.ShouldContain("MTRK_C"); state.Streams.Count.ShouldBe(3); } // Go ref: meta state survives a stepdown [Fact] public async Task Meta_state_streams_survive_meta_leader_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("SURVSD1", ["ss1.>"], replicas: 3); await cluster.CreateStreamAsync("SURVSD2", ["ss2.>"], replicas: 3); cluster.StepDownMetaLeader(); var state = cluster.GetMetaState(); state!.Streams.ShouldContain("SURVSD1"); state.Streams.ShouldContain("SURVSD2"); } // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamLeaderStepDown — data survives leader election // --------------------------------------------------------------- // Go ref: TestJetStreamClusterStreamLeaderStepDown line 4925 — all messages preserved [Fact] public async Task Messages_survive_stream_leader_election() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("ELECT_DATA", ["ed.>"], replicas: 3); for (var i = 0; i < 10; i++) await cluster.PublishAsync("ed.event", $"msg-{i}"); await cluster.StepDownStreamLeaderAsync("ELECT_DATA"); var state = await cluster.GetStreamStateAsync("ELECT_DATA"); state.Messages.ShouldBe(10UL); } // --------------------------------------------------------------- // Replica group structure after election // --------------------------------------------------------------- // Go ref: replica group has correct node count [Fact] public async Task R3_stream_replica_group_has_three_nodes() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("RG3", ["rg3.>"], replicas: 3); var group = cluster.GetReplicaGroup("RG3"); group.ShouldNotBeNull(); group!.Nodes.Count.ShouldBe(3); } // Go ref: replica group leader is marked as leader [Fact] public async Task R3_stream_replica_group_leader_is_marked_as_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("RGLDR", ["rgl.>"], replicas: 3); var group = cluster.GetReplicaGroup("RGLDR"); group.ShouldNotBeNull(); group!.Leader.IsLeader.ShouldBeTrue(); } // Go ref: replica group for unknown stream is null [Fact] public async Task Replica_group_for_unknown_stream_is_null() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var group = cluster.GetReplicaGroup("NONEXISTENT"); group.ShouldBeNull(); } // --------------------------------------------------------------- // Leadership version increments on each stepdown // --------------------------------------------------------------- // Go ref: leadership version tracks stepdown count [Fact] public async Task Leadership_version_increments_on_each_meta_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); cluster.GetMetaState()!.LeadershipVersion.ShouldBe(1); cluster.StepDownMetaLeader(); cluster.GetMetaState()!.LeadershipVersion.ShouldBe(2); cluster.StepDownMetaLeader(); cluster.GetMetaState()!.LeadershipVersion.ShouldBe(3); cluster.StepDownMetaLeader(); cluster.GetMetaState()!.LeadershipVersion.ShouldBe(4); } // Go ref: meta leader stepdown via API also increments version [Fact] public async Task Meta_leader_stepdown_via_api_increments_leadership_version() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("VERSIONAPI", ["va.>"], replicas: 3); var vBefore = cluster.GetMetaState()!.LeadershipVersion; (await cluster.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); cluster.GetMetaState()!.LeadershipVersion.ShouldBe(vBefore + 1); } // --------------------------------------------------------------- // Consumer leader ID is consistent with stream // --------------------------------------------------------------- // Go ref: consumerLeader — consumer leader ID includes consumer name [Fact] public async Task Consumer_leader_ids_are_distinct_for_different_consumers_on_same_stream() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("MULTICONS", ["mc.>"], replicas: 3); await cluster.CreateConsumerAsync("MULTICONS", "consA"); await cluster.CreateConsumerAsync("MULTICONS", "consB"); var leaderA = cluster.GetConsumerLeaderId("MULTICONS", "consA"); var leaderB = cluster.GetConsumerLeaderId("MULTICONS", "consB"); leaderA.ShouldNotBeNullOrWhiteSpace(); leaderB.ShouldNotBeNullOrWhiteSpace(); leaderA.ShouldNotBe(leaderB); } // Go ref: consumer leader ID for unknown stream returns empty [Fact] public async Task Consumer_leader_id_for_unknown_stream_is_empty() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); var leader = cluster.GetConsumerLeaderId("NO_SUCH_STREAM", "no_consumer"); leader.ShouldBeNullOrEmpty(); } // --------------------------------------------------------------- // Node lifecycle helpers do not affect stream state // --------------------------------------------------------------- // Go ref: shutdownServerAndRemoveStorage + restartServerAndWait [Fact] public async Task RemoveNode_and_restart_does_not_affect_stream_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("LIFECYCLE", ["lc.>"], replicas: 3); var leaderBefore = cluster.GetStreamLeaderId("LIFECYCLE"); cluster.RemoveNode(2); cluster.SimulateNodeRestart(2); var leaderAfter = cluster.GetStreamLeaderId("LIFECYCLE"); leaderBefore.ShouldNotBeNullOrWhiteSpace(); leaderAfter.ShouldNotBeNullOrWhiteSpace(); } }