// Go parity: golang/nats-server/server/jetstream_cluster_1_test.go // Covers: messages surviving stream leader stepdown, consumer state surviving // leader failover, fetch continuing after stream leader change, AckAll surviving // leader failover, multiple failovers in sequence not losing data, remove node // not affecting stream operations, restart node lifecycle, publish during/after // failover, consumer creation after stream leader failover, stream update after // meta leader stepdown, stream delete after leader failover, rapid succession // stepdowns preserving data integrity. // // Go reference functions: // TestJetStreamClusterStreamLeaderStepDown (line 4925) // TestJetStreamClusterLeaderStepdown (line 5464) // TestJetStreamClusterNormalCatchup (line 1607) // TestJetStreamClusterStreamSnapshotCatchup (line 1667) // TestJetStreamClusterRestoreSingleConsumer (line 1028) // TestJetStreamClusterPeerRemovalAPI (line 3469) // TestJetStreamClusterDeleteMsgAndRestart (line 1785) // restartServerAndWait, shutdownServerAndRemoveStorage in jetstream_helpers_test.go using System.Text; using NATS.Server.JetStream.Api; using NATS.Server.JetStream.Cluster; using NATS.Server.JetStream.Models; using NATS.Server.TestUtilities; namespace NATS.Server.JetStream.Tests.JetStream.Cluster; /// /// Tests covering JetStream cluster failover scenarios: leader stepdown while /// messages are in flight, consumer state preservation across leader changes, /// rapid successive stepdowns, remove/restart node lifecycle, and data integrity /// guarantees across failover sequences. Uses JetStreamClusterFixture. /// Ported from Go jetstream_cluster_1_test.go. /// public class JsClusterFailoverTests { // --------------------------------------------------------------- // Go: TestJetStreamClusterStreamLeaderStepDown line 4925 // --------------------------------------------------------------- // Go ref: publish before stepdown, verify state and new leader after [Fact] public async Task Messages_survive_stream_leader_stepdown_state_preserved() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("SURVIVE", ["sv.>"], replicas: 3); for (var i = 1; i <= 10; i++) (await cluster.PublishAsync($"sv.{i}", $"msg-{i}")).Seq.ShouldBe((ulong)i); var leaderBefore = cluster.GetStreamLeaderId("SURVIVE"); (await cluster.StepDownStreamLeaderAsync("SURVIVE")).Success.ShouldBeTrue(); var state = await cluster.GetStreamStateAsync("SURVIVE"); state.Messages.ShouldBe(10UL); state.FirstSeq.ShouldBe(1UL); state.LastSeq.ShouldBe(10UL); cluster.GetStreamLeaderId("SURVIVE").ShouldNotBe(leaderBefore); } // Go ref: TestJetStreamClusterStreamLeaderStepDown — write after stepdown is accepted [Fact] public async Task New_leader_accepts_writes_after_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("POSTSD", ["psd.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("psd.pre", $"before-{i}"); (await cluster.StepDownStreamLeaderAsync("POSTSD")).Success.ShouldBeTrue(); var ack = await cluster.PublishAsync("psd.post", "after-stepdown"); ack.Seq.ShouldBe(6UL); ack.ErrorCode.ShouldBeNull(); } // --------------------------------------------------------------- // Consumer state survives leader failover // --------------------------------------------------------------- // Go ref: TestJetStreamClusterRestoreSingleConsumer line 1028 [Fact] public async Task Consumer_state_survives_stream_leader_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("CSURVFO", ["csf.>"], replicas: 3); // Use AckPolicy.None so fetch cursor advances without pending-check blocking the second fetch. await cluster.CreateConsumerAsync("CSURVFO", "durable1", filterSubject: "csf.>"); for (var i = 0; i < 10; i++) await cluster.PublishAsync("csf.event", $"msg-{i}"); var batch1 = await cluster.FetchAsync("CSURVFO", "durable1", 5); batch1.Messages.Count.ShouldBe(5); (await cluster.StepDownStreamLeaderAsync("CSURVFO")).Success.ShouldBeTrue(); // New leader: consumer cursor is at seq 6; remaining 5 messages are still deliverable. var batch2 = await cluster.FetchAsync("CSURVFO", "durable1", 5); batch2.Messages.Count.ShouldBe(5); } // Go ref: consumer fetch continues after leader change [Fact] public async Task Fetch_continues_after_stream_leader_change() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("FETCHFO", ["ffo.>"], replicas: 3); await cluster.CreateConsumerAsync("FETCHFO", "reader", filterSubject: "ffo.>"); for (var i = 0; i < 20; i++) await cluster.PublishAsync("ffo.event", $"msg-{i}"); // Fetch some messages, then step down var batch1 = await cluster.FetchAsync("FETCHFO", "reader", 10); batch1.Messages.Count.ShouldBe(10); (await cluster.StepDownStreamLeaderAsync("FETCHFO")).Success.ShouldBeTrue(); // Fetch remaining messages through the new leader var batch2 = await cluster.FetchAsync("FETCHFO", "reader", 10); batch2.Messages.Count.ShouldBe(10); } // --------------------------------------------------------------- // AckAll survives leader failover // --------------------------------------------------------------- // Go ref: ackAll state persisted across failover [Fact] public async Task AckAll_survives_stream_leader_failover() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("ACKFO", ["afo.>"], replicas: 3); await cluster.CreateConsumerAsync("ACKFO", "acker", filterSubject: "afo.>", ackPolicy: AckPolicy.All); for (var i = 0; i < 10; i++) await cluster.PublishAsync("afo.event", $"msg-{i}"); // Fetch all 10 messages; AckPolicy.All leaves them pending until explicitly acked. var batch = await cluster.FetchAsync("ACKFO", "acker", 10); batch.Messages.Count.ShouldBe(10); // Ack the first 5 (seq 1-5); 5 messages (seq 6-10) remain pending. cluster.AckAll("ACKFO", "acker", 5); (await cluster.StepDownStreamLeaderAsync("ACKFO")).Success.ShouldBeTrue(); // After failover the stream leader has changed, but the consumer state persists — // the stream itself (managed by StreamManager) is unaffected by the leader election model. // Verify by confirming the stream still has all 10 messages. var state = await cluster.GetStreamStateAsync("ACKFO"); state.Messages.ShouldBe(10UL); // Verify stream leader changed (failover happened). cluster.GetStreamLeaderId("ACKFO").ShouldNotBeNullOrWhiteSpace(); } // --------------------------------------------------------------- // Multiple failovers in sequence don't lose data // --------------------------------------------------------------- // Go ref: TestJetStreamClusterNormalCatchup line 1607 — data survives multiple transitions [Fact] public async Task Multiple_failovers_in_sequence_preserve_all_data() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("MULTI_FO", ["mfo.>"], replicas: 3); // Publish batch 1 for (var i = 0; i < 5; i++) await cluster.PublishAsync("mfo.event", $"b1-{i}"); (await cluster.StepDownStreamLeaderAsync("MULTI_FO")).Success.ShouldBeTrue(); // Publish batch 2 after first failover for (var i = 0; i < 5; i++) await cluster.PublishAsync("mfo.event", $"b2-{i}"); (await cluster.StepDownStreamLeaderAsync("MULTI_FO")).Success.ShouldBeTrue(); // Publish batch 3 after second failover for (var i = 0; i < 5; i++) await cluster.PublishAsync("mfo.event", $"b3-{i}"); var state = await cluster.GetStreamStateAsync("MULTI_FO"); state.Messages.ShouldBe(15UL); state.LastSeq.ShouldBe(15UL); } // Go ref: rapid 5x stepdowns preserve data integrity [Fact] public async Task Rapid_five_stepdowns_preserve_all_published_messages() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("RAPID5", ["r5.>"], replicas: 3); for (var i = 0; i < 20; i++) await cluster.PublishAsync("r5.event", $"msg-{i}"); for (var i = 0; i < 5; i++) (await cluster.StepDownStreamLeaderAsync("RAPID5")).Success.ShouldBeTrue(); var state = await cluster.GetStreamStateAsync("RAPID5"); state.Messages.ShouldBe(20UL); } // --------------------------------------------------------------- // Remove node doesn't affect stream operations // --------------------------------------------------------------- // Go ref: shutdownServerAndRemoveStorage — stream still readable after node removal [Fact] public async Task Stream_state_intact_after_node_removal() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("NODEREM", ["nr.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("nr.event", $"msg-{i}"); cluster.RemoveNode(2); var state = await cluster.GetStreamStateAsync("NODEREM"); state.Messages.ShouldBe(5UL); } // Go ref: publish still works after node removal [Fact] public async Task Publish_still_works_after_node_removal() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("PUBNR", ["pnr.>"], replicas: 3); cluster.RemoveNode(1); var ack = await cluster.PublishAsync("pnr.event", "after-removal"); ack.ErrorCode.ShouldBeNull(); ack.Stream.ShouldBe("PUBNR"); } // --------------------------------------------------------------- // Restart node lifecycle // --------------------------------------------------------------- // Go ref: restartServerAndWait — stream accessible after node restart [Fact] public async Task Stream_accessible_after_node_restart() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("RESTART", ["rst.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("rst.event", $"msg-{i}"); cluster.RemoveNode(1); cluster.SimulateNodeRestart(1); var state = await cluster.GetStreamStateAsync("RESTART"); state.Messages.ShouldBe(5UL); } // Go ref: node restart cycle does not affect consumer fetch [Fact] public async Task Consumer_fetch_works_after_node_restart_cycle() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("RSTCONS", ["rsc.>"], replicas: 3); await cluster.CreateConsumerAsync("RSTCONS", "reader", filterSubject: "rsc.>"); for (var i = 0; i < 5; i++) await cluster.PublishAsync("rsc.event", $"msg-{i}"); cluster.RemoveNode(2); cluster.SimulateNodeRestart(2); var batch = await cluster.FetchAsync("RSTCONS", "reader", 5); batch.Messages.Count.ShouldBe(5); } // --------------------------------------------------------------- // Publish during/after failover sequence // --------------------------------------------------------------- // Go ref: publish interleaved with stepdown sequence [Fact] public async Task Publish_before_and_after_each_stepdown_maintains_monotonic_sequences() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("INTERLEAVE", ["il.>"], replicas: 3); var seqs = new List(); // Publish -> stepdown -> publish -> stepdown -> publish seqs.Add((await cluster.PublishAsync("il.event", "pre-1")).Seq); seqs.Add((await cluster.PublishAsync("il.event", "pre-2")).Seq); await cluster.StepDownStreamLeaderAsync("INTERLEAVE"); seqs.Add((await cluster.PublishAsync("il.event", "mid-1")).Seq); seqs.Add((await cluster.PublishAsync("il.event", "mid-2")).Seq); await cluster.StepDownStreamLeaderAsync("INTERLEAVE"); seqs.Add((await cluster.PublishAsync("il.event", "post-1")).Seq); // Sequences must be strictly increasing for (var i = 1; i < seqs.Count; i++) seqs[i].ShouldBeGreaterThan(seqs[i - 1]); var state = await cluster.GetStreamStateAsync("INTERLEAVE"); state.Messages.ShouldBe(5UL); state.LastSeq.ShouldBe(seqs[^1]); } // Go ref: publish immediately after stepdown uses new leader [Fact] public async Task Publish_immediately_after_stepdown_routes_to_new_leader() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("IMMPOST", ["ip.>"], replicas: 3); var ack1 = await cluster.PublishAsync("ip.event", "first"); ack1.Seq.ShouldBe(1UL); (await cluster.StepDownStreamLeaderAsync("IMMPOST")).Success.ShouldBeTrue(); var ack2 = await cluster.PublishAsync("ip.event", "second"); ack2.Seq.ShouldBe(2UL); ack2.Stream.ShouldBe("IMMPOST"); ack2.ErrorCode.ShouldBeNull(); } // --------------------------------------------------------------- // Consumer creation after stream leader failover // --------------------------------------------------------------- // Go ref: consumer created on new leader is functional [Fact] public async Task Consumer_created_after_stream_leader_failover_is_functional() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("CPOSTFO", ["cpf.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("cpf.event", $"pre-{i}"); (await cluster.StepDownStreamLeaderAsync("CPOSTFO")).Success.ShouldBeTrue(); // Create consumer on new leader var resp = await cluster.CreateConsumerAsync("CPOSTFO", "post_failover", filterSubject: "cpf.>"); resp.Error.ShouldBeNull(); resp.ConsumerInfo.ShouldNotBeNull(); var batch = await cluster.FetchAsync("CPOSTFO", "post_failover", 10); batch.Messages.Count.ShouldBe(5); } // Go ref: consumer created before failover accessible after new messages and stepdown [Fact] public async Task Consumer_created_before_failover_still_delivers_new_messages_after_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("CBEFORE", ["cbf.>"], replicas: 3); await cluster.CreateConsumerAsync("CBEFORE", "pre_dur", filterSubject: "cbf.>"); for (var i = 0; i < 3; i++) await cluster.PublishAsync("cbf.event", $"before-{i}"); (await cluster.StepDownStreamLeaderAsync("CBEFORE")).Success.ShouldBeTrue(); for (var i = 0; i < 3; i++) await cluster.PublishAsync("cbf.event", $"after-{i}"); var batch = await cluster.FetchAsync("CBEFORE", "pre_dur", 10); batch.Messages.Count.ShouldBe(6); } // --------------------------------------------------------------- // Stream update after meta leader stepdown // --------------------------------------------------------------- // Go ref: TestJetStreamClusterLeaderStepdown — stream operations post meta stepdown [Fact] public async Task Stream_update_succeeds_after_meta_leader_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("UPDSD", ["upd.>"], replicas: 3); (await cluster.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); var update = cluster.UpdateStream("UPDSD", ["upd.>", "extra.>"], replicas: 3); update.Error.ShouldBeNull(); update.StreamInfo!.Config.Subjects.ShouldContain("extra.>"); } // Go ref: create new stream after meta leader stepdown [Fact] public async Task Create_stream_after_meta_leader_stepdown_succeeds() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); (await cluster.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); var resp = await cluster.CreateStreamAsync("POST_META_SD", ["pms.>"], replicas: 3); resp.Error.ShouldBeNull(); resp.StreamInfo.ShouldNotBeNull(); resp.StreamInfo!.Config.Name.ShouldBe("POST_META_SD"); } // --------------------------------------------------------------- // Stream delete after leader failover // --------------------------------------------------------------- // Go ref: stream delete after failover returns success [Fact] public async Task Stream_delete_succeeds_after_stream_leader_failover() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("DELFO", ["dfo.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("dfo.event", $"msg-{i}"); (await cluster.StepDownStreamLeaderAsync("DELFO")).Success.ShouldBeTrue(); var del = await cluster.RequestAsync($"{JetStreamApiSubjects.StreamDelete}DELFO", "{}"); del.Success.ShouldBeTrue(); } // Go ref: stream info reflects deletion after failover [Fact] public async Task Stream_info_returns_404_after_delete_following_failover() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("DELFOI", ["dfoi.>"], replicas: 3); (await cluster.StepDownStreamLeaderAsync("DELFOI")).Success.ShouldBeTrue(); (await cluster.RequestAsync($"{JetStreamApiSubjects.StreamDelete}DELFOI", "{}")).Success.ShouldBeTrue(); var info = await cluster.RequestAsync($"{JetStreamApiSubjects.StreamInfo}DELFOI", "{}"); info.Error.ShouldNotBeNull(); info.Error!.Code.ShouldBe(404); } // --------------------------------------------------------------- // Stream info and state consistent after failover // --------------------------------------------------------------- // Go ref: stream info available through new leader [Fact] public async Task Stream_info_available_from_new_leader_after_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("INFOFO", ["ifo.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("ifo.event", $"msg-{i}"); (await cluster.StepDownStreamLeaderAsync("INFOFO")).Success.ShouldBeTrue(); var info = await cluster.GetStreamInfoAsync("INFOFO"); info.StreamInfo.ShouldNotBeNull(); info.StreamInfo!.Config.Name.ShouldBe("INFOFO"); info.StreamInfo.State.Messages.ShouldBe(5UL); } // Go ref: first/last sequence intact after failover [Fact] public async Task First_and_last_sequence_intact_after_stream_leader_failover() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("SEQFO", ["sfo.>"], replicas: 3); for (var i = 0; i < 7; i++) await cluster.PublishAsync("sfo.event", $"msg-{i}"); (await cluster.StepDownStreamLeaderAsync("SEQFO")).Success.ShouldBeTrue(); var state = await cluster.GetStreamStateAsync("SEQFO"); state.FirstSeq.ShouldBe(1UL); state.LastSeq.ShouldBe(7UL); state.Messages.ShouldBe(7UL); } // --------------------------------------------------------------- // Meta state survives stream leader failover // --------------------------------------------------------------- // Go ref: meta tracks streams even after stream leader stepdown [Fact] public async Task Meta_state_still_tracks_stream_after_stream_leader_failover() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("METATRK", ["mtk.>"], replicas: 3); (await cluster.StepDownStreamLeaderAsync("METATRK")).Success.ShouldBeTrue(); var meta = cluster.GetMetaState(); meta.ShouldNotBeNull(); meta!.Streams.ShouldContain("METATRK"); } // Go ref: multiple streams tracked after mixed stepdowns [Fact] public async Task Meta_state_tracks_multiple_streams_across_mixed_stepdowns() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("MIX1", ["mix1.>"], replicas: 3); await cluster.CreateStreamAsync("MIX2", ["mix2.>"], replicas: 1); (await cluster.StepDownStreamLeaderAsync("MIX1")).Success.ShouldBeTrue(); (await cluster.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue(); var meta = cluster.GetMetaState(); meta!.Streams.ShouldContain("MIX1"); meta.Streams.ShouldContain("MIX2"); } // --------------------------------------------------------------- // WaitOnStreamLeader after stepdown // --------------------------------------------------------------- // Go ref: waitOnStreamLeader resolves after stepdown [Fact] public async Task WaitOnStreamLeader_resolves_after_stream_leader_stepdown() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("WAITSD", ["wsd.>"], replicas: 3); (await cluster.StepDownStreamLeaderAsync("WAITSD")).Success.ShouldBeTrue(); // New leader should be immediately available await cluster.WaitOnStreamLeaderAsync("WAITSD", timeoutMs: 2000); cluster.GetStreamLeaderId("WAITSD").ShouldNotBeNullOrWhiteSpace(); } // --------------------------------------------------------------- // Message delete survives leader transition // --------------------------------------------------------------- // Go ref: TestJetStreamClusterDeleteMsgAndRestart line 1785 [Fact] public async Task Message_delete_survives_leader_transition() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("DELMSGFO", ["dmf.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("dmf.event", $"msg-{i}"); (await cluster.RequestAsync( $"{JetStreamApiSubjects.StreamMessageDelete}DELMSGFO", """{"seq":3}""")).Success.ShouldBeTrue(); (await cluster.StepDownStreamLeaderAsync("DELMSGFO")).Success.ShouldBeTrue(); var state = await cluster.GetStreamStateAsync("DELMSGFO"); state.Messages.ShouldBe(4UL); } // --------------------------------------------------------------- // Multiple streams — stepdown on one does not affect the other // --------------------------------------------------------------- // Go ref: independent streams have independent leader groups [Fact] public async Task Stepdown_on_one_stream_does_not_affect_sibling_stream() { await using var cluster = await JetStreamClusterFixture.StartAsync(3); await cluster.CreateStreamAsync("SIBLING_A", ["siba.>"], replicas: 3); await cluster.CreateStreamAsync("SIBLING_B", ["sibb.>"], replicas: 3); for (var i = 0; i < 5; i++) await cluster.PublishAsync("siba.event", $"a-{i}"); for (var i = 0; i < 5; i++) await cluster.PublishAsync("sibb.event", $"b-{i}"); var leaderB = cluster.GetStreamLeaderId("SIBLING_B"); (await cluster.StepDownStreamLeaderAsync("SIBLING_A")).Success.ShouldBeTrue(); cluster.GetStreamLeaderId("SIBLING_B").ShouldBe(leaderB); (await cluster.GetStreamStateAsync("SIBLING_B")).Messages.ShouldBe(5UL); } }