Files
natsdotnet/tests/NATS.Server.Tests/JetStream/Cluster/JetStreamMetaControllerTests.cs
Joseph Doherty d286349262 fix: re-enable leader check in API router and fix stepdown simulation
Re-enabled the leader check in JetStreamApiRouter.Route() that was
commented out during B8 agent work. Added BecomeLeader() method to
JetStreamMetaGroup for single-process test fixtures to simulate
winning the post-stepdown election. MetaControllerFixture now
auto-calls BecomeLeader() after a successful meta leader stepdown.
2026-02-24 18:02:54 -05:00

642 lines
24 KiB
C#

// Go parity: golang/nats-server/server/jetstream_cluster_1_test.go
// Covers: meta group leadership, API routing through meta leader,
// stream/consumer placement decisions, asset distribution,
// R1/R3 placement, preferred tags, cluster-wide operations.
using System.Collections.Concurrent;
using System.Reflection;
using System.Text;
using NATS.Server.JetStream;
using NATS.Server.JetStream.Api;
using NATS.Server.JetStream.Cluster;
using NATS.Server.JetStream.Consumers;
using NATS.Server.JetStream.Models;
using NATS.Server.JetStream.Publish;
namespace NATS.Server.Tests.JetStream.Cluster;
/// <summary>
/// Tests covering JetStream meta controller leadership, API routing through
/// the meta leader, stream/consumer placement decisions, asset distribution,
/// R1/R3 placement, and cluster-wide operations.
/// Ported from Go jetstream_cluster_1_test.go and jetstream_cluster_2_test.go.
/// </summary>
public class JetStreamMetaControllerTests
{
// ---------------------------------------------------------------
// Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73
// ---------------------------------------------------------------
[Fact]
public void Meta_group_initial_leader_is_meta_1()
{
var meta = new JetStreamMetaGroup(3);
var state = meta.GetState();
state.LeaderId.ShouldBe("meta-1");
state.ClusterSize.ShouldBe(3);
state.LeadershipVersion.ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73
// ---------------------------------------------------------------
[Fact]
public void Meta_group_stepdown_advances_leader_id()
{
var meta = new JetStreamMetaGroup(3);
meta.GetState().LeaderId.ShouldBe("meta-1");
meta.StepDown();
meta.GetState().LeaderId.ShouldBe("meta-2");
meta.StepDown();
meta.GetState().LeaderId.ShouldBe("meta-3");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73
// ---------------------------------------------------------------
[Fact]
public void Meta_group_stepdown_wraps_around_to_first_node()
{
var meta = new JetStreamMetaGroup(3);
meta.StepDown(); // meta-2
meta.StepDown(); // meta-3
meta.StepDown(); // meta-1 (wrap)
meta.GetState().LeaderId.ShouldBe("meta-1");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterLeader server/jetstream_cluster_1_test.go:73
// ---------------------------------------------------------------
[Fact]
public void Meta_group_leadership_version_increments_on_each_stepdown()
{
var meta = new JetStreamMetaGroup(3);
for (var i = 1; i <= 5; i++)
{
meta.GetState().LeadershipVersion.ShouldBe(i);
meta.StepDown();
}
meta.GetState().LeadershipVersion.ShouldBe(6);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterConfig server/jetstream_cluster_1_test.go:43
// ---------------------------------------------------------------
[Fact]
public async Task Meta_group_propose_creates_stream_record()
{
var meta = new JetStreamMetaGroup(3);
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "TEST" }, default);
var state = meta.GetState();
state.Streams.Count.ShouldBe(1);
state.Streams.ShouldContain("TEST");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299
// ---------------------------------------------------------------
[Fact]
public async Task Meta_group_tracks_multiple_stream_proposals()
{
var meta = new JetStreamMetaGroup(5);
for (var i = 0; i < 10; i++)
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = $"S{i}" }, default);
var state = meta.GetState();
state.Streams.Count.ShouldBe(10);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299
// ---------------------------------------------------------------
[Fact]
public async Task Meta_group_streams_are_sorted_alphabetically()
{
var meta = new JetStreamMetaGroup(3);
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "ZULU" }, default);
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "ALPHA" }, default);
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "MIKE" }, default);
var state = meta.GetState();
state.Streams[0].ShouldBe("ALPHA");
state.Streams[1].ShouldBe("MIKE");
state.Streams[2].ShouldBe("ZULU");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterConfig server/jetstream_cluster_1_test.go:43
// ---------------------------------------------------------------
[Fact]
public async Task Meta_group_duplicate_stream_proposal_is_idempotent()
{
var meta = new JetStreamMetaGroup(3);
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "DUP" }, default);
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = "DUP" }, default);
meta.GetState().Streams.Count.ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterExpandCluster server/jetstream_cluster_1_test.go:86
// ---------------------------------------------------------------
[Fact]
public void Meta_group_single_node_cluster_has_leader()
{
var meta = new JetStreamMetaGroup(1);
var state = meta.GetState();
state.ClusterSize.ShouldBe(1);
state.LeaderId.ShouldBe("meta-1");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterExpandCluster server/jetstream_cluster_1_test.go:86
// ---------------------------------------------------------------
[Fact]
public void Meta_group_single_node_stepdown_returns_to_same_leader()
{
var meta = new JetStreamMetaGroup(1);
meta.StepDown();
meta.GetState().LeaderId.ShouldBe("meta-1");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterLeaderStepdown server/jetstream_cluster_1_test.go:5464
// ---------------------------------------------------------------
[Fact]
public async Task Api_meta_leader_stepdown_changes_leader_and_preserves_streams()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("KEEPME", ["keep.>"], replicas: 3);
var before = fx.GetMetaState();
var leaderBefore = before.LeaderId;
var resp = await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}");
resp.Success.ShouldBeTrue();
var after = fx.GetMetaState();
after.LeaderId.ShouldNotBe(leaderBefore);
after.Streams.ShouldContain("KEEPME");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterAccountInfo server/jetstream_cluster_1_test.go:94
// ---------------------------------------------------------------
[Fact]
public async Task Api_routing_through_meta_leader_returns_account_info()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("A", ["a.>"], replicas: 3);
await fx.CreateStreamAsync("B", ["b.>"], replicas: 3);
await fx.CreateConsumerAsync("A", "c1");
var resp = await fx.RequestAsync(JetStreamApiSubjects.Info, "{}");
resp.AccountInfo.ShouldNotBeNull();
resp.AccountInfo!.Streams.ShouldBe(2);
resp.AccountInfo.Consumers.ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamLimitWithAccountDefaults server/jetstream_cluster_1_test.go:124
// ---------------------------------------------------------------
[Fact]
public async Task Placement_planner_r1_creates_single_node_placement()
{
var planner = new AssetPlacementPlanner(nodes: 5);
var placement = planner.PlanReplicas(replicas: 1);
placement.Count.ShouldBe(1);
placement[0].ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299
// ---------------------------------------------------------------
[Fact]
public void Placement_planner_r3_creates_three_node_placement()
{
var planner = new AssetPlacementPlanner(nodes: 5);
var placement = planner.PlanReplicas(replicas: 3);
placement.Count.ShouldBe(3);
placement[0].ShouldBe(1);
placement[1].ShouldBe(2);
placement[2].ShouldBe(3);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299
// ---------------------------------------------------------------
[Fact]
public void Placement_planner_caps_replicas_at_cluster_size()
{
var planner = new AssetPlacementPlanner(nodes: 3);
var placement = planner.PlanReplicas(replicas: 7);
placement.Count.ShouldBe(3);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMultiReplicaStreams server/jetstream_cluster_1_test.go:299
// ---------------------------------------------------------------
[Fact]
public void Placement_planner_negative_replicas_returns_one()
{
var planner = new AssetPlacementPlanner(nodes: 5);
var placement = planner.PlanReplicas(replicas: -1);
placement.Count.ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterConfig server/jetstream_cluster_1_test.go:43
// ---------------------------------------------------------------
[Fact]
public void Placement_planner_zero_nodes_returns_one()
{
var planner = new AssetPlacementPlanner(nodes: 0);
var placement = planner.PlanReplicas(replicas: 3);
placement.Count.ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160
// ---------------------------------------------------------------
[Fact]
public async Task Stream_create_via_meta_leader_sets_replica_group()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 5);
var resp = await fx.CreateStreamAsync("REPGRP", ["rg.>"], replicas: 3);
resp.Error.ShouldBeNull();
// The stream manager creates a replica group internally
var meta = fx.GetMetaState();
meta.Streams.ShouldContain("REPGRP");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMaxStreamsReached server/jetstream_cluster_1_test.go:3177
// ---------------------------------------------------------------
[Fact]
public async Task Multiple_stream_creates_all_tracked_in_meta_group()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
for (var i = 0; i < 20; i++)
await fx.CreateStreamAsync($"MS{i}", [$"ms{i}.>"], replicas: 3);
var meta = fx.GetMetaState();
meta.Streams.Count.ShouldBe(20);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamNames server/jetstream_cluster_1_test.go:1284
// ---------------------------------------------------------------
[Fact]
public async Task Stream_names_api_returns_all_streams_through_meta_leader()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("S1", ["s1.>"], replicas: 3);
await fx.CreateStreamAsync("S2", ["s2.>"], replicas: 1);
await fx.CreateStreamAsync("S3", ["s3.>"], replicas: 3);
var resp = await fx.RequestAsync(JetStreamApiSubjects.StreamNames, "{}");
resp.StreamNames.ShouldNotBeNull();
resp.StreamNames!.Count.ShouldBe(3);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterDelete server/jetstream_cluster_1_test.go:472
// ---------------------------------------------------------------
[Fact]
public async Task Stream_delete_removes_from_active_names()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("DEL1", ["d1.>"], replicas: 3);
await fx.CreateStreamAsync("DEL2", ["d2.>"], replicas: 3);
var del = await fx.RequestAsync($"{JetStreamApiSubjects.StreamDelete}DEL1", "{}");
del.Success.ShouldBeTrue();
var names = await fx.RequestAsync(JetStreamApiSubjects.StreamNames, "{}");
names.StreamNames!.Count.ShouldBe(1);
names.StreamNames.ShouldContain("DEL2");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterDoubleAdd server/jetstream_cluster_1_test.go:1551
// ---------------------------------------------------------------
[Fact]
public async Task Stream_create_idempotent_with_same_config()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
var first = await fx.CreateStreamAsync("IDEM", ["idem.>"], replicas: 3);
first.Error.ShouldBeNull();
var second = await fx.CreateStreamAsync("IDEM", ["idem.>"], replicas: 3);
second.Error.ShouldBeNull();
var meta = fx.GetMetaState();
meta.Streams.Count.ShouldBe(1);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamInfoList server/jetstream_cluster_1_test.go:1284
// ---------------------------------------------------------------
[Fact]
public async Task Consumer_create_tracked_in_cluster()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("CC", ["cc.>"], replicas: 3);
await fx.CreateConsumerAsync("CC", "d1");
await fx.CreateConsumerAsync("CC", "d2");
var names = await fx.RequestAsync($"{JetStreamApiSubjects.ConsumerNames}CC", "{}");
names.ConsumerNames.ShouldNotBeNull();
names.ConsumerNames!.Count.ShouldBe(2);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterPeerRemovalAPI server/jetstream_cluster_1_test.go:3469
// ---------------------------------------------------------------
[Fact]
public async Task Peer_removal_api_routed_through_meta()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("PR", ["pr.>"], replicas: 3);
var resp = await fx.RequestAsync($"{JetStreamApiSubjects.StreamPeerRemove}PR", """{"peer":"n2"}""");
resp.Success.ShouldBeTrue();
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMetaSnapshotsAndCatchup server/jetstream_cluster_1_test.go:833
// ---------------------------------------------------------------
[Fact]
public async Task Meta_state_preserved_across_multiple_stepdowns()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("M1", ["m1.>"], replicas: 3);
await fx.CreateStreamAsync("M2", ["m2.>"], replicas: 3);
(await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue();
(await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue();
(await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue();
var state = fx.GetMetaState();
state.Streams.ShouldContain("M1");
state.Streams.ShouldContain("M2");
state.LeadershipVersion.ShouldBe(4);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterMetaSnapshotsMultiChange server/jetstream_cluster_1_test.go:881
// ---------------------------------------------------------------
[Fact]
public async Task Create_and_delete_across_stepdowns_reflected_in_names()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("A", ["a.>"], replicas: 3);
(await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue();
await fx.CreateStreamAsync("B", ["b.>"], replicas: 3);
(await fx.RequestAsync($"{JetStreamApiSubjects.StreamDelete}A", "{}")).Success.ShouldBeTrue();
(await fx.RequestAsync(JetStreamApiSubjects.MetaLeaderStepdown, "{}")).Success.ShouldBeTrue();
var names = await fx.RequestAsync(JetStreamApiSubjects.StreamNames, "{}");
names.StreamNames!.Count.ShouldBe(1);
names.StreamNames.ShouldContain("B");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160
// ---------------------------------------------------------------
[Fact]
public async Task Stream_info_for_nonexistent_stream_returns_404()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
var resp = await fx.RequestAsync($"{JetStreamApiSubjects.StreamInfo}MISSING", "{}");
resp.Error.ShouldNotBeNull();
resp.Error!.Code.ShouldBe(404);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterConsumerCreate server/jetstream_cluster_1_test.go:700
// ---------------------------------------------------------------
[Fact]
public async Task Consumer_info_for_nonexistent_consumer_returns_404()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("NOCON", ["nc.>"], replicas: 3);
var resp = await fx.RequestAsync($"{JetStreamApiSubjects.ConsumerInfo}NOCON.MISSING", "{}");
resp.Error.ShouldNotBeNull();
resp.Error!.Code.ShouldBe(404);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160
// ---------------------------------------------------------------
[Fact]
public void Stream_create_without_name_returns_error()
{
var streamManager = new StreamManager();
var resp = streamManager.CreateOrUpdate(new StreamConfig { Name = "" });
resp.Error.ShouldNotBeNull();
resp.Error!.Description.ShouldContain("name");
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterStreamCreate server/jetstream_cluster_1_test.go:160
// ---------------------------------------------------------------
[Fact]
public async Task Unknown_api_subject_returns_404()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
var resp = await fx.RequestAsync("$JS.API.UNKNOWN.SUBJECT", "{}");
resp.Error.ShouldNotBeNull();
resp.Error!.Code.ShouldBe(404);
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterAccountPurge server/jetstream_cluster_1_test.go:3891
// ---------------------------------------------------------------
[Fact]
public async Task Account_purge_via_meta_returns_success()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
await fx.CreateStreamAsync("P", ["p.>"], replicas: 3);
var resp = await fx.RequestAsync($"{JetStreamApiSubjects.AccountPurge}GLOBAL", "{}");
resp.Success.ShouldBeTrue();
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterServerRemove server/jetstream_cluster_1_test.go:3620
// ---------------------------------------------------------------
[Fact]
public async Task Server_remove_via_meta_returns_success()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
var resp = await fx.RequestAsync(JetStreamApiSubjects.ServerRemove, "{}");
resp.Success.ShouldBeTrue();
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterAccountStreamMove server/jetstream_cluster_1_test.go:3750
// ---------------------------------------------------------------
[Fact]
public async Task Account_stream_move_via_meta_returns_success()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
var resp = await fx.RequestAsync($"{JetStreamApiSubjects.AccountStreamMove}TEST", "{}");
resp.Success.ShouldBeTrue();
}
// ---------------------------------------------------------------
// Go: TestJetStreamClusterAccountStreamMoveCancel server/jetstream_cluster_1_test.go:3780
// ---------------------------------------------------------------
[Fact]
public async Task Account_stream_move_cancel_via_meta_returns_success()
{
await using var fx = await MetaControllerFixture.StartAsync(nodes: 3);
var resp = await fx.RequestAsync($"{JetStreamApiSubjects.AccountStreamMoveCancel}TEST", "{}");
resp.Success.ShouldBeTrue();
}
}
/// <summary>
/// Self-contained fixture for JetStream meta controller tests.
/// </summary>
internal sealed class MetaControllerFixture : IAsyncDisposable
{
private readonly JetStreamMetaGroup _metaGroup;
private readonly StreamManager _streamManager;
private readonly ConsumerManager _consumerManager;
private readonly JetStreamApiRouter _router;
private readonly JetStreamPublisher _publisher;
private MetaControllerFixture(
JetStreamMetaGroup metaGroup,
StreamManager streamManager,
ConsumerManager consumerManager,
JetStreamApiRouter router,
JetStreamPublisher publisher)
{
_metaGroup = metaGroup;
_streamManager = streamManager;
_consumerManager = consumerManager;
_router = router;
_publisher = publisher;
}
public static Task<MetaControllerFixture> StartAsync(int nodes)
{
var meta = new JetStreamMetaGroup(nodes);
var consumerManager = new ConsumerManager(meta);
var streamManager = new StreamManager(meta, consumerManager: consumerManager);
var router = new JetStreamApiRouter(streamManager, consumerManager, meta);
var publisher = new JetStreamPublisher(streamManager);
return Task.FromResult(new MetaControllerFixture(meta, streamManager, consumerManager, router, publisher));
}
public Task<JetStreamApiResponse> CreateStreamAsync(string name, string[] subjects, int replicas)
{
var response = _streamManager.CreateOrUpdate(new StreamConfig
{
Name = name,
Subjects = [.. subjects],
Replicas = replicas,
});
return Task.FromResult(response);
}
public Task<JetStreamApiResponse> CreateConsumerAsync(string stream, string durableName)
{
return Task.FromResult(_consumerManager.CreateOrUpdate(stream, new ConsumerConfig
{
DurableName = durableName,
}));
}
public MetaGroupState GetMetaState() => _metaGroup.GetState();
public Task<JetStreamApiResponse> RequestAsync(string subject, string payload)
{
var response = _router.Route(subject, Encoding.UTF8.GetBytes(payload));
// In a real cluster, after stepdown a new leader is elected.
// Simulate this node becoming the new leader so subsequent mutating
// operations through the router succeed.
if (subject.Equals(JetStreamApiSubjects.MetaLeaderStepdown, StringComparison.Ordinal) && response.Success)
_metaGroup.BecomeLeader();
return Task.FromResult(response);
}
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}