// Go reference: jetstream_api.go:200-300 — API requests at non-leader nodes must be // forwarded to the current leader. Mutating operations return a not-leader error with // a leader_hint field; read-only operations are handled locally on any node. using System.Text; using NATS.Server.JetStream; using NATS.Server.JetStream.Api; using NATS.Server.JetStream.Cluster; namespace NATS.Server.JetStream.Tests.JetStream.Api; /// /// Simple test double for ILeaderForwarder. /// Returns a predetermined response or null depending on the constructor. /// file sealed class StubForwarder(JetStreamApiResponse? response) : ILeaderForwarder { public int CallCount { get; private set; } public string? LastSubject { get; private set; } public ReadOnlyMemory LastPayload { get; private set; } public string? LastLeaderName { get; private set; } public Task ForwardAsync( string subject, ReadOnlyMemory payload, string leaderName, CancellationToken ct) { CallCount++; LastSubject = subject; LastPayload = payload; LastLeaderName = leaderName; return Task.FromResult(response); } } /// /// Test double that throws OperationCanceledException to simulate a timeout. /// file sealed class TimeoutForwarder : ILeaderForwarder { public Task ForwardAsync( string subject, ReadOnlyMemory payload, string leaderName, CancellationToken ct) => Task.FromException(new OperationCanceledException("simulated timeout")); } public class LeaderForwardingTests { /// /// When this node IS the leader, mutating requests are handled locally. /// Go reference: jetstream_api.go — leader handles requests directly. /// [Fact] public void Route_WhenLeader_HandlesLocally() { // selfIndex=1 matches default leaderIndex=1, so this node is the leader. var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 1); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup); // Create a stream first so the purge has something to operate on. var createPayload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}"""); var createResult = router.Route("$JS.API.STREAM.CREATE.TEST", createPayload); createResult.Error.ShouldBeNull(); createResult.StreamInfo.ShouldNotBeNull(); // A mutating operation (delete) should succeed locally. var deleteResult = router.Route("$JS.API.STREAM.DELETE.TEST", ReadOnlySpan.Empty); deleteResult.Error.ShouldBeNull(); deleteResult.Success.ShouldBeTrue(); } /// /// When this node is NOT the leader, mutating operations return a not-leader error /// with the current leader's identifier in the leader_hint field. /// Go reference: jetstream_api.go:200-300 — not-leader response. /// [Fact] public void Route_WhenNotLeader_MutatingOp_ReturnsNotLeaderError() { // selfIndex=2, leaderIndex defaults to 1 — this node is NOT the leader. var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup); var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}"""); var result = router.Route("$JS.API.STREAM.CREATE.TEST", payload); result.Error.ShouldNotBeNull(); result.Error!.Code.ShouldBe(10003); result.Error.Description.ShouldBe("not leader"); result.Error.LeaderHint.ShouldNotBeNull(); result.Error.LeaderHint.ShouldBe("meta-1"); } /// /// Read-only operations (INFO, NAMES, LIST) are handled locally even when /// this node is not the leader. /// Go reference: jetstream_api.go — read operations do not require leadership. /// [Fact] public void Route_WhenNotLeader_ReadOp_HandlesLocally() { // selfIndex=2, leaderIndex defaults to 1 — this node is NOT the leader. var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup); // $JS.API.INFO is a read-only operation. var infoResult = router.Route("$JS.API.INFO", ReadOnlySpan.Empty); infoResult.Error.ShouldBeNull(); // $JS.API.STREAM.NAMES is a read-only operation. var namesResult = router.Route("$JS.API.STREAM.NAMES", ReadOnlySpan.Empty); namesResult.Error.ShouldBeNull(); namesResult.StreamNames.ShouldNotBeNull(); // $JS.API.STREAM.LIST is a read-only operation. var listResult = router.Route("$JS.API.STREAM.LIST", ReadOnlySpan.Empty); listResult.Error.ShouldBeNull(); listResult.StreamNames.ShouldNotBeNull(); } /// /// When there is no meta-group (single-server mode), all operations are handled /// locally regardless of the subject type. /// Go reference: jetstream_api.go — standalone servers have no meta-group. /// [Fact] public void Route_NoMetaGroup_HandlesLocally() { // No meta-group — single server mode. var streamManager = new StreamManager(); var consumerManager = new ConsumerManager(); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup: null); var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}"""); var result = router.Route("$JS.API.STREAM.CREATE.TEST", payload); // Should succeed — no leader check in single-server mode. result.Error.ShouldBeNull(); result.StreamInfo.ShouldNotBeNull(); result.StreamInfo!.Config.Name.ShouldBe("TEST"); } /// /// IsLeaderRequired returns true for Create, Update, Delete, and Purge operations. /// Go reference: jetstream_api.go:200-300 — mutating operations require leader. /// [Fact] public void IsLeaderRequired_CreateUpdate_ReturnsTrue() { JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.CREATE.TEST").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.UPDATE.TEST").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.DELETE.TEST").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.PURGE.TEST").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.RESTORE.TEST").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.MSG.DELETE.TEST").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.CREATE.STREAM.CON").ShouldBeTrue(); JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.DELETE.STREAM.CON").ShouldBeTrue(); } /// /// IsLeaderRequired returns false for Info, Names, List, and other read operations. /// Go reference: jetstream_api.go — read-only operations do not need leadership. /// [Fact] public void IsLeaderRequired_InfoList_ReturnsFalse() { JetStreamApiRouter.IsLeaderRequired("$JS.API.INFO").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.INFO.TEST").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.NAMES").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.LIST").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.MSG.GET.TEST").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.SNAPSHOT.TEST").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.INFO.STREAM.CON").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.NAMES.STREAM").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.LIST.STREAM").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.MSG.NEXT.STREAM.CON").ShouldBeFalse(); JetStreamApiRouter.IsLeaderRequired("$JS.API.DIRECT.GET.TEST").ShouldBeFalse(); } // --------------------------------------------------------------- // New tests for Task 19: Leader Forwarding (Gap 7.1) // Go reference: jetstream_api.go:200-300 — jsClusteredStreamXxxRequest helpers. // --------------------------------------------------------------- /// /// When not leader and a forwarder is provided, RouteAsync calls forward for mutating ops. /// Go reference: jetstream_api.go — non-leader nodes forward mutating ops to the leader. /// [Fact] public async Task Route_forwards_mutating_request_when_not_leader() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarded = JetStreamApiResponse.SuccessResponse(); var forwarder = new StubForwarder(forwarded); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); var payload = Encoding.UTF8.GetBytes("""{"name":"FWD","subjects":["fwd.>"]}"""); var result = await router.RouteAsync("$JS.API.STREAM.CREATE.FWD", payload.AsMemory()); forwarder.CallCount.ShouldBe(1); forwarder.LastSubject.ShouldBe("$JS.API.STREAM.CREATE.FWD"); forwarder.LastLeaderName.ShouldBe("meta-1"); result.ShouldBeSameAs(forwarded); } /// /// When not leader and no forwarder is provided, RouteAsync returns a NotLeader error. /// Go reference: jetstream_api.go — fallback to not-leader error when no forwarder. /// [Fact] public async Task Route_returns_not_leader_when_no_forwarder() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder: null); var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}"""); var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory()); result.Error.ShouldNotBeNull(); result.Error!.Code.ShouldBe(10003); result.Error.Description.ShouldBe("not leader"); result.Error.LeaderHint.ShouldBe("meta-1"); } /// /// Read-only operations are handled locally even when not leader and a forwarder is set. /// Go reference: jetstream_api.go — read ops do not require leadership, never forwarded. /// [Fact] public async Task Route_does_not_forward_read_only_requests() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse()); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); // $JS.API.INFO — read only var infoResult = await router.RouteAsync("$JS.API.INFO", ReadOnlyMemory.Empty); infoResult.Error.ShouldBeNull(); // $JS.API.STREAM.NAMES — read only var namesResult = await router.RouteAsync("$JS.API.STREAM.NAMES", ReadOnlyMemory.Empty); namesResult.Error.ShouldBeNull(); // Forwarder should never have been called for read-only subjects. forwarder.CallCount.ShouldBe(0); } /// /// When the forwarder returns null, RouteAsync falls back to a NotLeader response. /// Go reference: jetstream_api.go — null forward result means forwarding unavailable. /// [Fact] public async Task Route_handles_forward_returning_null_gracefully() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarder = new StubForwarder(null); // returns null var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}"""); var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory()); forwarder.CallCount.ShouldBe(1); result.Error.ShouldNotBeNull(); result.Error!.Code.ShouldBe(10003); result.Error.Description.ShouldBe("not leader"); } /// /// When the forwarder throws OperationCanceledException (timeout), RouteAsync falls back to NotLeader. /// Go reference: jetstream_api.go — timeout/cancellation during forwarding falls back gracefully. /// [Fact] public async Task Route_handles_forward_timeout() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarder = new TimeoutForwarder(); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}"""); var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory()); result.Error.ShouldNotBeNull(); result.Error!.Code.ShouldBe(10003); result.Error.Description.ShouldBe("not leader"); } /// /// ForwardedCount increments on each successful (non-null) forward result. /// Go reference: jetstream_api.go — monitoring/observability for forwarded requests. /// [Fact] public async Task ForwardedCount_increments_on_successful_forward() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse()); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); router.ForwardedCount.ShouldBe(0); var payload = Encoding.UTF8.GetBytes("""{"name":"A","subjects":["a.>"]}"""); await router.RouteAsync("$JS.API.STREAM.CREATE.A", payload.AsMemory()); router.ForwardedCount.ShouldBe(1); await router.RouteAsync("$JS.API.STREAM.DELETE.A", ReadOnlyMemory.Empty); router.ForwardedCount.ShouldBe(2); } /// /// When this node is the leader, RouteAsync handles requests locally and does not call the forwarder. /// Go reference: jetstream_api.go — leader handles requests directly. /// [Fact] public async Task Route_processes_locally_when_leader() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 1); // IS leader var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse()); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); var payload = Encoding.UTF8.GetBytes("""{"name":"LOCAL","subjects":["local.>"]}"""); var result = await router.RouteAsync("$JS.API.STREAM.CREATE.LOCAL", payload.AsMemory()); forwarder.CallCount.ShouldBe(0); result.Error.ShouldBeNull(); result.StreamInfo.ShouldNotBeNull(); result.StreamInfo!.Config.Name.ShouldBe("LOCAL"); } /// /// When no meta-group is configured (single-server), RouteAsync handles all requests locally. /// Go reference: jetstream_api.go — standalone servers have no meta-group. /// [Fact] public async Task Route_processes_locally_when_no_meta_group() { var streamManager = new StreamManager(); var consumerManager = new ConsumerManager(); var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse()); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup: null, forwarder); var payload = Encoding.UTF8.GetBytes("""{"name":"SOLO","subjects":["solo.>"]}"""); var result = await router.RouteAsync("$JS.API.STREAM.CREATE.SOLO", payload.AsMemory()); forwarder.CallCount.ShouldBe(0); result.Error.ShouldBeNull(); result.StreamInfo.ShouldNotBeNull(); result.StreamInfo!.Config.Name.ShouldBe("SOLO"); } /// /// RouteAsync passes the payload bytes verbatim to the forwarder. /// Go reference: jetstream_api.go — forwarded request includes the original payload. /// [Fact] public async Task RouteAsync_forwards_to_leader_with_payload() { var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2); var streamManager = new StreamManager(metaGroup); var consumerManager = new ConsumerManager(); var forwarded = JetStreamApiResponse.SuccessResponse(); var forwarder = new StubForwarder(forwarded); var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder); var payloadBytes = Encoding.UTF8.GetBytes("""{"name":"PAYLOAD","subjects":["p.>"]}"""); await router.RouteAsync("$JS.API.STREAM.CREATE.PAYLOAD", payloadBytes.AsMemory()); forwarder.LastPayload.Length.ShouldBe(payloadBytes.Length); var receivedBytes = forwarder.LastPayload.ToArray(); receivedBytes.ShouldBe(payloadBytes); } /// /// DefaultLeaderForwarder accepts a custom timeout value. /// Go reference: jetstream_api.go — configurable forward timeout for slow leader responses. /// [Fact] public void Forward_timeout_configurable() { var customTimeout = TimeSpan.FromSeconds(10); var forwarder = new DefaultLeaderForwarder(customTimeout); forwarder.Timeout.ShouldBe(customTimeout); } /// /// DefaultLeaderForwarder uses a 5-second default timeout when none is provided. /// Go reference: jetstream_api.go — default forward timeout. /// [Fact] public void Forward_timeout_defaults_to_five_seconds() { var forwarder = new DefaultLeaderForwarder(); forwarder.Timeout.ShouldBe(TimeSpan.FromSeconds(5)); } }