// Go reference: jetstream_api.go:200-300 — API requests at non-leader nodes must be
// forwarded to the current leader. Mutating operations return a not-leader error with
// a leader_hint field; read-only operations are handled locally on any node.
using System.Text;
using NATS.Server.JetStream;
using NATS.Server.JetStream.Api;
using NATS.Server.JetStream.Cluster;
namespace NATS.Server.JetStream.Tests.JetStream.Api;
///
/// Simple test double for ILeaderForwarder.
/// Returns a predetermined response or null depending on the constructor.
///
file sealed class StubForwarder(JetStreamApiResponse? response) : ILeaderForwarder
{
public int CallCount { get; private set; }
public string? LastSubject { get; private set; }
public ReadOnlyMemory LastPayload { get; private set; }
public string? LastLeaderName { get; private set; }
public Task ForwardAsync(
string subject, ReadOnlyMemory payload, string leaderName, CancellationToken ct)
{
CallCount++;
LastSubject = subject;
LastPayload = payload;
LastLeaderName = leaderName;
return Task.FromResult(response);
}
}
///
/// Test double that throws OperationCanceledException to simulate a timeout.
///
file sealed class TimeoutForwarder : ILeaderForwarder
{
public Task ForwardAsync(
string subject, ReadOnlyMemory payload, string leaderName, CancellationToken ct)
=> Task.FromException(new OperationCanceledException("simulated timeout"));
}
public class LeaderForwardingTests
{
///
/// When this node IS the leader, mutating requests are handled locally.
/// Go reference: jetstream_api.go — leader handles requests directly.
///
[Fact]
public void Route_WhenLeader_HandlesLocally()
{
// selfIndex=1 matches default leaderIndex=1, so this node is the leader.
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 1);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup);
// Create a stream first so the purge has something to operate on.
var createPayload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
var createResult = router.Route("$JS.API.STREAM.CREATE.TEST", createPayload);
createResult.Error.ShouldBeNull();
createResult.StreamInfo.ShouldNotBeNull();
// A mutating operation (delete) should succeed locally.
var deleteResult = router.Route("$JS.API.STREAM.DELETE.TEST", ReadOnlySpan.Empty);
deleteResult.Error.ShouldBeNull();
deleteResult.Success.ShouldBeTrue();
}
///
/// When this node is NOT the leader, mutating operations return a not-leader error
/// with the current leader's identifier in the leader_hint field.
/// Go reference: jetstream_api.go:200-300 — not-leader response.
///
[Fact]
public void Route_WhenNotLeader_MutatingOp_ReturnsNotLeaderError()
{
// selfIndex=2, leaderIndex defaults to 1 — this node is NOT the leader.
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup);
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
var result = router.Route("$JS.API.STREAM.CREATE.TEST", payload);
result.Error.ShouldNotBeNull();
result.Error!.Code.ShouldBe(10003);
result.Error.Description.ShouldBe("not leader");
result.Error.LeaderHint.ShouldNotBeNull();
result.Error.LeaderHint.ShouldBe("meta-1");
}
///
/// Read-only operations (INFO, NAMES, LIST) are handled locally even when
/// this node is not the leader.
/// Go reference: jetstream_api.go — read operations do not require leadership.
///
[Fact]
public void Route_WhenNotLeader_ReadOp_HandlesLocally()
{
// selfIndex=2, leaderIndex defaults to 1 — this node is NOT the leader.
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup);
// $JS.API.INFO is a read-only operation.
var infoResult = router.Route("$JS.API.INFO", ReadOnlySpan.Empty);
infoResult.Error.ShouldBeNull();
// $JS.API.STREAM.NAMES is a read-only operation.
var namesResult = router.Route("$JS.API.STREAM.NAMES", ReadOnlySpan.Empty);
namesResult.Error.ShouldBeNull();
namesResult.StreamNames.ShouldNotBeNull();
// $JS.API.STREAM.LIST is a read-only operation.
var listResult = router.Route("$JS.API.STREAM.LIST", ReadOnlySpan.Empty);
listResult.Error.ShouldBeNull();
listResult.StreamNames.ShouldNotBeNull();
}
///
/// When there is no meta-group (single-server mode), all operations are handled
/// locally regardless of the subject type.
/// Go reference: jetstream_api.go — standalone servers have no meta-group.
///
[Fact]
public void Route_NoMetaGroup_HandlesLocally()
{
// No meta-group — single server mode.
var streamManager = new StreamManager();
var consumerManager = new ConsumerManager();
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup: null);
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
var result = router.Route("$JS.API.STREAM.CREATE.TEST", payload);
// Should succeed — no leader check in single-server mode.
result.Error.ShouldBeNull();
result.StreamInfo.ShouldNotBeNull();
result.StreamInfo!.Config.Name.ShouldBe("TEST");
}
///
/// IsLeaderRequired returns true for Create, Update, Delete, and Purge operations.
/// Go reference: jetstream_api.go:200-300 — mutating operations require leader.
///
[Fact]
public void IsLeaderRequired_CreateUpdate_ReturnsTrue()
{
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.CREATE.TEST").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.UPDATE.TEST").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.DELETE.TEST").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.PURGE.TEST").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.RESTORE.TEST").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.MSG.DELETE.TEST").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.CREATE.STREAM.CON").ShouldBeTrue();
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.DELETE.STREAM.CON").ShouldBeTrue();
}
///
/// IsLeaderRequired returns false for Info, Names, List, and other read operations.
/// Go reference: jetstream_api.go — read-only operations do not need leadership.
///
[Fact]
public void IsLeaderRequired_InfoList_ReturnsFalse()
{
JetStreamApiRouter.IsLeaderRequired("$JS.API.INFO").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.INFO.TEST").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.NAMES").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.LIST").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.MSG.GET.TEST").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.SNAPSHOT.TEST").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.INFO.STREAM.CON").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.NAMES.STREAM").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.LIST.STREAM").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.MSG.NEXT.STREAM.CON").ShouldBeFalse();
JetStreamApiRouter.IsLeaderRequired("$JS.API.DIRECT.GET.TEST").ShouldBeFalse();
}
// ---------------------------------------------------------------
// New tests for Task 19: Leader Forwarding (Gap 7.1)
// Go reference: jetstream_api.go:200-300 — jsClusteredStreamXxxRequest helpers.
// ---------------------------------------------------------------
///
/// When not leader and a forwarder is provided, RouteAsync calls forward for mutating ops.
/// Go reference: jetstream_api.go — non-leader nodes forward mutating ops to the leader.
///
[Fact]
public async Task Route_forwards_mutating_request_when_not_leader()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarded = JetStreamApiResponse.SuccessResponse();
var forwarder = new StubForwarder(forwarded);
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
var payload = Encoding.UTF8.GetBytes("""{"name":"FWD","subjects":["fwd.>"]}""");
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.FWD", payload.AsMemory());
forwarder.CallCount.ShouldBe(1);
forwarder.LastSubject.ShouldBe("$JS.API.STREAM.CREATE.FWD");
forwarder.LastLeaderName.ShouldBe("meta-1");
result.ShouldBeSameAs(forwarded);
}
///
/// When not leader and no forwarder is provided, RouteAsync returns a NotLeader error.
/// Go reference: jetstream_api.go — fallback to not-leader error when no forwarder.
///
[Fact]
public async Task Route_returns_not_leader_when_no_forwarder()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder: null);
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory());
result.Error.ShouldNotBeNull();
result.Error!.Code.ShouldBe(10003);
result.Error.Description.ShouldBe("not leader");
result.Error.LeaderHint.ShouldBe("meta-1");
}
///
/// Read-only operations are handled locally even when not leader and a forwarder is set.
/// Go reference: jetstream_api.go — read ops do not require leadership, never forwarded.
///
[Fact]
public async Task Route_does_not_forward_read_only_requests()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
// $JS.API.INFO — read only
var infoResult = await router.RouteAsync("$JS.API.INFO", ReadOnlyMemory.Empty);
infoResult.Error.ShouldBeNull();
// $JS.API.STREAM.NAMES — read only
var namesResult = await router.RouteAsync("$JS.API.STREAM.NAMES", ReadOnlyMemory.Empty);
namesResult.Error.ShouldBeNull();
// Forwarder should never have been called for read-only subjects.
forwarder.CallCount.ShouldBe(0);
}
///
/// When the forwarder returns null, RouteAsync falls back to a NotLeader response.
/// Go reference: jetstream_api.go — null forward result means forwarding unavailable.
///
[Fact]
public async Task Route_handles_forward_returning_null_gracefully()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarder = new StubForwarder(null); // returns null
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory());
forwarder.CallCount.ShouldBe(1);
result.Error.ShouldNotBeNull();
result.Error!.Code.ShouldBe(10003);
result.Error.Description.ShouldBe("not leader");
}
///
/// When the forwarder throws OperationCanceledException (timeout), RouteAsync falls back to NotLeader.
/// Go reference: jetstream_api.go — timeout/cancellation during forwarding falls back gracefully.
///
[Fact]
public async Task Route_handles_forward_timeout()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarder = new TimeoutForwarder();
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory());
result.Error.ShouldNotBeNull();
result.Error!.Code.ShouldBe(10003);
result.Error.Description.ShouldBe("not leader");
}
///
/// ForwardedCount increments on each successful (non-null) forward result.
/// Go reference: jetstream_api.go — monitoring/observability for forwarded requests.
///
[Fact]
public async Task ForwardedCount_increments_on_successful_forward()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
router.ForwardedCount.ShouldBe(0);
var payload = Encoding.UTF8.GetBytes("""{"name":"A","subjects":["a.>"]}""");
await router.RouteAsync("$JS.API.STREAM.CREATE.A", payload.AsMemory());
router.ForwardedCount.ShouldBe(1);
await router.RouteAsync("$JS.API.STREAM.DELETE.A", ReadOnlyMemory.Empty);
router.ForwardedCount.ShouldBe(2);
}
///
/// When this node is the leader, RouteAsync handles requests locally and does not call the forwarder.
/// Go reference: jetstream_api.go — leader handles requests directly.
///
[Fact]
public async Task Route_processes_locally_when_leader()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 1); // IS leader
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
var payload = Encoding.UTF8.GetBytes("""{"name":"LOCAL","subjects":["local.>"]}""");
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.LOCAL", payload.AsMemory());
forwarder.CallCount.ShouldBe(0);
result.Error.ShouldBeNull();
result.StreamInfo.ShouldNotBeNull();
result.StreamInfo!.Config.Name.ShouldBe("LOCAL");
}
///
/// When no meta-group is configured (single-server), RouteAsync handles all requests locally.
/// Go reference: jetstream_api.go — standalone servers have no meta-group.
///
[Fact]
public async Task Route_processes_locally_when_no_meta_group()
{
var streamManager = new StreamManager();
var consumerManager = new ConsumerManager();
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup: null, forwarder);
var payload = Encoding.UTF8.GetBytes("""{"name":"SOLO","subjects":["solo.>"]}""");
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.SOLO", payload.AsMemory());
forwarder.CallCount.ShouldBe(0);
result.Error.ShouldBeNull();
result.StreamInfo.ShouldNotBeNull();
result.StreamInfo!.Config.Name.ShouldBe("SOLO");
}
///
/// RouteAsync passes the payload bytes verbatim to the forwarder.
/// Go reference: jetstream_api.go — forwarded request includes the original payload.
///
[Fact]
public async Task RouteAsync_forwards_to_leader_with_payload()
{
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
var streamManager = new StreamManager(metaGroup);
var consumerManager = new ConsumerManager();
var forwarded = JetStreamApiResponse.SuccessResponse();
var forwarder = new StubForwarder(forwarded);
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
var payloadBytes = Encoding.UTF8.GetBytes("""{"name":"PAYLOAD","subjects":["p.>"]}""");
await router.RouteAsync("$JS.API.STREAM.CREATE.PAYLOAD", payloadBytes.AsMemory());
forwarder.LastPayload.Length.ShouldBe(payloadBytes.Length);
var receivedBytes = forwarder.LastPayload.ToArray();
receivedBytes.ShouldBe(payloadBytes);
}
///
/// DefaultLeaderForwarder accepts a custom timeout value.
/// Go reference: jetstream_api.go — configurable forward timeout for slow leader responses.
///
[Fact]
public void Forward_timeout_configurable()
{
var customTimeout = TimeSpan.FromSeconds(10);
var forwarder = new DefaultLeaderForwarder(customTimeout);
forwarder.Timeout.ShouldBe(customTimeout);
}
///
/// DefaultLeaderForwarder uses a 5-second default timeout when none is provided.
/// Go reference: jetstream_api.go — default forward timeout.
///
[Fact]
public void Forward_timeout_defaults_to_five_seconds()
{
var forwarder = new DefaultLeaderForwarder();
forwarder.Timeout.ShouldBe(TimeSpan.FromSeconds(5));
}
}