78b4bc2486
Move 225 JetStream-related test files from NATS.Server.Tests into a dedicated NATS.Server.JetStream.Tests project. This includes root-level JetStream*.cs files, storage test files (FileStore, MemStore, StreamStoreContract), and the full JetStream/ subfolder tree (Api, Cluster, Consumers, MirrorSource, Snapshots, Storage, Streams). Updated all namespaces, added InternalsVisibleTo, registered in the solution file, and added the JETSTREAM_INTEGRATION_MATRIX define.
413 lines
19 KiB
C#
413 lines
19 KiB
C#
// Go reference: jetstream_api.go:200-300 — API requests at non-leader nodes must be
|
|
// forwarded to the current leader. Mutating operations return a not-leader error with
|
|
// a leader_hint field; read-only operations are handled locally on any node.
|
|
|
|
using System.Text;
|
|
using NATS.Server.JetStream;
|
|
using NATS.Server.JetStream.Api;
|
|
using NATS.Server.JetStream.Cluster;
|
|
|
|
namespace NATS.Server.JetStream.Tests.JetStream.Api;
|
|
|
|
/// <summary>
|
|
/// Simple test double for ILeaderForwarder.
|
|
/// Returns a predetermined response or null depending on the constructor.
|
|
/// </summary>
|
|
file sealed class StubForwarder(JetStreamApiResponse? response) : ILeaderForwarder
|
|
{
|
|
public int CallCount { get; private set; }
|
|
public string? LastSubject { get; private set; }
|
|
public ReadOnlyMemory<byte> LastPayload { get; private set; }
|
|
public string? LastLeaderName { get; private set; }
|
|
|
|
public Task<JetStreamApiResponse?> ForwardAsync(
|
|
string subject, ReadOnlyMemory<byte> payload, string leaderName, CancellationToken ct)
|
|
{
|
|
CallCount++;
|
|
LastSubject = subject;
|
|
LastPayload = payload;
|
|
LastLeaderName = leaderName;
|
|
return Task.FromResult(response);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Test double that throws OperationCanceledException to simulate a timeout.
|
|
/// </summary>
|
|
file sealed class TimeoutForwarder : ILeaderForwarder
|
|
{
|
|
public Task<JetStreamApiResponse?> ForwardAsync(
|
|
string subject, ReadOnlyMemory<byte> payload, string leaderName, CancellationToken ct)
|
|
=> Task.FromException<JetStreamApiResponse?>(new OperationCanceledException("simulated timeout"));
|
|
}
|
|
|
|
public class LeaderForwardingTests
|
|
{
|
|
/// <summary>
|
|
/// When this node IS the leader, mutating requests are handled locally.
|
|
/// Go reference: jetstream_api.go — leader handles requests directly.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Route_WhenLeader_HandlesLocally()
|
|
{
|
|
// selfIndex=1 matches default leaderIndex=1, so this node is the leader.
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 1);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup);
|
|
|
|
// Create a stream first so the purge has something to operate on.
|
|
var createPayload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
|
|
var createResult = router.Route("$JS.API.STREAM.CREATE.TEST", createPayload);
|
|
createResult.Error.ShouldBeNull();
|
|
createResult.StreamInfo.ShouldNotBeNull();
|
|
|
|
// A mutating operation (delete) should succeed locally.
|
|
var deleteResult = router.Route("$JS.API.STREAM.DELETE.TEST", ReadOnlySpan<byte>.Empty);
|
|
deleteResult.Error.ShouldBeNull();
|
|
deleteResult.Success.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>
|
|
/// When this node is NOT the leader, mutating operations return a not-leader error
|
|
/// with the current leader's identifier in the leader_hint field.
|
|
/// Go reference: jetstream_api.go:200-300 — not-leader response.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Route_WhenNotLeader_MutatingOp_ReturnsNotLeaderError()
|
|
{
|
|
// selfIndex=2, leaderIndex defaults to 1 — this node is NOT the leader.
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
|
|
var result = router.Route("$JS.API.STREAM.CREATE.TEST", payload);
|
|
|
|
result.Error.ShouldNotBeNull();
|
|
result.Error!.Code.ShouldBe(10003);
|
|
result.Error.Description.ShouldBe("not leader");
|
|
result.Error.LeaderHint.ShouldNotBeNull();
|
|
result.Error.LeaderHint.ShouldBe("meta-1");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read-only operations (INFO, NAMES, LIST) are handled locally even when
|
|
/// this node is not the leader.
|
|
/// Go reference: jetstream_api.go — read operations do not require leadership.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Route_WhenNotLeader_ReadOp_HandlesLocally()
|
|
{
|
|
// selfIndex=2, leaderIndex defaults to 1 — this node is NOT the leader.
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup);
|
|
|
|
// $JS.API.INFO is a read-only operation.
|
|
var infoResult = router.Route("$JS.API.INFO", ReadOnlySpan<byte>.Empty);
|
|
infoResult.Error.ShouldBeNull();
|
|
|
|
// $JS.API.STREAM.NAMES is a read-only operation.
|
|
var namesResult = router.Route("$JS.API.STREAM.NAMES", ReadOnlySpan<byte>.Empty);
|
|
namesResult.Error.ShouldBeNull();
|
|
namesResult.StreamNames.ShouldNotBeNull();
|
|
|
|
// $JS.API.STREAM.LIST is a read-only operation.
|
|
var listResult = router.Route("$JS.API.STREAM.LIST", ReadOnlySpan<byte>.Empty);
|
|
listResult.Error.ShouldBeNull();
|
|
listResult.StreamNames.ShouldNotBeNull();
|
|
}
|
|
|
|
/// <summary>
|
|
/// When there is no meta-group (single-server mode), all operations are handled
|
|
/// locally regardless of the subject type.
|
|
/// Go reference: jetstream_api.go — standalone servers have no meta-group.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Route_NoMetaGroup_HandlesLocally()
|
|
{
|
|
// No meta-group — single server mode.
|
|
var streamManager = new StreamManager();
|
|
var consumerManager = new ConsumerManager();
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup: null);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
|
|
var result = router.Route("$JS.API.STREAM.CREATE.TEST", payload);
|
|
|
|
// Should succeed — no leader check in single-server mode.
|
|
result.Error.ShouldBeNull();
|
|
result.StreamInfo.ShouldNotBeNull();
|
|
result.StreamInfo!.Config.Name.ShouldBe("TEST");
|
|
}
|
|
|
|
/// <summary>
|
|
/// IsLeaderRequired returns true for Create, Update, Delete, and Purge operations.
|
|
/// Go reference: jetstream_api.go:200-300 — mutating operations require leader.
|
|
/// </summary>
|
|
[Fact]
|
|
public void IsLeaderRequired_CreateUpdate_ReturnsTrue()
|
|
{
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.CREATE.TEST").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.UPDATE.TEST").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.DELETE.TEST").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.PURGE.TEST").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.RESTORE.TEST").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.MSG.DELETE.TEST").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.CREATE.STREAM.CON").ShouldBeTrue();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.DELETE.STREAM.CON").ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>
|
|
/// IsLeaderRequired returns false for Info, Names, List, and other read operations.
|
|
/// Go reference: jetstream_api.go — read-only operations do not need leadership.
|
|
/// </summary>
|
|
[Fact]
|
|
public void IsLeaderRequired_InfoList_ReturnsFalse()
|
|
{
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.INFO").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.INFO.TEST").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.NAMES").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.LIST").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.MSG.GET.TEST").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.STREAM.SNAPSHOT.TEST").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.INFO.STREAM.CON").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.NAMES.STREAM").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.LIST.STREAM").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.CONSUMER.MSG.NEXT.STREAM.CON").ShouldBeFalse();
|
|
JetStreamApiRouter.IsLeaderRequired("$JS.API.DIRECT.GET.TEST").ShouldBeFalse();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// New tests for Task 19: Leader Forwarding (Gap 7.1)
|
|
// Go reference: jetstream_api.go:200-300 — jsClusteredStreamXxxRequest helpers.
|
|
// ---------------------------------------------------------------
|
|
|
|
/// <summary>
|
|
/// When not leader and a forwarder is provided, RouteAsync calls forward for mutating ops.
|
|
/// Go reference: jetstream_api.go — non-leader nodes forward mutating ops to the leader.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_forwards_mutating_request_when_not_leader()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarded = JetStreamApiResponse.SuccessResponse();
|
|
var forwarder = new StubForwarder(forwarded);
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"FWD","subjects":["fwd.>"]}""");
|
|
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.FWD", payload.AsMemory());
|
|
|
|
forwarder.CallCount.ShouldBe(1);
|
|
forwarder.LastSubject.ShouldBe("$JS.API.STREAM.CREATE.FWD");
|
|
forwarder.LastLeaderName.ShouldBe("meta-1");
|
|
result.ShouldBeSameAs(forwarded);
|
|
}
|
|
|
|
/// <summary>
|
|
/// When not leader and no forwarder is provided, RouteAsync returns a NotLeader error.
|
|
/// Go reference: jetstream_api.go — fallback to not-leader error when no forwarder.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_returns_not_leader_when_no_forwarder()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder: null);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
|
|
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory());
|
|
|
|
result.Error.ShouldNotBeNull();
|
|
result.Error!.Code.ShouldBe(10003);
|
|
result.Error.Description.ShouldBe("not leader");
|
|
result.Error.LeaderHint.ShouldBe("meta-1");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read-only operations are handled locally even when not leader and a forwarder is set.
|
|
/// Go reference: jetstream_api.go — read ops do not require leadership, never forwarded.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_does_not_forward_read_only_requests()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
// $JS.API.INFO — read only
|
|
var infoResult = await router.RouteAsync("$JS.API.INFO", ReadOnlyMemory<byte>.Empty);
|
|
infoResult.Error.ShouldBeNull();
|
|
|
|
// $JS.API.STREAM.NAMES — read only
|
|
var namesResult = await router.RouteAsync("$JS.API.STREAM.NAMES", ReadOnlyMemory<byte>.Empty);
|
|
namesResult.Error.ShouldBeNull();
|
|
|
|
// Forwarder should never have been called for read-only subjects.
|
|
forwarder.CallCount.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// When the forwarder returns null, RouteAsync falls back to a NotLeader response.
|
|
/// Go reference: jetstream_api.go — null forward result means forwarding unavailable.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_handles_forward_returning_null_gracefully()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarder = new StubForwarder(null); // returns null
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
|
|
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory());
|
|
|
|
forwarder.CallCount.ShouldBe(1);
|
|
result.Error.ShouldNotBeNull();
|
|
result.Error!.Code.ShouldBe(10003);
|
|
result.Error.Description.ShouldBe("not leader");
|
|
}
|
|
|
|
/// <summary>
|
|
/// When the forwarder throws OperationCanceledException (timeout), RouteAsync falls back to NotLeader.
|
|
/// Go reference: jetstream_api.go — timeout/cancellation during forwarding falls back gracefully.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_handles_forward_timeout()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarder = new TimeoutForwarder();
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"TEST","subjects":["test.>"]}""");
|
|
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.TEST", payload.AsMemory());
|
|
|
|
result.Error.ShouldNotBeNull();
|
|
result.Error!.Code.ShouldBe(10003);
|
|
result.Error.Description.ShouldBe("not leader");
|
|
}
|
|
|
|
/// <summary>
|
|
/// ForwardedCount increments on each successful (non-null) forward result.
|
|
/// Go reference: jetstream_api.go — monitoring/observability for forwarded requests.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task ForwardedCount_increments_on_successful_forward()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
router.ForwardedCount.ShouldBe(0);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"A","subjects":["a.>"]}""");
|
|
await router.RouteAsync("$JS.API.STREAM.CREATE.A", payload.AsMemory());
|
|
router.ForwardedCount.ShouldBe(1);
|
|
|
|
await router.RouteAsync("$JS.API.STREAM.DELETE.A", ReadOnlyMemory<byte>.Empty);
|
|
router.ForwardedCount.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>
|
|
/// When this node is the leader, RouteAsync handles requests locally and does not call the forwarder.
|
|
/// Go reference: jetstream_api.go — leader handles requests directly.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_processes_locally_when_leader()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 1); // IS leader
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"LOCAL","subjects":["local.>"]}""");
|
|
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.LOCAL", payload.AsMemory());
|
|
|
|
forwarder.CallCount.ShouldBe(0);
|
|
result.Error.ShouldBeNull();
|
|
result.StreamInfo.ShouldNotBeNull();
|
|
result.StreamInfo!.Config.Name.ShouldBe("LOCAL");
|
|
}
|
|
|
|
/// <summary>
|
|
/// When no meta-group is configured (single-server), RouteAsync handles all requests locally.
|
|
/// Go reference: jetstream_api.go — standalone servers have no meta-group.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Route_processes_locally_when_no_meta_group()
|
|
{
|
|
var streamManager = new StreamManager();
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarder = new StubForwarder(JetStreamApiResponse.SuccessResponse());
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup: null, forwarder);
|
|
|
|
var payload = Encoding.UTF8.GetBytes("""{"name":"SOLO","subjects":["solo.>"]}""");
|
|
var result = await router.RouteAsync("$JS.API.STREAM.CREATE.SOLO", payload.AsMemory());
|
|
|
|
forwarder.CallCount.ShouldBe(0);
|
|
result.Error.ShouldBeNull();
|
|
result.StreamInfo.ShouldNotBeNull();
|
|
result.StreamInfo!.Config.Name.ShouldBe("SOLO");
|
|
}
|
|
|
|
/// <summary>
|
|
/// RouteAsync passes the payload bytes verbatim to the forwarder.
|
|
/// Go reference: jetstream_api.go — forwarded request includes the original payload.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task RouteAsync_forwards_to_leader_with_payload()
|
|
{
|
|
var metaGroup = new JetStreamMetaGroup(nodes: 3, selfIndex: 2);
|
|
var streamManager = new StreamManager(metaGroup);
|
|
var consumerManager = new ConsumerManager();
|
|
var forwarded = JetStreamApiResponse.SuccessResponse();
|
|
var forwarder = new StubForwarder(forwarded);
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, metaGroup, forwarder);
|
|
|
|
var payloadBytes = Encoding.UTF8.GetBytes("""{"name":"PAYLOAD","subjects":["p.>"]}""");
|
|
await router.RouteAsync("$JS.API.STREAM.CREATE.PAYLOAD", payloadBytes.AsMemory());
|
|
|
|
forwarder.LastPayload.Length.ShouldBe(payloadBytes.Length);
|
|
var receivedBytes = forwarder.LastPayload.ToArray();
|
|
receivedBytes.ShouldBe(payloadBytes);
|
|
}
|
|
|
|
/// <summary>
|
|
/// DefaultLeaderForwarder accepts a custom timeout value.
|
|
/// Go reference: jetstream_api.go — configurable forward timeout for slow leader responses.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Forward_timeout_configurable()
|
|
{
|
|
var customTimeout = TimeSpan.FromSeconds(10);
|
|
var forwarder = new DefaultLeaderForwarder(customTimeout);
|
|
|
|
forwarder.Timeout.ShouldBe(customTimeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// DefaultLeaderForwarder uses a 5-second default timeout when none is provided.
|
|
/// Go reference: jetstream_api.go — default forward timeout.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Forward_timeout_defaults_to_five_seconds()
|
|
{
|
|
var forwarder = new DefaultLeaderForwarder();
|
|
|
|
forwarder.Timeout.ShouldBe(TimeSpan.FromSeconds(5));
|
|
}
|
|
}
|