using Microsoft.Extensions.Logging.Abstractions; using NATS.Client.Core; using NATS.Server.Configuration; using NATS.Server.TestUtilities; namespace NATS.Server.LeafNodes.Tests.LeafNodes; /// /// Tests for message forwarding through leaf node connections (hub-to-leaf, leaf-to-hub, leaf-to-leaf). /// Reference: golang/nats-server/server/leafnode_test.go /// public class LeafNodeForwardingTests { // Go: TestLeafNodeRemoteIsHub server/leafnode_test.go:1177 [Fact] public async Task Hub_publishes_message_reaches_leaf_subscriber() { await using var fixture = await LeafFixture.StartAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var sub = await leafConn.SubscribeCoreAsync("forward.test"); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("forward.test"); await hubConn.PublishAsync("forward.test", "from-hub"); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldBe("from-hub"); } // Go: TestLeafNodeRemoteIsHub server/leafnode_test.go:1177 [Fact] public async Task Leaf_publishes_message_reaches_hub_subscriber() { await using var fixture = await LeafFixture.StartAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var sub = await hubConn.SubscribeCoreAsync("forward.hub"); await hubConn.PingAsync(); await fixture.WaitForRemoteInterestOnSpokeAsync("forward.hub"); await leafConn.PublishAsync("forward.hub", "from-leaf"); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldBe("from-leaf"); } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 [Fact] public async Task Message_published_on_leaf_does_not_loop_back_via_hub() { await using var fixture = await LeafFixture.StartAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var sub = await leafConn.SubscribeCoreAsync("noloop.test"); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("noloop.test"); await leafConn.PublishAsync("noloop.test", "from-leaf"); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldBe("from-leaf"); using var leakCts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); await Should.ThrowAsync(async () => await sub.Msgs.ReadAsync(leakCts.Token)); } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 [Fact] public async Task Multiple_messages_forwarded_from_hub_each_arrive_once() { await using var fixture = await LeafFixture.StartAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var sub = await leafConn.SubscribeCoreAsync("multi.test"); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("multi.test"); const int count = 10; for (var i = 0; i < count; i++) await hubConn.PublishAsync("multi.test", $"msg-{i}"); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var received = new List(); for (var i = 0; i < count; i++) { var msg = await sub.Msgs.ReadAsync(cts.Token); received.Add(msg.Data!); } received.Count.ShouldBe(count); for (var i = 0; i < count; i++) received.ShouldContain($"msg-{i}"); } // Go: TestLeafNodeRemoteIsHub server/leafnode_test.go:1177 [Fact] public async Task Bidirectional_forwarding_hub_and_leaf_can_exchange_messages() { await using var fixture = await LeafFixture.StartAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var hubSub = await hubConn.SubscribeCoreAsync("bidir.hub"); await using var leafSub = await leafConn.SubscribeCoreAsync("bidir.leaf"); await hubConn.PingAsync(); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnSpokeAsync("bidir.hub"); await fixture.WaitForRemoteInterestOnHubAsync("bidir.leaf"); await leafConn.PublishAsync("bidir.hub", "leaf-to-hub"); using var cts1 = new CancellationTokenSource(TimeSpan.FromSeconds(5)); (await hubSub.Msgs.ReadAsync(cts1.Token)).Data.ShouldBe("leaf-to-hub"); await hubConn.PublishAsync("bidir.leaf", "hub-to-leaf"); using var cts2 = new CancellationTokenSource(TimeSpan.FromSeconds(5)); (await leafSub.Msgs.ReadAsync(cts2.Token)).Data.ShouldBe("hub-to-leaf"); } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 [Fact] public async Task Two_spokes_interest_propagates_to_hub() { await using var fixture = await TwoSpokeFixture.StartAsync(); await using var spoke1Conn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke1.Port}" }); await spoke1Conn.ConnectAsync(); await using var spoke2Conn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke2.Port}" }); await spoke2Conn.ConnectAsync(); await using var sub1 = await spoke1Conn.SubscribeCoreAsync("spoke1.interest"); await using var sub2 = await spoke2Conn.SubscribeCoreAsync("spoke2.interest"); await spoke1Conn.PingAsync(); await spoke2Conn.PingAsync(); // Both spokes' interests should propagate to the hub await PollHelper.WaitUntilAsync(() => !((!fixture.Hub.HasRemoteInterest("spoke1.interest") || !fixture.Hub.HasRemoteInterest("spoke2.interest"))), timeoutMs: 5000); fixture.Hub.HasRemoteInterest("spoke1.interest").ShouldBeTrue(); fixture.Hub.HasRemoteInterest("spoke2.interest").ShouldBeTrue(); } // Go: TestLeafNodeRemoteIsHub server/leafnode_test.go:1177 [Fact] public async Task Large_payload_forwarded_correctly_through_leaf_node() { await using var fixture = await LeafFixture.StartAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var sub = await leafConn.SubscribeCoreAsync("large.payload"); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("large.payload"); var largePayload = new byte[10240]; Random.Shared.NextBytes(largePayload); await hubConn.PublishAsync("large.payload", largePayload); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldNotBeNull(); msg.Data!.Length.ShouldBe(largePayload.Length); msg.Data.ShouldBe(largePayload); } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 // Note: Request-reply across leaf nodes requires _INBOX reply subject // interest propagation which needs the hub to forward reply-to messages // back to the requester. This is a more complex scenario tested at // the integration level when full reply routing is implemented. [Fact] public async Task Reply_subject_from_hub_reaches_leaf_subscriber() { await using var fixture = await LeafFixture.StartAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var requestSub = await leafConn.SubscribeCoreAsync("request.test"); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("request.test"); // Publish with a reply-to from hub await hubConn.PublishAsync("request.test", "hello", replyTo: "reply.subject"); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await requestSub.Msgs.ReadAsync(cts.Token); msg.Data.ShouldBe("hello"); // The reply-to may or may not be propagated depending on implementation // At minimum, the message itself should arrive } // Go: TestLeafNodeDuplicateMsg server/leafnode_test.go:6513 [Fact] public async Task Subscriber_on_both_hub_and_leaf_receives_message_once_each() { await using var fixture = await LeafFixture.StartAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var hubSub = await hubConn.SubscribeCoreAsync("both.test"); await using var leafSub = await leafConn.SubscribeCoreAsync("both.test"); await hubConn.PingAsync(); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("both.test"); await using var pubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await pubConn.ConnectAsync(); await pubConn.PublishAsync("both.test", "dual"); using var cts1 = new CancellationTokenSource(TimeSpan.FromSeconds(5)); (await hubSub.Msgs.ReadAsync(cts1.Token)).Data.ShouldBe("dual"); using var cts2 = new CancellationTokenSource(TimeSpan.FromSeconds(5)); (await leafSub.Msgs.ReadAsync(cts2.Token)).Data.ShouldBe("dual"); } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 [Fact] public async Task Hub_subscriber_receives_leaf_message_with_correct_subject() { await using var fixture = await LeafFixture.StartAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var sub = await hubConn.SubscribeCoreAsync("subject.check"); await hubConn.PingAsync(); await fixture.WaitForRemoteInterestOnSpokeAsync("subject.check"); await leafConn.PublishAsync("subject.check", "payload"); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Subject.ShouldBe("subject.check"); msg.Data.ShouldBe("payload"); } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 [Fact] public async Task No_message_received_when_no_subscriber_on_leaf() { await using var fixture = await LeafFixture.StartAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await hubConn.PublishAsync("no.subscriber", "lost"); await PollHelper.YieldForAsync(200); true.ShouldBeTrue(); // No crash = success } // Go: TestLeafNodeNoMsgLoop server/leafnode_test.go:3800 [Fact] public async Task Empty_payload_forwarded_correctly_through_leaf_node() { await using var fixture = await LeafFixture.StartAsync(); await using var leafConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Spoke.Port}" }); await leafConn.ConnectAsync(); await using var hubConn = new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{fixture.Hub.Port}" }); await hubConn.ConnectAsync(); await using var sub = await leafConn.SubscribeCoreAsync("empty.payload"); await leafConn.PingAsync(); await fixture.WaitForRemoteInterestOnHubAsync("empty.payload"); await hubConn.PublishAsync("empty.payload", []); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var msg = await sub.Msgs.ReadAsync(cts.Token); msg.Subject.ShouldBe("empty.payload"); } } internal sealed class TwoSpokeFixture : IAsyncDisposable { private readonly CancellationTokenSource _hubCts; private readonly CancellationTokenSource _spoke1Cts; private readonly CancellationTokenSource _spoke2Cts; private TwoSpokeFixture(NatsServer hub, NatsServer spoke1, NatsServer spoke2, CancellationTokenSource hubCts, CancellationTokenSource spoke1Cts, CancellationTokenSource spoke2Cts) { Hub = hub; Spoke1 = spoke1; Spoke2 = spoke2; _hubCts = hubCts; _spoke1Cts = spoke1Cts; _spoke2Cts = spoke2Cts; } public NatsServer Hub { get; } public NatsServer Spoke1 { get; } public NatsServer Spoke2 { get; } public static async Task StartAsync() { var hubOptions = new NatsOptions { Host = "127.0.0.1", Port = 0, LeafNode = new LeafNodeOptions { Host = "127.0.0.1", Port = 0 }, }; var hub = new NatsServer(hubOptions, NullLoggerFactory.Instance); var hubCts = new CancellationTokenSource(); _ = hub.StartAsync(hubCts.Token); await hub.WaitForReadyAsync(); var spoke1Options = new NatsOptions { Host = "127.0.0.1", Port = 0, LeafNode = new LeafNodeOptions { Host = "127.0.0.1", Port = 0, Remotes = [hub.LeafListen!] }, }; var spoke1 = new NatsServer(spoke1Options, NullLoggerFactory.Instance); var spoke1Cts = new CancellationTokenSource(); _ = spoke1.StartAsync(spoke1Cts.Token); await spoke1.WaitForReadyAsync(); var spoke2Options = new NatsOptions { Host = "127.0.0.1", Port = 0, LeafNode = new LeafNodeOptions { Host = "127.0.0.1", Port = 0, Remotes = [hub.LeafListen!] }, }; var spoke2 = new NatsServer(spoke2Options, NullLoggerFactory.Instance); var spoke2Cts = new CancellationTokenSource(); _ = spoke2.StartAsync(spoke2Cts.Token); await spoke2.WaitForReadyAsync(); await PollHelper.WaitUntilAsync(() => Interlocked.Read(ref hub.Stats.Leafs) >= 2 && spoke1.Stats.Leafs > 0 && spoke2.Stats.Leafs > 0); return new TwoSpokeFixture(hub, spoke1, spoke2, hubCts, spoke1Cts, spoke2Cts); } public async ValueTask DisposeAsync() { await _spoke2Cts.CancelAsync(); await _spoke1Cts.CancelAsync(); await _hubCts.CancelAsync(); Spoke2.Dispose(); Spoke1.Dispose(); Hub.Dispose(); _spoke2Cts.Dispose(); _spoke1Cts.Dispose(); _hubCts.Dispose(); } }