natsdotnet/tests/NATS.Server.Clustering.Tests/Routes/RouteConfigTests.cs

using Microsoft.Extensions.Logging.Abstractions;
using NATS.Client.Core;
using NATS.Server.Configuration;

namespace NATS.Server.Clustering.Tests.Routes;

/// <summary>
/// Tests cluster route formation and message forwarding between servers.
/// Ported from Go: server/routes_test.go — TestRouteConfig, TestSeedSolicitWorks.
/// </summary>
public class RouteConfigTests
{
    [Fact]
    public async Task Two_servers_form_full_mesh_cluster()
    {
        // Reference: Go TestSeedSolicitWorks — verifies that two servers
        // with one pointing Routes at the other form a connected cluster.
        var clusterName = Guid.NewGuid().ToString("N");

        var optsA = new NatsOptions
        {
            Host = "127.0.0.1",
            Port = 0,
            Cluster = new ClusterOptions
            {
                Name = clusterName,
                Host = "127.0.0.1",
                Port = 0,
            },
        };

        var serverA = new NatsServer(optsA, NullLoggerFactory.Instance);
        var ctsA = new CancellationTokenSource();
        _ = serverA.StartAsync(ctsA.Token);
        await serverA.WaitForReadyAsync();

        var optsB = new NatsOptions
        {
            Host = "127.0.0.1",
            Port = 0,
            Cluster = new ClusterOptions
            {
                Name = clusterName,
                Host = "127.0.0.1",
                Port = 0,
                Routes = [serverA.ClusterListen!],
            },
        };

        var serverB = new NatsServer(optsB, NullLoggerFactory.Instance);
        var ctsB = new CancellationTokenSource();
        _ = serverB.StartAsync(ctsB.Token);
        await serverB.WaitForReadyAsync();

        try
        {
            // Wait for both servers to see a route connection
            using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!timeout.IsCancellationRequested
                   && (Interlocked.Read(ref serverA.Stats.Routes) == 0
                       || Interlocked.Read(ref serverB.Stats.Routes) == 0))
            {
                await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            Interlocked.Read(ref serverA.Stats.Routes).ShouldBeGreaterThan(0);
            Interlocked.Read(ref serverB.Stats.Routes).ShouldBeGreaterThan(0);
        }
        finally
        {
            await ctsA.CancelAsync();
            await ctsB.CancelAsync();
            serverA.Dispose();
            serverB.Dispose();
            ctsA.Dispose();
            ctsB.Dispose();
        }
    }

    [Fact]
    public async Task Route_forwards_messages_between_clusters()
    {
        // Reference: Go TestSeedSolicitWorks — sets up a seed + one server,
        // subscribes on one, publishes on the other, verifies delivery.
        var clusterName = Guid.NewGuid().ToString("N");

        var optsA = new NatsOptions
        {
            Host = "127.0.0.1",
            Port = 0,
            Cluster = new ClusterOptions
            {
                Name = clusterName,
                Host = "127.0.0.1",
                Port = 0,
            },
        };

        var serverA = new NatsServer(optsA, NullLoggerFactory.Instance);
        var ctsA = new CancellationTokenSource();
        _ = serverA.StartAsync(ctsA.Token);
        await serverA.WaitForReadyAsync();

        var optsB = new NatsOptions
        {
            Host = "127.0.0.1",
            Port = 0,
            Cluster = new ClusterOptions
            {
                Name = clusterName,
                Host = "127.0.0.1",
                Port = 0,
                Routes = [serverA.ClusterListen!],
            },
        };

        var serverB = new NatsServer(optsB, NullLoggerFactory.Instance);
        var ctsB = new CancellationTokenSource();
        _ = serverB.StartAsync(ctsB.Token);
        await serverB.WaitForReadyAsync();

        try
        {
            // Wait for route formation
            using var routeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!routeTimeout.IsCancellationRequested
                   && (Interlocked.Read(ref serverA.Stats.Routes) == 0
                       || Interlocked.Read(ref serverB.Stats.Routes) == 0))
            {
                await Task.Delay(50, routeTimeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            // Connect subscriber to server A
            await using var subscriber = new NatsConnection(new NatsOpts
            {
                Url = $"nats://127.0.0.1:{serverA.Port}",
            });
            await subscriber.ConnectAsync();

            await using var sub = await subscriber.SubscribeCoreAsync<string>("foo");
            await subscriber.PingAsync();

            // Wait for remote interest to propagate from A to B
            using var interestTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!interestTimeout.IsCancellationRequested
                   && !serverB.HasRemoteInterest("foo"))
            {
                await Task.Delay(50, interestTimeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            // Connect publisher to server B and publish
            await using var publisher = new NatsConnection(new NatsOpts
            {
                Url = $"nats://127.0.0.1:{serverB.Port}",
            });
            await publisher.ConnectAsync();
            await publisher.PublishAsync("foo", "Hello");

            // Verify message arrives on server A's subscriber
            using var receiveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            var msg = await sub.Msgs.ReadAsync(receiveTimeout.Token);
            msg.Data.ShouldBe("Hello");
        }
        finally
        {
            await ctsA.CancelAsync();
            await ctsB.CancelAsync();
            serverA.Dispose();
            serverB.Dispose();
            ctsA.Dispose();
            ctsB.Dispose();
        }
    }

    [Fact]
    public async Task Route_reconnects_after_peer_restart()
    {
        // Verifies that when a peer is stopped and restarted, the route
        // re-forms and message forwarding resumes.
        var clusterName = Guid.NewGuid().ToString("N");

        var optsA = new NatsOptions
        {
            Host = "127.0.0.1",
            Port = 0,
            Cluster = new ClusterOptions
            {
                Name = clusterName,
                Host = "127.0.0.1",
                Port = 0,
            },
        };

        var serverA = new NatsServer(optsA, NullLoggerFactory.Instance);
        var ctsA = new CancellationTokenSource();
        _ = serverA.StartAsync(ctsA.Token);
        await serverA.WaitForReadyAsync();

        var clusterListenA = serverA.ClusterListen!;

        var optsB = new NatsOptions
        {
            Host = "127.0.0.1",
            Port = 0,
            Cluster = new ClusterOptions
            {
                Name = clusterName,
                Host = "127.0.0.1",
                Port = 0,
                Routes = [clusterListenA],
            },
        };

        var serverB = new NatsServer(optsB, NullLoggerFactory.Instance);
        var ctsB = new CancellationTokenSource();
        _ = serverB.StartAsync(ctsB.Token);
        await serverB.WaitForReadyAsync();

        try
        {
            // Wait for initial route formation
            using var timeout1 = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!timeout1.IsCancellationRequested
                   && (Interlocked.Read(ref serverA.Stats.Routes) == 0
                       || Interlocked.Read(ref serverB.Stats.Routes) == 0))
            {
                await Task.Delay(50, timeout1.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            Interlocked.Read(ref serverA.Stats.Routes).ShouldBeGreaterThan(0);

            // Stop server B
            await ctsB.CancelAsync();
            serverB.Dispose();
            ctsB.Dispose();

            // Wait for server A to notice the route drop
            using var dropTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!dropTimeout.IsCancellationRequested
                   && Interlocked.Read(ref serverA.Stats.Routes) != 0)
            {
                await Task.Delay(50, dropTimeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            // Restart server B with the same cluster route target
            var optsB2 = new NatsOptions
            {
                Host = "127.0.0.1",
                Port = 0,
                Cluster = new ClusterOptions
                {
                    Name = clusterName,
                    Host = "127.0.0.1",
                    Port = 0,
                    Routes = [clusterListenA],
                },
            };

            serverB = new NatsServer(optsB2, NullLoggerFactory.Instance);
            ctsB = new CancellationTokenSource();
            _ = serverB.StartAsync(ctsB.Token);
            await serverB.WaitForReadyAsync();

            // Wait for route to re-form
            using var timeout2 = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!timeout2.IsCancellationRequested
                   && (Interlocked.Read(ref serverA.Stats.Routes) == 0
                       || Interlocked.Read(ref serverB.Stats.Routes) == 0))
            {
                await Task.Delay(50, timeout2.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            Interlocked.Read(ref serverA.Stats.Routes).ShouldBeGreaterThan(0);
            Interlocked.Read(ref serverB.Stats.Routes).ShouldBeGreaterThan(0);

            // Verify message forwarding works after reconnect
            await using var subscriber = new NatsConnection(new NatsOpts
            {
                Url = $"nats://127.0.0.1:{serverA.Port}",
            });
            await subscriber.ConnectAsync();

            await using var sub = await subscriber.SubscribeCoreAsync<string>("bar");
            await subscriber.PingAsync();

            // Wait for remote interest to propagate
            using var interestTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            while (!interestTimeout.IsCancellationRequested
                   && !serverB.HasRemoteInterest("bar"))
            {
                await Task.Delay(50, interestTimeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
            }

            await using var publisher = new NatsConnection(new NatsOpts
            {
                Url = $"nats://127.0.0.1:{serverB.Port}",
            });
            await publisher.ConnectAsync();
            await publisher.PublishAsync("bar", "AfterReconnect");

            using var receiveTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
            var msg = await sub.Msgs.ReadAsync(receiveTimeout.Token);
            msg.Data.ShouldBe("AfterReconnect");
        }
        finally
        {
            await ctsA.CancelAsync();
            await ctsB.CancelAsync();
            serverA.Dispose();
            serverB.Dispose();
            ctsA.Dispose();
            ctsB.Dispose();
        }
    }
}