natsdotnet/tests/NATS.E2E.Tests/Infrastructure/ClusterFixture.cs

using System.Net.Http.Json;
using NATS.Client.Core;

namespace NATS.E2E.Tests.Infrastructure;

public sealed class ClusterFixture : IAsyncLifetime
{
    private NatsServerProcess _server1 = null!;
    private NatsServerProcess _server2 = null!;
    private NatsServerProcess _server3 = null!;

    public int Port1 => _server1.Port;
    public int Port2 => _server2.Port;
    public int Port3 => _server3.Port;

    public async Task InitializeAsync()
    {
        var clusterPort1 = NatsServerProcess.AllocateFreePort();
        var clusterPort2 = NatsServerProcess.AllocateFreePort();
        var clusterPort3 = NatsServerProcess.AllocateFreePort();

        var routes = $"""
            nats-route://127.0.0.1:{clusterPort1}
            nats-route://127.0.0.1:{clusterPort2}
            nats-route://127.0.0.1:{clusterPort3}
            """;

        string MakeConfig(string name, int clusterPort) => $$"""
            server_name: {{name}}
            cluster {
                name: e2e-cluster
                listen: 127.0.0.1:{{clusterPort}}
                pool_size: 1
                routes: [
                    {{routes}}
                ]
            }
            """;

        _server1 = NatsServerProcess.WithConfig(MakeConfig("node1", clusterPort1), enableMonitoring: true, extraArgs: ["-DV"]);
        _server2 = NatsServerProcess.WithConfig(MakeConfig("node2", clusterPort2), enableMonitoring: true, extraArgs: ["-DV"]);
        _server3 = NatsServerProcess.WithConfig(MakeConfig("node3", clusterPort3), enableMonitoring: true, extraArgs: ["-DV"]);

        await Task.WhenAll(
            _server1.StartAsync(),
            _server2.StartAsync(),
            _server3.StartAsync());

        // Poll until all 3 nodes each report 2 connected routes (full mesh)
        await WaitForFullMeshAsync();
    }

    private async Task WaitForFullMeshAsync()
    {
        using var http = new HttpClient();
        using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30));
        using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(200));

        var monitorPorts = new[] { _server1.MonitorPort!.Value, _server2.MonitorPort!.Value, _server3.MonitorPort!.Value };

        while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
        {
            var allConnected = true;
            foreach (var monitorPort in monitorPorts)
            {
                try
                {
                    var routez = await http.GetFromJsonAsync<Routez>(
                        $"http://127.0.0.1:{monitorPort}/routez",
                        timeout.Token);

                    if (routez?.NumRoutes < 2)
                    {
                        allConnected = false;
                        break;
                    }
                }
                catch
                {
                    allConnected = false;
                    break;
                }
            }

            if (allConnected)
                return;
        }

        throw new TimeoutException("Cluster did not form a full mesh within 30s.");
    }

    public async Task DisposeAsync()
    {
        await Task.WhenAll(
            _server1.DisposeAsync().AsTask(),
            _server2.DisposeAsync().AsTask(),
            _server3.DisposeAsync().AsTask());
    }

    public string GetServerOutput(int nodeIndex) => nodeIndex switch
    {
        0 => _server1.Output,
        1 => _server2.Output,
        2 => _server3.Output,
        _ => throw new ArgumentOutOfRangeException(nameof(nodeIndex)),
    };

    public NatsConnection CreateClient(int nodeIndex = 0)
    {
        var port = nodeIndex switch
        {
            0 => Port1,
            1 => Port2,
            2 => Port3,
            _ => throw new ArgumentOutOfRangeException(nameof(nodeIndex)),
        };
        return new NatsConnection(new NatsOpts { Url = $"nats://127.0.0.1:{port}" });
    }

    private sealed class Routez
    {
        [System.Text.Json.Serialization.JsonPropertyName("num_routes")]
        public int NumRoutes { get; init; }
    }
}

[CollectionDefinition("E2E-Cluster")]
public class ClusterCollection : ICollectionFixture<ClusterFixture>;