feat: add benchmark test project for Go vs .NET server comparison
Side-by-side performance benchmarks using NATS.Client.Core against both servers on ephemeral ports. Includes core pub/sub, request/reply latency, and JetStream throughput tests with comparison output and benchmarks_comparison.md results. Also fixes timestamp flakiness in StoreInterfaceTests by using explicit timestamps.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,7 @@
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
[CollectionDefinition("Benchmark-Core")]
|
||||
public class BenchmarkCoreCollection : ICollectionFixture<CoreServerPairFixture>;
|
||||
|
||||
[CollectionDefinition("Benchmark-JetStream")]
|
||||
public class BenchmarkJetStreamCollection : ICollectionFixture<JetStreamServerPairFixture>;
|
||||
@@ -0,0 +1,47 @@
|
||||
using NATS.Client.Core;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Starts both a Go and .NET NATS server for core pub/sub benchmarks.
|
||||
/// Shared across all tests in the "Benchmark-Core" collection.
|
||||
/// </summary>
|
||||
public sealed class CoreServerPairFixture : IAsyncLifetime
|
||||
{
|
||||
private GoServerProcess? _goServer;
|
||||
private DotNetServerProcess? _dotNetServer;
|
||||
|
||||
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||
public bool GoAvailable => _goServer is not null;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_dotNetServer = new DotNetServerProcess();
|
||||
var dotNetTask = _dotNetServer.StartAsync();
|
||||
|
||||
if (GoServerProcess.IsAvailable())
|
||||
{
|
||||
_goServer = new GoServerProcess();
|
||||
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||
}
|
||||
else
|
||||
{
|
||||
await dotNetTask;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_goServer is not null)
|
||||
await _goServer.DisposeAsync();
|
||||
if (_dotNetServer is not null)
|
||||
await _dotNetServer.DisposeAsync();
|
||||
}
|
||||
|
||||
public NatsConnection CreateGoClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoPort}" });
|
||||
|
||||
public NatsConnection CreateDotNetClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetPort}" });
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Manages a NATS.Server.Host child process for benchmark testing.
|
||||
/// </summary>
|
||||
public sealed class DotNetServerProcess : IAsyncDisposable
|
||||
{
|
||||
private Process? _process;
|
||||
private readonly StringBuilder _output = new();
|
||||
private readonly object _outputLock = new();
|
||||
private readonly string? _configContent;
|
||||
private string? _configFilePath;
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public string Output
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_outputLock)
|
||||
return _output.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public DotNetServerProcess(string? configContent = null)
|
||||
{
|
||||
Port = PortAllocator.AllocateFreePort();
|
||||
_configContent = configContent;
|
||||
}
|
||||
|
||||
public async Task StartAsync()
|
||||
{
|
||||
var hostDll = ResolveHostDll();
|
||||
|
||||
if (_configContent is not null)
|
||||
{
|
||||
_configFilePath = Path.Combine(Path.GetTempPath(), $"nats-bench-dotnet-{Guid.NewGuid():N}.conf");
|
||||
await File.WriteAllTextAsync(_configFilePath, _configContent);
|
||||
}
|
||||
|
||||
var args = new StringBuilder($"exec \"{hostDll}\" -p {Port}");
|
||||
if (_configFilePath is not null)
|
||||
args.Append($" -c \"{_configFilePath}\"");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = "dotnet",
|
||||
Arguments = args.ToString(),
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
_process = new Process { StartInfo = psi, EnableRaisingEvents = true };
|
||||
|
||||
_process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
_process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
_process.Start();
|
||||
_process.BeginOutputReadLine();
|
||||
_process.BeginErrorReadLine();
|
||||
|
||||
await WaitForTcpReadyAsync();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_process is not null)
|
||||
{
|
||||
if (!_process.HasExited)
|
||||
{
|
||||
_process.Kill(entireProcessTree: true);
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
try
|
||||
{
|
||||
await _process.WaitForExitAsync(cts.Token);
|
||||
}
|
||||
catch (OperationCanceledException ex) when (!_process.HasExited)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"NATS .NET server process did not exit within 5s after kill.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
_process.Dispose();
|
||||
_process = null;
|
||||
}
|
||||
|
||||
if (_configFilePath is not null && File.Exists(_configFilePath))
|
||||
{
|
||||
File.Delete(_configFilePath);
|
||||
_configFilePath = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForTcpReadyAsync()
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(100));
|
||||
SocketException? lastError = null;
|
||||
|
||||
while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
|
||||
{
|
||||
try
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(new IPEndPoint(IPAddress.Loopback, Port), timeout.Token);
|
||||
return;
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
lastError = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"NATS .NET server did not become ready on port {Port} within 10s. Last error: {lastError?.Message}\n\nServer output:\n{Output}");
|
||||
}
|
||||
|
||||
private static string ResolveHostDll()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir is not null)
|
||||
{
|
||||
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
|
||||
{
|
||||
var dll = Path.Combine(dir.FullName, "src", "NATS.Server.Host", "bin", "Debug", "net10.0", "NATS.Server.Host.dll");
|
||||
if (File.Exists(dll))
|
||||
return dll;
|
||||
|
||||
var build = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "dotnet",
|
||||
Arguments = "build src/NATS.Server.Host/NATS.Server.Host.csproj -c Debug",
|
||||
WorkingDirectory = dir.FullName,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
});
|
||||
build!.WaitForExit();
|
||||
|
||||
if (build.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to build NATS.Server.Host:\n{build.StandardError.ReadToEnd()}");
|
||||
|
||||
if (File.Exists(dll))
|
||||
return dll;
|
||||
|
||||
throw new FileNotFoundException($"Built NATS.Server.Host but DLL not found at: {dll}");
|
||||
}
|
||||
|
||||
dir = dir.Parent;
|
||||
}
|
||||
|
||||
throw new FileNotFoundException(
|
||||
"Could not find solution root (NatsDotNet.slnx) walking up from " + AppContext.BaseDirectory);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Manages a Go nats-server child process for benchmark testing.
|
||||
/// Builds the Go binary if not present and skips gracefully if Go is not installed.
|
||||
/// </summary>
|
||||
public sealed class GoServerProcess : IAsyncDisposable
|
||||
{
|
||||
private Process? _process;
|
||||
private readonly StringBuilder _output = new();
|
||||
private readonly object _outputLock = new();
|
||||
private readonly string? _configContent;
|
||||
private string? _configFilePath;
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public string Output
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_outputLock)
|
||||
return _output.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public GoServerProcess(string? configContent = null)
|
||||
{
|
||||
Port = PortAllocator.AllocateFreePort();
|
||||
_configContent = configContent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if Go is installed and the Go NATS server source exists.
|
||||
/// </summary>
|
||||
public static bool IsAvailable()
|
||||
{
|
||||
try
|
||||
{
|
||||
var goVersion = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "go",
|
||||
Arguments = "version",
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
goVersion!.WaitForExit();
|
||||
if (goVersion.ExitCode != 0)
|
||||
return false;
|
||||
|
||||
var solutionRoot = FindSolutionRoot();
|
||||
return solutionRoot is not null &&
|
||||
Directory.Exists(Path.Combine(solutionRoot, "golang", "nats-server"));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StartAsync()
|
||||
{
|
||||
var binary = ResolveGoBinary();
|
||||
|
||||
if (_configContent is not null)
|
||||
{
|
||||
_configFilePath = Path.Combine(Path.GetTempPath(), $"nats-bench-go-{Guid.NewGuid():N}.conf");
|
||||
await File.WriteAllTextAsync(_configFilePath, _configContent);
|
||||
}
|
||||
|
||||
var args = new StringBuilder($"-p {Port}");
|
||||
if (_configFilePath is not null)
|
||||
args.Append($" -c \"{_configFilePath}\"");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = binary,
|
||||
Arguments = args.ToString(),
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
_process = new Process { StartInfo = psi, EnableRaisingEvents = true };
|
||||
|
||||
_process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
_process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
_process.Start();
|
||||
_process.BeginOutputReadLine();
|
||||
_process.BeginErrorReadLine();
|
||||
|
||||
await WaitForTcpReadyAsync();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_process is not null)
|
||||
{
|
||||
if (!_process.HasExited)
|
||||
{
|
||||
_process.Kill(entireProcessTree: true);
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
try
|
||||
{
|
||||
await _process.WaitForExitAsync(cts.Token);
|
||||
}
|
||||
catch (OperationCanceledException ex) when (!_process.HasExited)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Go nats-server process did not exit within 5s after kill.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
_process.Dispose();
|
||||
_process = null;
|
||||
}
|
||||
|
||||
if (_configFilePath is not null && File.Exists(_configFilePath))
|
||||
{
|
||||
File.Delete(_configFilePath);
|
||||
_configFilePath = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForTcpReadyAsync()
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(100));
|
||||
SocketException? lastError = null;
|
||||
|
||||
while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
|
||||
{
|
||||
try
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(new IPEndPoint(IPAddress.Loopback, Port), timeout.Token);
|
||||
return;
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
lastError = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"Go nats-server did not become ready on port {Port} within 15s. Last error: {lastError?.Message}\n\nServer output:\n{Output}");
|
||||
}
|
||||
|
||||
private static string ResolveGoBinary()
|
||||
{
|
||||
var solutionRoot = FindSolutionRoot()
|
||||
?? throw new FileNotFoundException(
|
||||
"Could not find solution root (NatsDotNet.slnx) walking up from " + AppContext.BaseDirectory);
|
||||
|
||||
var goServerDir = Path.Combine(solutionRoot, "golang", "nats-server");
|
||||
if (!Directory.Exists(goServerDir))
|
||||
throw new DirectoryNotFoundException($"Go nats-server source not found at: {goServerDir}");
|
||||
|
||||
var binaryName = OperatingSystem.IsWindows() ? "nats-server.exe" : "nats-server";
|
||||
var binaryPath = Path.Combine(goServerDir, binaryName);
|
||||
|
||||
if (File.Exists(binaryPath))
|
||||
return binaryPath;
|
||||
|
||||
// Build the Go binary
|
||||
var build = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "go",
|
||||
Arguments = "build -o " + binaryName,
|
||||
WorkingDirectory = goServerDir,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
build!.WaitForExit();
|
||||
|
||||
if (build.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to build Go nats-server:\n{build.StandardError.ReadToEnd()}");
|
||||
|
||||
if (File.Exists(binaryPath))
|
||||
return binaryPath;
|
||||
|
||||
throw new FileNotFoundException($"Built Go nats-server but binary not found at: {binaryPath}");
|
||||
}
|
||||
|
||||
internal static string? FindSolutionRoot()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir is not null)
|
||||
{
|
||||
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
|
||||
return dir.FullName;
|
||||
dir = dir.Parent;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using NATS.Client.Core;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Starts both a Go and .NET NATS server with JetStream enabled for benchmark testing.
|
||||
/// Shared across all tests in the "Benchmark-JetStream" collection.
|
||||
/// </summary>
|
||||
public sealed class JetStreamServerPairFixture : IAsyncLifetime
|
||||
{
|
||||
private GoServerProcess? _goServer;
|
||||
private DotNetServerProcess? _dotNetServer;
|
||||
private string? _goStoreDir;
|
||||
private string? _dotNetStoreDir;
|
||||
|
||||
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||
public bool GoAvailable => _goServer is not null;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_dotNetStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-dotnet-js-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
Directory.CreateDirectory(_dotNetStoreDir);
|
||||
|
||||
var dotNetConfig = $$"""
|
||||
jetstream {
|
||||
store_dir: "{{_dotNetStoreDir}}"
|
||||
max_mem_store: 256mb
|
||||
max_file_store: 1gb
|
||||
}
|
||||
""";
|
||||
|
||||
_dotNetServer = new DotNetServerProcess(dotNetConfig);
|
||||
var dotNetTask = _dotNetServer.StartAsync();
|
||||
|
||||
if (GoServerProcess.IsAvailable())
|
||||
{
|
||||
_goStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-go-js-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
Directory.CreateDirectory(_goStoreDir);
|
||||
|
||||
var goConfig = $$"""
|
||||
jetstream {
|
||||
store_dir: "{{_goStoreDir}}"
|
||||
max_mem_store: 256mb
|
||||
max_file_store: 1gb
|
||||
}
|
||||
""";
|
||||
|
||||
_goServer = new GoServerProcess(goConfig);
|
||||
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||
}
|
||||
else
|
||||
{
|
||||
await dotNetTask;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_goServer is not null)
|
||||
await _goServer.DisposeAsync();
|
||||
if (_dotNetServer is not null)
|
||||
await _dotNetServer.DisposeAsync();
|
||||
|
||||
CleanupDir(_goStoreDir);
|
||||
CleanupDir(_dotNetStoreDir);
|
||||
}
|
||||
|
||||
public NatsConnection CreateGoClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoPort}" });
|
||||
|
||||
public NatsConnection CreateDotNetClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetPort}" });
|
||||
|
||||
private static void CleanupDir(string? dir)
|
||||
{
|
||||
if (dir is not null && Directory.Exists(dir))
|
||||
{
|
||||
try { Directory.Delete(dir, recursive: true); }
|
||||
catch { /* best-effort cleanup */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
internal static class PortAllocator
|
||||
{
|
||||
public static int AllocateFreePort()
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
return ((IPEndPoint)socket.LocalEndPoint!).Port;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user