feat: add benchmark test project for Go vs .NET server comparison

Side-by-side performance benchmarks using NATS.Client.Core against both
servers on ephemeral ports. Includes core pub/sub, request/reply latency,
and JetStream throughput tests with comparison output and
benchmarks_comparison.md results. Also fixes timestamp flakiness in
StoreInterfaceTests by using explicit timestamps.
This commit is contained in:
Joseph Doherty
2026-03-13 01:23:31 -04:00
parent e9c86c51c3
commit 37575dc41c
28 changed files with 2264 additions and 12 deletions

View File

@@ -0,0 +1 @@
[assembly: CollectionBehavior(DisableTestParallelization = true)]

View File

@@ -0,0 +1,99 @@
using NATS.Client.Core;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.CorePubSub;
[Collection("Benchmark-Core")]
public class FanOutTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task FanOut1To4_128B()
{
const int payloadSize = 128;
const int messageCount = 10_000;
const int subscriberCount = 4;
var dotnetResult = await RunFanOut("Fan-Out 1:4 (128B)", "DotNet", payloadSize, messageCount, subscriberCount, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunFanOut("Fan-Out 1:4 (128B)", "Go", payloadSize, messageCount, subscriberCount, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunFanOut(string name, string serverType, int payloadSize, int messageCount, int subscriberCount, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var subject = $"bench.fanout.{serverType.ToLowerInvariant()}.{Guid.NewGuid():N}";
await using var pubClient = createClient();
await pubClient.ConnectAsync();
var subClients = new NatsConnection[subscriberCount];
var subs = new INatsSub<byte[]>[subscriberCount];
var subTasks = new Task[subscriberCount];
var totalExpected = messageCount * subscriberCount;
var totalReceived = 0;
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
for (var i = 0; i < subscriberCount; i++)
{
subClients[i] = createClient();
await subClients[i].ConnectAsync();
subs[i] = await subClients[i].SubscribeCoreAsync<byte[]>(subject);
}
// Flush to ensure all subscriptions are propagated
foreach (var client in subClients)
await client.PingAsync();
await pubClient.PingAsync();
// Start reading after subscriptions are confirmed
for (var i = 0; i < subscriberCount; i++)
{
var sub = subs[i];
subTasks[i] = Task.Run(async () =>
{
await foreach (var _ in sub.Msgs.ReadAllAsync())
{
if (Interlocked.Increment(ref totalReceived) >= totalExpected)
{
tcs.TrySetResult();
return;
}
}
});
}
var sw = System.Diagnostics.Stopwatch.StartNew();
for (var i = 0; i < messageCount; i++)
await pubClient.PublishAsync(subject, payload);
await pubClient.PingAsync();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();
foreach (var sub in subs)
await sub.UnsubscribeAsync();
foreach (var client in subClients)
await client.DisposeAsync();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = totalExpected,
TotalBytes = (long)totalExpected * payloadSize,
Duration = sw.Elapsed,
};
}
}

View File

@@ -0,0 +1,124 @@
using NATS.Client.Core;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.CorePubSub;
[Collection("Benchmark-Core")]
public class MultiPubSubTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task MultiPubSub4x4_128B()
{
const int payloadSize = 128;
const int messagesPerPublisher = 2_000;
const int pubCount = 4;
const int subCount = 4;
var dotnetResult = await RunMultiPubSub("Multi 4Px4S (128B)", "DotNet", payloadSize, messagesPerPublisher, pubCount, subCount, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunMultiPubSub("Multi 4Px4S (128B)", "Go", payloadSize, messagesPerPublisher, pubCount, subCount, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunMultiPubSub(
string name, string serverType, int payloadSize, int messagesPerPublisher,
int pubCount, int subCount, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var totalMessages = messagesPerPublisher * pubCount;
var runId = Guid.NewGuid().ToString("N")[..8];
var subjects = Enumerable.Range(0, pubCount).Select(i => $"bench.multi.{serverType.ToLowerInvariant()}.{runId}.{i}").ToArray();
// Create subscribers — one per subject
var subClients = new NatsConnection[subCount];
var subs = new INatsSub<byte[]>[subCount];
var subTasks = new Task[subCount];
var totalReceived = 0;
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
for (var i = 0; i < subCount; i++)
{
subClients[i] = createClient();
await subClients[i].ConnectAsync();
subs[i] = await subClients[i].SubscribeCoreAsync<byte[]>(subjects[i % subjects.Length]);
}
// Flush to ensure all subscriptions are propagated
foreach (var client in subClients)
await client.PingAsync();
// Start reading
for (var i = 0; i < subCount; i++)
{
var sub = subs[i];
subTasks[i] = Task.Run(async () =>
{
await foreach (var _ in sub.Msgs.ReadAllAsync())
{
if (Interlocked.Increment(ref totalReceived) >= totalMessages)
{
tcs.TrySetResult();
return;
}
}
});
}
// Create publishers
var pubClients = new NatsConnection[pubCount];
for (var i = 0; i < pubCount; i++)
{
pubClients[i] = createClient();
await pubClients[i].ConnectAsync();
}
var sw = System.Diagnostics.Stopwatch.StartNew();
var pubTasks = new Task[pubCount];
for (var p = 0; p < pubCount; p++)
{
var client = pubClients[p];
var subject = subjects[p];
pubTasks[p] = Task.Run(async () =>
{
for (var i = 0; i < messagesPerPublisher; i++)
await client.PublishAsync(subject, payload);
});
}
await Task.WhenAll(pubTasks);
// Flush all publishers
foreach (var client in pubClients)
await client.PingAsync();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();
foreach (var sub in subs)
await sub.UnsubscribeAsync();
foreach (var client in subClients)
await client.DisposeAsync();
foreach (var client in pubClients)
await client.DisposeAsync();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = totalMessages,
TotalBytes = (long)totalMessages * payloadSize,
Duration = sw.Elapsed,
};
}
}

View File

@@ -0,0 +1,105 @@
using NATS.Client.Core;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.CorePubSub;
[Collection("Benchmark-Core")]
public class PubSubOneToOneTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task PubSub1To1_16B()
{
const int payloadSize = 16;
const int messageCount = 10_000;
var dotnetResult = await RunPubSub("PubSub 1:1 (16B)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunPubSub("PubSub 1:1 (16B)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
[Fact]
[Trait("Category", "Benchmark")]
public async Task PubSub1To1_16KB()
{
const int payloadSize = 16 * 1024;
const int messageCount = 1_000;
var dotnetResult = await RunPubSub("PubSub 1:1 (16KB)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunPubSub("PubSub 1:1 (16KB)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunPubSub(string name, string serverType, int payloadSize, int messageCount, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var subject = $"bench.pubsub.{serverType.ToLowerInvariant()}.{Guid.NewGuid():N}";
await using var pubClient = createClient();
await using var subClient = createClient();
await pubClient.ConnectAsync();
await subClient.ConnectAsync();
var received = 0;
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
// Start subscriber
var sub = await subClient.SubscribeCoreAsync<byte[]>(subject);
// Flush to ensure subscription is propagated
await subClient.PingAsync();
await pubClient.PingAsync();
var subTask = Task.Run(async () =>
{
await foreach (var msg in sub.Msgs.ReadAllAsync())
{
if (Interlocked.Increment(ref received) >= messageCount)
{
tcs.TrySetResult();
return;
}
}
});
// Publish and measure
var sw = System.Diagnostics.Stopwatch.StartNew();
for (var i = 0; i < messageCount; i++)
await pubClient.PublishAsync(subject, payload);
await pubClient.PingAsync(); // Flush all pending writes
// Wait for all messages received
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();
await sub.UnsubscribeAsync();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = messageCount,
TotalBytes = (long)messageCount * payloadSize,
Duration = sw.Elapsed,
};
}
}

View File

@@ -0,0 +1,63 @@
using NATS.Client.Core;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.CorePubSub;
[Collection("Benchmark-Core")]
public class SinglePublisherThroughputTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
private readonly BenchmarkRunner _runner = new() { WarmupCount = 1_000, MeasurementCount = 100_000 };
[Fact]
[Trait("Category", "Benchmark")]
public async Task PubNoSub_16B()
{
const int payloadSize = 16;
var payload = new byte[payloadSize];
const string subject = "bench.pub.nosub.16";
var dotnetResult = await RunThroughput("Single Publisher (16B)", "DotNet", subject, payload, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunThroughput("Single Publisher (16B)", "Go", subject, payload, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
[Fact]
[Trait("Category", "Benchmark")]
public async Task PubNoSub_128B()
{
const int payloadSize = 128;
var payload = new byte[payloadSize];
const string subject = "bench.pub.nosub.128";
var dotnetResult = await RunThroughput("Single Publisher (128B)", "DotNet", subject, payload, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunThroughput("Single Publisher (128B)", "Go", subject, payload, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private async Task<BenchmarkResult> RunThroughput(string name, string serverType, string subject, byte[] payload, Func<NatsConnection> createClient)
{
await using var client = createClient();
await client.ConnectAsync();
return await _runner.MeasureThroughputAsync(name, serverType, payload.Length,
async _ => await client.PublishAsync(subject, payload));
}
}

View File

@@ -0,0 +1,27 @@
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Captures the results of a single benchmark run against one server.
/// </summary>
public sealed record BenchmarkResult
{
public required string Name { get; init; }
public required string ServerType { get; init; }
public required long TotalMessages { get; init; }
public required long TotalBytes { get; init; }
public required TimeSpan Duration { get; init; }
/// <summary>Latency percentiles in microseconds, if measured.</summary>
public LatencyPercentiles? Latencies { get; init; }
public double MessagesPerSecond => TotalMessages / Duration.TotalSeconds;
public double BytesPerSecond => TotalBytes / Duration.TotalSeconds;
public double MegabytesPerSecond => BytesPerSecond / (1024.0 * 1024.0);
}
public sealed record LatencyPercentiles(
double P50Us,
double P95Us,
double P99Us,
double MinUs,
double MaxUs);

View File

@@ -0,0 +1,55 @@
using System.Globalization;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Writes side-by-side benchmark comparison output to xUnit's ITestOutputHelper.
/// </summary>
public static class BenchmarkResultWriter
{
public static void WriteComparison(ITestOutputHelper output, BenchmarkResult goResult, BenchmarkResult dotnetResult)
{
var ratio = dotnetResult.MessagesPerSecond / goResult.MessagesPerSecond;
output.WriteLine($"=== {goResult.Name} ===");
output.WriteLine($"Go: {FormatRate(goResult.MessagesPerSecond)} msg/s | {goResult.MegabytesPerSecond:F1} MB/s | {goResult.Duration.TotalMilliseconds:F0} ms");
output.WriteLine($".NET: {FormatRate(dotnetResult.MessagesPerSecond)} msg/s | {dotnetResult.MegabytesPerSecond:F1} MB/s | {dotnetResult.Duration.TotalMilliseconds:F0} ms");
output.WriteLine($"Ratio: {ratio:F2}x (.NET / Go)");
if (goResult.Latencies is not null && dotnetResult.Latencies is not null)
{
output.WriteLine("");
output.WriteLine("Latency (us):");
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
WriteLatencyRow(output, "Go", goResult.Latencies);
WriteLatencyRow(output, ".NET", dotnetResult.Latencies);
}
output.WriteLine("");
}
public static void WriteSingle(ITestOutputHelper output, BenchmarkResult result)
{
output.WriteLine($"=== {result.Name} ({result.ServerType}) ===");
output.WriteLine($"{FormatRate(result.MessagesPerSecond)} msg/s | {result.MegabytesPerSecond:F1} MB/s | {result.Duration.TotalMilliseconds:F0} ms");
if (result.Latencies is not null)
{
output.WriteLine("");
output.WriteLine("Latency (us):");
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
WriteLatencyRow(output, result.ServerType, result.Latencies);
}
output.WriteLine("");
}
private static void WriteLatencyRow(ITestOutputHelper output, string label, LatencyPercentiles p)
{
output.WriteLine($" {label,8} {p.P50Us,10:F1} {p.P95Us,10:F1} {p.P99Us,10:F1} {p.MinUs,10:F1} {p.MaxUs,10:F1}");
}
private static string FormatRate(double rate)
=> rate.ToString("N0", CultureInfo.InvariantCulture).PadLeft(15);
}

View File

@@ -0,0 +1,80 @@
using System.Diagnostics;
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Lightweight benchmark runner with warmup + timed measurement.
/// </summary>
public sealed class BenchmarkRunner
{
public int WarmupCount { get; init; } = 1_000;
public int MeasurementCount { get; init; } = 100_000;
/// <summary>
/// Measures throughput for a fire-and-forget style workload (pub-only or pub+sub).
/// The <paramref name="action"/> is called <see cref="MeasurementCount"/> times.
/// </summary>
public async Task<BenchmarkResult> MeasureThroughputAsync(
string name,
string serverType,
int payloadSize,
Func<int, Task> action)
{
// Warmup
for (var i = 0; i < WarmupCount; i++)
await action(i);
// Measurement
var sw = Stopwatch.StartNew();
for (var i = 0; i < MeasurementCount; i++)
await action(i);
sw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = MeasurementCount,
TotalBytes = (long)MeasurementCount * payloadSize,
Duration = sw.Elapsed,
};
}
/// <summary>
/// Measures latency for a request-reply style workload.
/// Records per-iteration round-trip time and computes percentiles.
/// </summary>
public async Task<BenchmarkResult> MeasureLatencyAsync(
string name,
string serverType,
int payloadSize,
Func<int, Task> roundTripAction)
{
// Warmup
for (var i = 0; i < WarmupCount; i++)
await roundTripAction(i);
// Measurement with per-iteration timing
var tracker = new LatencyTracker(MeasurementCount);
var overallSw = Stopwatch.StartNew();
for (var i = 0; i < MeasurementCount; i++)
{
var start = Stopwatch.GetTimestamp();
await roundTripAction(i);
tracker.Record(Stopwatch.GetTimestamp() - start);
}
overallSw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = MeasurementCount,
TotalBytes = (long)MeasurementCount * payloadSize,
Duration = overallSw.Elapsed,
Latencies = tracker.ComputePercentiles(),
};
}
}

View File

@@ -0,0 +1,51 @@
using System.Diagnostics;
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Pre-allocated latency recording buffer with percentile computation.
/// Records elapsed ticks from Stopwatch for each sample.
/// </summary>
public sealed class LatencyTracker
{
private readonly long[] _samples;
private int _count;
private bool _sorted;
public LatencyTracker(int capacity)
{
_samples = new long[capacity];
}
public void Record(long elapsedTicks)
{
if (_count < _samples.Length)
{
_samples[_count++] = elapsedTicks;
_sorted = false;
}
}
public LatencyPercentiles ComputePercentiles()
{
if (_count == 0)
return new LatencyPercentiles(0, 0, 0, 0, 0);
if (!_sorted)
{
Array.Sort(_samples, 0, _count);
_sorted = true;
}
var ticksPerUs = Stopwatch.Frequency / 1_000_000.0;
return new LatencyPercentiles(
P50Us: _samples[Percentile(50)] / ticksPerUs,
P95Us: _samples[Percentile(95)] / ticksPerUs,
P99Us: _samples[Percentile(99)] / ticksPerUs,
MinUs: _samples[0] / ticksPerUs,
MaxUs: _samples[_count - 1] / ticksPerUs);
}
private int Percentile(int p) => Math.Min((int)(_count * (p / 100.0)), _count - 1);
}

View File

@@ -0,0 +1,7 @@
namespace NATS.Server.Benchmark.Tests.Infrastructure;
[CollectionDefinition("Benchmark-Core")]
public class BenchmarkCoreCollection : ICollectionFixture<CoreServerPairFixture>;
[CollectionDefinition("Benchmark-JetStream")]
public class BenchmarkJetStreamCollection : ICollectionFixture<JetStreamServerPairFixture>;

View File

@@ -0,0 +1,47 @@
using NATS.Client.Core;
namespace NATS.Server.Benchmark.Tests.Infrastructure;
/// <summary>
/// Starts both a Go and .NET NATS server for core pub/sub benchmarks.
/// Shared across all tests in the "Benchmark-Core" collection.
/// </summary>
public sealed class CoreServerPairFixture : IAsyncLifetime
{
private GoServerProcess? _goServer;
private DotNetServerProcess? _dotNetServer;
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
public bool GoAvailable => _goServer is not null;
public async Task InitializeAsync()
{
_dotNetServer = new DotNetServerProcess();
var dotNetTask = _dotNetServer.StartAsync();
if (GoServerProcess.IsAvailable())
{
_goServer = new GoServerProcess();
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
}
else
{
await dotNetTask;
}
}
public async Task DisposeAsync()
{
if (_goServer is not null)
await _goServer.DisposeAsync();
if (_dotNetServer is not null)
await _dotNetServer.DisposeAsync();
}
public NatsConnection CreateGoClient()
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoPort}" });
public NatsConnection CreateDotNetClient()
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetPort}" });
}

View File

@@ -0,0 +1,173 @@
using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using System.Text;
namespace NATS.Server.Benchmark.Tests.Infrastructure;
/// <summary>
/// Manages a NATS.Server.Host child process for benchmark testing.
/// </summary>
public sealed class DotNetServerProcess : IAsyncDisposable
{
private Process? _process;
private readonly StringBuilder _output = new();
private readonly object _outputLock = new();
private readonly string? _configContent;
private string? _configFilePath;
public int Port { get; }
public string Output
{
get
{
lock (_outputLock)
return _output.ToString();
}
}
public DotNetServerProcess(string? configContent = null)
{
Port = PortAllocator.AllocateFreePort();
_configContent = configContent;
}
public async Task StartAsync()
{
var hostDll = ResolveHostDll();
if (_configContent is not null)
{
_configFilePath = Path.Combine(Path.GetTempPath(), $"nats-bench-dotnet-{Guid.NewGuid():N}.conf");
await File.WriteAllTextAsync(_configFilePath, _configContent);
}
var args = new StringBuilder($"exec \"{hostDll}\" -p {Port}");
if (_configFilePath is not null)
args.Append($" -c \"{_configFilePath}\"");
var psi = new ProcessStartInfo
{
FileName = "dotnet",
Arguments = args.ToString(),
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
};
_process = new Process { StartInfo = psi, EnableRaisingEvents = true };
_process.OutputDataReceived += (_, e) =>
{
if (e.Data is not null)
lock (_outputLock) _output.AppendLine(e.Data);
};
_process.ErrorDataReceived += (_, e) =>
{
if (e.Data is not null)
lock (_outputLock) _output.AppendLine(e.Data);
};
_process.Start();
_process.BeginOutputReadLine();
_process.BeginErrorReadLine();
await WaitForTcpReadyAsync();
}
public async ValueTask DisposeAsync()
{
if (_process is not null)
{
if (!_process.HasExited)
{
_process.Kill(entireProcessTree: true);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
try
{
await _process.WaitForExitAsync(cts.Token);
}
catch (OperationCanceledException ex) when (!_process.HasExited)
{
throw new InvalidOperationException(
"NATS .NET server process did not exit within 5s after kill.", ex);
}
}
_process.Dispose();
_process = null;
}
if (_configFilePath is not null && File.Exists(_configFilePath))
{
File.Delete(_configFilePath);
_configFilePath = null;
}
}
private async Task WaitForTcpReadyAsync()
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(100));
SocketException? lastError = null;
while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
{
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(new IPEndPoint(IPAddress.Loopback, Port), timeout.Token);
return;
}
catch (SocketException ex)
{
lastError = ex;
}
}
throw new TimeoutException(
$"NATS .NET server did not become ready on port {Port} within 10s. Last error: {lastError?.Message}\n\nServer output:\n{Output}");
}
private static string ResolveHostDll()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir is not null)
{
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
{
var dll = Path.Combine(dir.FullName, "src", "NATS.Server.Host", "bin", "Debug", "net10.0", "NATS.Server.Host.dll");
if (File.Exists(dll))
return dll;
var build = Process.Start(new ProcessStartInfo
{
FileName = "dotnet",
Arguments = "build src/NATS.Server.Host/NATS.Server.Host.csproj -c Debug",
WorkingDirectory = dir.FullName,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
});
build!.WaitForExit();
if (build.ExitCode != 0)
throw new InvalidOperationException(
$"Failed to build NATS.Server.Host:\n{build.StandardError.ReadToEnd()}");
if (File.Exists(dll))
return dll;
throw new FileNotFoundException($"Built NATS.Server.Host but DLL not found at: {dll}");
}
dir = dir.Parent;
}
throw new FileNotFoundException(
"Could not find solution root (NatsDotNet.slnx) walking up from " + AppContext.BaseDirectory);
}
}

View File

@@ -0,0 +1,217 @@
using System.Diagnostics;
using System.Net;
using System.Net.Sockets;
using System.Text;
namespace NATS.Server.Benchmark.Tests.Infrastructure;
/// <summary>
/// Manages a Go nats-server child process for benchmark testing.
/// Builds the Go binary if not present and skips gracefully if Go is not installed.
/// </summary>
public sealed class GoServerProcess : IAsyncDisposable
{
private Process? _process;
private readonly StringBuilder _output = new();
private readonly object _outputLock = new();
private readonly string? _configContent;
private string? _configFilePath;
public int Port { get; }
public string Output
{
get
{
lock (_outputLock)
return _output.ToString();
}
}
public GoServerProcess(string? configContent = null)
{
Port = PortAllocator.AllocateFreePort();
_configContent = configContent;
}
/// <summary>
/// Returns true if Go is installed and the Go NATS server source exists.
/// </summary>
public static bool IsAvailable()
{
try
{
var goVersion = Process.Start(new ProcessStartInfo
{
FileName = "go",
Arguments = "version",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
});
goVersion!.WaitForExit();
if (goVersion.ExitCode != 0)
return false;
var solutionRoot = FindSolutionRoot();
return solutionRoot is not null &&
Directory.Exists(Path.Combine(solutionRoot, "golang", "nats-server"));
}
catch
{
return false;
}
}
public async Task StartAsync()
{
var binary = ResolveGoBinary();
if (_configContent is not null)
{
_configFilePath = Path.Combine(Path.GetTempPath(), $"nats-bench-go-{Guid.NewGuid():N}.conf");
await File.WriteAllTextAsync(_configFilePath, _configContent);
}
var args = new StringBuilder($"-p {Port}");
if (_configFilePath is not null)
args.Append($" -c \"{_configFilePath}\"");
var psi = new ProcessStartInfo
{
FileName = binary,
Arguments = args.ToString(),
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
};
_process = new Process { StartInfo = psi, EnableRaisingEvents = true };
_process.OutputDataReceived += (_, e) =>
{
if (e.Data is not null)
lock (_outputLock) _output.AppendLine(e.Data);
};
_process.ErrorDataReceived += (_, e) =>
{
if (e.Data is not null)
lock (_outputLock) _output.AppendLine(e.Data);
};
_process.Start();
_process.BeginOutputReadLine();
_process.BeginErrorReadLine();
await WaitForTcpReadyAsync();
}
public async ValueTask DisposeAsync()
{
if (_process is not null)
{
if (!_process.HasExited)
{
_process.Kill(entireProcessTree: true);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
try
{
await _process.WaitForExitAsync(cts.Token);
}
catch (OperationCanceledException ex) when (!_process.HasExited)
{
throw new InvalidOperationException(
"Go nats-server process did not exit within 5s after kill.", ex);
}
}
_process.Dispose();
_process = null;
}
if (_configFilePath is not null && File.Exists(_configFilePath))
{
File.Delete(_configFilePath);
_configFilePath = null;
}
}
private async Task WaitForTcpReadyAsync()
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15));
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(100));
SocketException? lastError = null;
while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
{
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(new IPEndPoint(IPAddress.Loopback, Port), timeout.Token);
return;
}
catch (SocketException ex)
{
lastError = ex;
}
}
throw new TimeoutException(
$"Go nats-server did not become ready on port {Port} within 15s. Last error: {lastError?.Message}\n\nServer output:\n{Output}");
}
private static string ResolveGoBinary()
{
var solutionRoot = FindSolutionRoot()
?? throw new FileNotFoundException(
"Could not find solution root (NatsDotNet.slnx) walking up from " + AppContext.BaseDirectory);
var goServerDir = Path.Combine(solutionRoot, "golang", "nats-server");
if (!Directory.Exists(goServerDir))
throw new DirectoryNotFoundException($"Go nats-server source not found at: {goServerDir}");
var binaryName = OperatingSystem.IsWindows() ? "nats-server.exe" : "nats-server";
var binaryPath = Path.Combine(goServerDir, binaryName);
if (File.Exists(binaryPath))
return binaryPath;
// Build the Go binary
var build = Process.Start(new ProcessStartInfo
{
FileName = "go",
Arguments = "build -o " + binaryName,
WorkingDirectory = goServerDir,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
});
build!.WaitForExit();
if (build.ExitCode != 0)
throw new InvalidOperationException(
$"Failed to build Go nats-server:\n{build.StandardError.ReadToEnd()}");
if (File.Exists(binaryPath))
return binaryPath;
throw new FileNotFoundException($"Built Go nats-server but binary not found at: {binaryPath}");
}
internal static string? FindSolutionRoot()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir is not null)
{
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
return dir.FullName;
dir = dir.Parent;
}
return null;
}
}

View File

@@ -0,0 +1,83 @@
using NATS.Client.Core;
namespace NATS.Server.Benchmark.Tests.Infrastructure;
/// <summary>
/// Starts both a Go and .NET NATS server with JetStream enabled for benchmark testing.
/// Shared across all tests in the "Benchmark-JetStream" collection.
/// </summary>
public sealed class JetStreamServerPairFixture : IAsyncLifetime
{
private GoServerProcess? _goServer;
private DotNetServerProcess? _dotNetServer;
private string? _goStoreDir;
private string? _dotNetStoreDir;
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
public bool GoAvailable => _goServer is not null;
public async Task InitializeAsync()
{
_dotNetStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-dotnet-js-" + Guid.NewGuid().ToString("N")[..8]);
Directory.CreateDirectory(_dotNetStoreDir);
var dotNetConfig = $$"""
jetstream {
store_dir: "{{_dotNetStoreDir}}"
max_mem_store: 256mb
max_file_store: 1gb
}
""";
_dotNetServer = new DotNetServerProcess(dotNetConfig);
var dotNetTask = _dotNetServer.StartAsync();
if (GoServerProcess.IsAvailable())
{
_goStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-go-js-" + Guid.NewGuid().ToString("N")[..8]);
Directory.CreateDirectory(_goStoreDir);
var goConfig = $$"""
jetstream {
store_dir: "{{_goStoreDir}}"
max_mem_store: 256mb
max_file_store: 1gb
}
""";
_goServer = new GoServerProcess(goConfig);
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
}
else
{
await dotNetTask;
}
}
public async Task DisposeAsync()
{
if (_goServer is not null)
await _goServer.DisposeAsync();
if (_dotNetServer is not null)
await _dotNetServer.DisposeAsync();
CleanupDir(_goStoreDir);
CleanupDir(_dotNetStoreDir);
}
public NatsConnection CreateGoClient()
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoPort}" });
public NatsConnection CreateDotNetClient()
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetPort}" });
private static void CleanupDir(string? dir)
{
if (dir is not null && Directory.Exists(dir))
{
try { Directory.Delete(dir, recursive: true); }
catch { /* best-effort cleanup */ }
}
}
}

View File

@@ -0,0 +1,14 @@
using System.Net;
using System.Net.Sockets;
namespace NATS.Server.Benchmark.Tests.Infrastructure;
internal static class PortAllocator
{
public static int AllocateFreePort()
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
return ((IPEndPoint)socket.LocalEndPoint!).Port;
}
}

View File

@@ -0,0 +1,93 @@
using System.Diagnostics;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.JetStream;
[Collection("Benchmark-JetStream")]
public class AsyncPublishTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task JSAsyncPublish_128B_FileStore()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int batchSize = 100;
var dotnetResult = await RunAsyncPublish("JS Async Publish (128B File)", "DotNet", payloadSize, messageCount, batchSize, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunAsyncPublish("JS Async Publish (128B File)", "Go", payloadSize, messageCount, batchSize, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunAsyncPublish(string name, string serverType, int payloadSize, int messageCount, int batchSize, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var streamName = $"BENCH_ASYNC_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
var subject = $"bench.js.async.{serverType.ToLowerInvariant()}";
await using var nats = createClient();
await nats.ConnectAsync();
var js = new NatsJSContext(nats);
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
{
Storage = StreamConfigStorage.File,
Retention = StreamConfigRetention.Limits,
MaxMsgs = 10_000_000,
});
try
{
// Warmup
for (var i = 0; i < 500; i++)
await js.PublishAsync(subject, payload);
// Measurement — fire-and-gather in batches
var sw = Stopwatch.StartNew();
var tasks = new List<ValueTask<PubAckResponse>>(batchSize);
for (var i = 0; i < messageCount; i++)
{
tasks.Add(js.PublishAsync(subject, payload));
if (tasks.Count >= batchSize)
{
foreach (var t in tasks)
await t;
tasks.Clear();
}
}
foreach (var t in tasks)
await t;
sw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = messageCount,
TotalBytes = (long)messageCount * payloadSize,
Duration = sw.Elapsed,
};
}
finally
{
await js.DeleteStreamAsync(streamName);
}
}
}

View File

@@ -0,0 +1,109 @@
using System.Diagnostics;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.JetStream;
[Collection("Benchmark-JetStream")]
public class DurableConsumerFetchTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task JSDurableFetch_Throughput()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int fetchBatchSize = 500;
var dotnetResult = await RunDurableFetch("JS Durable Fetch (128B)", "DotNet", payloadSize, messageCount, fetchBatchSize, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunDurableFetch("JS Durable Fetch (128B)", "Go", payloadSize, messageCount, fetchBatchSize, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunDurableFetch(
string name, string serverType, int payloadSize, int messageCount, int fetchBatchSize,
Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var streamName = $"BENCH_DUR_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
var subject = $"bench.js.durable.{serverType.ToLowerInvariant()}";
var consumerName = $"bench-dur-{serverType.ToLowerInvariant()}";
await using var nats = createClient();
await nats.ConnectAsync();
var js = new NatsJSContext(nats);
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
{
Storage = StreamConfigStorage.Memory,
Retention = StreamConfigRetention.Limits,
MaxMsgs = 10_000_000,
});
try
{
// Pre-populate stream
var pubTasks = new List<ValueTask<PubAckResponse>>(1000);
for (var i = 0; i < messageCount; i++)
{
pubTasks.Add(js.PublishAsync(subject, payload));
if (pubTasks.Count >= 1000)
{
foreach (var t in pubTasks)
await t;
pubTasks.Clear();
}
}
foreach (var t in pubTasks)
await t;
// Create durable consumer
var consumer = await js.CreateOrUpdateConsumerAsync(streamName, new ConsumerConfig(consumerName)
{
AckPolicy = ConsumerConfigAckPolicy.None,
});
// Fetch in batches
var received = 0;
var sw = Stopwatch.StartNew();
while (received < messageCount)
{
await foreach (var msg in consumer.FetchAsync<byte[]>(new NatsJSFetchOpts { MaxMsgs = fetchBatchSize }))
{
received++;
if (received >= messageCount)
break;
}
}
sw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = received,
TotalBytes = (long)received * payloadSize,
Duration = sw.Elapsed,
};
}
finally
{
await js.DeleteStreamAsync(streamName);
}
}
}

View File

@@ -0,0 +1,108 @@
using System.Diagnostics;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.JetStream;
[Collection("Benchmark-JetStream")]
public class OrderedConsumerTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task JSOrderedConsumer_Throughput()
{
const int payloadSize = 128;
const int messageCount = 5_000;
BenchmarkResult? dotnetResult = null;
try
{
dotnetResult = await RunOrderedConsume("JS Ordered Consumer (128B)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
}
catch (Exception ex) when (ex.GetType().Name.Contains("NatsJS"))
{
output.WriteLine($"[DotNet] Ordered consumer not fully supported: {ex.Message}");
}
if (fixture.GoAvailable)
{
var goResult = await RunOrderedConsume("JS Ordered Consumer (128B)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
if (dotnetResult is not null)
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
else
BenchmarkResultWriter.WriteSingle(output, goResult);
}
else if (dotnetResult is not null)
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunOrderedConsume(string name, string serverType, int payloadSize, int messageCount, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var streamName = $"BENCH_ORD_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
var subject = $"bench.js.ordered.{serverType.ToLowerInvariant()}";
await using var nats = createClient();
await nats.ConnectAsync();
var js = new NatsJSContext(nats);
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
{
Storage = StreamConfigStorage.Memory,
Retention = StreamConfigRetention.Limits,
MaxMsgs = 10_000_000,
});
try
{
// Pre-populate stream
var pubTasks = new List<ValueTask<PubAckResponse>>(1000);
for (var i = 0; i < messageCount; i++)
{
pubTasks.Add(js.PublishAsync(subject, payload));
if (pubTasks.Count >= 1000)
{
foreach (var t in pubTasks)
await t;
pubTasks.Clear();
}
}
foreach (var t in pubTasks)
await t;
// Consume via ordered consumer
var consumer = await js.CreateOrderedConsumerAsync(streamName);
var received = 0;
var sw = Stopwatch.StartNew();
await foreach (var msg in consumer.ConsumeAsync<byte[]>())
{
received++;
if (received >= messageCount)
break;
}
sw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = received,
TotalBytes = (long)received * payloadSize,
Duration = sw.Elapsed,
};
}
finally
{
await js.DeleteStreamAsync(streamName);
}
}
}

View File

@@ -0,0 +1,62 @@
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.JetStream;
[Collection("Benchmark-JetStream")]
public class SyncPublishTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
{
private readonly BenchmarkRunner _runner = new() { WarmupCount = 500, MeasurementCount = 10_000 };
[Fact]
[Trait("Category", "Benchmark")]
public async Task JSSyncPublish_16B_MemoryStore()
{
const int payloadSize = 16;
var dotnetResult = await RunSyncPublish("JS Sync Publish (16B Memory)", "DotNet", payloadSize, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunSyncPublish("JS Sync Publish (16B Memory)", "Go", payloadSize, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private async Task<BenchmarkResult> RunSyncPublish(string name, string serverType, int payloadSize, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
var streamName = $"BENCH_SYNC_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
var subject = $"bench.js.sync.{serverType.ToLowerInvariant()}";
await using var nats = createClient();
await nats.ConnectAsync();
var js = new NatsJSContext(nats);
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
{
Storage = StreamConfigStorage.Memory,
Retention = StreamConfigRetention.Limits,
MaxMsgs = 1_000_000,
});
try
{
var result = await _runner.MeasureThroughputAsync(name, serverType, payloadSize,
async _ => await js.PublishAsync(subject, payload));
return result;
}
finally
{
await js.DeleteStreamAsync(streamName);
}
}
}

View File

@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" />
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="NATS.Client.Core" />
<PackageReference Include="NATS.Client.JetStream" />
<PackageReference Include="Shouldly" />
<PackageReference Include="xunit" />
<PackageReference Include="xunit.runner.visualstudio" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
<Using Include="Shouldly" />
</ItemGroup>
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,113 @@
# NATS.Server.Benchmark.Tests
Side-by-side performance comparison of the Go and .NET NATS server implementations. Both servers are launched as child processes on ephemeral ports and exercised with identical workloads using the `NATS.Client.Core` / `NATS.Client.JetStream` NuGet client libraries.
## Prerequisites
- .NET 10 SDK
- Go toolchain (for Go server comparison; benchmarks still run .NET-only if Go is unavailable)
- The Go NATS server source at `golang/nats-server/` (the Go binary is built automatically on first run)
## Running Benchmarks
All benchmark tests are tagged with `[Trait("Category", "Benchmark")]`, so a plain `dotnet test` against the solution will **not** run them. Use the `--filter` flag.
```bash
# Run all benchmarks
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark" -v normal
# Core pub/sub only
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~CorePubSub" -v normal
# Request/reply only
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~RequestReply" -v normal
# JetStream only
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~JetStream" -v normal
# A single benchmark by name
dotnet test tests/NATS.Server.Benchmark.Tests --filter "FullyQualifiedName=NATS.Server.Benchmark.Tests.CorePubSub.SinglePublisherThroughputTests.PubNoSub_16B" -v normal
```
Use `-v normal` or `--logger "console;verbosity=detailed"` to see the comparison output in the console. Without verbosity, xUnit suppresses `ITestOutputHelper` output for passing tests.
## Benchmark List
| Test Class | Test Method | What It Measures |
|---|---|---|
| `SinglePublisherThroughputTests` | `PubNoSub_16B` | Publish-only throughput, 16-byte payload |
| `SinglePublisherThroughputTests` | `PubNoSub_128B` | Publish-only throughput, 128-byte payload |
| `PubSubOneToOneTests` | `PubSub1To1_16B` | 1 publisher, 1 subscriber, 16-byte payload |
| `PubSubOneToOneTests` | `PubSub1To1_16KB` | 1 publisher, 1 subscriber, 16 KB payload |
| `FanOutTests` | `FanOut1To4_128B` | 1 publisher, 4 subscribers, 128-byte payload |
| `MultiPubSubTests` | `MultiPubSub4x4_128B` | 4 publishers, 4 subscribers, 128-byte payload |
| `SingleClientLatencyTests` | `RequestReply_SingleClient_128B` | Request/reply round-trip latency, 1 client, 1 service |
| `MultiClientLatencyTests` | `RequestReply_10Clients2Services_16B` | Request/reply latency, 10 concurrent clients, 2 queue-group services |
| `SyncPublishTests` | `JSSyncPublish_16B_MemoryStore` | JetStream synchronous publish, memory-backed stream |
| `AsyncPublishTests` | `JSAsyncPublish_128B_FileStore` | JetStream async batch publish, file-backed stream |
| `OrderedConsumerTests` | `JSOrderedConsumer_Throughput` | JetStream ordered ephemeral consumer read throughput |
| `DurableConsumerFetchTests` | `JSDurableFetch_Throughput` | JetStream durable consumer fetch-in-batches throughput |
## Output Format
Each test writes a side-by-side comparison to xUnit's test output:
```
=== Single Publisher (16B) ===
Go: 2,436,416 msg/s | 37.2 MB/s | 41 ms
.NET: 1,425,767 msg/s | 21.8 MB/s | 70 ms
Ratio: 0.59x (.NET / Go)
```
Request/reply tests also include latency percentiles:
```
Latency (us):
P50 P95 P99 Min Max
Go 104.5 124.2 146.2 82.8 1204.5
.NET 134.7 168.0 190.5 91.6 3469.5
```
If Go is not available, only the .NET result is printed.
## Updating benchmarks_comparison.md
After running the full benchmark suite, update `benchmarks_comparison.md` in the repository root with the new numbers:
1. Run all benchmarks with detailed output:
```bash
dotnet test tests/NATS.Server.Benchmark.Tests \
--filter "Category=Benchmark" \
-v normal \
--logger "console;verbosity=detailed" 2>&1 | tee /tmp/bench-output.txt
```
2. Open `/tmp/bench-output.txt` and extract the comparison blocks from the "Standard Output Messages" sections.
3. Update the tables in `benchmarks_comparison.md` with the new msg/s, MB/s, ratio, and latency values. Update the date on the first line and the environment description.
4. Review the Summary table and Key Observations — update assessments if ratios have changed significantly.
## Architecture
```
Infrastructure/
PortAllocator.cs # Ephemeral port allocation (bind to port 0)
DotNetServerProcess.cs # Builds + launches NATS.Server.Host
GoServerProcess.cs # Builds + launches golang/nats-server
CoreServerPairFixture.cs # IAsyncLifetime: Go + .NET servers for core tests
JetStreamServerPairFixture # IAsyncLifetime: Go + .NET servers with JetStream
Collections.cs # xUnit collection definitions
Harness/
BenchmarkResult.cs # Result record (msg/s, MB/s, latencies)
LatencyTracker.cs # Pre-allocated sample buffer, percentile math
BenchmarkRunner.cs # Warmup + timed measurement loop
BenchmarkResultWriter.cs # Formats side-by-side comparison output
```
Server pair fixtures start both servers once per xUnit collection and expose `CreateGoClient()` / `CreateDotNetClient()` factory methods. Test parallelization is disabled at the assembly level (`AssemblyInfo.cs`) to prevent resource contention between collections.
## Notes
- The Go binary at `golang/nats-server/nats-server` is built automatically on first run via `go build`. Subsequent runs reuse the cached binary.
- If Go is not installed or `golang/nats-server/` does not exist, Go benchmarks are skipped and only .NET results are reported.
- JetStream tests create temporary store directories under `$TMPDIR` and clean them up on teardown.
- Message counts are intentionally conservative (2K10K per test) to keep the full suite under 2 minutes while still producing meaningful throughput ratios. For higher-fidelity numbers, increase the counts in individual test methods or the `BenchmarkRunner` configuration.

View File

@@ -0,0 +1,106 @@
using System.Diagnostics;
using NATS.Client.Core;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.RequestReply;
[Collection("Benchmark-Core")]
public class MultiClientLatencyTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
[Fact]
[Trait("Category", "Benchmark")]
public async Task RequestReply_10Clients2Services_16B()
{
const int payloadSize = 16;
const int requestsPerClient = 1_000;
const int clientCount = 10;
const int serviceCount = 2;
var dotnetResult = await RunMultiLatency("Request-Reply 10Cx2S (16B)", "DotNet", payloadSize, requestsPerClient, clientCount, serviceCount, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunMultiLatency("Request-Reply 10Cx2S (16B)", "Go", payloadSize, requestsPerClient, clientCount, serviceCount, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private static async Task<BenchmarkResult> RunMultiLatency(
string name, string serverType, int payloadSize, int requestsPerClient,
int clientCount, int serviceCount, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
const string subject = "bench.reqrep.multi";
var queueGroup = $"bench-svc-{serverType.ToLowerInvariant()}";
// Start service responders on a queue group
var serviceClients = new NatsConnection[serviceCount];
var serviceSubs = new INatsSub<byte[]>[serviceCount];
var serviceTasks = new Task[serviceCount];
for (var i = 0; i < serviceCount; i++)
{
serviceClients[i] = createClient();
await serviceClients[i].ConnectAsync();
serviceSubs[i] = await serviceClients[i].SubscribeCoreAsync<byte[]>(subject, queueGroup: queueGroup);
var client = serviceClients[i];
var sub = serviceSubs[i];
serviceTasks[i] = Task.Run(async () =>
{
await foreach (var msg in sub.Msgs.ReadAllAsync())
{
if (msg.ReplyTo is not null)
await client.PublishAsync(msg.ReplyTo, payload);
}
});
}
await Task.Delay(50);
// Run concurrent clients
var totalMessages = requestsPerClient * clientCount;
var tracker = new LatencyTracker(totalMessages);
var sw = Stopwatch.StartNew();
var clientTasks = new Task[clientCount];
for (var c = 0; c < clientCount; c++)
{
clientTasks[c] = Task.Run(async () =>
{
await using var client = createClient();
await client.ConnectAsync();
for (var i = 0; i < requestsPerClient; i++)
{
var start = Stopwatch.GetTimestamp();
await client.RequestAsync<byte[], byte[]>(subject, payload);
tracker.Record(Stopwatch.GetTimestamp() - start);
}
});
}
await Task.WhenAll(clientTasks);
sw.Stop();
foreach (var sub in serviceSubs)
await sub.UnsubscribeAsync();
foreach (var client in serviceClients)
await client.DisposeAsync();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = totalMessages,
TotalBytes = (long)totalMessages * payloadSize,
Duration = sw.Elapsed,
Latencies = tracker.ComputePercentiles(),
};
}
}

View File

@@ -0,0 +1,64 @@
using NATS.Client.Core;
using NATS.Server.Benchmark.Tests.Harness;
using NATS.Server.Benchmark.Tests.Infrastructure;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.RequestReply;
[Collection("Benchmark-Core")]
public class SingleClientLatencyTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
private readonly BenchmarkRunner _runner = new() { WarmupCount = 500, MeasurementCount = 10_000 };
[Fact]
[Trait("Category", "Benchmark")]
public async Task RequestReply_SingleClient_128B()
{
const int payloadSize = 128;
const string subject = "bench.reqrep.single";
var dotnetResult = await RunLatency("Request-Reply Single (128B)", "DotNet", subject, payloadSize, fixture.CreateDotNetClient);
if (fixture.GoAvailable)
{
var goResult = await RunLatency("Request-Reply Single (128B)", "Go", subject, payloadSize, fixture.CreateGoClient);
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
}
else
{
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
}
}
private async Task<BenchmarkResult> RunLatency(string name, string serverType, string subject, int payloadSize, Func<NatsConnection> createClient)
{
var payload = new byte[payloadSize];
await using var serviceClient = createClient();
await using var requestClient = createClient();
await serviceClient.ConnectAsync();
await requestClient.ConnectAsync();
// Start service responder
var sub = await serviceClient.SubscribeCoreAsync<byte[]>(subject);
var responderTask = Task.Run(async () =>
{
await foreach (var msg in sub.Msgs.ReadAllAsync())
{
if (msg.ReplyTo is not null)
await serviceClient.PublishAsync(msg.ReplyTo, payload);
}
});
await Task.Delay(50);
var result = await _runner.MeasureLatencyAsync(name, serverType, payloadSize,
async _ =>
{
await requestClient.RequestAsync<byte[], byte[]>(subject, payload);
});
await sub.UnsubscribeAsync();
return result;
}
}

View File

@@ -0,0 +1,5 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"parallelizeAssembly": false,
"parallelizeTestCollections": false
}