feat: add benchmark test project for Go vs .NET server comparison

Side-by-side performance benchmarks using NATS.Client.Core against both
servers on ephemeral ports. Includes core pub/sub, request/reply latency,
and JetStream throughput tests with comparison output and
benchmarks_comparison.md results. Also fixes timestamp flakiness in
StoreInterfaceTests by using explicit timestamps.
This commit is contained in:
Joseph Doherty
2026-03-13 01:23:31 -04:00
parent e9c86c51c3
commit 37575dc41c
28 changed files with 2264 additions and 12 deletions

View File

@@ -0,0 +1,27 @@
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Captures the results of a single benchmark run against one server.
/// </summary>
public sealed record BenchmarkResult
{
public required string Name { get; init; }
public required string ServerType { get; init; }
public required long TotalMessages { get; init; }
public required long TotalBytes { get; init; }
public required TimeSpan Duration { get; init; }
/// <summary>Latency percentiles in microseconds, if measured.</summary>
public LatencyPercentiles? Latencies { get; init; }
public double MessagesPerSecond => TotalMessages / Duration.TotalSeconds;
public double BytesPerSecond => TotalBytes / Duration.TotalSeconds;
public double MegabytesPerSecond => BytesPerSecond / (1024.0 * 1024.0);
}
public sealed record LatencyPercentiles(
double P50Us,
double P95Us,
double P99Us,
double MinUs,
double MaxUs);

View File

@@ -0,0 +1,55 @@
using System.Globalization;
using Xunit.Abstractions;
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Writes side-by-side benchmark comparison output to xUnit's ITestOutputHelper.
/// </summary>
public static class BenchmarkResultWriter
{
public static void WriteComparison(ITestOutputHelper output, BenchmarkResult goResult, BenchmarkResult dotnetResult)
{
var ratio = dotnetResult.MessagesPerSecond / goResult.MessagesPerSecond;
output.WriteLine($"=== {goResult.Name} ===");
output.WriteLine($"Go: {FormatRate(goResult.MessagesPerSecond)} msg/s | {goResult.MegabytesPerSecond:F1} MB/s | {goResult.Duration.TotalMilliseconds:F0} ms");
output.WriteLine($".NET: {FormatRate(dotnetResult.MessagesPerSecond)} msg/s | {dotnetResult.MegabytesPerSecond:F1} MB/s | {dotnetResult.Duration.TotalMilliseconds:F0} ms");
output.WriteLine($"Ratio: {ratio:F2}x (.NET / Go)");
if (goResult.Latencies is not null && dotnetResult.Latencies is not null)
{
output.WriteLine("");
output.WriteLine("Latency (us):");
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
WriteLatencyRow(output, "Go", goResult.Latencies);
WriteLatencyRow(output, ".NET", dotnetResult.Latencies);
}
output.WriteLine("");
}
public static void WriteSingle(ITestOutputHelper output, BenchmarkResult result)
{
output.WriteLine($"=== {result.Name} ({result.ServerType}) ===");
output.WriteLine($"{FormatRate(result.MessagesPerSecond)} msg/s | {result.MegabytesPerSecond:F1} MB/s | {result.Duration.TotalMilliseconds:F0} ms");
if (result.Latencies is not null)
{
output.WriteLine("");
output.WriteLine("Latency (us):");
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
WriteLatencyRow(output, result.ServerType, result.Latencies);
}
output.WriteLine("");
}
private static void WriteLatencyRow(ITestOutputHelper output, string label, LatencyPercentiles p)
{
output.WriteLine($" {label,8} {p.P50Us,10:F1} {p.P95Us,10:F1} {p.P99Us,10:F1} {p.MinUs,10:F1} {p.MaxUs,10:F1}");
}
private static string FormatRate(double rate)
=> rate.ToString("N0", CultureInfo.InvariantCulture).PadLeft(15);
}

View File

@@ -0,0 +1,80 @@
using System.Diagnostics;
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Lightweight benchmark runner with warmup + timed measurement.
/// </summary>
public sealed class BenchmarkRunner
{
public int WarmupCount { get; init; } = 1_000;
public int MeasurementCount { get; init; } = 100_000;
/// <summary>
/// Measures throughput for a fire-and-forget style workload (pub-only or pub+sub).
/// The <paramref name="action"/> is called <see cref="MeasurementCount"/> times.
/// </summary>
public async Task<BenchmarkResult> MeasureThroughputAsync(
string name,
string serverType,
int payloadSize,
Func<int, Task> action)
{
// Warmup
for (var i = 0; i < WarmupCount; i++)
await action(i);
// Measurement
var sw = Stopwatch.StartNew();
for (var i = 0; i < MeasurementCount; i++)
await action(i);
sw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = MeasurementCount,
TotalBytes = (long)MeasurementCount * payloadSize,
Duration = sw.Elapsed,
};
}
/// <summary>
/// Measures latency for a request-reply style workload.
/// Records per-iteration round-trip time and computes percentiles.
/// </summary>
public async Task<BenchmarkResult> MeasureLatencyAsync(
string name,
string serverType,
int payloadSize,
Func<int, Task> roundTripAction)
{
// Warmup
for (var i = 0; i < WarmupCount; i++)
await roundTripAction(i);
// Measurement with per-iteration timing
var tracker = new LatencyTracker(MeasurementCount);
var overallSw = Stopwatch.StartNew();
for (var i = 0; i < MeasurementCount; i++)
{
var start = Stopwatch.GetTimestamp();
await roundTripAction(i);
tracker.Record(Stopwatch.GetTimestamp() - start);
}
overallSw.Stop();
return new BenchmarkResult
{
Name = name,
ServerType = serverType,
TotalMessages = MeasurementCount,
TotalBytes = (long)MeasurementCount * payloadSize,
Duration = overallSw.Elapsed,
Latencies = tracker.ComputePercentiles(),
};
}
}

View File

@@ -0,0 +1,51 @@
using System.Diagnostics;
namespace NATS.Server.Benchmark.Tests.Harness;
/// <summary>
/// Pre-allocated latency recording buffer with percentile computation.
/// Records elapsed ticks from Stopwatch for each sample.
/// </summary>
public sealed class LatencyTracker
{
private readonly long[] _samples;
private int _count;
private bool _sorted;
public LatencyTracker(int capacity)
{
_samples = new long[capacity];
}
public void Record(long elapsedTicks)
{
if (_count < _samples.Length)
{
_samples[_count++] = elapsedTicks;
_sorted = false;
}
}
public LatencyPercentiles ComputePercentiles()
{
if (_count == 0)
return new LatencyPercentiles(0, 0, 0, 0, 0);
if (!_sorted)
{
Array.Sort(_samples, 0, _count);
_sorted = true;
}
var ticksPerUs = Stopwatch.Frequency / 1_000_000.0;
return new LatencyPercentiles(
P50Us: _samples[Percentile(50)] / ticksPerUs,
P95Us: _samples[Percentile(95)] / ticksPerUs,
P99Us: _samples[Percentile(99)] / ticksPerUs,
MinUs: _samples[0] / ticksPerUs,
MaxUs: _samples[_count - 1] / ticksPerUs);
}
private int Percentile(int p) => Math.Min((int)(_count * (p / 100.0)), _count - 1);
}