feat: add benchmark test project for Go vs .NET server comparison
Side-by-side performance benchmarks using NATS.Client.Core against both servers on ephemeral ports. Includes core pub/sub, request/reply latency, and JetStream throughput tests with comparison output and benchmarks_comparison.md results. Also fixes timestamp flakiness in StoreInterfaceTests by using explicit timestamps.
This commit is contained in:
27
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkResult.cs
Normal file
27
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkResult.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Captures the results of a single benchmark run against one server.
|
||||
/// </summary>
|
||||
public sealed record BenchmarkResult
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required string ServerType { get; init; }
|
||||
public required long TotalMessages { get; init; }
|
||||
public required long TotalBytes { get; init; }
|
||||
public required TimeSpan Duration { get; init; }
|
||||
|
||||
/// <summary>Latency percentiles in microseconds, if measured.</summary>
|
||||
public LatencyPercentiles? Latencies { get; init; }
|
||||
|
||||
public double MessagesPerSecond => TotalMessages / Duration.TotalSeconds;
|
||||
public double BytesPerSecond => TotalBytes / Duration.TotalSeconds;
|
||||
public double MegabytesPerSecond => BytesPerSecond / (1024.0 * 1024.0);
|
||||
}
|
||||
|
||||
public sealed record LatencyPercentiles(
|
||||
double P50Us,
|
||||
double P95Us,
|
||||
double P99Us,
|
||||
double MinUs,
|
||||
double MaxUs);
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Globalization;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Writes side-by-side benchmark comparison output to xUnit's ITestOutputHelper.
|
||||
/// </summary>
|
||||
public static class BenchmarkResultWriter
|
||||
{
|
||||
public static void WriteComparison(ITestOutputHelper output, BenchmarkResult goResult, BenchmarkResult dotnetResult)
|
||||
{
|
||||
var ratio = dotnetResult.MessagesPerSecond / goResult.MessagesPerSecond;
|
||||
|
||||
output.WriteLine($"=== {goResult.Name} ===");
|
||||
output.WriteLine($"Go: {FormatRate(goResult.MessagesPerSecond)} msg/s | {goResult.MegabytesPerSecond:F1} MB/s | {goResult.Duration.TotalMilliseconds:F0} ms");
|
||||
output.WriteLine($".NET: {FormatRate(dotnetResult.MessagesPerSecond)} msg/s | {dotnetResult.MegabytesPerSecond:F1} MB/s | {dotnetResult.Duration.TotalMilliseconds:F0} ms");
|
||||
output.WriteLine($"Ratio: {ratio:F2}x (.NET / Go)");
|
||||
|
||||
if (goResult.Latencies is not null && dotnetResult.Latencies is not null)
|
||||
{
|
||||
output.WriteLine("");
|
||||
output.WriteLine("Latency (us):");
|
||||
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
|
||||
WriteLatencyRow(output, "Go", goResult.Latencies);
|
||||
WriteLatencyRow(output, ".NET", dotnetResult.Latencies);
|
||||
}
|
||||
|
||||
output.WriteLine("");
|
||||
}
|
||||
|
||||
public static void WriteSingle(ITestOutputHelper output, BenchmarkResult result)
|
||||
{
|
||||
output.WriteLine($"=== {result.Name} ({result.ServerType}) ===");
|
||||
output.WriteLine($"{FormatRate(result.MessagesPerSecond)} msg/s | {result.MegabytesPerSecond:F1} MB/s | {result.Duration.TotalMilliseconds:F0} ms");
|
||||
|
||||
if (result.Latencies is not null)
|
||||
{
|
||||
output.WriteLine("");
|
||||
output.WriteLine("Latency (us):");
|
||||
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
|
||||
WriteLatencyRow(output, result.ServerType, result.Latencies);
|
||||
}
|
||||
|
||||
output.WriteLine("");
|
||||
}
|
||||
|
||||
private static void WriteLatencyRow(ITestOutputHelper output, string label, LatencyPercentiles p)
|
||||
{
|
||||
output.WriteLine($" {label,8} {p.P50Us,10:F1} {p.P95Us,10:F1} {p.P99Us,10:F1} {p.MinUs,10:F1} {p.MaxUs,10:F1}");
|
||||
}
|
||||
|
||||
private static string FormatRate(double rate)
|
||||
=> rate.ToString("N0", CultureInfo.InvariantCulture).PadLeft(15);
|
||||
}
|
||||
80
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkRunner.cs
Normal file
80
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkRunner.cs
Normal file
@@ -0,0 +1,80 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight benchmark runner with warmup + timed measurement.
|
||||
/// </summary>
|
||||
public sealed class BenchmarkRunner
|
||||
{
|
||||
public int WarmupCount { get; init; } = 1_000;
|
||||
public int MeasurementCount { get; init; } = 100_000;
|
||||
|
||||
/// <summary>
|
||||
/// Measures throughput for a fire-and-forget style workload (pub-only or pub+sub).
|
||||
/// The <paramref name="action"/> is called <see cref="MeasurementCount"/> times.
|
||||
/// </summary>
|
||||
public async Task<BenchmarkResult> MeasureThroughputAsync(
|
||||
string name,
|
||||
string serverType,
|
||||
int payloadSize,
|
||||
Func<int, Task> action)
|
||||
{
|
||||
// Warmup
|
||||
for (var i = 0; i < WarmupCount; i++)
|
||||
await action(i);
|
||||
|
||||
// Measurement
|
||||
var sw = Stopwatch.StartNew();
|
||||
for (var i = 0; i < MeasurementCount; i++)
|
||||
await action(i);
|
||||
sw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = MeasurementCount,
|
||||
TotalBytes = (long)MeasurementCount * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures latency for a request-reply style workload.
|
||||
/// Records per-iteration round-trip time and computes percentiles.
|
||||
/// </summary>
|
||||
public async Task<BenchmarkResult> MeasureLatencyAsync(
|
||||
string name,
|
||||
string serverType,
|
||||
int payloadSize,
|
||||
Func<int, Task> roundTripAction)
|
||||
{
|
||||
// Warmup
|
||||
for (var i = 0; i < WarmupCount; i++)
|
||||
await roundTripAction(i);
|
||||
|
||||
// Measurement with per-iteration timing
|
||||
var tracker = new LatencyTracker(MeasurementCount);
|
||||
var overallSw = Stopwatch.StartNew();
|
||||
|
||||
for (var i = 0; i < MeasurementCount; i++)
|
||||
{
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
await roundTripAction(i);
|
||||
tracker.Record(Stopwatch.GetTimestamp() - start);
|
||||
}
|
||||
|
||||
overallSw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = MeasurementCount,
|
||||
TotalBytes = (long)MeasurementCount * payloadSize,
|
||||
Duration = overallSw.Elapsed,
|
||||
Latencies = tracker.ComputePercentiles(),
|
||||
};
|
||||
}
|
||||
}
|
||||
51
tests/NATS.Server.Benchmark.Tests/Harness/LatencyTracker.cs
Normal file
51
tests/NATS.Server.Benchmark.Tests/Harness/LatencyTracker.cs
Normal file
@@ -0,0 +1,51 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Pre-allocated latency recording buffer with percentile computation.
|
||||
/// Records elapsed ticks from Stopwatch for each sample.
|
||||
/// </summary>
|
||||
public sealed class LatencyTracker
|
||||
{
|
||||
private readonly long[] _samples;
|
||||
private int _count;
|
||||
private bool _sorted;
|
||||
|
||||
public LatencyTracker(int capacity)
|
||||
{
|
||||
_samples = new long[capacity];
|
||||
}
|
||||
|
||||
public void Record(long elapsedTicks)
|
||||
{
|
||||
if (_count < _samples.Length)
|
||||
{
|
||||
_samples[_count++] = elapsedTicks;
|
||||
_sorted = false;
|
||||
}
|
||||
}
|
||||
|
||||
public LatencyPercentiles ComputePercentiles()
|
||||
{
|
||||
if (_count == 0)
|
||||
return new LatencyPercentiles(0, 0, 0, 0, 0);
|
||||
|
||||
if (!_sorted)
|
||||
{
|
||||
Array.Sort(_samples, 0, _count);
|
||||
_sorted = true;
|
||||
}
|
||||
|
||||
var ticksPerUs = Stopwatch.Frequency / 1_000_000.0;
|
||||
|
||||
return new LatencyPercentiles(
|
||||
P50Us: _samples[Percentile(50)] / ticksPerUs,
|
||||
P95Us: _samples[Percentile(95)] / ticksPerUs,
|
||||
P99Us: _samples[Percentile(99)] / ticksPerUs,
|
||||
MinUs: _samples[0] / ticksPerUs,
|
||||
MaxUs: _samples[_count - 1] / ticksPerUs);
|
||||
}
|
||||
|
||||
private int Percentile(int p) => Math.Min((int)(_count * (p / 100.0)), _count - 1);
|
||||
}
|
||||
Reference in New Issue
Block a user