feat: add benchmark test project for Go vs .NET server comparison
Side-by-side performance benchmarks using NATS.Client.Core against both servers on ephemeral ports. Includes core pub/sub, request/reply latency, and JetStream throughput tests with comparison output and benchmarks_comparison.md results. Also fixes timestamp flakiness in StoreInterfaceTests by using explicit timestamps.
This commit is contained in:
1
tests/NATS.Server.Benchmark.Tests/AssemblyInfo.cs
Normal file
1
tests/NATS.Server.Benchmark.Tests/AssemblyInfo.cs
Normal file
@@ -0,0 +1 @@
|
||||
[assembly: CollectionBehavior(DisableTestParallelization = true)]
|
||||
99
tests/NATS.Server.Benchmark.Tests/CorePubSub/FanOutTests.cs
Normal file
99
tests/NATS.Server.Benchmark.Tests/CorePubSub/FanOutTests.cs
Normal file
@@ -0,0 +1,99 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class FanOutTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task FanOut1To4_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messageCount = 10_000;
|
||||
const int subscriberCount = 4;
|
||||
|
||||
var dotnetResult = await RunFanOut("Fan-Out 1:4 (128B)", "DotNet", payloadSize, messageCount, subscriberCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunFanOut("Fan-Out 1:4 (128B)", "Go", payloadSize, messageCount, subscriberCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunFanOut(string name, string serverType, int payloadSize, int messageCount, int subscriberCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var subject = $"bench.fanout.{serverType.ToLowerInvariant()}.{Guid.NewGuid():N}";
|
||||
|
||||
await using var pubClient = createClient();
|
||||
await pubClient.ConnectAsync();
|
||||
|
||||
var subClients = new NatsConnection[subscriberCount];
|
||||
var subs = new INatsSub<byte[]>[subscriberCount];
|
||||
var subTasks = new Task[subscriberCount];
|
||||
var totalExpected = messageCount * subscriberCount;
|
||||
var totalReceived = 0;
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
for (var i = 0; i < subscriberCount; i++)
|
||||
{
|
||||
subClients[i] = createClient();
|
||||
await subClients[i].ConnectAsync();
|
||||
subs[i] = await subClients[i].SubscribeCoreAsync<byte[]>(subject);
|
||||
}
|
||||
|
||||
// Flush to ensure all subscriptions are propagated
|
||||
foreach (var client in subClients)
|
||||
await client.PingAsync();
|
||||
await pubClient.PingAsync();
|
||||
|
||||
// Start reading after subscriptions are confirmed
|
||||
for (var i = 0; i < subscriberCount; i++)
|
||||
{
|
||||
var sub = subs[i];
|
||||
subTasks[i] = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var _ in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (Interlocked.Increment(ref totalReceived) >= totalExpected)
|
||||
{
|
||||
tcs.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
await pubClient.PublishAsync(subject, payload);
|
||||
await pubClient.PingAsync();
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||
await tcs.Task.WaitAsync(cts.Token);
|
||||
sw.Stop();
|
||||
|
||||
foreach (var sub in subs)
|
||||
await sub.UnsubscribeAsync();
|
||||
foreach (var client in subClients)
|
||||
await client.DisposeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = totalExpected,
|
||||
TotalBytes = (long)totalExpected * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
124
tests/NATS.Server.Benchmark.Tests/CorePubSub/MultiPubSubTests.cs
Normal file
124
tests/NATS.Server.Benchmark.Tests/CorePubSub/MultiPubSubTests.cs
Normal file
@@ -0,0 +1,124 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class MultiPubSubTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task MultiPubSub4x4_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messagesPerPublisher = 2_000;
|
||||
const int pubCount = 4;
|
||||
const int subCount = 4;
|
||||
|
||||
var dotnetResult = await RunMultiPubSub("Multi 4Px4S (128B)", "DotNet", payloadSize, messagesPerPublisher, pubCount, subCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunMultiPubSub("Multi 4Px4S (128B)", "Go", payloadSize, messagesPerPublisher, pubCount, subCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunMultiPubSub(
|
||||
string name, string serverType, int payloadSize, int messagesPerPublisher,
|
||||
int pubCount, int subCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var totalMessages = messagesPerPublisher * pubCount;
|
||||
var runId = Guid.NewGuid().ToString("N")[..8];
|
||||
var subjects = Enumerable.Range(0, pubCount).Select(i => $"bench.multi.{serverType.ToLowerInvariant()}.{runId}.{i}").ToArray();
|
||||
|
||||
// Create subscribers — one per subject
|
||||
var subClients = new NatsConnection[subCount];
|
||||
var subs = new INatsSub<byte[]>[subCount];
|
||||
var subTasks = new Task[subCount];
|
||||
var totalReceived = 0;
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
for (var i = 0; i < subCount; i++)
|
||||
{
|
||||
subClients[i] = createClient();
|
||||
await subClients[i].ConnectAsync();
|
||||
subs[i] = await subClients[i].SubscribeCoreAsync<byte[]>(subjects[i % subjects.Length]);
|
||||
}
|
||||
|
||||
// Flush to ensure all subscriptions are propagated
|
||||
foreach (var client in subClients)
|
||||
await client.PingAsync();
|
||||
|
||||
// Start reading
|
||||
for (var i = 0; i < subCount; i++)
|
||||
{
|
||||
var sub = subs[i];
|
||||
subTasks[i] = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var _ in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (Interlocked.Increment(ref totalReceived) >= totalMessages)
|
||||
{
|
||||
tcs.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Create publishers
|
||||
var pubClients = new NatsConnection[pubCount];
|
||||
for (var i = 0; i < pubCount; i++)
|
||||
{
|
||||
pubClients[i] = createClient();
|
||||
await pubClients[i].ConnectAsync();
|
||||
}
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
var pubTasks = new Task[pubCount];
|
||||
for (var p = 0; p < pubCount; p++)
|
||||
{
|
||||
var client = pubClients[p];
|
||||
var subject = subjects[p];
|
||||
pubTasks[p] = Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < messagesPerPublisher; i++)
|
||||
await client.PublishAsync(subject, payload);
|
||||
});
|
||||
}
|
||||
|
||||
await Task.WhenAll(pubTasks);
|
||||
|
||||
// Flush all publishers
|
||||
foreach (var client in pubClients)
|
||||
await client.PingAsync();
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||
await tcs.Task.WaitAsync(cts.Token);
|
||||
sw.Stop();
|
||||
|
||||
foreach (var sub in subs)
|
||||
await sub.UnsubscribeAsync();
|
||||
foreach (var client in subClients)
|
||||
await client.DisposeAsync();
|
||||
foreach (var client in pubClients)
|
||||
await client.DisposeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = totalMessages,
|
||||
TotalBytes = (long)totalMessages * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class PubSubOneToOneTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubSub1To1_16B()
|
||||
{
|
||||
const int payloadSize = 16;
|
||||
const int messageCount = 10_000;
|
||||
|
||||
var dotnetResult = await RunPubSub("PubSub 1:1 (16B)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunPubSub("PubSub 1:1 (16B)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubSub1To1_16KB()
|
||||
{
|
||||
const int payloadSize = 16 * 1024;
|
||||
const int messageCount = 1_000;
|
||||
|
||||
var dotnetResult = await RunPubSub("PubSub 1:1 (16KB)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunPubSub("PubSub 1:1 (16KB)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunPubSub(string name, string serverType, int payloadSize, int messageCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var subject = $"bench.pubsub.{serverType.ToLowerInvariant()}.{Guid.NewGuid():N}";
|
||||
|
||||
await using var pubClient = createClient();
|
||||
await using var subClient = createClient();
|
||||
await pubClient.ConnectAsync();
|
||||
await subClient.ConnectAsync();
|
||||
|
||||
var received = 0;
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
// Start subscriber
|
||||
var sub = await subClient.SubscribeCoreAsync<byte[]>(subject);
|
||||
|
||||
// Flush to ensure subscription is propagated
|
||||
await subClient.PingAsync();
|
||||
await pubClient.PingAsync();
|
||||
|
||||
var subTask = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var msg in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (Interlocked.Increment(ref received) >= messageCount)
|
||||
{
|
||||
tcs.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Publish and measure
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
await pubClient.PublishAsync(subject, payload);
|
||||
await pubClient.PingAsync(); // Flush all pending writes
|
||||
|
||||
// Wait for all messages received
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||
await tcs.Task.WaitAsync(cts.Token);
|
||||
sw.Stop();
|
||||
|
||||
await sub.UnsubscribeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = messageCount,
|
||||
TotalBytes = (long)messageCount * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class SinglePublisherThroughputTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
private readonly BenchmarkRunner _runner = new() { WarmupCount = 1_000, MeasurementCount = 100_000 };
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubNoSub_16B()
|
||||
{
|
||||
const int payloadSize = 16;
|
||||
var payload = new byte[payloadSize];
|
||||
const string subject = "bench.pub.nosub.16";
|
||||
|
||||
var dotnetResult = await RunThroughput("Single Publisher (16B)", "DotNet", subject, payload, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunThroughput("Single Publisher (16B)", "Go", subject, payload, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubNoSub_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
var payload = new byte[payloadSize];
|
||||
const string subject = "bench.pub.nosub.128";
|
||||
|
||||
var dotnetResult = await RunThroughput("Single Publisher (128B)", "DotNet", subject, payload, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunThroughput("Single Publisher (128B)", "Go", subject, payload, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<BenchmarkResult> RunThroughput(string name, string serverType, string subject, byte[] payload, Func<NatsConnection> createClient)
|
||||
{
|
||||
await using var client = createClient();
|
||||
await client.ConnectAsync();
|
||||
|
||||
return await _runner.MeasureThroughputAsync(name, serverType, payload.Length,
|
||||
async _ => await client.PublishAsync(subject, payload));
|
||||
}
|
||||
}
|
||||
27
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkResult.cs
Normal file
27
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkResult.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Captures the results of a single benchmark run against one server.
|
||||
/// </summary>
|
||||
public sealed record BenchmarkResult
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required string ServerType { get; init; }
|
||||
public required long TotalMessages { get; init; }
|
||||
public required long TotalBytes { get; init; }
|
||||
public required TimeSpan Duration { get; init; }
|
||||
|
||||
/// <summary>Latency percentiles in microseconds, if measured.</summary>
|
||||
public LatencyPercentiles? Latencies { get; init; }
|
||||
|
||||
public double MessagesPerSecond => TotalMessages / Duration.TotalSeconds;
|
||||
public double BytesPerSecond => TotalBytes / Duration.TotalSeconds;
|
||||
public double MegabytesPerSecond => BytesPerSecond / (1024.0 * 1024.0);
|
||||
}
|
||||
|
||||
public sealed record LatencyPercentiles(
|
||||
double P50Us,
|
||||
double P95Us,
|
||||
double P99Us,
|
||||
double MinUs,
|
||||
double MaxUs);
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Globalization;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Writes side-by-side benchmark comparison output to xUnit's ITestOutputHelper.
|
||||
/// </summary>
|
||||
public static class BenchmarkResultWriter
|
||||
{
|
||||
public static void WriteComparison(ITestOutputHelper output, BenchmarkResult goResult, BenchmarkResult dotnetResult)
|
||||
{
|
||||
var ratio = dotnetResult.MessagesPerSecond / goResult.MessagesPerSecond;
|
||||
|
||||
output.WriteLine($"=== {goResult.Name} ===");
|
||||
output.WriteLine($"Go: {FormatRate(goResult.MessagesPerSecond)} msg/s | {goResult.MegabytesPerSecond:F1} MB/s | {goResult.Duration.TotalMilliseconds:F0} ms");
|
||||
output.WriteLine($".NET: {FormatRate(dotnetResult.MessagesPerSecond)} msg/s | {dotnetResult.MegabytesPerSecond:F1} MB/s | {dotnetResult.Duration.TotalMilliseconds:F0} ms");
|
||||
output.WriteLine($"Ratio: {ratio:F2}x (.NET / Go)");
|
||||
|
||||
if (goResult.Latencies is not null && dotnetResult.Latencies is not null)
|
||||
{
|
||||
output.WriteLine("");
|
||||
output.WriteLine("Latency (us):");
|
||||
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
|
||||
WriteLatencyRow(output, "Go", goResult.Latencies);
|
||||
WriteLatencyRow(output, ".NET", dotnetResult.Latencies);
|
||||
}
|
||||
|
||||
output.WriteLine("");
|
||||
}
|
||||
|
||||
public static void WriteSingle(ITestOutputHelper output, BenchmarkResult result)
|
||||
{
|
||||
output.WriteLine($"=== {result.Name} ({result.ServerType}) ===");
|
||||
output.WriteLine($"{FormatRate(result.MessagesPerSecond)} msg/s | {result.MegabytesPerSecond:F1} MB/s | {result.Duration.TotalMilliseconds:F0} ms");
|
||||
|
||||
if (result.Latencies is not null)
|
||||
{
|
||||
output.WriteLine("");
|
||||
output.WriteLine("Latency (us):");
|
||||
output.WriteLine($" {"",8} {"P50",10} {"P95",10} {"P99",10} {"Min",10} {"Max",10}");
|
||||
WriteLatencyRow(output, result.ServerType, result.Latencies);
|
||||
}
|
||||
|
||||
output.WriteLine("");
|
||||
}
|
||||
|
||||
private static void WriteLatencyRow(ITestOutputHelper output, string label, LatencyPercentiles p)
|
||||
{
|
||||
output.WriteLine($" {label,8} {p.P50Us,10:F1} {p.P95Us,10:F1} {p.P99Us,10:F1} {p.MinUs,10:F1} {p.MaxUs,10:F1}");
|
||||
}
|
||||
|
||||
private static string FormatRate(double rate)
|
||||
=> rate.ToString("N0", CultureInfo.InvariantCulture).PadLeft(15);
|
||||
}
|
||||
80
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkRunner.cs
Normal file
80
tests/NATS.Server.Benchmark.Tests/Harness/BenchmarkRunner.cs
Normal file
@@ -0,0 +1,80 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight benchmark runner with warmup + timed measurement.
|
||||
/// </summary>
|
||||
public sealed class BenchmarkRunner
|
||||
{
|
||||
public int WarmupCount { get; init; } = 1_000;
|
||||
public int MeasurementCount { get; init; } = 100_000;
|
||||
|
||||
/// <summary>
|
||||
/// Measures throughput for a fire-and-forget style workload (pub-only or pub+sub).
|
||||
/// The <paramref name="action"/> is called <see cref="MeasurementCount"/> times.
|
||||
/// </summary>
|
||||
public async Task<BenchmarkResult> MeasureThroughputAsync(
|
||||
string name,
|
||||
string serverType,
|
||||
int payloadSize,
|
||||
Func<int, Task> action)
|
||||
{
|
||||
// Warmup
|
||||
for (var i = 0; i < WarmupCount; i++)
|
||||
await action(i);
|
||||
|
||||
// Measurement
|
||||
var sw = Stopwatch.StartNew();
|
||||
for (var i = 0; i < MeasurementCount; i++)
|
||||
await action(i);
|
||||
sw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = MeasurementCount,
|
||||
TotalBytes = (long)MeasurementCount * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures latency for a request-reply style workload.
|
||||
/// Records per-iteration round-trip time and computes percentiles.
|
||||
/// </summary>
|
||||
public async Task<BenchmarkResult> MeasureLatencyAsync(
|
||||
string name,
|
||||
string serverType,
|
||||
int payloadSize,
|
||||
Func<int, Task> roundTripAction)
|
||||
{
|
||||
// Warmup
|
||||
for (var i = 0; i < WarmupCount; i++)
|
||||
await roundTripAction(i);
|
||||
|
||||
// Measurement with per-iteration timing
|
||||
var tracker = new LatencyTracker(MeasurementCount);
|
||||
var overallSw = Stopwatch.StartNew();
|
||||
|
||||
for (var i = 0; i < MeasurementCount; i++)
|
||||
{
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
await roundTripAction(i);
|
||||
tracker.Record(Stopwatch.GetTimestamp() - start);
|
||||
}
|
||||
|
||||
overallSw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = MeasurementCount,
|
||||
TotalBytes = (long)MeasurementCount * payloadSize,
|
||||
Duration = overallSw.Elapsed,
|
||||
Latencies = tracker.ComputePercentiles(),
|
||||
};
|
||||
}
|
||||
}
|
||||
51
tests/NATS.Server.Benchmark.Tests/Harness/LatencyTracker.cs
Normal file
51
tests/NATS.Server.Benchmark.Tests/Harness/LatencyTracker.cs
Normal file
@@ -0,0 +1,51 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Pre-allocated latency recording buffer with percentile computation.
|
||||
/// Records elapsed ticks from Stopwatch for each sample.
|
||||
/// </summary>
|
||||
public sealed class LatencyTracker
|
||||
{
|
||||
private readonly long[] _samples;
|
||||
private int _count;
|
||||
private bool _sorted;
|
||||
|
||||
public LatencyTracker(int capacity)
|
||||
{
|
||||
_samples = new long[capacity];
|
||||
}
|
||||
|
||||
public void Record(long elapsedTicks)
|
||||
{
|
||||
if (_count < _samples.Length)
|
||||
{
|
||||
_samples[_count++] = elapsedTicks;
|
||||
_sorted = false;
|
||||
}
|
||||
}
|
||||
|
||||
public LatencyPercentiles ComputePercentiles()
|
||||
{
|
||||
if (_count == 0)
|
||||
return new LatencyPercentiles(0, 0, 0, 0, 0);
|
||||
|
||||
if (!_sorted)
|
||||
{
|
||||
Array.Sort(_samples, 0, _count);
|
||||
_sorted = true;
|
||||
}
|
||||
|
||||
var ticksPerUs = Stopwatch.Frequency / 1_000_000.0;
|
||||
|
||||
return new LatencyPercentiles(
|
||||
P50Us: _samples[Percentile(50)] / ticksPerUs,
|
||||
P95Us: _samples[Percentile(95)] / ticksPerUs,
|
||||
P99Us: _samples[Percentile(99)] / ticksPerUs,
|
||||
MinUs: _samples[0] / ticksPerUs,
|
||||
MaxUs: _samples[_count - 1] / ticksPerUs);
|
||||
}
|
||||
|
||||
private int Percentile(int p) => Math.Min((int)(_count * (p / 100.0)), _count - 1);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
[CollectionDefinition("Benchmark-Core")]
|
||||
public class BenchmarkCoreCollection : ICollectionFixture<CoreServerPairFixture>;
|
||||
|
||||
[CollectionDefinition("Benchmark-JetStream")]
|
||||
public class BenchmarkJetStreamCollection : ICollectionFixture<JetStreamServerPairFixture>;
|
||||
@@ -0,0 +1,47 @@
|
||||
using NATS.Client.Core;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Starts both a Go and .NET NATS server for core pub/sub benchmarks.
|
||||
/// Shared across all tests in the "Benchmark-Core" collection.
|
||||
/// </summary>
|
||||
public sealed class CoreServerPairFixture : IAsyncLifetime
|
||||
{
|
||||
private GoServerProcess? _goServer;
|
||||
private DotNetServerProcess? _dotNetServer;
|
||||
|
||||
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||
public bool GoAvailable => _goServer is not null;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_dotNetServer = new DotNetServerProcess();
|
||||
var dotNetTask = _dotNetServer.StartAsync();
|
||||
|
||||
if (GoServerProcess.IsAvailable())
|
||||
{
|
||||
_goServer = new GoServerProcess();
|
||||
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||
}
|
||||
else
|
||||
{
|
||||
await dotNetTask;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_goServer is not null)
|
||||
await _goServer.DisposeAsync();
|
||||
if (_dotNetServer is not null)
|
||||
await _dotNetServer.DisposeAsync();
|
||||
}
|
||||
|
||||
public NatsConnection CreateGoClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoPort}" });
|
||||
|
||||
public NatsConnection CreateDotNetClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetPort}" });
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Manages a NATS.Server.Host child process for benchmark testing.
|
||||
/// </summary>
|
||||
public sealed class DotNetServerProcess : IAsyncDisposable
|
||||
{
|
||||
private Process? _process;
|
||||
private readonly StringBuilder _output = new();
|
||||
private readonly object _outputLock = new();
|
||||
private readonly string? _configContent;
|
||||
private string? _configFilePath;
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public string Output
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_outputLock)
|
||||
return _output.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public DotNetServerProcess(string? configContent = null)
|
||||
{
|
||||
Port = PortAllocator.AllocateFreePort();
|
||||
_configContent = configContent;
|
||||
}
|
||||
|
||||
public async Task StartAsync()
|
||||
{
|
||||
var hostDll = ResolveHostDll();
|
||||
|
||||
if (_configContent is not null)
|
||||
{
|
||||
_configFilePath = Path.Combine(Path.GetTempPath(), $"nats-bench-dotnet-{Guid.NewGuid():N}.conf");
|
||||
await File.WriteAllTextAsync(_configFilePath, _configContent);
|
||||
}
|
||||
|
||||
var args = new StringBuilder($"exec \"{hostDll}\" -p {Port}");
|
||||
if (_configFilePath is not null)
|
||||
args.Append($" -c \"{_configFilePath}\"");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = "dotnet",
|
||||
Arguments = args.ToString(),
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
_process = new Process { StartInfo = psi, EnableRaisingEvents = true };
|
||||
|
||||
_process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
_process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
_process.Start();
|
||||
_process.BeginOutputReadLine();
|
||||
_process.BeginErrorReadLine();
|
||||
|
||||
await WaitForTcpReadyAsync();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_process is not null)
|
||||
{
|
||||
if (!_process.HasExited)
|
||||
{
|
||||
_process.Kill(entireProcessTree: true);
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
try
|
||||
{
|
||||
await _process.WaitForExitAsync(cts.Token);
|
||||
}
|
||||
catch (OperationCanceledException ex) when (!_process.HasExited)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"NATS .NET server process did not exit within 5s after kill.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
_process.Dispose();
|
||||
_process = null;
|
||||
}
|
||||
|
||||
if (_configFilePath is not null && File.Exists(_configFilePath))
|
||||
{
|
||||
File.Delete(_configFilePath);
|
||||
_configFilePath = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForTcpReadyAsync()
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(100));
|
||||
SocketException? lastError = null;
|
||||
|
||||
while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
|
||||
{
|
||||
try
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(new IPEndPoint(IPAddress.Loopback, Port), timeout.Token);
|
||||
return;
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
lastError = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"NATS .NET server did not become ready on port {Port} within 10s. Last error: {lastError?.Message}\n\nServer output:\n{Output}");
|
||||
}
|
||||
|
||||
private static string ResolveHostDll()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir is not null)
|
||||
{
|
||||
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
|
||||
{
|
||||
var dll = Path.Combine(dir.FullName, "src", "NATS.Server.Host", "bin", "Debug", "net10.0", "NATS.Server.Host.dll");
|
||||
if (File.Exists(dll))
|
||||
return dll;
|
||||
|
||||
var build = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "dotnet",
|
||||
Arguments = "build src/NATS.Server.Host/NATS.Server.Host.csproj -c Debug",
|
||||
WorkingDirectory = dir.FullName,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
});
|
||||
build!.WaitForExit();
|
||||
|
||||
if (build.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to build NATS.Server.Host:\n{build.StandardError.ReadToEnd()}");
|
||||
|
||||
if (File.Exists(dll))
|
||||
return dll;
|
||||
|
||||
throw new FileNotFoundException($"Built NATS.Server.Host but DLL not found at: {dll}");
|
||||
}
|
||||
|
||||
dir = dir.Parent;
|
||||
}
|
||||
|
||||
throw new FileNotFoundException(
|
||||
"Could not find solution root (NatsDotNet.slnx) walking up from " + AppContext.BaseDirectory);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Manages a Go nats-server child process for benchmark testing.
|
||||
/// Builds the Go binary if not present and skips gracefully if Go is not installed.
|
||||
/// </summary>
|
||||
public sealed class GoServerProcess : IAsyncDisposable
|
||||
{
|
||||
private Process? _process;
|
||||
private readonly StringBuilder _output = new();
|
||||
private readonly object _outputLock = new();
|
||||
private readonly string? _configContent;
|
||||
private string? _configFilePath;
|
||||
|
||||
public int Port { get; }
|
||||
|
||||
public string Output
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_outputLock)
|
||||
return _output.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public GoServerProcess(string? configContent = null)
|
||||
{
|
||||
Port = PortAllocator.AllocateFreePort();
|
||||
_configContent = configContent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if Go is installed and the Go NATS server source exists.
|
||||
/// </summary>
|
||||
public static bool IsAvailable()
|
||||
{
|
||||
try
|
||||
{
|
||||
var goVersion = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "go",
|
||||
Arguments = "version",
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
goVersion!.WaitForExit();
|
||||
if (goVersion.ExitCode != 0)
|
||||
return false;
|
||||
|
||||
var solutionRoot = FindSolutionRoot();
|
||||
return solutionRoot is not null &&
|
||||
Directory.Exists(Path.Combine(solutionRoot, "golang", "nats-server"));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StartAsync()
|
||||
{
|
||||
var binary = ResolveGoBinary();
|
||||
|
||||
if (_configContent is not null)
|
||||
{
|
||||
_configFilePath = Path.Combine(Path.GetTempPath(), $"nats-bench-go-{Guid.NewGuid():N}.conf");
|
||||
await File.WriteAllTextAsync(_configFilePath, _configContent);
|
||||
}
|
||||
|
||||
var args = new StringBuilder($"-p {Port}");
|
||||
if (_configFilePath is not null)
|
||||
args.Append($" -c \"{_configFilePath}\"");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = binary,
|
||||
Arguments = args.ToString(),
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
_process = new Process { StartInfo = psi, EnableRaisingEvents = true };
|
||||
|
||||
_process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
_process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is not null)
|
||||
lock (_outputLock) _output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
_process.Start();
|
||||
_process.BeginOutputReadLine();
|
||||
_process.BeginErrorReadLine();
|
||||
|
||||
await WaitForTcpReadyAsync();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_process is not null)
|
||||
{
|
||||
if (!_process.HasExited)
|
||||
{
|
||||
_process.Kill(entireProcessTree: true);
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
try
|
||||
{
|
||||
await _process.WaitForExitAsync(cts.Token);
|
||||
}
|
||||
catch (OperationCanceledException ex) when (!_process.HasExited)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Go nats-server process did not exit within 5s after kill.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
_process.Dispose();
|
||||
_process = null;
|
||||
}
|
||||
|
||||
if (_configFilePath is not null && File.Exists(_configFilePath))
|
||||
{
|
||||
File.Delete(_configFilePath);
|
||||
_configFilePath = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForTcpReadyAsync()
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(100));
|
||||
SocketException? lastError = null;
|
||||
|
||||
while (await timer.WaitForNextTickAsync(timeout.Token).ConfigureAwait(false))
|
||||
{
|
||||
try
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(new IPEndPoint(IPAddress.Loopback, Port), timeout.Token);
|
||||
return;
|
||||
}
|
||||
catch (SocketException ex)
|
||||
{
|
||||
lastError = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"Go nats-server did not become ready on port {Port} within 15s. Last error: {lastError?.Message}\n\nServer output:\n{Output}");
|
||||
}
|
||||
|
||||
private static string ResolveGoBinary()
|
||||
{
|
||||
var solutionRoot = FindSolutionRoot()
|
||||
?? throw new FileNotFoundException(
|
||||
"Could not find solution root (NatsDotNet.slnx) walking up from " + AppContext.BaseDirectory);
|
||||
|
||||
var goServerDir = Path.Combine(solutionRoot, "golang", "nats-server");
|
||||
if (!Directory.Exists(goServerDir))
|
||||
throw new DirectoryNotFoundException($"Go nats-server source not found at: {goServerDir}");
|
||||
|
||||
var binaryName = OperatingSystem.IsWindows() ? "nats-server.exe" : "nats-server";
|
||||
var binaryPath = Path.Combine(goServerDir, binaryName);
|
||||
|
||||
if (File.Exists(binaryPath))
|
||||
return binaryPath;
|
||||
|
||||
// Build the Go binary
|
||||
var build = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "go",
|
||||
Arguments = "build -o " + binaryName,
|
||||
WorkingDirectory = goServerDir,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
build!.WaitForExit();
|
||||
|
||||
if (build.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to build Go nats-server:\n{build.StandardError.ReadToEnd()}");
|
||||
|
||||
if (File.Exists(binaryPath))
|
||||
return binaryPath;
|
||||
|
||||
throw new FileNotFoundException($"Built Go nats-server but binary not found at: {binaryPath}");
|
||||
}
|
||||
|
||||
internal static string? FindSolutionRoot()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir is not null)
|
||||
{
|
||||
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
|
||||
return dir.FullName;
|
||||
dir = dir.Parent;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using NATS.Client.Core;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Starts both a Go and .NET NATS server with JetStream enabled for benchmark testing.
|
||||
/// Shared across all tests in the "Benchmark-JetStream" collection.
|
||||
/// </summary>
|
||||
public sealed class JetStreamServerPairFixture : IAsyncLifetime
|
||||
{
|
||||
private GoServerProcess? _goServer;
|
||||
private DotNetServerProcess? _dotNetServer;
|
||||
private string? _goStoreDir;
|
||||
private string? _dotNetStoreDir;
|
||||
|
||||
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||
public bool GoAvailable => _goServer is not null;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_dotNetStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-dotnet-js-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
Directory.CreateDirectory(_dotNetStoreDir);
|
||||
|
||||
var dotNetConfig = $$"""
|
||||
jetstream {
|
||||
store_dir: "{{_dotNetStoreDir}}"
|
||||
max_mem_store: 256mb
|
||||
max_file_store: 1gb
|
||||
}
|
||||
""";
|
||||
|
||||
_dotNetServer = new DotNetServerProcess(dotNetConfig);
|
||||
var dotNetTask = _dotNetServer.StartAsync();
|
||||
|
||||
if (GoServerProcess.IsAvailable())
|
||||
{
|
||||
_goStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-go-js-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
Directory.CreateDirectory(_goStoreDir);
|
||||
|
||||
var goConfig = $$"""
|
||||
jetstream {
|
||||
store_dir: "{{_goStoreDir}}"
|
||||
max_mem_store: 256mb
|
||||
max_file_store: 1gb
|
||||
}
|
||||
""";
|
||||
|
||||
_goServer = new GoServerProcess(goConfig);
|
||||
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||
}
|
||||
else
|
||||
{
|
||||
await dotNetTask;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_goServer is not null)
|
||||
await _goServer.DisposeAsync();
|
||||
if (_dotNetServer is not null)
|
||||
await _dotNetServer.DisposeAsync();
|
||||
|
||||
CleanupDir(_goStoreDir);
|
||||
CleanupDir(_dotNetStoreDir);
|
||||
}
|
||||
|
||||
public NatsConnection CreateGoClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoPort}" });
|
||||
|
||||
public NatsConnection CreateDotNetClient()
|
||||
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetPort}" });
|
||||
|
||||
private static void CleanupDir(string? dir)
|
||||
{
|
||||
if (dir is not null && Directory.Exists(dir))
|
||||
{
|
||||
try { Directory.Delete(dir, recursive: true); }
|
||||
catch { /* best-effort cleanup */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
|
||||
internal static class PortAllocator
|
||||
{
|
||||
public static int AllocateFreePort()
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
return ((IPEndPoint)socket.LocalEndPoint!).Port;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using System.Diagnostics;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.JetStream;
|
||||
|
||||
[Collection("Benchmark-JetStream")]
|
||||
public class AsyncPublishTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task JSAsyncPublish_128B_FileStore()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messageCount = 5_000;
|
||||
const int batchSize = 100;
|
||||
|
||||
var dotnetResult = await RunAsyncPublish("JS Async Publish (128B File)", "DotNet", payloadSize, messageCount, batchSize, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunAsyncPublish("JS Async Publish (128B File)", "Go", payloadSize, messageCount, batchSize, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunAsyncPublish(string name, string serverType, int payloadSize, int messageCount, int batchSize, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var streamName = $"BENCH_ASYNC_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
|
||||
var subject = $"bench.js.async.{serverType.ToLowerInvariant()}";
|
||||
|
||||
await using var nats = createClient();
|
||||
await nats.ConnectAsync();
|
||||
var js = new NatsJSContext(nats);
|
||||
|
||||
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
|
||||
{
|
||||
Storage = StreamConfigStorage.File,
|
||||
Retention = StreamConfigRetention.Limits,
|
||||
MaxMsgs = 10_000_000,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
// Warmup
|
||||
for (var i = 0; i < 500; i++)
|
||||
await js.PublishAsync(subject, payload);
|
||||
|
||||
// Measurement — fire-and-gather in batches
|
||||
var sw = Stopwatch.StartNew();
|
||||
var tasks = new List<ValueTask<PubAckResponse>>(batchSize);
|
||||
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
{
|
||||
tasks.Add(js.PublishAsync(subject, payload));
|
||||
|
||||
if (tasks.Count >= batchSize)
|
||||
{
|
||||
foreach (var t in tasks)
|
||||
await t;
|
||||
tasks.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var t in tasks)
|
||||
await t;
|
||||
|
||||
sw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = messageCount,
|
||||
TotalBytes = (long)messageCount * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
await js.DeleteStreamAsync(streamName);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
using System.Diagnostics;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.JetStream;
|
||||
|
||||
[Collection("Benchmark-JetStream")]
|
||||
public class DurableConsumerFetchTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task JSDurableFetch_Throughput()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messageCount = 5_000;
|
||||
const int fetchBatchSize = 500;
|
||||
|
||||
var dotnetResult = await RunDurableFetch("JS Durable Fetch (128B)", "DotNet", payloadSize, messageCount, fetchBatchSize, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunDurableFetch("JS Durable Fetch (128B)", "Go", payloadSize, messageCount, fetchBatchSize, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunDurableFetch(
|
||||
string name, string serverType, int payloadSize, int messageCount, int fetchBatchSize,
|
||||
Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var streamName = $"BENCH_DUR_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
|
||||
var subject = $"bench.js.durable.{serverType.ToLowerInvariant()}";
|
||||
var consumerName = $"bench-dur-{serverType.ToLowerInvariant()}";
|
||||
|
||||
await using var nats = createClient();
|
||||
await nats.ConnectAsync();
|
||||
var js = new NatsJSContext(nats);
|
||||
|
||||
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
|
||||
{
|
||||
Storage = StreamConfigStorage.Memory,
|
||||
Retention = StreamConfigRetention.Limits,
|
||||
MaxMsgs = 10_000_000,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
// Pre-populate stream
|
||||
var pubTasks = new List<ValueTask<PubAckResponse>>(1000);
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
{
|
||||
pubTasks.Add(js.PublishAsync(subject, payload));
|
||||
if (pubTasks.Count >= 1000)
|
||||
{
|
||||
foreach (var t in pubTasks)
|
||||
await t;
|
||||
pubTasks.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var t in pubTasks)
|
||||
await t;
|
||||
|
||||
// Create durable consumer
|
||||
var consumer = await js.CreateOrUpdateConsumerAsync(streamName, new ConsumerConfig(consumerName)
|
||||
{
|
||||
AckPolicy = ConsumerConfigAckPolicy.None,
|
||||
});
|
||||
|
||||
// Fetch in batches
|
||||
var received = 0;
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
while (received < messageCount)
|
||||
{
|
||||
await foreach (var msg in consumer.FetchAsync<byte[]>(new NatsJSFetchOpts { MaxMsgs = fetchBatchSize }))
|
||||
{
|
||||
received++;
|
||||
if (received >= messageCount)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = received,
|
||||
TotalBytes = (long)received * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
await js.DeleteStreamAsync(streamName);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using System.Diagnostics;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.JetStream;
|
||||
|
||||
[Collection("Benchmark-JetStream")]
|
||||
public class OrderedConsumerTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task JSOrderedConsumer_Throughput()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messageCount = 5_000;
|
||||
|
||||
BenchmarkResult? dotnetResult = null;
|
||||
try
|
||||
{
|
||||
dotnetResult = await RunOrderedConsume("JS Ordered Consumer (128B)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
|
||||
}
|
||||
catch (Exception ex) when (ex.GetType().Name.Contains("NatsJS"))
|
||||
{
|
||||
output.WriteLine($"[DotNet] Ordered consumer not fully supported: {ex.Message}");
|
||||
}
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunOrderedConsume("JS Ordered Consumer (128B)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
|
||||
if (dotnetResult is not null)
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
else
|
||||
BenchmarkResultWriter.WriteSingle(output, goResult);
|
||||
}
|
||||
else if (dotnetResult is not null)
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunOrderedConsume(string name, string serverType, int payloadSize, int messageCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var streamName = $"BENCH_ORD_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
|
||||
var subject = $"bench.js.ordered.{serverType.ToLowerInvariant()}";
|
||||
|
||||
await using var nats = createClient();
|
||||
await nats.ConnectAsync();
|
||||
var js = new NatsJSContext(nats);
|
||||
|
||||
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
|
||||
{
|
||||
Storage = StreamConfigStorage.Memory,
|
||||
Retention = StreamConfigRetention.Limits,
|
||||
MaxMsgs = 10_000_000,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
// Pre-populate stream
|
||||
var pubTasks = new List<ValueTask<PubAckResponse>>(1000);
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
{
|
||||
pubTasks.Add(js.PublishAsync(subject, payload));
|
||||
if (pubTasks.Count >= 1000)
|
||||
{
|
||||
foreach (var t in pubTasks)
|
||||
await t;
|
||||
pubTasks.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var t in pubTasks)
|
||||
await t;
|
||||
|
||||
// Consume via ordered consumer
|
||||
var consumer = await js.CreateOrderedConsumerAsync(streamName);
|
||||
var received = 0;
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
await foreach (var msg in consumer.ConsumeAsync<byte[]>())
|
||||
{
|
||||
received++;
|
||||
if (received >= messageCount)
|
||||
break;
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = received,
|
||||
TotalBytes = (long)received * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
await js.DeleteStreamAsync(streamName);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using NATS.Client.JetStream.Models;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.JetStream;
|
||||
|
||||
[Collection("Benchmark-JetStream")]
|
||||
public class SyncPublishTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
private readonly BenchmarkRunner _runner = new() { WarmupCount = 500, MeasurementCount = 10_000 };
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task JSSyncPublish_16B_MemoryStore()
|
||||
{
|
||||
const int payloadSize = 16;
|
||||
|
||||
var dotnetResult = await RunSyncPublish("JS Sync Publish (16B Memory)", "DotNet", payloadSize, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunSyncPublish("JS Sync Publish (16B Memory)", "Go", payloadSize, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<BenchmarkResult> RunSyncPublish(string name, string serverType, int payloadSize, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var streamName = $"BENCH_SYNC_{serverType.ToUpperInvariant()}_{Guid.NewGuid():N}"[..30];
|
||||
var subject = $"bench.js.sync.{serverType.ToLowerInvariant()}";
|
||||
|
||||
await using var nats = createClient();
|
||||
await nats.ConnectAsync();
|
||||
var js = new NatsJSContext(nats);
|
||||
|
||||
await js.CreateStreamAsync(new StreamConfig(streamName, [subject])
|
||||
{
|
||||
Storage = StreamConfigStorage.Memory,
|
||||
Retention = StreamConfigRetention.Limits,
|
||||
MaxMsgs = 1_000_000,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _runner.MeasureThroughputAsync(name, serverType, payloadSize,
|
||||
async _ => await js.PublishAsync(subject, payload));
|
||||
return result;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await js.DeleteStreamAsync(streamName);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="NATS.Client.Core" />
|
||||
<PackageReference Include="NATS.Client.JetStream" />
|
||||
<PackageReference Include="Shouldly" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
<Using Include="Shouldly" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
113
tests/NATS.Server.Benchmark.Tests/README.md
Normal file
113
tests/NATS.Server.Benchmark.Tests/README.md
Normal file
@@ -0,0 +1,113 @@
|
||||
# NATS.Server.Benchmark.Tests
|
||||
|
||||
Side-by-side performance comparison of the Go and .NET NATS server implementations. Both servers are launched as child processes on ephemeral ports and exercised with identical workloads using the `NATS.Client.Core` / `NATS.Client.JetStream` NuGet client libraries.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK
|
||||
- Go toolchain (for Go server comparison; benchmarks still run .NET-only if Go is unavailable)
|
||||
- The Go NATS server source at `golang/nats-server/` (the Go binary is built automatically on first run)
|
||||
|
||||
## Running Benchmarks
|
||||
|
||||
All benchmark tests are tagged with `[Trait("Category", "Benchmark")]`, so a plain `dotnet test` against the solution will **not** run them. Use the `--filter` flag.
|
||||
|
||||
```bash
|
||||
# Run all benchmarks
|
||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark" -v normal
|
||||
|
||||
# Core pub/sub only
|
||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~CorePubSub" -v normal
|
||||
|
||||
# Request/reply only
|
||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~RequestReply" -v normal
|
||||
|
||||
# JetStream only
|
||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~JetStream" -v normal
|
||||
|
||||
# A single benchmark by name
|
||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "FullyQualifiedName=NATS.Server.Benchmark.Tests.CorePubSub.SinglePublisherThroughputTests.PubNoSub_16B" -v normal
|
||||
```
|
||||
|
||||
Use `-v normal` or `--logger "console;verbosity=detailed"` to see the comparison output in the console. Without verbosity, xUnit suppresses `ITestOutputHelper` output for passing tests.
|
||||
|
||||
## Benchmark List
|
||||
|
||||
| Test Class | Test Method | What It Measures |
|
||||
|---|---|---|
|
||||
| `SinglePublisherThroughputTests` | `PubNoSub_16B` | Publish-only throughput, 16-byte payload |
|
||||
| `SinglePublisherThroughputTests` | `PubNoSub_128B` | Publish-only throughput, 128-byte payload |
|
||||
| `PubSubOneToOneTests` | `PubSub1To1_16B` | 1 publisher, 1 subscriber, 16-byte payload |
|
||||
| `PubSubOneToOneTests` | `PubSub1To1_16KB` | 1 publisher, 1 subscriber, 16 KB payload |
|
||||
| `FanOutTests` | `FanOut1To4_128B` | 1 publisher, 4 subscribers, 128-byte payload |
|
||||
| `MultiPubSubTests` | `MultiPubSub4x4_128B` | 4 publishers, 4 subscribers, 128-byte payload |
|
||||
| `SingleClientLatencyTests` | `RequestReply_SingleClient_128B` | Request/reply round-trip latency, 1 client, 1 service |
|
||||
| `MultiClientLatencyTests` | `RequestReply_10Clients2Services_16B` | Request/reply latency, 10 concurrent clients, 2 queue-group services |
|
||||
| `SyncPublishTests` | `JSSyncPublish_16B_MemoryStore` | JetStream synchronous publish, memory-backed stream |
|
||||
| `AsyncPublishTests` | `JSAsyncPublish_128B_FileStore` | JetStream async batch publish, file-backed stream |
|
||||
| `OrderedConsumerTests` | `JSOrderedConsumer_Throughput` | JetStream ordered ephemeral consumer read throughput |
|
||||
| `DurableConsumerFetchTests` | `JSDurableFetch_Throughput` | JetStream durable consumer fetch-in-batches throughput |
|
||||
|
||||
## Output Format
|
||||
|
||||
Each test writes a side-by-side comparison to xUnit's test output:
|
||||
|
||||
```
|
||||
=== Single Publisher (16B) ===
|
||||
Go: 2,436,416 msg/s | 37.2 MB/s | 41 ms
|
||||
.NET: 1,425,767 msg/s | 21.8 MB/s | 70 ms
|
||||
Ratio: 0.59x (.NET / Go)
|
||||
```
|
||||
|
||||
Request/reply tests also include latency percentiles:
|
||||
|
||||
```
|
||||
Latency (us):
|
||||
P50 P95 P99 Min Max
|
||||
Go 104.5 124.2 146.2 82.8 1204.5
|
||||
.NET 134.7 168.0 190.5 91.6 3469.5
|
||||
```
|
||||
|
||||
If Go is not available, only the .NET result is printed.
|
||||
|
||||
## Updating benchmarks_comparison.md
|
||||
|
||||
After running the full benchmark suite, update `benchmarks_comparison.md` in the repository root with the new numbers:
|
||||
|
||||
1. Run all benchmarks with detailed output:
|
||||
```bash
|
||||
dotnet test tests/NATS.Server.Benchmark.Tests \
|
||||
--filter "Category=Benchmark" \
|
||||
-v normal \
|
||||
--logger "console;verbosity=detailed" 2>&1 | tee /tmp/bench-output.txt
|
||||
```
|
||||
2. Open `/tmp/bench-output.txt` and extract the comparison blocks from the "Standard Output Messages" sections.
|
||||
3. Update the tables in `benchmarks_comparison.md` with the new msg/s, MB/s, ratio, and latency values. Update the date on the first line and the environment description.
|
||||
4. Review the Summary table and Key Observations — update assessments if ratios have changed significantly.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Infrastructure/
|
||||
PortAllocator.cs # Ephemeral port allocation (bind to port 0)
|
||||
DotNetServerProcess.cs # Builds + launches NATS.Server.Host
|
||||
GoServerProcess.cs # Builds + launches golang/nats-server
|
||||
CoreServerPairFixture.cs # IAsyncLifetime: Go + .NET servers for core tests
|
||||
JetStreamServerPairFixture # IAsyncLifetime: Go + .NET servers with JetStream
|
||||
Collections.cs # xUnit collection definitions
|
||||
|
||||
Harness/
|
||||
BenchmarkResult.cs # Result record (msg/s, MB/s, latencies)
|
||||
LatencyTracker.cs # Pre-allocated sample buffer, percentile math
|
||||
BenchmarkRunner.cs # Warmup + timed measurement loop
|
||||
BenchmarkResultWriter.cs # Formats side-by-side comparison output
|
||||
```
|
||||
|
||||
Server pair fixtures start both servers once per xUnit collection and expose `CreateGoClient()` / `CreateDotNetClient()` factory methods. Test parallelization is disabled at the assembly level (`AssemblyInfo.cs`) to prevent resource contention between collections.
|
||||
|
||||
## Notes
|
||||
|
||||
- The Go binary at `golang/nats-server/nats-server` is built automatically on first run via `go build`. Subsequent runs reuse the cached binary.
|
||||
- If Go is not installed or `golang/nats-server/` does not exist, Go benchmarks are skipped and only .NET results are reported.
|
||||
- JetStream tests create temporary store directories under `$TMPDIR` and clean them up on teardown.
|
||||
- Message counts are intentionally conservative (2K–10K per test) to keep the full suite under 2 minutes while still producing meaningful throughput ratios. For higher-fidelity numbers, increase the counts in individual test methods or the `BenchmarkRunner` configuration.
|
||||
@@ -0,0 +1,106 @@
|
||||
using System.Diagnostics;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.RequestReply;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class MultiClientLatencyTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task RequestReply_10Clients2Services_16B()
|
||||
{
|
||||
const int payloadSize = 16;
|
||||
const int requestsPerClient = 1_000;
|
||||
const int clientCount = 10;
|
||||
const int serviceCount = 2;
|
||||
|
||||
var dotnetResult = await RunMultiLatency("Request-Reply 10Cx2S (16B)", "DotNet", payloadSize, requestsPerClient, clientCount, serviceCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunMultiLatency("Request-Reply 10Cx2S (16B)", "Go", payloadSize, requestsPerClient, clientCount, serviceCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunMultiLatency(
|
||||
string name, string serverType, int payloadSize, int requestsPerClient,
|
||||
int clientCount, int serviceCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
const string subject = "bench.reqrep.multi";
|
||||
var queueGroup = $"bench-svc-{serverType.ToLowerInvariant()}";
|
||||
|
||||
// Start service responders on a queue group
|
||||
var serviceClients = new NatsConnection[serviceCount];
|
||||
var serviceSubs = new INatsSub<byte[]>[serviceCount];
|
||||
var serviceTasks = new Task[serviceCount];
|
||||
|
||||
for (var i = 0; i < serviceCount; i++)
|
||||
{
|
||||
serviceClients[i] = createClient();
|
||||
await serviceClients[i].ConnectAsync();
|
||||
serviceSubs[i] = await serviceClients[i].SubscribeCoreAsync<byte[]>(subject, queueGroup: queueGroup);
|
||||
var client = serviceClients[i];
|
||||
var sub = serviceSubs[i];
|
||||
serviceTasks[i] = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var msg in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (msg.ReplyTo is not null)
|
||||
await client.PublishAsync(msg.ReplyTo, payload);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await Task.Delay(50);
|
||||
|
||||
// Run concurrent clients
|
||||
var totalMessages = requestsPerClient * clientCount;
|
||||
var tracker = new LatencyTracker(totalMessages);
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
var clientTasks = new Task[clientCount];
|
||||
for (var c = 0; c < clientCount; c++)
|
||||
{
|
||||
clientTasks[c] = Task.Run(async () =>
|
||||
{
|
||||
await using var client = createClient();
|
||||
await client.ConnectAsync();
|
||||
|
||||
for (var i = 0; i < requestsPerClient; i++)
|
||||
{
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
await client.RequestAsync<byte[], byte[]>(subject, payload);
|
||||
tracker.Record(Stopwatch.GetTimestamp() - start);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await Task.WhenAll(clientTasks);
|
||||
sw.Stop();
|
||||
|
||||
foreach (var sub in serviceSubs)
|
||||
await sub.UnsubscribeAsync();
|
||||
foreach (var client in serviceClients)
|
||||
await client.DisposeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = totalMessages,
|
||||
TotalBytes = (long)totalMessages * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
Latencies = tracker.ComputePercentiles(),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.RequestReply;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class SingleClientLatencyTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
private readonly BenchmarkRunner _runner = new() { WarmupCount = 500, MeasurementCount = 10_000 };
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task RequestReply_SingleClient_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const string subject = "bench.reqrep.single";
|
||||
|
||||
var dotnetResult = await RunLatency("Request-Reply Single (128B)", "DotNet", subject, payloadSize, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunLatency("Request-Reply Single (128B)", "Go", subject, payloadSize, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<BenchmarkResult> RunLatency(string name, string serverType, string subject, int payloadSize, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
|
||||
await using var serviceClient = createClient();
|
||||
await using var requestClient = createClient();
|
||||
await serviceClient.ConnectAsync();
|
||||
await requestClient.ConnectAsync();
|
||||
|
||||
// Start service responder
|
||||
var sub = await serviceClient.SubscribeCoreAsync<byte[]>(subject);
|
||||
var responderTask = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var msg in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (msg.ReplyTo is not null)
|
||||
await serviceClient.PublishAsync(msg.ReplyTo, payload);
|
||||
}
|
||||
});
|
||||
|
||||
await Task.Delay(50);
|
||||
|
||||
var result = await _runner.MeasureLatencyAsync(name, serverType, payloadSize,
|
||||
async _ =>
|
||||
{
|
||||
await requestClient.RequestAsync<byte[], byte[]>(subject, payload);
|
||||
});
|
||||
|
||||
await sub.UnsubscribeAsync();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
5
tests/NATS.Server.Benchmark.Tests/xunit.runner.json
Normal file
5
tests/NATS.Server.Benchmark.Tests/xunit.runner.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
|
||||
"parallelizeAssembly": false,
|
||||
"parallelizeTestCollections": false
|
||||
}
|
||||
Reference in New Issue
Block a user