feat: add benchmark test project for Go vs .NET server comparison
Side-by-side performance benchmarks using NATS.Client.Core against both servers on ephemeral ports. Includes core pub/sub, request/reply latency, and JetStream throughput tests with comparison output and benchmarks_comparison.md results. Also fixes timestamp flakiness in StoreInterfaceTests by using explicit timestamps.
This commit is contained in:
Joseph Doherty
99
tests/NATS.Server.Benchmark.Tests/CorePubSub/FanOutTests.cs
Normal file
99
tests/NATS.Server.Benchmark.Tests/CorePubSub/FanOutTests.cs
Normal file
@@ -0,0 +1,99 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class FanOutTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task FanOut1To4_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messageCount = 10_000;
|
||||
const int subscriberCount = 4;
|
||||
|
||||
var dotnetResult = await RunFanOut("Fan-Out 1:4 (128B)", "DotNet", payloadSize, messageCount, subscriberCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunFanOut("Fan-Out 1:4 (128B)", "Go", payloadSize, messageCount, subscriberCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunFanOut(string name, string serverType, int payloadSize, int messageCount, int subscriberCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var subject = $"bench.fanout.{serverType.ToLowerInvariant()}.{Guid.NewGuid():N}";
|
||||
|
||||
await using var pubClient = createClient();
|
||||
await pubClient.ConnectAsync();
|
||||
|
||||
var subClients = new NatsConnection[subscriberCount];
|
||||
var subs = new INatsSub<byte[]>[subscriberCount];
|
||||
var subTasks = new Task[subscriberCount];
|
||||
var totalExpected = messageCount * subscriberCount;
|
||||
var totalReceived = 0;
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
for (var i = 0; i < subscriberCount; i++)
|
||||
{
|
||||
subClients[i] = createClient();
|
||||
await subClients[i].ConnectAsync();
|
||||
subs[i] = await subClients[i].SubscribeCoreAsync<byte[]>(subject);
|
||||
}
|
||||
|
||||
// Flush to ensure all subscriptions are propagated
|
||||
foreach (var client in subClients)
|
||||
await client.PingAsync();
|
||||
await pubClient.PingAsync();
|
||||
|
||||
// Start reading after subscriptions are confirmed
|
||||
for (var i = 0; i < subscriberCount; i++)
|
||||
{
|
||||
var sub = subs[i];
|
||||
subTasks[i] = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var _ in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (Interlocked.Increment(ref totalReceived) >= totalExpected)
|
||||
{
|
||||
tcs.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
await pubClient.PublishAsync(subject, payload);
|
||||
await pubClient.PingAsync();
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||
await tcs.Task.WaitAsync(cts.Token);
|
||||
sw.Stop();
|
||||
|
||||
foreach (var sub in subs)
|
||||
await sub.UnsubscribeAsync();
|
||||
foreach (var client in subClients)
|
||||
await client.DisposeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = totalExpected,
|
||||
TotalBytes = (long)totalExpected * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
124
tests/NATS.Server.Benchmark.Tests/CorePubSub/MultiPubSubTests.cs
Normal file
124
tests/NATS.Server.Benchmark.Tests/CorePubSub/MultiPubSubTests.cs
Normal file
@@ -0,0 +1,124 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class MultiPubSubTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task MultiPubSub4x4_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
const int messagesPerPublisher = 2_000;
|
||||
const int pubCount = 4;
|
||||
const int subCount = 4;
|
||||
|
||||
var dotnetResult = await RunMultiPubSub("Multi 4Px4S (128B)", "DotNet", payloadSize, messagesPerPublisher, pubCount, subCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunMultiPubSub("Multi 4Px4S (128B)", "Go", payloadSize, messagesPerPublisher, pubCount, subCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunMultiPubSub(
|
||||
string name, string serverType, int payloadSize, int messagesPerPublisher,
|
||||
int pubCount, int subCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var totalMessages = messagesPerPublisher * pubCount;
|
||||
var runId = Guid.NewGuid().ToString("N")[..8];
|
||||
var subjects = Enumerable.Range(0, pubCount).Select(i => $"bench.multi.{serverType.ToLowerInvariant()}.{runId}.{i}").ToArray();
|
||||
|
||||
// Create subscribers — one per subject
|
||||
var subClients = new NatsConnection[subCount];
|
||||
var subs = new INatsSub<byte[]>[subCount];
|
||||
var subTasks = new Task[subCount];
|
||||
var totalReceived = 0;
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
for (var i = 0; i < subCount; i++)
|
||||
{
|
||||
subClients[i] = createClient();
|
||||
await subClients[i].ConnectAsync();
|
||||
subs[i] = await subClients[i].SubscribeCoreAsync<byte[]>(subjects[i % subjects.Length]);
|
||||
}
|
||||
|
||||
// Flush to ensure all subscriptions are propagated
|
||||
foreach (var client in subClients)
|
||||
await client.PingAsync();
|
||||
|
||||
// Start reading
|
||||
for (var i = 0; i < subCount; i++)
|
||||
{
|
||||
var sub = subs[i];
|
||||
subTasks[i] = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var _ in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (Interlocked.Increment(ref totalReceived) >= totalMessages)
|
||||
{
|
||||
tcs.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Create publishers
|
||||
var pubClients = new NatsConnection[pubCount];
|
||||
for (var i = 0; i < pubCount; i++)
|
||||
{
|
||||
pubClients[i] = createClient();
|
||||
await pubClients[i].ConnectAsync();
|
||||
}
|
||||
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
var pubTasks = new Task[pubCount];
|
||||
for (var p = 0; p < pubCount; p++)
|
||||
{
|
||||
var client = pubClients[p];
|
||||
var subject = subjects[p];
|
||||
pubTasks[p] = Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < messagesPerPublisher; i++)
|
||||
await client.PublishAsync(subject, payload);
|
||||
});
|
||||
}
|
||||
|
||||
await Task.WhenAll(pubTasks);
|
||||
|
||||
// Flush all publishers
|
||||
foreach (var client in pubClients)
|
||||
await client.PingAsync();
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||
await tcs.Task.WaitAsync(cts.Token);
|
||||
sw.Stop();
|
||||
|
||||
foreach (var sub in subs)
|
||||
await sub.UnsubscribeAsync();
|
||||
foreach (var client in subClients)
|
||||
await client.DisposeAsync();
|
||||
foreach (var client in pubClients)
|
||||
await client.DisposeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = totalMessages,
|
||||
TotalBytes = (long)totalMessages * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class PubSubOneToOneTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubSub1To1_16B()
|
||||
{
|
||||
const int payloadSize = 16;
|
||||
const int messageCount = 10_000;
|
||||
|
||||
var dotnetResult = await RunPubSub("PubSub 1:1 (16B)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunPubSub("PubSub 1:1 (16B)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubSub1To1_16KB()
|
||||
{
|
||||
const int payloadSize = 16 * 1024;
|
||||
const int messageCount = 1_000;
|
||||
|
||||
var dotnetResult = await RunPubSub("PubSub 1:1 (16KB)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunPubSub("PubSub 1:1 (16KB)", "Go", payloadSize, messageCount, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<BenchmarkResult> RunPubSub(string name, string serverType, int payloadSize, int messageCount, Func<NatsConnection> createClient)
|
||||
{
|
||||
var payload = new byte[payloadSize];
|
||||
var subject = $"bench.pubsub.{serverType.ToLowerInvariant()}.{Guid.NewGuid():N}";
|
||||
|
||||
await using var pubClient = createClient();
|
||||
await using var subClient = createClient();
|
||||
await pubClient.ConnectAsync();
|
||||
await subClient.ConnectAsync();
|
||||
|
||||
var received = 0;
|
||||
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
// Start subscriber
|
||||
var sub = await subClient.SubscribeCoreAsync<byte[]>(subject);
|
||||
|
||||
// Flush to ensure subscription is propagated
|
||||
await subClient.PingAsync();
|
||||
await pubClient.PingAsync();
|
||||
|
||||
var subTask = Task.Run(async () =>
|
||||
{
|
||||
await foreach (var msg in sub.Msgs.ReadAllAsync())
|
||||
{
|
||||
if (Interlocked.Increment(ref received) >= messageCount)
|
||||
{
|
||||
tcs.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Publish and measure
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < messageCount; i++)
|
||||
await pubClient.PublishAsync(subject, payload);
|
||||
await pubClient.PingAsync(); // Flush all pending writes
|
||||
|
||||
// Wait for all messages received
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||
await tcs.Task.WaitAsync(cts.Token);
|
||||
sw.Stop();
|
||||
|
||||
await sub.UnsubscribeAsync();
|
||||
|
||||
return new BenchmarkResult
|
||||
{
|
||||
Name = name,
|
||||
ServerType = serverType,
|
||||
TotalMessages = messageCount,
|
||||
TotalBytes = (long)messageCount * payloadSize,
|
||||
Duration = sw.Elapsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using NATS.Client.Core;
|
||||
using NATS.Server.Benchmark.Tests.Harness;
|
||||
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace NATS.Server.Benchmark.Tests.CorePubSub;
|
||||
|
||||
[Collection("Benchmark-Core")]
|
||||
public class SinglePublisherThroughputTests(CoreServerPairFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
private readonly BenchmarkRunner _runner = new() { WarmupCount = 1_000, MeasurementCount = 100_000 };
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubNoSub_16B()
|
||||
{
|
||||
const int payloadSize = 16;
|
||||
var payload = new byte[payloadSize];
|
||||
const string subject = "bench.pub.nosub.16";
|
||||
|
||||
var dotnetResult = await RunThroughput("Single Publisher (16B)", "DotNet", subject, payload, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunThroughput("Single Publisher (16B)", "Go", subject, payload, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Benchmark")]
|
||||
public async Task PubNoSub_128B()
|
||||
{
|
||||
const int payloadSize = 128;
|
||||
var payload = new byte[payloadSize];
|
||||
const string subject = "bench.pub.nosub.128";
|
||||
|
||||
var dotnetResult = await RunThroughput("Single Publisher (128B)", "DotNet", subject, payload, fixture.CreateDotNetClient);
|
||||
|
||||
if (fixture.GoAvailable)
|
||||
{
|
||||
var goResult = await RunThroughput("Single Publisher (128B)", "Go", subject, payload, fixture.CreateGoClient);
|
||||
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||
}
|
||||
else
|
||||
{
|
||||
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<BenchmarkResult> RunThroughput(string name, string serverType, string subject, byte[] payload, Func<NatsConnection> createClient)
|
||||
{
|
||||
await using var client = createClient();
|
||||
await client.ConnectAsync();
|
||||
|
||||
return await _runner.MeasureThroughputAsync(name, serverType, payload.Length,
|
||||
async _ => await client.PublishAsync(subject, payload));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user