dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: refresh benchmark comparison with increased message counts

Browse Source 
Increase message counts across all 14 benchmark test files to reduce
run-to-run variance (e.g. PubSub 16B: 10K→50K, FanOut: 10K→15K,
SinglePub: 100K→500K, JS tests: 5K→25K). Rewrite benchmarks_comparison.md
with fresh numbers from two-batch runs. Key changes: multi 4x4 reached
parity (1.01x), fan-out improved to 0.84x, TLS pub/sub shows 4.70x .NET
advantage, previous small-count anomalies corrected.
This commit is contained in:
Joseph Doherty
2026-03-13 17:52:03 -04:00
parent 660a897234
commit 1d4b87e5f9
14 changed files with 94 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/NATS.Server.Benchmark.Tests/CorePubSub/FanOutTests.cs
View File

@@ -13,7 +13,7 @@ public class FanOutTests(CoreServerPairFixture fixture, ITestOutputHelper output
public async Task FanOut1To4_128B()
{
const int payloadSize = 128;
const int messageCount = 10_000;
const int messageCount = 15_000;
const int subscriberCount = 4;
var dotnetResult = await RunFanOut("Fan-Out 1:4 (128B)", "DotNet", payloadSize, messageCount, subscriberCount, fixture.CreateDotNetClient);
@@ -78,7 +78,7 @@ public class FanOutTests(CoreServerPairFixture fixture, ITestOutputHelper output
await pubClient.PublishAsync(subject, payload);
await pubClient.PingAsync();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(180));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();

4
tests/NATS.Server.Benchmark.Tests/CorePubSub/MultiPubSubTests.cs
View File

@@ -13,7 +13,7 @@ public class MultiPubSubTests(CoreServerPairFixture fixture, ITestOutputHelper o
public async Task MultiPubSub4x4_128B()
{
const int payloadSize = 128;
const int messagesPerPublisher = 2_000;
const int messagesPerPublisher = 5_000;
const int pubCount = 4;
const int subCount = 4;
@@ -101,7 +101,7 @@ public class MultiPubSubTests(CoreServerPairFixture fixture, ITestOutputHelper o
foreach (var client in pubClients)
await client.PingAsync();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();

6
tests/NATS.Server.Benchmark.Tests/CorePubSub/PubSubOneToOneTests.cs
View File

@@ -13,7 +13,7 @@ public class PubSubOneToOneTests(CoreServerPairFixture fixture, ITestOutputHelpe
public async Task PubSub1To1_16B()
{
const int payloadSize = 16;
const int messageCount = 10_000;
const int messageCount = 50_000;
var dotnetResult = await RunPubSub("PubSub 1:1 (16B)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
@@ -33,7 +33,7 @@ public class PubSubOneToOneTests(CoreServerPairFixture fixture, ITestOutputHelpe
public async Task PubSub1To1_16KB()
{
const int payloadSize = 16 * 1024;
const int messageCount = 1_000;
const int messageCount = 5_000;
var dotnetResult = await RunPubSub("PubSub 1:1 (16KB)", "DotNet", payloadSize, messageCount, fixture.CreateDotNetClient);
@@ -87,7 +87,7 @@ public class PubSubOneToOneTests(CoreServerPairFixture fixture, ITestOutputHelpe
await pubClient.PingAsync(); // Flush all pending writes
// Wait for all messages received
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();

2
tests/NATS.Server.Benchmark.Tests/CorePubSub/SinglePublisherThroughputTests.cs
View File

@@ -8,7 +8,7 @@ namespace NATS.Server.Benchmark.Tests.CorePubSub;
[Collection("Benchmark-Core")]
public class SinglePublisherThroughputTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
private readonly BenchmarkRunner _runner = new() { WarmupCount = 1_000, MeasurementCount = 100_000 };
private readonly BenchmarkRunner _runner = new() { WarmupCount = 10_000, MeasurementCount = 500_000 };
[Fact]
[Trait("Category", "Benchmark")]

2
tests/NATS.Server.Benchmark.Tests/JetStream/AsyncPublishTests.cs
View File

@@ -16,7 +16,7 @@ public class AsyncPublishTests(JetStreamServerPairFixture fixture, ITestOutputHe
public async Task JSAsyncPublish_128B_FileStore()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int messageCount = 25_000;
const int batchSize = 100;
var dotnetResult = await RunAsyncPublish("JS Async Publish (128B File)", "DotNet", payloadSize, messageCount, batchSize, fixture.CreateDotNetClient);

2
tests/NATS.Server.Benchmark.Tests/JetStream/DurableConsumerFetchTests.cs
View File

@@ -16,7 +16,7 @@ public class DurableConsumerFetchTests(JetStreamServerPairFixture fixture, ITest
public async Task JSDurableFetch_Throughput()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int messageCount = 25_000;
const int fetchBatchSize = 500;
var dotnetResult = await RunDurableFetch("JS Durable Fetch (128B)", "DotNet", payloadSize, messageCount, fetchBatchSize, fixture.CreateDotNetClient);

2
tests/NATS.Server.Benchmark.Tests/JetStream/OrderedConsumerTests.cs
View File

@@ -16,7 +16,7 @@ public class OrderedConsumerTests(JetStreamServerPairFixture fixture, ITestOutpu
public async Task JSOrderedConsumer_Throughput()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int messageCount = 25_000;
BenchmarkResult? dotnetResult = null;
try

2
tests/NATS.Server.Benchmark.Tests/JetStream/SyncPublishTests.cs
View File

@@ -10,7 +10,7 @@ namespace NATS.Server.Benchmark.Tests.JetStream;
[Collection("Benchmark-JetStream")]
public class SyncPublishTests(JetStreamServerPairFixture fixture, ITestOutputHelper output)
{
private readonly BenchmarkRunner _runner = new() { WarmupCount = 500, MeasurementCount = 10_000 };
private readonly BenchmarkRunner _runner = new() { WarmupCount = 1_000, MeasurementCount = 50_000 };
[Fact]
[Trait("Category", "Benchmark")]

8
tests/NATS.Server.Benchmark.Tests/Mqtt/MqttThroughputTests.cs
View File

@@ -15,7 +15,7 @@ public class MqttThroughputTests(MqttServerFixture fixture, ITestOutputHelper ou
public async Task MqttPubSub_128B()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int messageCount = 25_000;
var dotnetResult = await RunMqttPubSub("MQTT PubSub (128B)", "DotNet", fixture.DotNetMqttPort, payloadSize, messageCount);
@@ -35,7 +35,7 @@ public class MqttThroughputTests(MqttServerFixture fixture, ITestOutputHelper ou
public async Task MqttCrossProtocol_NatsPub_MqttSub_128B()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int messageCount = 25_000;
var dotnetResult = await RunCrossProtocol("Cross-Protocol NATS→MQTT (128B)", "DotNet", fixture.DotNetMqttPort, fixture.CreateDotNetNatsClient, payloadSize, messageCount);
@@ -55,7 +55,7 @@ public class MqttThroughputTests(MqttServerFixture fixture, ITestOutputHelper ou
var payload = new byte[payloadSize];
var topic = $"bench/mqtt/pubsub/{Guid.NewGuid():N}";
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
var factory = new MqttFactory();
using var subscriber = factory.CreateMqttClient();
@@ -127,7 +127,7 @@ public class MqttThroughputTests(MqttServerFixture fixture, ITestOutputHelper ou
var natsSubject = $"bench.mqtt.cross.{Guid.NewGuid():N}";
var mqttTopic = natsSubject.Replace('.', '/');
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
var factory = new MqttFactory();
using var mqttSub = factory.CreateMqttClient();

2
tests/NATS.Server.Benchmark.Tests/RequestReply/MultiClientLatencyTests.cs
View File

@@ -14,7 +14,7 @@ public class MultiClientLatencyTests(CoreServerPairFixture fixture, ITestOutputH
public async Task RequestReply_10Clients2Services_16B()
{
const int payloadSize = 16;
const int requestsPerClient = 1_000;
const int requestsPerClient = 5_000;
const int clientCount = 10;
const int serviceCount = 2;

2
tests/NATS.Server.Benchmark.Tests/RequestReply/SingleClientLatencyTests.cs
View File

@@ -8,7 +8,7 @@ namespace NATS.Server.Benchmark.Tests.RequestReply;
[Collection("Benchmark-Core")]
public class SingleClientLatencyTests(CoreServerPairFixture fixture, ITestOutputHelper output)
{
private readonly BenchmarkRunner _runner = new() { WarmupCount = 500, MeasurementCount = 10_000 };
private readonly BenchmarkRunner _runner = new() { WarmupCount = 1_000, MeasurementCount = 50_000 };
[Fact]
[Trait("Category", "Benchmark")]

6
tests/NATS.Server.Benchmark.Tests/Transport/TlsPubSubTests.cs
View File

@@ -13,7 +13,7 @@ public class TlsPubSubTests(TlsServerFixture fixture, ITestOutputHelper output)
public async Task TlsPubSub1To1_128B()
{
const int payloadSize = 128;
const int messageCount = 10_000;
const int messageCount = 50_000;
var dotnetResult = await RunTlsPubSub("TLS PubSub 1:1 (128B)", "DotNet", fixture.CreateDotNetTlsClient, payloadSize, messageCount);
@@ -82,7 +82,7 @@ public class TlsPubSubTests(TlsServerFixture fixture, ITestOutputHelper output)
await pubClient.PublishAsync(subject, payload);
await pubClient.PingAsync();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
await tcs.Task.WaitAsync(cts.Token);
sw.Stop();
@@ -105,7 +105,7 @@ public class TlsPubSubTests(TlsServerFixture fixture, ITestOutputHelper output)
await using var client = createClient();
await client.ConnectAsync();
var runner = new BenchmarkRunner { WarmupCount = 1_000, MeasurementCount = 100_000 };
var runner = new BenchmarkRunner { WarmupCount = 10_000, MeasurementCount = 500_000 };
return await runner.MeasureThroughputAsync(
name,

8
tests/NATS.Server.Benchmark.Tests/Transport/WebSocketPubSubTests.cs
View File

@@ -15,7 +15,7 @@ public class WebSocketPubSubTests(WebSocketServerFixture fixture, ITestOutputHel
public async Task WsPubSub1To1_128B()
{
const int payloadSize = 128;
const int messageCount = 5_000;
const int messageCount = 25_000;
var dotnetResult = await RunWsPubSub("WebSocket PubSub 1:1 (128B)", "DotNet", fixture.DotNetWsPort, fixture.CreateDotNetNatsClient, payloadSize, messageCount);
@@ -35,7 +35,7 @@ public class WebSocketPubSubTests(WebSocketServerFixture fixture, ITestOutputHel
public async Task WsPubNoSub_128B()
{
const int payloadSize = 128;
const int messageCount = 10_000;
const int messageCount = 50_000;
var dotnetResult = await RunWsPubOnly("WebSocket Pub-Only (128B)", "DotNet", fixture.DotNetWsPort, payloadSize, messageCount);
@@ -55,7 +55,7 @@ public class WebSocketPubSubTests(WebSocketServerFixture fixture, ITestOutputHel
var payload = new byte[payloadSize];
var subject = $"bench.ws.pubsub.{Guid.NewGuid():N}";
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
using var ws = new ClientWebSocket();
await ws.ConnectAsync(new Uri($"ws://127.0.0.1:{wsPort}"), cts.Token);
@@ -110,7 +110,7 @@ public class WebSocketPubSubTests(WebSocketServerFixture fixture, ITestOutputHel
private static async Task<BenchmarkResult> RunWsPubOnly(string name, string serverType, int wsPort, int payloadSize, int messageCount)
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120));
using var ws = new ClientWebSocket();
await ws.ConnectAsync(new Uri($"ws://127.0.0.1:{wsPort}"), cts.Token);