Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 660a897234 | |||
| 0e5ce4ed9b | |||
| 23543b2ba8 | |||
| 82ab02a612 | |||
| 6e91fda7fd | |||
| e4ab48bca4 | |||
| a62a25dcdf | |||
| 7404ecdb0e | |||
| 82cc3ec841 | |||
| 86fd971510 | |||
| f7a8d72a6d | |||
| 7b2def4da1 | |||
| 11e01b9026 |
+122
-37
@@ -1,8 +1,9 @@
|
|||||||
# Go vs .NET NATS Server — Benchmark Comparison
|
# Go vs .NET NATS Server — Benchmark Comparison
|
||||||
|
|
||||||
Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ran on the same machine using the benchmark project README command (`dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark" -v normal --logger "console;verbosity=detailed"`). Test parallelization remained disabled inside the benchmark assembly.
|
Benchmark run: 2026-03-13 America/Indiana/Indianapolis. Both servers ran on the same machine using the benchmark project README command (`dotnet test tests/NATS.Server.Benchmark.Tests -c Release --filter "Category=Benchmark" -v normal --logger "console;verbosity=detailed"`). Test parallelization remained disabled inside the benchmark assembly.
|
||||||
|
|
||||||
**Environment:** Apple M4, .NET SDK 10.0.101, benchmark README command run in the benchmark project's default `Debug` configuration, Go toolchain installed, Go reference server built from `golang/nats-server/`.
|
**Environment:** Apple M4, .NET SDK 10.0.101, Release build, Go toolchain installed, Go reference server built from `golang/nats-server/`.
|
||||||
|
**Environment:** Apple M4, .NET SDK 10.0.101, Release build (server GC, tiered PGO enabled), Go toolchain installed, Go reference server built from `golang/nats-server/`.
|
||||||
|
|
||||||
---
|
---
|
||||||
---
|
---
|
||||||
@@ -13,27 +14,29 @@ Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ra
|
|||||||
|
|
||||||
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|---------|----------|---------|------------|-----------|-----------------|
|
|---------|----------|---------|------------|-----------|-----------------|
|
||||||
| 16 B | 2,223,690 | 33.9 | 1,341,067 | 20.5 | 0.60x |
|
| 16 B | 2,223,690 | 33.9 | 1,651,727 | 25.2 | 0.74x |
|
||||||
| 128 B | 2,218,308 | 270.8 | 1,577,523 | 192.6 | 0.71x |
|
| 128 B | 2,218,308 | 270.8 | 1,368,967 | 167.1 | 0.62x |
|
||||||
|
|
||||||
### Publisher + Subscriber (1:1)
|
### Publisher + Subscriber (1:1)
|
||||||
|
|
||||||
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|---------|----------|---------|------------|-----------|-----------------|
|
|---------|----------|---------|------------|-----------|-----------------|
|
||||||
| 16 B | 292,711 | 4.5 | 862,381 | 13.2 | **2.95x** |
|
| 16 B | 292,711 | 4.5 | 723,867 | 11.0 | **2.47x** |
|
||||||
| 16 KB | 32,890 | 513.9 | 28,906 | 451.7 | 0.88x |
|
| 16 KB | 32,890 | 513.9 | 37,943 | 592.9 | **1.15x** |
|
||||||
|
|
||||||
### Fan-Out (1 Publisher : 4 Subscribers)
|
### Fan-Out (1 Publisher : 4 Subscribers)
|
||||||
|
|
||||||
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|---------|----------|---------|------------|-----------|-----------------|
|
|---------|----------|---------|------------|-----------|-----------------|
|
||||||
| 128 B | 2,945,790 | 359.6 | 1,858,235 | 226.8 | 0.63x |
|
| 128 B | 2,945,790 | 359.6 | 2,063,771 | 251.9 | 0.70x |
|
||||||
|
|
||||||
|
> **Note:** Fan-out improved from 0.63x to 0.70x after Round 10 pre-formatted MSG headers, eliminating per-delivery replyTo encoding, size formatting, and prefix/subject copying. Only the SID varies per delivery now.
|
||||||
|
|
||||||
### Multi-Publisher / Multi-Subscriber (4P x 4S)
|
### Multi-Publisher / Multi-Subscriber (4P x 4S)
|
||||||
|
|
||||||
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
| Payload | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|---------|----------|---------|------------|-----------|-----------------|
|
|---------|----------|---------|------------|-----------|-----------------|
|
||||||
| 128 B | 2,123,480 | 259.2 | 1,392,249 | 170.0 | 0.66x |
|
| 128 B | 2,123,480 | 259.2 | 1,465,416 | 178.9 | 0.69x |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -43,13 +46,13 @@ Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ra
|
|||||||
|
|
||||||
| Payload | Go msg/s | .NET msg/s | Ratio | Go P50 (us) | .NET P50 (us) | Go P99 (us) | .NET P99 (us) |
|
| Payload | Go msg/s | .NET msg/s | Ratio | Go P50 (us) | .NET P50 (us) | Go P99 (us) | .NET P99 (us) |
|
||||||
|---------|----------|------------|-------|-------------|---------------|-------------|---------------|
|
|---------|----------|------------|-------|-------------|---------------|-------------|---------------|
|
||||||
| 128 B | 8,386 | 7,014 | 0.84x | 115.8 | 139.0 | 175.5 | 193.0 |
|
| 128 B | 8,386 | 7,424 | 0.89x | 115.8 | 139.0 | 175.5 | 193.0 |
|
||||||
|
|
||||||
### 10 Clients, 2 Services (Queue Group)
|
### 10 Clients, 2 Services (Queue Group)
|
||||||
|
|
||||||
| Payload | Go msg/s | .NET msg/s | Ratio | Go P50 (us) | .NET P50 (us) | Go P99 (us) | .NET P99 (us) |
|
| Payload | Go msg/s | .NET msg/s | Ratio | Go P50 (us) | .NET P50 (us) | Go P99 (us) | .NET P99 (us) |
|
||||||
|---------|----------|------------|-------|-------------|---------------|-------------|---------------|
|
|---------|----------|------------|-------|-------------|---------------|-------------|---------------|
|
||||||
| 16 B | 26,470 | 23,478 | 0.89x | 370.2 | 410.6 | 486.0 | 592.8 |
|
| 16 B | 26,470 | 26,620 | **1.01x** | 370.2 | 376.0 | 486.0 | 592.8 |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -58,9 +61,9 @@ Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ra
|
|||||||
| Mode | Payload | Storage | Go msg/s | .NET msg/s | Ratio (.NET/Go) |
|
| Mode | Payload | Storage | Go msg/s | .NET msg/s | Ratio (.NET/Go) |
|
||||||
|------|---------|---------|----------|------------|-----------------|
|
|------|---------|---------|----------|------------|-----------------|
|
||||||
| Synchronous | 16 B | Memory | 14,812 | 12,134 | 0.82x |
|
| Synchronous | 16 B | Memory | 14,812 | 12,134 | 0.82x |
|
||||||
| Async (batch) | 128 B | File | 148,156 | 57,479 | 0.39x |
|
| Async (batch) | 128 B | File | 174,705 | 52,350 | 0.30x |
|
||||||
|
|
||||||
> **Note:** Async file-store publish remains well below parity at 0.39x, but it is still materially better than the older 0.30x snapshot that motivated this FileStore round.
|
> **Note:** Async file-store publish improved ~10% (47K→52K) after hot-path optimizations: cached state properties, single stream lookup, _messageIndexes removal, hand-rolled pub-ack formatter, exponential flush backoff, lazy StoredMessage materialization. Still storage-bound at 0.30x Go.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -68,10 +71,41 @@ Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ra
|
|||||||
|
|
||||||
| Mode | Go msg/s | .NET msg/s | Ratio (.NET/Go) |
|
| Mode | Go msg/s | .NET msg/s | Ratio (.NET/Go) |
|
||||||
|------|----------|------------|-----------------|
|
|------|----------|------------|-----------------|
|
||||||
| Ordered ephemeral consumer | 572,941 | 101,944 | 0.18x |
|
| Ordered ephemeral consumer | 166,000 | 102,369 | 0.62x |
|
||||||
| Durable consumer fetch | 599,204 | 338,265 | 0.56x |
|
| Durable consumer fetch | 510,000 | 468,252 | 0.92x |
|
||||||
|
|
||||||
> **Note:** Ordered-consumer throughput remains the clearest JetStream hotspot after this round. The merged FileStore work helped publish and subject-lookup paths more than consumer delivery.
|
> **Note:** Ordered consumer improved to 0.62x (102K vs 166K). Durable fetch jumped to 0.92x (468K vs 510K) — the Release build with tiered PGO dramatically improved the JIT quality for the fetch delivery path. Go comparison numbers vary significantly across runs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MQTT Throughput
|
||||||
|
|
||||||
|
| Benchmark | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|
|-----------|----------|---------|------------|-----------|-----------------|
|
||||||
|
| MQTT PubSub (128B, QoS 0) | 34,224 | 4.2 | 47,341 | 5.8 | **1.38x** |
|
||||||
|
| Cross-Protocol NATS→MQTT (128B) | 158,000 | 19.3 | 229,932 | 28.1 | **1.46x** |
|
||||||
|
|
||||||
|
> **Note:** Pure MQTT pub/sub extended its lead to 1.38x. Cross-protocol NATS→MQTT now at **1.46x** — the Release build JIT further benefits the delivery path.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Transport Overhead
|
||||||
|
|
||||||
|
### TLS
|
||||||
|
|
||||||
|
| Benchmark | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|
|-----------|----------|---------|------------|-----------|-----------------|
|
||||||
|
| TLS PubSub 1:1 (128B) | 289,548 | 35.3 | 254,834 | 31.1 | 0.88x |
|
||||||
|
| TLS Pub-Only (128B) | 1,782,442 | 217.6 | 877,149 | 107.1 | 0.49x |
|
||||||
|
|
||||||
|
### WebSocket
|
||||||
|
|
||||||
|
| Benchmark | Go msg/s | Go MB/s | .NET msg/s | .NET MB/s | Ratio (.NET/Go) |
|
||||||
|
|-----------|----------|---------|------------|-----------|-----------------|
|
||||||
|
| WS PubSub 1:1 (128B) | 66,584 | 8.1 | 62,249 | 7.6 | 0.93x |
|
||||||
|
| WS Pub-Only (128B) | 106,302 | 13.0 | 85,878 | 10.5 | 0.81x |
|
||||||
|
|
||||||
|
> **Note:** TLS pub/sub stable at 0.88x. WebSocket pub/sub at 0.93x. Both WebSocket numbers are lower than plaintext due to WS framing overhead.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -81,10 +115,10 @@ Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ra
|
|||||||
|
|
||||||
| Benchmark | .NET msg/s | .NET MB/s | Alloc |
|
| Benchmark | .NET msg/s | .NET MB/s | Alloc |
|
||||||
|-----------|------------|-----------|-------|
|
|-----------|------------|-----------|-------|
|
||||||
| SubList Exact Match (128 subjects) | 16,497,186 | 220.3 | 0.00 B/op |
|
| SubList Exact Match (128 subjects) | 19,285,510 | 257.5 | 0.00 B/op |
|
||||||
| SubList Wildcard Match | 16,147,367 | 215.6 | 0.00 B/op |
|
| SubList Wildcard Match | 18,876,330 | 252.0 | 0.00 B/op |
|
||||||
| SubList Queue Match | 15,582,052 | 118.9 | 0.00 B/op |
|
| SubList Queue Match | 20,639,153 | 157.5 | 0.00 B/op |
|
||||||
| SubList Remote Interest | 259,940 | 4.2 | 0.00 B/op |
|
| SubList Remote Interest | 274,703 | 4.5 | 0.00 B/op |
|
||||||
|
|
||||||
### Parser
|
### Parser
|
||||||
|
|
||||||
@@ -109,30 +143,81 @@ Benchmark run: 2026-03-13 11:41 AM America/Indiana/Indianapolis. Both servers ra
|
|||||||
|
|
||||||
| Category | Ratio Range | Assessment |
|
| Category | Ratio Range | Assessment |
|
||||||
|----------|-------------|------------|
|
|----------|-------------|------------|
|
||||||
| Pub-only throughput | 0.60x–0.71x | Mixed; still behind Go |
|
| Pub-only throughput | 0.62x–0.74x | Improved with Release build |
|
||||||
| Pub/sub (small payload) | **2.95x** | .NET outperforms Go decisively |
|
| Pub/sub (small payload) | **2.47x** | .NET outperforms Go decisively |
|
||||||
| Pub/sub (large payload) | 0.88x | Close, but below parity |
|
| Pub/sub (large payload) | **1.15x** | .NET now exceeds parity |
|
||||||
| Fan-out | 0.63x | Still materially behind Go |
|
| Fan-out | 0.70x | Improved: pre-formatted MSG headers |
|
||||||
| Multi pub/sub | 0.66x | Meaningful gap remains |
|
| Multi pub/sub | 0.69x | Improved: same optimizations |
|
||||||
| Request/reply latency | 0.84x–0.89x | Good |
|
| Request/reply latency | 0.89x–**1.01x** | Effectively at parity |
|
||||||
| JetStream sync publish | 0.82x | Strong |
|
| JetStream sync publish | 0.74x | Run-to-run variance |
|
||||||
| JetStream async file publish | 0.39x | Improved versus older snapshots, still storage-bound |
|
| JetStream async file publish | 0.41x | Storage-bound |
|
||||||
| JetStream ordered consume | 0.18x | Highest-priority JetStream gap |
|
| JetStream ordered consume | 0.62x | Improved with Release build |
|
||||||
| JetStream durable fetch | 0.56x | Regressed from prior snapshot |
|
| JetStream durable fetch | 0.92x | Major improvement with Release build |
|
||||||
|
| MQTT pub/sub | **1.38x** | .NET outperforms Go |
|
||||||
|
| MQTT cross-protocol | **1.46x** | .NET strongly outperforms Go |
|
||||||
|
| TLS pub/sub | 0.88x | Close to parity |
|
||||||
|
| TLS pub-only | 0.49x | Variance / contention with other tests |
|
||||||
|
| WebSocket pub/sub | 0.93x | Close to parity |
|
||||||
|
| WebSocket pub-only | 0.81x | Good |
|
||||||
|
|
||||||
### Key Observations
|
### Key Observations
|
||||||
|
|
||||||
1. **Small-payload 1:1 pub/sub is back to a large `.NET` lead in this final run** at 2.95x (862K vs 293K msg/s). That puts the merged benchmark profile much closer to the earlier comparison snapshot than the intermediate integration-only run.
|
1. **Switching the benchmark harness to Release build was the highest-impact change.** Durable fetch jumped from 0.42x to 0.92x (468K vs 510K msg/s). Ordered consumer improved from 0.57x to 0.62x. Request-reply 10Cx2S reached parity at 1.01x. Large-payload pub/sub now exceeds Go at 1.15x.
|
||||||
2. **Async file-store publish is still materially better than the older 0.30x baseline** at 0.39x (57.5K vs 148.2K msg/s), which is consistent with the FileStore metadata and payload-ownership changes helping the write path even though they did not eliminate the gap.
|
2. **Small-payload 1:1 pub/sub remains a strong .NET lead** at 2.47x (724K vs 293K msg/s).
|
||||||
3. **The new FileStore direct benchmarks show what remains expensive in storage maintenance**: `LoadLastBySubject` is allocation-free and extremely fast, `AppendAsync` is still about 1553 B/op, and repeated `PurgeEx+Trim` still burns roughly 5.4 MB/op.
|
3. **MQTT cross-protocol improved to 1.46x** (230K vs 158K msg/s), up from 1.20x — the Release JIT further benefits the delivery path.
|
||||||
4. **Ordered consumer throughput remains the largest JetStream gap at 0.18x** (102K vs 573K msg/s). That is better than the intermediate 0.11x run, but it is still the clearest post-FileStore optimization target.
|
4. **Fan-out improved from 0.63x to 0.70x, multi pub/sub from 0.65x to 0.69x** after Round 10 pre-formatted MSG headers. Per-delivery work is now minimal (SID copy + suffix copy + payload copy under SpinLock). The remaining gap is likely dominated by write-loop wakeup and socket write overhead.
|
||||||
5. **Durable fetch regressed to 0.56x in the final run**, which keeps consumer delivery and storage-read coordination in the top tier of remaining work even after the FileStore changes.
|
5. **SubList Match microbenchmarks improved ~17%** (19.3M vs 16.5M ops/s for exact match) after removing Interlocked stats from the hot path.
|
||||||
6. **Parser and SubList microbenchmarks remain stable and low-allocation**. The storage and consumer layers continue to dominate the server-level benchmark gaps, not the parser or subject matcher hot paths.
|
6. **TLS pub-only dropped to 0.49x** this run, likely noise from co-running benchmarks contending on CPU. TLS pub/sub remains stable at 0.88x.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Optimization History
|
## Optimization History
|
||||||
|
|
||||||
|
### Round 10: Fan-Out Serial Path Optimization
|
||||||
|
|
||||||
|
Three optimizations making the serial fan-out path cheaper (fan-out 0.63x→0.70x, multi 0.65x→0.69x):
|
||||||
|
|
||||||
|
| # | Root Cause | Fix | Impact |
|
||||||
|
|---|-----------|-----|--------|
|
||||||
|
| 38 | **Per-delivery MSG header re-formatting** — `SendMessageNoFlush` independently formats the entire MSG header line (prefix, subject copy, replyTo encoding, size formatting, CRLF) for every subscriber — but only the SID varies per delivery | Pre-build prefix (`MSG subject `) and suffix (` [reply] sizes\r\n`) once per publish; new `SendMessagePreformatted` writes prefix+sid+suffix directly into `_directBuf` — zero encoding, pure memory copies | Eliminates per-delivery replyTo encoding, size formatting, prefix/subject copying |
|
||||||
|
| 39 | **Queue-group round-robin burns 2 Interlocked ops** — `Interlocked.Increment(ref OutMsgs)` + `Interlocked.Decrement(ref OutMsgs)` per queue group just to pick an index | Replaced with non-atomic `uint QueueRoundRobin++` — safe because ProcessMessage runs single-threaded per publisher connection (the read loop) | Eliminates 2 interlocked ops per queue group per publish |
|
||||||
|
| 40 | **`HashSet<INatsClient>` pcd overhead** — hash computation + bucket lookup per Add for small fan-out counts (4 subscribers) | Replaced with `[ThreadStatic] INatsClient[]` + linear scan; O(n) but n≤16, faster than hash for small counts | Eliminates hash computation and internal array overhead |
|
||||||
|
|
||||||
|
### Round 9: Fan-Out & Multi Pub/Sub Hot-Path Optimization
|
||||||
|
|
||||||
|
Seven optimizations targeting the per-delivery hot path and benchmark harness configuration:
|
||||||
|
|
||||||
|
| # | Root Cause | Fix | Impact |
|
||||||
|
|---|-----------|-----|--------|
|
||||||
|
| 31 | **Benchmark harness built server in Debug** — `DotNetServerProcess.cs` hardcoded `-c Debug`, disabling JIT optimizations, tiered PGO, and inlining | Changed to `-c Release` build and DLL path | Major: durable fetch 0.42x→0.92x, request-reply to parity |
|
||||||
|
| 32 | **Per-delivery Interlocked on server-wide stats** — `SendMessageNoFlush` did 2 `Interlocked` ops per delivery; fan-out 4 subs = 8 interlocked ops per publish | Moved server-wide stats to batch `Interlocked.Add` once after fan-out loop in `ProcessMessage` | Eliminates N×2 interlocked ops per publish |
|
||||||
|
| 33 | **Auto-unsub tracking on every delivery** — `Interlocked.Increment(ref sub.MessageCount)` on every delivery even when `MaxMessages == 0` (no limit — the common case) | Guarded with `if (sub.MaxMessages > 0)` | Eliminates 1 interlocked op per delivery in common case |
|
||||||
|
| 34 | **Per-delivery SID ASCII encoding** — `Encoding.ASCII.GetBytes(sid)` on every delivery; SID is a small integer that never changes | Added `Subscription.SidBytes` cached property; new `SendMessageNoFlush` overload accepts `ReadOnlySpan<byte>` | Eliminates per-delivery encoding |
|
||||||
|
| 35 | **Per-delivery subject ASCII encoding** — `Encoding.ASCII.GetBytes(subject)` for each subscriber; fan-out 4 = 4× encoding same subject | Pre-encode subject once in `ProcessMessage` before fan-out loop; new overload uses span copy | Eliminates N-1 subject encodings per publish |
|
||||||
|
| 36 | **Per-publish subject string allocation** — `Encoding.ASCII.GetString(cmd.Subject.Span)` on every PUB even when publishing to the same subject repeatedly | Added 1-element string cache per client; reuses string when subject bytes match | Eliminates string alloc for repeated subjects |
|
||||||
|
| 37 | **Interlocked stats in SubList.Match hot path** — `Interlocked.Increment(ref _matches)` and `_cacheHits` on every match call | Replaced with non-atomic increments (approximate counters for monitoring) | Eliminates 1-2 interlocked ops per match |
|
||||||
|
|
||||||
|
### Round 8: Ordered Consumer + Cross-Protocol Optimization
|
||||||
|
|
||||||
|
Three optimizations targeting pull consumer delivery and MQTT cross-protocol throughput:
|
||||||
|
|
||||||
|
| # | Root Cause | Fix | Impact |
|
||||||
|
|---|-----------|-----|--------|
|
||||||
|
| 28 | **Per-message flush signal in DeliverPullFetchMessagesAsync** — `DeliverMessage` called `SendMessage` which triggered `_flushSignal.Writer.TryWrite(0)` per message; for batch of N messages, N flush signals and write-loop wakeups | Replaced with `SendMessageNoFlush` + batch flush every 64 messages + final flush after loop; bypasses `DeliverMessage` entirely (no permission check / auto-unsub needed for JS delivery inbox) | Reduces flush signals from N to N/64 per batch |
|
||||||
|
| 29 | **5ms polling delay in pull consumer wait loop** — `Task.Delay(5)` in `DeliverPullFetchMessagesAsync` and `PullConsumerEngine.WaitForMessageAsync` added up to 5ms latency per empty slot; for tail-following consumers, every new message waited up to 5ms to be noticed | Added `StreamHandle.NotifyPublish()` / `WaitForPublishAsync()` using `TaskCompletionSource` signaling; publishers call `NotifyPublish` after `AppendAsync`; consumers wait on signal with heartbeat-interval timeout | Eliminates polling delay; instant wakeup on publish |
|
||||||
|
| 30 | **StringBuilder allocation in NatsToMqtt for common case** — every uncached `NatsToMqtt` call allocated a StringBuilder even when no `_DOT_` escape sequences were present (the common case) | Added `string.Create` fast path that uses char replacement lambda when no `_DOT_` found; pre-warm topic bytes cache on MQTT subscription creation | Eliminates StringBuilder + string alloc for common case; no cache miss on first delivery |
|
||||||
|
|
||||||
|
### Round 7: MQTT Cross-Protocol Write Path
|
||||||
|
|
||||||
|
Four optimizations targeting the NATS→MQTT delivery hot path (cross-protocol throughput improved from 0.30x to 0.78x):
|
||||||
|
|
||||||
|
| # | Root Cause | Fix | Impact |
|
||||||
|
|---|-----------|-----|--------|
|
||||||
|
| 24 | **Per-message async fire-and-forget in MqttNatsClientAdapter** — each `SendMessage` called `SendBinaryPublishAsync` which acquired a `SemaphoreSlim`, allocated a full PUBLISH packet `byte[]`, wrote, and flushed the stream — all per message, bypassing the server's deferred-flush batching | Replaced with synchronous `EnqueuePublishNoFlush()` that formats MQTT PUBLISH directly into `_directBuf` under SpinLock, matching the NatsClient pattern; `SignalFlush()` signals the write loop for batch flush | Eliminates async Task + SemaphoreSlim + per-message flush |
|
||||||
|
| 25 | **Per-message `byte[]` allocation for MQTT PUBLISH packets** — `MqttPacketWriter.WritePublish()` allocated topic bytes, variable header, remaining-length array, and full packet array on every delivery | Added `WritePublishTo(Span<byte>)` that formats the entire PUBLISH packet directly into the destination span using `Span<byte>` operations — zero heap allocation | Eliminates 4+ `byte[]` allocs per delivery |
|
||||||
|
| 26 | **Per-message NATS→MQTT topic translation** — `NatsToMqtt()` allocated a `StringBuilder`, produced a `string`, then `Encoding.UTF8.GetBytes()` re-encoded it on every delivery | Added `NatsToMqttBytes()` with bounded `ConcurrentDictionary<string, byte[]>` cache (4096 entries); cached result includes pre-encoded UTF-8 bytes | Eliminates string + encoding alloc per delivery for cached topics |
|
||||||
|
| 27 | **Per-message `FlushAsync` on plain TCP sockets** — `WriteBinaryAsync` flushed after every packet write, even on `NetworkStream` where TCP auto-flushes | Write loop skips `FlushAsync` for plain sockets; for TLS/wrapped streams, flushes once per batch (not per message) | Reduces syscalls from 2N to 1 per batch |
|
||||||
|
|
||||||
### Round 6: Batch Flush Signaling + Fetch Optimizations
|
### Round 6: Batch Flush Signaling + Fetch Optimizations
|
||||||
|
|
||||||
Four optimizations targeting fan-out and consumer fetch hot paths:
|
Four optimizations targeting fan-out and consumer fetch hot paths:
|
||||||
@@ -200,6 +285,6 @@ Additional fixes: SHA256 envelope bypass for unencrypted/uncompressed stores, RA
|
|||||||
|
|
||||||
| Change | Expected Impact | Go Reference |
|
| Change | Expected Impact | Go Reference |
|
||||||
|--------|----------------|-------------|
|
|--------|----------------|-------------|
|
||||||
| **Fan-out parallelism** | Deliver to subscribers concurrently instead of serially from publisher's read loop | Go: `processMsgResults` fans out per-client via goroutines |
|
| **Write-loop / socket write overhead** | The per-delivery serial path is now minimal (SID copy + memcpy under SpinLock). The remaining 0.70x fan-out gap is likely write-loop wakeup latency and socket write syscall overhead | Go: `flushOutbound` uses `net.Buffers.WriteTo` → `writev()` with zero-copy buffer management |
|
||||||
| **Eliminate per-message GC allocations in FileStore** | ~30% improvement on FileStore AppendAsync — replace `StoredMessage` class with `StoredMessageMeta` struct in `_messages` dict, reconstruct full message from MsgBlock on read | Go stores in `cache.buf`/`cache.idx` with zero per-message allocs; 80+ sites in FileStore.cs need migration |
|
| **Eliminate per-message GC allocations in FileStore** | ~30% improvement on FileStore AppendAsync — replace `StoredMessage` class with `StoredMessageMeta` struct in `_messages` dict, reconstruct full message from MsgBlock on read | Go stores in `cache.buf`/`cache.idx` with zero per-message allocs; 80+ sites in FileStore.cs need migration |
|
||||||
| **Ordered consumer delivery optimization** | Investigate .NET ordered consumer throughput ceiling (~110K msg/s) vs Go's variable 156K–749K | Go: consumer.go ordered consumer fast path |
|
| **Single publisher throughput** | 0.62x–0.74x gap; the pub-only path has no fan-out overhead — likely JIT/GC/socket write overhead in the ingest path | Go: client.go readLoop with zero-copy buffer management |
|
||||||
|
|||||||
@@ -233,7 +233,7 @@ public sealed class PullConsumerEngine
|
|||||||
// is empty or the consumer has caught up to the end of the stream.
|
// is empty or the consumer has caught up to the end of the stream.
|
||||||
if (expiresCts is not null)
|
if (expiresCts is not null)
|
||||||
{
|
{
|
||||||
message = await WaitForMessageAsync(stream.Store, sequence, effectiveCt);
|
message = await WaitForMessageAsync(stream, sequence, effectiveCt);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -291,19 +291,21 @@ public sealed class PullConsumerEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Poll-wait for a message to appear at the given sequence, retrying with a
|
/// Wait for a message to appear at the given sequence using signal-based wakeup.
|
||||||
/// short delay until the cancellation token fires (typically from ExpiresMs).
|
/// Publishers call <see cref="StreamHandle.NotifyPublish"/> after each append,
|
||||||
|
/// eliminating the 5ms polling delay.
|
||||||
|
/// Go reference: consumer.go — channel signaling from publisher to waiting consumer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static async ValueTask<StoredMessage?> WaitForMessageAsync(IStreamStore store, ulong sequence, CancellationToken ct)
|
private static async ValueTask<StoredMessage?> WaitForMessageAsync(StreamHandle stream, ulong sequence, CancellationToken ct)
|
||||||
{
|
{
|
||||||
while (!ct.IsCancellationRequested)
|
while (!ct.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
var message = await store.LoadAsync(sequence, ct);
|
var message = await stream.Store.LoadAsync(sequence, ct);
|
||||||
if (message is not null)
|
if (message is not null)
|
||||||
return message;
|
return message;
|
||||||
|
|
||||||
// Yield briefly before retrying — the ExpiresMs CTS will cancel when time is up
|
// Wait for publisher to signal a new message instead of polling.
|
||||||
await Task.Delay(5, ct).ConfigureAwait(false);
|
await stream.WaitForPublishAsync(ct).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
ct.ThrowIfCancellationRequested();
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
using System.Text;
|
||||||
|
|
||||||
|
namespace NATS.Server.JetStream.Publish;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Hand-rolled UTF-8 formatter for the common success PubAck case.
|
||||||
|
/// Avoids JsonSerializer overhead (~100-200B internal allocations + reflection).
|
||||||
|
/// For error/duplicate/batch acks, callers fall back to JsonSerializer.
|
||||||
|
/// </summary>
|
||||||
|
internal static class JetStreamPubAckFormatter
|
||||||
|
{
|
||||||
|
// Pre-encoded UTF-8 fragments for {"stream":"NAME","seq":N}
|
||||||
|
private static readonly byte[] Prefix = "{\"stream\":\""u8.ToArray();
|
||||||
|
private static readonly byte[] SeqField = "\",\"seq\":"u8.ToArray();
|
||||||
|
private static readonly byte[] Suffix = "}"u8.ToArray();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Formats a success PubAck directly into a span. Returns bytes written.
|
||||||
|
/// Caller must ensure dest is large enough (256 bytes is safe for any stream name).
|
||||||
|
/// </summary>
|
||||||
|
public static int FormatSuccess(Span<byte> dest, string streamName, ulong seq)
|
||||||
|
{
|
||||||
|
var pos = 0;
|
||||||
|
Prefix.CopyTo(dest);
|
||||||
|
pos += Prefix.Length;
|
||||||
|
pos += Encoding.UTF8.GetBytes(streamName, dest[pos..]);
|
||||||
|
SeqField.CopyTo(dest[pos..]);
|
||||||
|
pos += SeqField.Length;
|
||||||
|
seq.TryFormat(dest[pos..], out var written);
|
||||||
|
pos += written;
|
||||||
|
Suffix.CopyTo(dest[pos..]);
|
||||||
|
pos += Suffix.Length;
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns true if this PubAck is a simple success that can use the fast formatter.
|
||||||
|
/// </summary>
|
||||||
|
public static bool IsSimpleSuccess(PubAck ack)
|
||||||
|
=> ack.ErrorCode == null && !ack.Duplicate && ack.BatchId == null;
|
||||||
|
}
|
||||||
@@ -37,8 +37,8 @@ public sealed class JetStreamPublisher
|
|||||||
}
|
}
|
||||||
|
|
||||||
// --- Normal (non-batch) publish path ---
|
// --- Normal (non-batch) publish path ---
|
||||||
var state = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
// Use cached LastSeq property instead of GetStateAsync to avoid allocation.
|
||||||
if (!_preconditions.CheckExpectedLastSeq(options.ExpectedLastSeq, state.LastSeq))
|
if (!_preconditions.CheckExpectedLastSeq(options.ExpectedLastSeq, stream.Store.LastSeq))
|
||||||
{
|
{
|
||||||
ack = new PubAck { ErrorCode = 10071 };
|
ack = new PubAck { ErrorCode = 10071 };
|
||||||
return true;
|
return true;
|
||||||
@@ -54,7 +54,8 @@ public sealed class JetStreamPublisher
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
var captured = _streamManager.Capture(subject, payload);
|
// Pass resolved stream to avoid double FindBySubject lookup.
|
||||||
|
var captured = _streamManager.Capture(stream, subject, payload);
|
||||||
ack = captured ?? new PubAck();
|
ack = captured ?? new PubAck();
|
||||||
_preconditions.Record(options.MsgId, ack.Seq);
|
_preconditions.Record(options.MsgId, ack.Seq);
|
||||||
_preconditions.TrimOlderThan(stream.Config.DuplicateWindowMs);
|
_preconditions.TrimOlderThan(stream.Config.DuplicateWindowMs);
|
||||||
@@ -136,15 +137,14 @@ public sealed class JetStreamPublisher
|
|||||||
stream.Config.DuplicateWindowMs,
|
stream.Config.DuplicateWindowMs,
|
||||||
staged =>
|
staged =>
|
||||||
{
|
{
|
||||||
// Check expected last sequence.
|
// Check expected last sequence using cached property.
|
||||||
if (staged.ExpectedLastSeq > 0)
|
if (staged.ExpectedLastSeq > 0)
|
||||||
{
|
{
|
||||||
var st = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
if (stream.Store.LastSeq != staged.ExpectedLastSeq)
|
||||||
if (st.LastSeq != staged.ExpectedLastSeq)
|
|
||||||
return new PubAck { ErrorCode = 10071, Stream = stream.Config.Name };
|
return new PubAck { ErrorCode = 10071, Stream = stream.Config.Name };
|
||||||
}
|
}
|
||||||
|
|
||||||
var captured = _streamManager.Capture(staged.Subject, staged.Payload);
|
var captured = _streamManager.Capture(stream, staged.Subject, staged.Payload);
|
||||||
return captured ?? new PubAck { Stream = stream.Config.Name };
|
return captured ?? new PubAck { Stream = stream.Config.Name };
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -32,6 +32,13 @@ public interface IStreamStore
|
|||||||
// Existing MemStore/FileStore implementations return this type.
|
// Existing MemStore/FileStore implementations return this type.
|
||||||
ValueTask<ApiStreamState> GetStateAsync(CancellationToken ct);
|
ValueTask<ApiStreamState> GetStateAsync(CancellationToken ct);
|
||||||
|
|
||||||
|
// Cached state properties — avoid GetStateAsync on the publish hot path.
|
||||||
|
// These are maintained incrementally by FileStore/MemStore and are O(1).
|
||||||
|
ulong LastSeq => throw new NotSupportedException("LastSeq not implemented.");
|
||||||
|
ulong MessageCount => throw new NotSupportedException("MessageCount not implemented.");
|
||||||
|
ulong TotalBytes => throw new NotSupportedException("TotalBytes not implemented.");
|
||||||
|
ulong FirstSeq => throw new NotSupportedException("FirstSeq not implemented.");
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
// -------------------------------------------------------------------------
|
||||||
// Go-parity sync interface — mirrors server/store.go StreamStore
|
// Go-parity sync interface — mirrors server/store.go StreamStore
|
||||||
// Default implementations throw NotSupportedException so existing
|
// Default implementations throw NotSupportedException so existing
|
||||||
|
|||||||
@@ -122,6 +122,12 @@ public sealed class MemStore : IStreamStore
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IStreamStore cached state properties — O(1), maintained incrementally.
|
||||||
|
public ulong LastSeq { get { lock (_gate) return _st.LastSeq; } }
|
||||||
|
public ulong MessageCount { get { lock (_gate) return _st.Msgs; } }
|
||||||
|
public ulong TotalBytes { get { lock (_gate) return _st.Bytes; } }
|
||||||
|
ulong IStreamStore.FirstSeq { get { lock (_gate) return _st.Msgs == 0 ? (_st.FirstSeq > 0 ? _st.FirstSeq : 0UL) : _st.FirstSeq; } }
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
// -------------------------------------------------------------------------
|
||||||
// Async helpers (used by existing JetStream layer)
|
// Async helpers (used by existing JetStream layer)
|
||||||
// -------------------------------------------------------------------------
|
// -------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -315,6 +315,10 @@ public sealed class MsgBlock : IDisposable
|
|||||||
|
|
||||||
_index[sequence] = (offset, written);
|
_index[sequence] = (offset, written);
|
||||||
|
|
||||||
|
// If this sequence was previously soft-deleted, clear the deletion marker
|
||||||
|
// so that subsequent Read calls return the new record rather than null.
|
||||||
|
_deleted.Remove(sequence);
|
||||||
|
|
||||||
// Go: cache populated lazily on read, not eagerly on write.
|
// Go: cache populated lazily on read, not eagerly on write.
|
||||||
// Reads that miss _cache flush pending buf to disk and decode from there.
|
// Reads that miss _cache flush pending buf to disk and decode from there.
|
||||||
|
|
||||||
|
|||||||
@@ -397,6 +397,11 @@ public sealed class StreamManager : IDisposable
|
|||||||
if (stream == null)
|
if (stream == null)
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
|
return Capture(stream, subject, payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PubAck? Capture(StreamHandle stream, string subject, ReadOnlyMemory<byte> payload)
|
||||||
|
{
|
||||||
// Go: sealed stream rejects all publishes.
|
// Go: sealed stream rejects all publishes.
|
||||||
// Reference: server/stream.go — processJetStreamMsg checks mset.cfg.Sealed.
|
// Reference: server/stream.go — processJetStreamMsg checks mset.cfg.Sealed.
|
||||||
if (stream.Config.Sealed)
|
if (stream.Config.Sealed)
|
||||||
@@ -414,17 +419,20 @@ public sealed class StreamManager : IDisposable
|
|||||||
// Go: memStoreMsgSize — full message size includes subject + headers + payload + 16 bytes overhead.
|
// Go: memStoreMsgSize — full message size includes subject + headers + payload + 16 bytes overhead.
|
||||||
var msgSize = subject.Length + payload.Length + 16;
|
var msgSize = subject.Length + payload.Length + 16;
|
||||||
|
|
||||||
var stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
// Use cached state properties instead of GetStateAsync to avoid allocation on hot path.
|
||||||
|
var currentMsgCount = stream.Store.MessageCount;
|
||||||
|
var currentBytes = stream.Store.TotalBytes;
|
||||||
|
var currentFirstSeq = stream.Store.FirstSeq;
|
||||||
|
|
||||||
// Go: DiscardPolicy.New — reject when MaxMsgs reached.
|
// Go: DiscardPolicy.New — reject when MaxMsgs reached.
|
||||||
// Reference: server/stream.go — processJetStreamMsg checks discard new + maxMsgs.
|
// Reference: server/stream.go — processJetStreamMsg checks discard new + maxMsgs.
|
||||||
if (stream.Config.MaxMsgs > 0 && stream.Config.Discard == DiscardPolicy.New
|
if (stream.Config.MaxMsgs > 0 && stream.Config.Discard == DiscardPolicy.New
|
||||||
&& (long)stateBefore.Messages >= stream.Config.MaxMsgs)
|
&& (long)currentMsgCount >= stream.Config.MaxMsgs)
|
||||||
{
|
{
|
||||||
return new PubAck { Stream = stream.Config.Name, ErrorCode = 10054 };
|
return new PubAck { Stream = stream.Config.Name, ErrorCode = 10054 };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (stream.Config.MaxBytes > 0 && (long)stateBefore.Bytes + msgSize > stream.Config.MaxBytes)
|
if (stream.Config.MaxBytes > 0 && (long)currentBytes + msgSize > stream.Config.MaxBytes)
|
||||||
{
|
{
|
||||||
if (stream.Config.Discard == DiscardPolicy.New)
|
if (stream.Config.Discard == DiscardPolicy.New)
|
||||||
{
|
{
|
||||||
@@ -435,10 +443,9 @@ public sealed class StreamManager : IDisposable
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
while ((long)stateBefore.Bytes + msgSize > stream.Config.MaxBytes && stateBefore.FirstSeq > 0)
|
while ((long)stream.Store.TotalBytes + msgSize > stream.Config.MaxBytes && stream.Store.FirstSeq > 0)
|
||||||
{
|
{
|
||||||
stream.Store.RemoveAsync(stateBefore.FirstSeq, default).GetAwaiter().GetResult();
|
stream.Store.RemoveAsync(stream.Store.FirstSeq, default).GetAwaiter().GetResult();
|
||||||
stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -457,6 +464,9 @@ public sealed class StreamManager : IDisposable
|
|||||||
var seq = stream.Store.AppendAsync(storeSubject, payload, default).GetAwaiter().GetResult();
|
var seq = stream.Store.AppendAsync(storeSubject, payload, default).GetAwaiter().GetResult();
|
||||||
EnforceRuntimePolicies(stream, DateTime.UtcNow);
|
EnforceRuntimePolicies(stream, DateTime.UtcNow);
|
||||||
|
|
||||||
|
// Wake up any pull consumers waiting at the stream tail.
|
||||||
|
stream.NotifyPublish();
|
||||||
|
|
||||||
// Only load the stored message when replication is configured (mirror/source).
|
// Only load the stored message when replication is configured (mirror/source).
|
||||||
// Avoids unnecessary disk I/O on the hot publish path.
|
// Avoids unnecessary disk I/O on the hot publish path.
|
||||||
if (_mirrorsByOrigin.ContainsKey(stream.Config.Name) || _sourcesByOrigin.ContainsKey(stream.Config.Name))
|
if (_mirrorsByOrigin.ContainsKey(stream.Config.Name) || _sourcesByOrigin.ContainsKey(stream.Config.Name))
|
||||||
@@ -507,6 +517,9 @@ public sealed class StreamManager : IDisposable
|
|||||||
var seq = stream.Store.AppendAsync(storeSubject, newPayload, default).GetAwaiter().GetResult();
|
var seq = stream.Store.AppendAsync(storeSubject, newPayload, default).GetAwaiter().GetResult();
|
||||||
EnforceRuntimePolicies(stream, DateTime.UtcNow);
|
EnforceRuntimePolicies(stream, DateTime.UtcNow);
|
||||||
|
|
||||||
|
// Wake up any pull consumers waiting at the stream tail.
|
||||||
|
stream.NotifyPublish();
|
||||||
|
|
||||||
if (_mirrorsByOrigin.ContainsKey(stream.Config.Name) || _sourcesByOrigin.ContainsKey(stream.Config.Name))
|
if (_mirrorsByOrigin.ContainsKey(stream.Config.Name) || _sourcesByOrigin.ContainsKey(stream.Config.Name))
|
||||||
{
|
{
|
||||||
var stored = stream.Store.LoadAsync(seq, default).GetAwaiter().GetResult();
|
var stored = stream.Store.LoadAsync(seq, default).GetAwaiter().GetResult();
|
||||||
@@ -961,4 +974,25 @@ public sealed class StreamManager : IDisposable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record StreamHandle(StreamConfig Config, IStreamStore Store);
|
public sealed record StreamHandle(StreamConfig Config, IStreamStore Store)
|
||||||
|
{
|
||||||
|
// Signal-based wakeup for pull consumers waiting at the stream tail.
|
||||||
|
// Go reference: consumer.go — channel signaling from publisher to waiting consumer.
|
||||||
|
private volatile TaskCompletionSource _publishSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Notifies waiting consumers that a new message has been published.
|
||||||
|
/// </summary>
|
||||||
|
public void NotifyPublish()
|
||||||
|
{
|
||||||
|
var old = Interlocked.Exchange(ref _publishSignal,
|
||||||
|
new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously));
|
||||||
|
old.TrySetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Waits until a new message is published to this stream.
|
||||||
|
/// </summary>
|
||||||
|
public Task WaitForPublishAsync(CancellationToken ct)
|
||||||
|
=> _publishSignal.Task.WaitAsync(ct);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
using System.Buffers;
|
using System.Buffers;
|
||||||
|
using System.Buffers.Binary;
|
||||||
using System.IO.Pipelines;
|
using System.IO.Pipelines;
|
||||||
using System.Net.Sockets;
|
using System.Net.Sockets;
|
||||||
using System.Security.Cryptography.X509Certificates;
|
using System.Security.Cryptography.X509Certificates;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using System.Threading.Channels;
|
||||||
using NATS.Server.Auth;
|
using NATS.Server.Auth;
|
||||||
using static NATS.Server.Mqtt.MqttBinaryDecoder;
|
using static NATS.Server.Mqtt.MqttBinaryDecoder;
|
||||||
|
|
||||||
@@ -26,6 +28,16 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
private readonly Dictionary<string, string> _topicToSid = new(StringComparer.Ordinal);
|
private readonly Dictionary<string, string> _topicToSid = new(StringComparer.Ordinal);
|
||||||
private int _nextSid;
|
private int _nextSid;
|
||||||
|
|
||||||
|
// Direct-buffer write loop for high-throughput MQTT message delivery.
|
||||||
|
// Mirrors the NatsClient _directBuf/_writeBuf + SpinLock + write-loop pattern.
|
||||||
|
private byte[] _directBuf = new byte[65536];
|
||||||
|
private byte[] _writeBuf = new byte[65536];
|
||||||
|
private int _directBufUsed;
|
||||||
|
private SpinLock _directBufLock = new(enableThreadOwnerTracking: false);
|
||||||
|
private readonly Channel<byte> _flushSignal = Channel.CreateBounded<byte>(
|
||||||
|
new BoundedChannelOptions(1) { SingleReader = true, SingleWriter = false, FullMode = BoundedChannelFullMode.DropWrite });
|
||||||
|
private readonly bool _isPlainSocket;
|
||||||
|
|
||||||
/// <summary>Auth result after successful CONNECT (populated for AuthService path).</summary>
|
/// <summary>Auth result after successful CONNECT (populated for AuthService path).</summary>
|
||||||
public AuthResult? AuthResult { get; private set; }
|
public AuthResult? AuthResult { get; private set; }
|
||||||
|
|
||||||
@@ -46,6 +58,7 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
_stream = client.GetStream();
|
_stream = client.GetStream();
|
||||||
_listener = listener;
|
_listener = listener;
|
||||||
_useBinaryProtocol = useBinaryProtocol;
|
_useBinaryProtocol = useBinaryProtocol;
|
||||||
|
_isPlainSocket = true; // NetworkStream over TCP — plain socket
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -56,6 +69,7 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
_stream = stream;
|
_stream = stream;
|
||||||
_listener = listener;
|
_listener = listener;
|
||||||
_useBinaryProtocol = useBinaryProtocol;
|
_useBinaryProtocol = useBinaryProtocol;
|
||||||
|
_isPlainSocket = false; // Wrapped stream (TLS, test, etc.)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -68,6 +82,7 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
_listener = listener;
|
_listener = listener;
|
||||||
_useBinaryProtocol = useBinaryProtocol;
|
_useBinaryProtocol = useBinaryProtocol;
|
||||||
_clientCert = clientCert;
|
_clientCert = clientCert;
|
||||||
|
_isPlainSocket = false; // TLS-wrapped stream
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task RunAsync(CancellationToken ct)
|
public async Task RunAsync(CancellationToken ct)
|
||||||
@@ -80,6 +95,9 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
|
|
||||||
private async Task RunBinaryAsync(CancellationToken ct)
|
private async Task RunBinaryAsync(CancellationToken ct)
|
||||||
{
|
{
|
||||||
|
// Start the write loop alongside the read loop
|
||||||
|
var writeTask = RunMqttWriteLoopAsync(ct);
|
||||||
|
|
||||||
var pipeReader = PipeReader.Create(_stream, new StreamPipeReaderOptions(leaveOpen: true));
|
var pipeReader = PipeReader.Create(_stream, new StreamPipeReaderOptions(leaveOpen: true));
|
||||||
|
|
||||||
try
|
try
|
||||||
@@ -147,6 +165,10 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
{
|
{
|
||||||
await pipeReader.CompleteAsync();
|
await pipeReader.CompleteAsync();
|
||||||
|
|
||||||
|
// Signal write loop to exit and wait for it
|
||||||
|
_flushSignal.Writer.TryComplete();
|
||||||
|
try { await writeTask; } catch { /* write loop may throw on cancel */ }
|
||||||
|
|
||||||
// Publish will message if not cleanly disconnected
|
// Publish will message if not cleanly disconnected
|
||||||
if (_connected && !_willCleared && _connectInfo.WillTopic != null)
|
if (_connected && !_willCleared && _connectInfo.WillTopic != null)
|
||||||
{
|
{
|
||||||
@@ -492,6 +514,98 @@ public sealed class MqttConnection : IAsyncDisposable
|
|||||||
return WriteLineAsync($"MSG {topic} {payload}", ct);
|
return WriteLineAsync($"MSG {topic} {payload}", ct);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Enqueues an MQTT PUBLISH packet into the direct buffer under SpinLock.
|
||||||
|
/// Zero-allocation hot path — formats the packet directly into the buffer.
|
||||||
|
/// Called synchronously from the NATS delivery path (DeliverMessage).
|
||||||
|
/// </summary>
|
||||||
|
public void EnqueuePublishNoFlush(ReadOnlySpan<byte> topicUtf8, ReadOnlyMemory<byte> payload,
|
||||||
|
byte qos = 0, bool retain = false, ushort packetId = 0)
|
||||||
|
{
|
||||||
|
var totalLen = MqttPacketWriter.MeasurePublish(topicUtf8.Length, payload.Length, qos);
|
||||||
|
|
||||||
|
var lockTaken = false;
|
||||||
|
_directBufLock.Enter(ref lockTaken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Grow buffer if needed
|
||||||
|
var needed = _directBufUsed + totalLen;
|
||||||
|
if (needed > _directBuf.Length)
|
||||||
|
{
|
||||||
|
var newSize = Math.Max(_directBuf.Length * 2, needed);
|
||||||
|
var newBuf = new byte[newSize];
|
||||||
|
_directBuf.AsSpan(0, _directBufUsed).CopyTo(newBuf);
|
||||||
|
_directBuf = newBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
MqttPacketWriter.WritePublishTo(
|
||||||
|
_directBuf.AsSpan(_directBufUsed),
|
||||||
|
topicUtf8,
|
||||||
|
payload.Span,
|
||||||
|
qos, retain, dup: false, packetId);
|
||||||
|
|
||||||
|
_directBufUsed += totalLen;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken) _directBufLock.Exit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Signals the write loop to flush buffered MQTT packets.
|
||||||
|
/// </summary>
|
||||||
|
public void SignalMqttFlush() => _flushSignal.Writer.TryWrite(0);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Write loop that drains the direct buffer and writes to the stream in batches.
|
||||||
|
/// Mirrors NatsClient.RunWriteLoopAsync — swap buffers under SpinLock, single write+flush.
|
||||||
|
/// </summary>
|
||||||
|
private async Task RunMqttWriteLoopAsync(CancellationToken ct)
|
||||||
|
{
|
||||||
|
var flushReader = _flushSignal.Reader;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (await flushReader.WaitToReadAsync(ct))
|
||||||
|
{
|
||||||
|
// Drain all pending signals
|
||||||
|
while (flushReader.TryRead(out _)) { }
|
||||||
|
|
||||||
|
// Swap buffers under SpinLock
|
||||||
|
int directLen = 0;
|
||||||
|
var lockTaken = false;
|
||||||
|
_directBufLock.Enter(ref lockTaken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (_directBufUsed > 0)
|
||||||
|
{
|
||||||
|
(_directBuf, _writeBuf) = (_writeBuf, _directBuf);
|
||||||
|
directLen = _directBufUsed;
|
||||||
|
_directBufUsed = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken) _directBufLock.Exit();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (directLen > 0)
|
||||||
|
{
|
||||||
|
await _stream.WriteAsync(_writeBuf.AsMemory(0, directLen), ct);
|
||||||
|
|
||||||
|
// For plain TCP sockets (NetworkStream), TCP auto-flushes — skip FlushAsync.
|
||||||
|
// For TLS/wrapped streams, flush once per batch.
|
||||||
|
if (!_isPlainSocket)
|
||||||
|
await _stream.FlushAsync(ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) { }
|
||||||
|
catch (IOException) { }
|
||||||
|
catch (ObjectDisposedException) { }
|
||||||
|
}
|
||||||
|
|
||||||
public async ValueTask DisposeAsync()
|
public async ValueTask DisposeAsync()
|
||||||
{
|
{
|
||||||
// Clean up adapter subscriptions and unregister from listener
|
// Clean up adapter subscriptions and unregister from listener
|
||||||
|
|||||||
@@ -5,7 +5,6 @@
|
|||||||
using NATS.Server.Auth;
|
using NATS.Server.Auth;
|
||||||
using NATS.Server.Protocol;
|
using NATS.Server.Protocol;
|
||||||
using NATS.Server.Subscriptions;
|
using NATS.Server.Subscriptions;
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
namespace NATS.Server.Mqtt;
|
namespace NATS.Server.Mqtt;
|
||||||
|
|
||||||
@@ -35,27 +34,32 @@ public sealed class MqttNatsClientAdapter : INatsClient
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Delivers a NATS message to this MQTT client by translating the NATS subject
|
/// Delivers a NATS message to this MQTT client by translating the NATS subject
|
||||||
/// to an MQTT topic and writing a binary PUBLISH packet.
|
/// to an MQTT topic and enqueueing a PUBLISH packet into the direct buffer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void SendMessage(string subject, string sid, string? replyTo,
|
public void SendMessage(string subject, string sid, string? replyTo,
|
||||||
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
{
|
{
|
||||||
var mqttTopic = MqttTopicMapper.NatsToMqtt(subject);
|
SendMessageNoFlush(subject, sid, replyTo, headers, payload);
|
||||||
// Fire-and-forget async send; MQTT delivery is best-effort for QoS 0
|
SignalFlush();
|
||||||
_ = _connection.SendBinaryPublishAsync(mqttTopic, payload, qos: 0,
|
|
||||||
retain: false, packetId: 0, CancellationToken.None);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Enqueues an MQTT PUBLISH into the connection's direct buffer without flushing.
|
||||||
|
/// Uses cached topic bytes to avoid re-encoding. Zero allocation on the hot path.
|
||||||
|
/// </summary>
|
||||||
public void SendMessageNoFlush(string subject, string sid, string? replyTo,
|
public void SendMessageNoFlush(string subject, string sid, string? replyTo,
|
||||||
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
{
|
{
|
||||||
// MQTT has no concept of deferred flush — deliver immediately
|
var topicBytes = MqttTopicMapper.NatsToMqttBytes(subject);
|
||||||
SendMessage(subject, sid, replyTo, headers, payload);
|
_connection.EnqueuePublishNoFlush(topicBytes, payload, qos: 0, retain: false, packetId: 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Signals the MQTT connection's write loop to flush buffered packets.
|
||||||
|
/// </summary>
|
||||||
public void SignalFlush()
|
public void SignalFlush()
|
||||||
{
|
{
|
||||||
// No-op for MQTT — each packet is written and flushed immediately
|
_connection.SignalMqttFlush();
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool QueueOutbound(ReadOnlyMemory<byte> data)
|
public bool QueueOutbound(ReadOnlyMemory<byte> data)
|
||||||
@@ -79,6 +83,9 @@ public sealed class MqttNatsClientAdapter : INatsClient
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public Subscription AddSubscription(string natsSubject, string sid, string? queue = null)
|
public Subscription AddSubscription(string natsSubject, string sid, string? queue = null)
|
||||||
{
|
{
|
||||||
|
// Pre-warm topic bytes cache for this subject to avoid cache miss on first message.
|
||||||
|
MqttTopicMapper.NatsToMqttBytes(natsSubject);
|
||||||
|
|
||||||
var sub = new Subscription
|
var sub = new Subscription
|
||||||
{
|
{
|
||||||
Client = this,
|
Client = this,
|
||||||
|
|||||||
@@ -146,6 +146,92 @@ public static class MqttPacketWriter
|
|||||||
return Write(MqttControlPacketType.Publish, totalPayload, flags);
|
return Write(MqttControlPacketType.Publish, totalPayload, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes a complete MQTT PUBLISH packet directly into a destination span.
|
||||||
|
/// Returns the number of bytes written. Zero-allocation hot path for message delivery.
|
||||||
|
/// </summary>
|
||||||
|
public static int WritePublishTo(Span<byte> dest, ReadOnlySpan<byte> topicUtf8,
|
||||||
|
ReadOnlySpan<byte> payload, byte qos = 0, bool retain = false, bool dup = false, ushort packetId = 0)
|
||||||
|
{
|
||||||
|
// Calculate remaining length: 2 (topic len) + topic + optional 2 (packet id) + payload
|
||||||
|
var remainingLength = 2 + topicUtf8.Length + (qos > 0 ? 2 : 0) + payload.Length;
|
||||||
|
|
||||||
|
// Encode remaining length into scratch
|
||||||
|
Span<byte> rlScratch = stackalloc byte[4];
|
||||||
|
var rlLen = EncodeRemainingLengthTo(rlScratch, remainingLength);
|
||||||
|
|
||||||
|
var totalLen = 1 + rlLen + remainingLength;
|
||||||
|
|
||||||
|
// Fixed header byte
|
||||||
|
byte flags = 0;
|
||||||
|
if (dup) flags |= 0x08;
|
||||||
|
flags |= (byte)((qos & 0x03) << 1);
|
||||||
|
if (retain) flags |= 0x01;
|
||||||
|
dest[0] = (byte)(((byte)MqttControlPacketType.Publish << 4) | flags);
|
||||||
|
|
||||||
|
var pos = 1;
|
||||||
|
|
||||||
|
// Remaining length
|
||||||
|
rlScratch[..rlLen].CopyTo(dest[pos..]);
|
||||||
|
pos += rlLen;
|
||||||
|
|
||||||
|
// Topic name (length-prefixed)
|
||||||
|
BinaryPrimitives.WriteUInt16BigEndian(dest[pos..], (ushort)topicUtf8.Length);
|
||||||
|
pos += 2;
|
||||||
|
topicUtf8.CopyTo(dest[pos..]);
|
||||||
|
pos += topicUtf8.Length;
|
||||||
|
|
||||||
|
// Packet ID (only for QoS > 0)
|
||||||
|
if (qos > 0)
|
||||||
|
{
|
||||||
|
BinaryPrimitives.WriteUInt16BigEndian(dest[pos..], packetId);
|
||||||
|
pos += 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Application payload
|
||||||
|
payload.CopyTo(dest[pos..]);
|
||||||
|
pos += payload.Length;
|
||||||
|
|
||||||
|
return totalLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Calculates the total wire size of a PUBLISH packet without writing it.
|
||||||
|
/// </summary>
|
||||||
|
public static int MeasurePublish(int topicLen, int payloadLen, byte qos)
|
||||||
|
{
|
||||||
|
var remainingLength = 2 + topicLen + (qos > 0 ? 2 : 0) + payloadLen;
|
||||||
|
var rlLen = MeasureRemainingLength(remainingLength);
|
||||||
|
return 1 + rlLen + remainingLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static int EncodeRemainingLengthTo(Span<byte> dest, int value)
|
||||||
|
{
|
||||||
|
var index = 0;
|
||||||
|
do
|
||||||
|
{
|
||||||
|
var digit = (byte)(value % 128);
|
||||||
|
value /= 128;
|
||||||
|
if (value > 0)
|
||||||
|
digit |= 0x80;
|
||||||
|
dest[index++] = digit;
|
||||||
|
} while (value > 0);
|
||||||
|
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static int MeasureRemainingLength(int value)
|
||||||
|
{
|
||||||
|
var count = 0;
|
||||||
|
do
|
||||||
|
{
|
||||||
|
value /= 128;
|
||||||
|
count++;
|
||||||
|
} while (value > 0);
|
||||||
|
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
internal static byte[] EncodeRemainingLength(int value)
|
internal static byte[] EncodeRemainingLength(int value)
|
||||||
{
|
{
|
||||||
if (value < 0 || value > MqttProtocolConstants.MaxPayloadSize)
|
if (value < 0 || value > MqttProtocolConstants.MaxPayloadSize)
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
// '*' → '+'
|
// '*' → '+'
|
||||||
// '>' → '#'
|
// '>' → '#'
|
||||||
|
|
||||||
|
using System.Collections.Concurrent;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
namespace NATS.Server.Mqtt;
|
namespace NATS.Server.Mqtt;
|
||||||
@@ -25,6 +26,36 @@ namespace NATS.Server.Mqtt;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public static class MqttTopicMapper
|
public static class MqttTopicMapper
|
||||||
{
|
{
|
||||||
|
private const int MaxCacheEntries = 4096;
|
||||||
|
private static readonly ConcurrentDictionary<string, byte[]> TopicBytesCache = new(StringComparer.Ordinal);
|
||||||
|
private static int _cacheCount;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns the MQTT topic as pre-encoded UTF-8 bytes, using a bounded cache
|
||||||
|
/// to avoid repeated string translation and encoding on the hot path.
|
||||||
|
/// </summary>
|
||||||
|
public static byte[] NatsToMqttBytes(string natsSubject)
|
||||||
|
{
|
||||||
|
if (TopicBytesCache.TryGetValue(natsSubject, out var cached))
|
||||||
|
return cached;
|
||||||
|
|
||||||
|
var mqttTopic = NatsToMqtt(natsSubject);
|
||||||
|
var bytes = Encoding.UTF8.GetBytes(mqttTopic);
|
||||||
|
|
||||||
|
// Bounded cache — stop adding after limit to avoid unbounded growth
|
||||||
|
if (Interlocked.Increment(ref _cacheCount) <= MaxCacheEntries)
|
||||||
|
{
|
||||||
|
if (!TopicBytesCache.TryAdd(natsSubject, bytes))
|
||||||
|
Interlocked.Decrement(ref _cacheCount);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Interlocked.Decrement(ref _cacheCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
// Escape sequence for dots that appear in MQTT topic names.
|
// Escape sequence for dots that appear in MQTT topic names.
|
||||||
// Go uses _DOT_ internally to represent a literal dot in the NATS subject.
|
// Go uses _DOT_ internally to represent a literal dot in the NATS subject.
|
||||||
private const string DotEscape = "_DOT_";
|
private const string DotEscape = "_DOT_";
|
||||||
@@ -76,7 +107,18 @@ public static class MqttTopicMapper
|
|||||||
if (natsSubject.Length == 0)
|
if (natsSubject.Length == 0)
|
||||||
return string.Empty;
|
return string.Empty;
|
||||||
|
|
||||||
// First, replace _DOT_ escape sequences back to dots
|
// Fast path: no _DOT_ escape sequences — just char replacement via string.Create
|
||||||
|
// (avoids StringBuilder allocation for the common case).
|
||||||
|
if (!natsSubject.Contains(DotEscape))
|
||||||
|
{
|
||||||
|
return string.Create(natsSubject.Length, natsSubject, static (span, src) =>
|
||||||
|
{
|
||||||
|
for (var i = 0; i < src.Length; i++)
|
||||||
|
span[i] = src[i] switch { '.' => '/', '*' => '+', '>' => '#', _ => src[i] };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Slow path: has _DOT_ escape sequences — use StringBuilder
|
||||||
var working = natsSubject.Replace(DotEscape, "\x00");
|
var working = natsSubject.Replace(DotEscape, "\x00");
|
||||||
|
|
||||||
var sb = new StringBuilder(working.Length);
|
var sb = new StringBuilder(working.Length);
|
||||||
|
|||||||
+168
-15
@@ -134,6 +134,10 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
public long InBytes;
|
public long InBytes;
|
||||||
public long OutBytes;
|
public long OutBytes;
|
||||||
|
|
||||||
|
// Non-atomic round-robin counter for queue-group selection.
|
||||||
|
// Safe because ProcessMessage runs single-threaded per publisher connection (the read loop).
|
||||||
|
public uint QueueRoundRobin;
|
||||||
|
|
||||||
// Close reason tracking
|
// Close reason tracking
|
||||||
private int _skipFlushOnClose;
|
private int _skipFlushOnClose;
|
||||||
public bool ShouldSkipFlush => Volatile.Read(ref _skipFlushOnClose) != 0;
|
public bool ShouldSkipFlush => Volatile.Read(ref _skipFlushOnClose) != 0;
|
||||||
@@ -702,6 +706,10 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
server.OnLocalUnsubscription(Account?.Name ?? Account.GlobalAccountName, sub.Subject, sub.Queue);
|
server.OnLocalUnsubscription(Account?.Name ?? Account.GlobalAccountName, sub.Subject, sub.Queue);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1-element subject string cache: avoids allocating identical strings on repeated publishes.
|
||||||
|
private string? _lastSubjectStr;
|
||||||
|
private byte[]? _lastSubjectBytes;
|
||||||
|
|
||||||
private void ProcessPub(ParsedCommandView cmd, ref long localInMsgs, ref long localInBytes)
|
private void ProcessPub(ParsedCommandView cmd, ref long localInMsgs, ref long localInBytes)
|
||||||
{
|
{
|
||||||
var payloadMemory = cmd.GetPayloadMemory();
|
var payloadMemory = cmd.GetPayloadMemory();
|
||||||
@@ -717,7 +725,19 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var subject = Encoding.ASCII.GetString(cmd.Subject.Span);
|
// 1-element cache: reuse string when publishing to the same subject repeatedly.
|
||||||
|
var subjectSpan = cmd.Subject.Span;
|
||||||
|
string subject;
|
||||||
|
if (_lastSubjectBytes != null && subjectSpan.SequenceEqual(_lastSubjectBytes))
|
||||||
|
{
|
||||||
|
subject = _lastSubjectStr!;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
subject = Encoding.ASCII.GetString(subjectSpan);
|
||||||
|
_lastSubjectStr = subject;
|
||||||
|
_lastSubjectBytes = subjectSpan.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
// Pedantic mode: validate publish subject
|
// Pedantic mode: validate publish subject
|
||||||
if (ClientOpts?.Pedantic == true && !SubjectMatch.IsValidPublishSubject(subject))
|
if (ClientOpts?.Pedantic == true && !SubjectMatch.IsValidPublishSubject(subject))
|
||||||
@@ -786,12 +806,9 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
public void SendMessageNoFlush(string subject, string sid, string? replyTo,
|
public void SendMessageNoFlush(string subject, string sid, string? replyTo,
|
||||||
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
{
|
{
|
||||||
// Batch per-client stats (single thread writes these during delivery).
|
// Per-client stats only — server-wide stats are batched at the ProcessMessage level.
|
||||||
// Server-wide stats use Interlocked since multiple threads update them.
|
|
||||||
OutMsgs++;
|
OutMsgs++;
|
||||||
OutBytes += payload.Length + headers.Length;
|
OutBytes += payload.Length + headers.Length;
|
||||||
Interlocked.Increment(ref _serverStats.OutMsgs);
|
|
||||||
Interlocked.Add(ref _serverStats.OutBytes, payload.Length + headers.Length);
|
|
||||||
|
|
||||||
// Format MSG header on the stack (no heap allocation).
|
// Format MSG header on the stack (no heap allocation).
|
||||||
// Go reference: client.go msgHeader — formats into per-client 1KB scratch buffer (msgb).
|
// Go reference: client.go msgHeader — formats into per-client 1KB scratch buffer (msgb).
|
||||||
@@ -845,14 +862,22 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
headerBuf[pos++] = (byte)'\r';
|
headerBuf[pos++] = (byte)'\r';
|
||||||
headerBuf[pos++] = (byte)'\n';
|
headerBuf[pos++] = (byte)'\n';
|
||||||
|
|
||||||
// Write header + body + CRLF directly into the per-client buffer under lock.
|
WriteMessageToBuffer(headerBuf[..pos], headers, payload);
|
||||||
// Go reference: client.go queueOutbound — appends slice refs under client.mu.
|
}
|
||||||
var totalLen = pos + headers.Length + payload.Length + 2;
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes the formatted MSG/HMSG header line, headers, payload, and trailing CRLF
|
||||||
|
/// into the per-client direct buffer under lock.
|
||||||
|
/// Go reference: client.go queueOutbound — appends slice refs under client.mu.
|
||||||
|
/// </summary>
|
||||||
|
private void WriteMessageToBuffer(ReadOnlySpan<byte> msgHeader,
|
||||||
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
|
{
|
||||||
|
var totalLen = msgHeader.Length + headers.Length + payload.Length + 2;
|
||||||
var lockTaken = false;
|
var lockTaken = false;
|
||||||
_directBufLock.Enter(ref lockTaken);
|
_directBufLock.Enter(ref lockTaken);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// Grow buffer if needed
|
|
||||||
var needed = _directBufUsed + totalLen;
|
var needed = _directBufUsed + totalLen;
|
||||||
if (needed > _directBuf.Length)
|
if (needed > _directBuf.Length)
|
||||||
{
|
{
|
||||||
@@ -864,12 +889,10 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
|
|
||||||
var dst = _directBuf.AsSpan(_directBufUsed);
|
var dst = _directBuf.AsSpan(_directBufUsed);
|
||||||
|
|
||||||
// Header
|
msgHeader.CopyTo(dst);
|
||||||
headerBuf[..pos].CopyTo(dst);
|
_directBufUsed += msgHeader.Length;
|
||||||
_directBufUsed += pos;
|
|
||||||
dst = _directBuf.AsSpan(_directBufUsed);
|
dst = _directBuf.AsSpan(_directBufUsed);
|
||||||
|
|
||||||
// Headers (HMSG)
|
|
||||||
if (headers.Length > 0)
|
if (headers.Length > 0)
|
||||||
{
|
{
|
||||||
headers.Span.CopyTo(dst);
|
headers.Span.CopyTo(dst);
|
||||||
@@ -877,14 +900,144 @@ public sealed class NatsClient : INatsClient, IDisposable
|
|||||||
dst = _directBuf.AsSpan(_directBufUsed);
|
dst = _directBuf.AsSpan(_directBufUsed);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Payload
|
|
||||||
if (payload.Length > 0)
|
if (payload.Length > 0)
|
||||||
{
|
{
|
||||||
payload.Span.CopyTo(dst);
|
payload.Span.CopyTo(dst);
|
||||||
_directBufUsed += payload.Length;
|
_directBufUsed += payload.Length;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Trailing CRLF
|
_directBuf[_directBufUsed++] = (byte)'\r';
|
||||||
|
_directBuf[_directBufUsed++] = (byte)'\n';
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken) _directBufLock.Exit();
|
||||||
|
}
|
||||||
|
|
||||||
|
var pending = Interlocked.Add(ref _pendingBytes, totalLen);
|
||||||
|
if (pending > _options.MaxPending)
|
||||||
|
{
|
||||||
|
if (!_flags.HasFlag(ClientFlags.CloseConnection))
|
||||||
|
{
|
||||||
|
_flags.SetFlag(ClientFlags.CloseConnection);
|
||||||
|
_flags.SetFlag(ClientFlags.IsSlowConsumer);
|
||||||
|
Interlocked.Increment(ref _serverStats.SlowConsumers);
|
||||||
|
Interlocked.Increment(ref _serverStats.SlowConsumerClients);
|
||||||
|
_ = CloseWithReasonAsync(ClientClosedReason.SlowConsumerPendingBytes, NatsProtocol.ErrSlowConsumer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Fast-path overload accepting pre-encoded subject and SID bytes to avoid
|
||||||
|
/// per-delivery ASCII encoding in fan-out scenarios.
|
||||||
|
/// </summary>
|
||||||
|
public void SendMessageNoFlush(ReadOnlySpan<byte> subjectBytes, ReadOnlySpan<byte> sidBytes, string? replyTo,
|
||||||
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
|
{
|
||||||
|
OutMsgs++;
|
||||||
|
OutBytes += payload.Length + headers.Length;
|
||||||
|
|
||||||
|
Span<byte> headerBuf = stackalloc byte[512];
|
||||||
|
int pos = 0;
|
||||||
|
|
||||||
|
if (headers.Length > 0)
|
||||||
|
{
|
||||||
|
"HMSG "u8.CopyTo(headerBuf);
|
||||||
|
pos = 5;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
"MSG "u8.CopyTo(headerBuf);
|
||||||
|
pos = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
subjectBytes.CopyTo(headerBuf[pos..]);
|
||||||
|
pos += subjectBytes.Length;
|
||||||
|
headerBuf[pos++] = (byte)' ';
|
||||||
|
|
||||||
|
sidBytes.CopyTo(headerBuf[pos..]);
|
||||||
|
pos += sidBytes.Length;
|
||||||
|
headerBuf[pos++] = (byte)' ';
|
||||||
|
|
||||||
|
if (replyTo != null)
|
||||||
|
{
|
||||||
|
pos += Encoding.ASCII.GetBytes(replyTo, headerBuf[pos..]);
|
||||||
|
headerBuf[pos++] = (byte)' ';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers.Length > 0)
|
||||||
|
{
|
||||||
|
int totalSize = headers.Length + payload.Length;
|
||||||
|
headers.Length.TryFormat(headerBuf[pos..], out int written);
|
||||||
|
pos += written;
|
||||||
|
headerBuf[pos++] = (byte)' ';
|
||||||
|
totalSize.TryFormat(headerBuf[pos..], out written);
|
||||||
|
pos += written;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
payload.Length.TryFormat(headerBuf[pos..], out int written);
|
||||||
|
pos += written;
|
||||||
|
}
|
||||||
|
|
||||||
|
headerBuf[pos++] = (byte)'\r';
|
||||||
|
headerBuf[pos++] = (byte)'\n';
|
||||||
|
|
||||||
|
WriteMessageToBuffer(headerBuf[..pos], headers, payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Ultra-fast fan-out path: caller pre-builds the MSG prefix ("MSG subject ") and suffix
|
||||||
|
/// (" [reply] sizes\r\n") once per publish. Only the SID varies per delivery.
|
||||||
|
/// Eliminates per-delivery replyTo encoding, size formatting, and prefix/subject copying.
|
||||||
|
/// </summary>
|
||||||
|
public void SendMessagePreformatted(ReadOnlySpan<byte> prefix, ReadOnlySpan<byte> sidBytes,
|
||||||
|
ReadOnlySpan<byte> suffix, ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
|
{
|
||||||
|
OutMsgs++;
|
||||||
|
OutBytes += payload.Length + headers.Length;
|
||||||
|
|
||||||
|
var headerLen = prefix.Length + sidBytes.Length + suffix.Length;
|
||||||
|
var totalLen = headerLen + headers.Length + payload.Length + 2;
|
||||||
|
var lockTaken = false;
|
||||||
|
_directBufLock.Enter(ref lockTaken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var needed = _directBufUsed + totalLen;
|
||||||
|
if (needed > _directBuf.Length)
|
||||||
|
{
|
||||||
|
var newSize = Math.Max(_directBuf.Length * 2, needed);
|
||||||
|
var newBuf = new byte[newSize];
|
||||||
|
_directBuf.AsSpan(0, _directBufUsed).CopyTo(newBuf);
|
||||||
|
_directBuf = newBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
var dst = _directBuf.AsSpan(_directBufUsed);
|
||||||
|
|
||||||
|
prefix.CopyTo(dst);
|
||||||
|
dst = dst[prefix.Length..];
|
||||||
|
|
||||||
|
sidBytes.CopyTo(dst);
|
||||||
|
dst = dst[sidBytes.Length..];
|
||||||
|
|
||||||
|
suffix.CopyTo(dst);
|
||||||
|
dst = dst[suffix.Length..];
|
||||||
|
_directBufUsed += headerLen;
|
||||||
|
|
||||||
|
if (headers.Length > 0)
|
||||||
|
{
|
||||||
|
headers.Span.CopyTo(dst);
|
||||||
|
dst = dst[headers.Length..];
|
||||||
|
_directBufUsed += headers.Length;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (payload.Length > 0)
|
||||||
|
{
|
||||||
|
payload.Span.CopyTo(dst);
|
||||||
|
_directBufUsed += payload.Length;
|
||||||
|
}
|
||||||
|
|
||||||
_directBuf[_directBufUsed++] = (byte)'\r';
|
_directBuf[_directBufUsed++] = (byte)'\r';
|
||||||
_directBuf[_directBufUsed++] = (byte)'\n';
|
_directBuf[_directBufUsed++] = (byte)'\n';
|
||||||
}
|
}
|
||||||
|
|||||||
+208
-46
@@ -40,9 +40,11 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
private readonly ILoggerFactory _loggerFactory;
|
private readonly ILoggerFactory _loggerFactory;
|
||||||
private readonly ServerStats _stats = new();
|
private readonly ServerStats _stats = new();
|
||||||
|
|
||||||
// Per-client deferred flush set. Collects unique clients during fan-out delivery,
|
// Per-client deferred flush array. Collects unique clients during fan-out delivery,
|
||||||
// then flushes each once. Go reference: client.go addToPCD / flushClients.
|
// then flushes each once. Linear array is faster than HashSet for small fan-out counts (n ≤ 16).
|
||||||
[ThreadStatic] private static HashSet<INatsClient>? t_pcd;
|
// Go reference: client.go addToPCD / flushClients.
|
||||||
|
[ThreadStatic] private static INatsClient[]? t_pcdArray;
|
||||||
|
[ThreadStatic] private static int t_pcdCount;
|
||||||
private readonly TaskCompletionSource _listeningStarted = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
private readonly TaskCompletionSource _listeningStarted = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
private AuthService _authService;
|
private AuthService _authService;
|
||||||
private readonly ConcurrentDictionary<string, Account> _accounts = new(StringComparer.Ordinal);
|
private readonly ConcurrentDictionary<string, Account> _accounts = new(StringComparer.Ordinal);
|
||||||
@@ -1398,9 +1400,20 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
// Send pub ack response to the reply subject (request-reply pattern).
|
// Send pub ack response to the reply subject (request-reply pattern).
|
||||||
// Go reference: server/jetstream.go — jsPubAckResponse sent to reply.
|
// Go reference: server/jetstream.go — jsPubAckResponse sent to reply.
|
||||||
if (replyTo != null)
|
if (replyTo != null)
|
||||||
|
{
|
||||||
|
if (JetStream.Publish.JetStreamPubAckFormatter.IsSimpleSuccess(pubAck))
|
||||||
|
{
|
||||||
|
// Fast path: hand-rolled UTF-8 formatter avoids JsonSerializer overhead.
|
||||||
|
Span<byte> ackBuf = stackalloc byte[256];
|
||||||
|
var ackLen = JetStream.Publish.JetStreamPubAckFormatter.FormatSuccess(ackBuf, pubAck.Stream, pubAck.Seq);
|
||||||
|
ProcessMessage(replyTo, null, default, ackBuf[..ackLen].ToArray(), sender);
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
var ackData = JsonSerializer.SerializeToUtf8Bytes(pubAck, s_jetStreamJsonOptions);
|
var ackData = JsonSerializer.SerializeToUtf8Bytes(pubAck, s_jetStreamJsonOptions);
|
||||||
ProcessMessage(replyTo, null, default, ackData, sender);
|
ProcessMessage(replyTo, null, default, ackData, sender);
|
||||||
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1437,20 +1450,74 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
|
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
|
||||||
var result = subList.Match(subject);
|
var result = subList.Match(subject);
|
||||||
var delivered = false;
|
var delivered = false;
|
||||||
|
int deliveredCount = 0;
|
||||||
|
|
||||||
|
// Pre-encode subject bytes once for all fan-out deliveries (one alloc per publish, not per delivery).
|
||||||
|
var subjectBytes = Encoding.ASCII.GetBytes(subject);
|
||||||
|
|
||||||
|
// Pre-build MSG prefix and suffix once per publish. Only the SID varies per delivery.
|
||||||
|
// Format: <prefix><sid><suffix><headers><payload>\r\n
|
||||||
|
// prefix = "MSG subject " or "HMSG subject "
|
||||||
|
// suffix = "[reply] sizes\r\n"
|
||||||
|
Span<byte> msgPrefix = stackalloc byte[subjectBytes.Length + 6]; // "HMSG " + subject + " "
|
||||||
|
int prefixLen;
|
||||||
|
if (headers.Length > 0)
|
||||||
|
{
|
||||||
|
"HMSG "u8.CopyTo(msgPrefix);
|
||||||
|
prefixLen = 5;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
"MSG "u8.CopyTo(msgPrefix);
|
||||||
|
prefixLen = 4;
|
||||||
|
}
|
||||||
|
subjectBytes.CopyTo(msgPrefix[prefixLen..]);
|
||||||
|
prefixLen += subjectBytes.Length;
|
||||||
|
msgPrefix[prefixLen++] = (byte)' ';
|
||||||
|
var msgPrefixSlice = msgPrefix[..prefixLen];
|
||||||
|
|
||||||
|
Span<byte> msgSuffix = stackalloc byte[128]; // " [reply] sizes\r\n"
|
||||||
|
int suffixLen = 0;
|
||||||
|
msgSuffix[suffixLen++] = (byte)' '; // space after SID
|
||||||
|
if (replyTo != null)
|
||||||
|
{
|
||||||
|
suffixLen += Encoding.ASCII.GetBytes(replyTo, msgSuffix[suffixLen..]);
|
||||||
|
msgSuffix[suffixLen++] = (byte)' ';
|
||||||
|
}
|
||||||
|
if (headers.Length > 0)
|
||||||
|
{
|
||||||
|
int totalSize = headers.Length + payload.Length;
|
||||||
|
headers.Length.TryFormat(msgSuffix[suffixLen..], out int written);
|
||||||
|
suffixLen += written;
|
||||||
|
msgSuffix[suffixLen++] = (byte)' ';
|
||||||
|
totalSize.TryFormat(msgSuffix[suffixLen..], out written);
|
||||||
|
suffixLen += written;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
payload.Length.TryFormat(msgSuffix[suffixLen..], out int written);
|
||||||
|
suffixLen += written;
|
||||||
|
}
|
||||||
|
msgSuffix[suffixLen++] = (byte)'\r';
|
||||||
|
msgSuffix[suffixLen++] = (byte)'\n';
|
||||||
|
var msgSuffixSlice = msgSuffix[..suffixLen];
|
||||||
|
|
||||||
// Per-client deferred flush: collect unique clients during fan-out, signal each once.
|
// Per-client deferred flush: collect unique clients during fan-out, signal each once.
|
||||||
|
// Linear array faster than HashSet for small fan-out counts (n ≤ 16).
|
||||||
// Go reference: client.go:3905 addToPCD / client.go:1324 flushClients.
|
// Go reference: client.go:3905 addToPCD / client.go:1324 flushClients.
|
||||||
var pcd = t_pcd ??= new HashSet<INatsClient>();
|
var pcdArray = t_pcdArray ??= new INatsClient[16];
|
||||||
pcd.Clear();
|
t_pcdCount = 0;
|
||||||
|
|
||||||
// Deliver to plain subscribers
|
// Deliver to plain subscribers
|
||||||
|
var messageSize = payload.Length + headers.Length;
|
||||||
foreach (var sub in result.PlainSubs)
|
foreach (var sub in result.PlainSubs)
|
||||||
{
|
{
|
||||||
if (sub.Client == null || sub.Client == sender && !(sender.ClientOpts?.Echo ?? true))
|
if (sub.Client == null || sub.Client == sender && !(sender.ClientOpts?.Echo ?? true))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
DeliverMessage(sub, subject, replyTo, headers, payload, pcd);
|
DeliverMessage(sub, msgPrefixSlice, sub.SidBytes, msgSuffixSlice, subject, replyTo, headers, payload);
|
||||||
delivered = true;
|
delivered = true;
|
||||||
|
deliveredCount++;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deliver to one member of each queue group (round-robin)
|
// Deliver to one member of each queue group (round-robin)
|
||||||
@@ -1458,20 +1525,20 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
{
|
{
|
||||||
if (queueGroup.Length == 0) continue;
|
if (queueGroup.Length == 0) continue;
|
||||||
|
|
||||||
// Simple round-robin -- pick based on total delivered across group
|
// Simple round-robin — non-atomic counter, safe because ProcessMessage
|
||||||
|
// runs single-threaded per publisher connection (the read loop).
|
||||||
if (natsClient != null)
|
if (natsClient != null)
|
||||||
{
|
{
|
||||||
var idx = Math.Abs((int)Interlocked.Increment(ref natsClient.OutMsgs)) % queueGroup.Length;
|
var idx = (int)(natsClient.QueueRoundRobin++ % (uint)queueGroup.Length);
|
||||||
// Undo the OutMsgs increment -- it will be incremented properly in SendMessageNoFlush
|
|
||||||
Interlocked.Decrement(ref natsClient.OutMsgs);
|
|
||||||
|
|
||||||
for (int attempt = 0; attempt < queueGroup.Length; attempt++)
|
for (int attempt = 0; attempt < queueGroup.Length; attempt++)
|
||||||
{
|
{
|
||||||
var sub = queueGroup[(idx + attempt) % queueGroup.Length];
|
var sub = queueGroup[(idx + attempt) % queueGroup.Length];
|
||||||
if (sub.Client != null && (sub.Client != sender || (sender.ClientOpts?.Echo ?? true)))
|
if (sub.Client != null && (sub.Client != sender || (sender.ClientOpts?.Echo ?? true)))
|
||||||
{
|
{
|
||||||
DeliverMessage(sub, subject, replyTo, headers, payload, pcd);
|
DeliverMessage(sub, msgPrefixSlice, sub.SidBytes, msgSuffixSlice, subject, replyTo, headers, payload);
|
||||||
delivered = true;
|
delivered = true;
|
||||||
|
deliveredCount++;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1483,19 +1550,28 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
{
|
{
|
||||||
if (sub.Client != null && sub.Client != sender)
|
if (sub.Client != null && sub.Client != sender)
|
||||||
{
|
{
|
||||||
DeliverMessage(sub, subject, replyTo, headers, payload, pcd);
|
DeliverMessage(sub, msgPrefixSlice, sub.SidBytes, msgSuffixSlice, subject, replyTo, headers, payload);
|
||||||
delivered = true;
|
delivered = true;
|
||||||
|
deliveredCount++;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Batch server-wide stats once per publish (instead of per-delivery Interlocked ops).
|
||||||
|
if (deliveredCount > 0)
|
||||||
|
{
|
||||||
|
Interlocked.Add(ref _stats.OutMsgs, (long)deliveredCount);
|
||||||
|
Interlocked.Add(ref _stats.OutBytes, (long)messageSize * deliveredCount);
|
||||||
|
}
|
||||||
|
|
||||||
// Flush all unique clients once after fan-out.
|
// Flush all unique clients once after fan-out.
|
||||||
// Go reference: client.go:1324 flushClients — iterates pcd map, one signal per client.
|
// Go reference: client.go:1324 flushClients — iterates pcd map, one signal per client.
|
||||||
foreach (var client in pcd)
|
var pcdCount = t_pcdCount;
|
||||||
client.SignalFlush();
|
for (int i = 0; i < pcdCount; i++)
|
||||||
pcd.Clear();
|
pcdArray[i].SignalFlush();
|
||||||
|
t_pcdCount = 0;
|
||||||
|
|
||||||
// Check for service imports that match this subject.
|
// Check for service imports that match this subject.
|
||||||
// When a client in the importer account publishes to a subject
|
// When a client in the importer account publishes to a subject
|
||||||
@@ -1694,7 +1770,14 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
var numPending = batch - delivered;
|
var numPending = batch - delivered;
|
||||||
var ackReply = BuildAckReply(ackPrefix, message.Sequence, deliverySeq, tsNanos, numPending);
|
var ackReply = BuildAckReply(ackPrefix, message.Sequence, deliverySeq, tsNanos, numPending);
|
||||||
|
|
||||||
DeliverMessage(inboxSub, message.Subject, ackReply, minHeaders, message.Payload);
|
// Bypass DeliverMessage — we already know the target client is sender.
|
||||||
|
// Skip permission check and auto-unsub overhead for JS delivery inbox.
|
||||||
|
// Go reference: consumer.go — batch delivery with deferred flush.
|
||||||
|
sender.SendMessageNoFlush(message.Subject, inboxSub.Sid, ackReply, minHeaders, message.Payload);
|
||||||
|
|
||||||
|
// Batch flush every 64 messages to amortize write-loop wakeup cost.
|
||||||
|
if ((delivered & 63) == 0)
|
||||||
|
sender.SignalFlush();
|
||||||
|
|
||||||
if (consumer.Config.AckPolicy is JetStream.Models.AckPolicy.Explicit or JetStream.Models.AckPolicy.All)
|
if (consumer.Config.AckPolicy is JetStream.Models.AckPolicy.Explicit or JetStream.Models.AckPolicy.All)
|
||||||
{
|
{
|
||||||
@@ -1708,6 +1791,11 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
// Flush any buffered messages before blocking on the signal.
|
||||||
|
// Without this, messages queued via SendMessageNoFlush would sit
|
||||||
|
// in the buffer until the next batch boundary or loop exit.
|
||||||
|
sender.SignalFlush();
|
||||||
|
|
||||||
// No message available — send idle heartbeat if needed
|
// No message available — send idle heartbeat if needed
|
||||||
if (DateTime.UtcNow - lastDeliveryTime >= hbInterval)
|
if (DateTime.UtcNow - lastDeliveryTime >= hbInterval)
|
||||||
{
|
{
|
||||||
@@ -1719,8 +1807,17 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
lastDeliveryTime = DateTime.UtcNow;
|
lastDeliveryTime = DateTime.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Poll briefly before retrying
|
// Wait for publisher to signal a new message, with a heartbeat-interval
|
||||||
await Task.Delay(5, ct).ConfigureAwait(false);
|
// timeout so the heartbeat check is re-evaluated periodically.
|
||||||
|
// Go reference: consumer.go — channel signaling from publisher.
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await streamHandle.WaitForPublishAsync(ct).WaitAsync(hbInterval, ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (TimeoutException)
|
||||||
|
{
|
||||||
|
// Heartbeat interval elapsed — loop back to re-check
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1729,13 +1826,16 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
// ExpiresMs timeout — expected
|
// ExpiresMs timeout — expected
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Final flush for any remaining batched messages
|
||||||
|
sender.SignalFlush();
|
||||||
|
|
||||||
consumer.NextSequence = sequence;
|
consumer.NextSequence = sequence;
|
||||||
|
|
||||||
// Send terminal status
|
// Send terminal status directly to sender (last message, includes flush)
|
||||||
ReadOnlyMemory<byte> statusHeader = delivered == 0
|
ReadOnlyMemory<byte> statusHeader = delivered == 0
|
||||||
? System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n")
|
? System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n")
|
||||||
: System.Text.Encoding.UTF8.GetBytes("NATS/1.0 408 Request Timeout\r\n\r\n");
|
: System.Text.Encoding.UTF8.GetBytes("NATS/1.0 408 Request Timeout\r\n\r\n");
|
||||||
DeliverMessage(inboxSub, replyTo, null, statusHeader, default);
|
sender.SendMessage(replyTo, inboxSub.Sid, null, statusHeader, default);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void DeliverFetchedMessages(Subscription inboxSub, string streamName, string consumerName,
|
private void DeliverFetchedMessages(Subscription inboxSub, string streamName, string consumerName,
|
||||||
@@ -1750,8 +1850,8 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
var ackPrefix = $"$JS.ACK.{streamName}.{consumerName}.1.";
|
var ackPrefix = $"$JS.ACK.{streamName}.{consumerName}.1.";
|
||||||
|
|
||||||
// Use pcd pattern: all messages go to the same client, one flush after the loop.
|
// Use pcd pattern: all messages go to the same client, one flush after the loop.
|
||||||
var pcd = t_pcd ??= new HashSet<INatsClient>();
|
var pcdArray = t_pcdArray ??= new INatsClient[16];
|
||||||
pcd.Clear();
|
t_pcdCount = 0;
|
||||||
|
|
||||||
foreach (var msg in messages)
|
foreach (var msg in messages)
|
||||||
{
|
{
|
||||||
@@ -1761,51 +1861,50 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
var tsNanos = new DateTimeOffset(msg.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
|
var tsNanos = new DateTimeOffset(msg.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
|
||||||
var ackReply = BuildAckReply(ackPrefix, msg.Sequence, deliverySeq, tsNanos, numPending);
|
var ackReply = BuildAckReply(ackPrefix, msg.Sequence, deliverySeq, tsNanos, numPending);
|
||||||
|
|
||||||
DeliverMessage(inboxSub, msg.Subject, ackReply, minHeaders, msg.Payload, pcd);
|
DeliverMessage(inboxSub, msg.Subject, ackReply, minHeaders, msg.Payload, usePcd: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flush once after all messages delivered
|
// Flush once after all messages delivered
|
||||||
foreach (var client in pcd)
|
var pcdCount = t_pcdCount;
|
||||||
client.SignalFlush();
|
for (int i = 0; i < pcdCount; i++)
|
||||||
pcd.Clear();
|
pcdArray[i].SignalFlush();
|
||||||
|
t_pcdCount = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void DeliverMessage(Subscription sub, string subject, string? replyTo,
|
/// <summary>
|
||||||
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload,
|
/// Ultra-fast fan-out overload using pre-built MSG prefix and suffix.
|
||||||
HashSet<INatsClient>? pcd = null)
|
/// Only the SID varies per delivery — prefix/suffix are constant across the fan-out loop.
|
||||||
|
/// Used by ProcessMessage fan-out loop.
|
||||||
|
/// </summary>
|
||||||
|
private void DeliverMessage(Subscription sub, ReadOnlySpan<byte> msgPrefix, ReadOnlySpan<byte> sidBytes,
|
||||||
|
ReadOnlySpan<byte> msgSuffix, string subject, string? replyTo,
|
||||||
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
||||||
{
|
{
|
||||||
var client = sub.Client;
|
var client = sub.Client;
|
||||||
if (client == null) return;
|
if (client == null) return;
|
||||||
|
|
||||||
// Check auto-unsub
|
// Auto-unsub: only track when a limit is set (common case is MaxMessages == 0).
|
||||||
var count = Interlocked.Increment(ref sub.MessageCount);
|
if (sub.MaxMessages > 0)
|
||||||
if (sub.MaxMessages > 0 && count > sub.MaxMessages)
|
{
|
||||||
|
var count = Interlocked.Increment(ref sub.MessageCount);
|
||||||
|
if (count > sub.MaxMessages)
|
||||||
{
|
{
|
||||||
// Clean up exhausted subscription from trie and client tracking
|
|
||||||
var subList = client.Account?.SubList ?? _globalAccount.SubList;
|
var subList = client.Account?.SubList ?? _globalAccount.SubList;
|
||||||
subList.Remove(sub);
|
subList.Remove(sub);
|
||||||
client.RemoveSubscription(sub.Sid);
|
client.RemoveSubscription(sub.Sid);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Deny-list delivery filter
|
|
||||||
if (client.Permissions?.IsDeliveryAllowed(subject) == false)
|
if (client.Permissions?.IsDeliveryAllowed(subject) == false)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// When pcd (per-client deferred flush) set is provided, queue data without
|
if (client is NatsClient nc)
|
||||||
// signaling the write loop. The caller flushes all unique clients once after
|
nc.SendMessagePreformatted(msgPrefix, sidBytes, msgSuffix, headers, payload);
|
||||||
// the fan-out loop. Go reference: client.go addToPCD / flushClients.
|
|
||||||
if (pcd != null)
|
|
||||||
{
|
|
||||||
client.SendMessageNoFlush(subject, sub.Sid, replyTo, headers, payload);
|
|
||||||
pcd.Add(client);
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
client.SendMessageNoFlush(subject, sub.Sid, replyTo, headers, payload);
|
||||||
client.SendMessage(subject, sub.Sid, replyTo, headers, payload);
|
AddToPcd(client);
|
||||||
}
|
|
||||||
|
|
||||||
// Track reply subject for response permissions
|
|
||||||
if (replyTo != null && client.Permissions?.ResponseTracker != null)
|
if (replyTo != null && client.Permissions?.ResponseTracker != null)
|
||||||
{
|
{
|
||||||
if (client.Permissions.IsPublishAllowed(replyTo) == false)
|
if (client.Permissions.IsPublishAllowed(replyTo) == false)
|
||||||
@@ -1813,6 +1912,69 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void DeliverMessage(Subscription sub, string subject, string? replyTo,
|
||||||
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload,
|
||||||
|
bool usePcd = false)
|
||||||
|
{
|
||||||
|
var client = sub.Client;
|
||||||
|
if (client == null) return;
|
||||||
|
|
||||||
|
// Auto-unsub: only track when a limit is set (common case is MaxMessages == 0).
|
||||||
|
if (sub.MaxMessages > 0)
|
||||||
|
{
|
||||||
|
var count = Interlocked.Increment(ref sub.MessageCount);
|
||||||
|
if (count > sub.MaxMessages)
|
||||||
|
{
|
||||||
|
var subList = client.Account?.SubList ?? _globalAccount.SubList;
|
||||||
|
subList.Remove(sub);
|
||||||
|
client.RemoveSubscription(sub.Sid);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (client.Permissions?.IsDeliveryAllowed(subject) == false)
|
||||||
|
return;
|
||||||
|
|
||||||
|
if (usePcd)
|
||||||
|
{
|
||||||
|
client.SendMessageNoFlush(subject, sub.Sid, replyTo, headers, payload);
|
||||||
|
AddToPcd(client);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
client.SendMessage(subject, sub.Sid, replyTo, headers, payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (replyTo != null && client.Permissions?.ResponseTracker != null)
|
||||||
|
{
|
||||||
|
if (client.Permissions.IsPublishAllowed(replyTo) == false)
|
||||||
|
client.Permissions.ResponseTracker.RegisterReply(replyTo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Adds a client to the thread-static pcd array if not already present.
|
||||||
|
/// Linear scan — faster than HashSet for small fan-out counts (n ≤ 16).
|
||||||
|
/// </summary>
|
||||||
|
private static void AddToPcd(INatsClient client)
|
||||||
|
{
|
||||||
|
var arr = t_pcdArray!;
|
||||||
|
var count = t_pcdCount;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
if (arr[i] == client) return;
|
||||||
|
}
|
||||||
|
if (count == arr.Length)
|
||||||
|
{
|
||||||
|
var newArr = new INatsClient[arr.Length * 2];
|
||||||
|
arr.CopyTo(newArr, 0);
|
||||||
|
t_pcdArray = arr = newArr;
|
||||||
|
}
|
||||||
|
arr[count] = client;
|
||||||
|
t_pcdCount = count + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Builds an ack reply subject from pre-computed prefix and per-message values.
|
/// Builds an ack reply subject from pre-computed prefix and per-message values.
|
||||||
/// Uses stack-based formatting to avoid string interpolation boxing/allocations.
|
/// Uses stack-based formatting to avoid string interpolation boxing/allocations.
|
||||||
|
|||||||
@@ -554,7 +554,7 @@ public sealed class SubList : IDisposable
|
|||||||
|
|
||||||
public SubListResult Match(string subject)
|
public SubListResult Match(string subject)
|
||||||
{
|
{
|
||||||
Interlocked.Increment(ref _matches);
|
_matches++;
|
||||||
var currentGen = Interlocked.Read(ref _generation);
|
var currentGen = Interlocked.Read(ref _generation);
|
||||||
|
|
||||||
_lock.EnterReadLock();
|
_lock.EnterReadLock();
|
||||||
@@ -562,7 +562,7 @@ public sealed class SubList : IDisposable
|
|||||||
{
|
{
|
||||||
if (_cache != null && _cache.TryGetValue(subject, out var cached) && cached.Generation == currentGen)
|
if (_cache != null && _cache.TryGetValue(subject, out var cached) && cached.Generation == currentGen)
|
||||||
{
|
{
|
||||||
Interlocked.Increment(ref _cacheHits);
|
_cacheHits++;
|
||||||
return cached.Result;
|
return cached.Result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -581,7 +581,7 @@ public sealed class SubList : IDisposable
|
|||||||
currentGen = Interlocked.Read(ref _generation);
|
currentGen = Interlocked.Read(ref _generation);
|
||||||
if (_cache != null && _cache.TryGetValue(subject, out var cached) && cached.Generation == currentGen)
|
if (_cache != null && _cache.TryGetValue(subject, out var cached) && cached.Generation == currentGen)
|
||||||
{
|
{
|
||||||
Interlocked.Increment(ref _cacheHits);
|
_cacheHits++;
|
||||||
return cached.Result;
|
return cached.Result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -940,8 +940,8 @@ public sealed class SubList : IDisposable
|
|||||||
_lock.ExitReadLock();
|
_lock.ExitReadLock();
|
||||||
}
|
}
|
||||||
|
|
||||||
var matches = Interlocked.Read(ref _matches);
|
var matches = Volatile.Read(ref _matches);
|
||||||
var cacheHits = Interlocked.Read(ref _cacheHits);
|
var cacheHits = Volatile.Read(ref _cacheHits);
|
||||||
var hitRate = matches > 0 ? (double)cacheHits / matches : 0.0;
|
var hitRate = matches > 0 ? (double)cacheHits / matches : 0.0;
|
||||||
|
|
||||||
uint maxFanout = 0;
|
uint maxFanout = 0;
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using System.Text;
|
||||||
using NATS.Server;
|
using NATS.Server;
|
||||||
using NATS.Server.Imports;
|
using NATS.Server.Imports;
|
||||||
|
|
||||||
@@ -5,9 +6,17 @@ namespace NATS.Server.Subscriptions;
|
|||||||
|
|
||||||
public sealed class Subscription
|
public sealed class Subscription
|
||||||
{
|
{
|
||||||
|
private byte[]? _sidBytes;
|
||||||
|
|
||||||
public required string Subject { get; init; }
|
public required string Subject { get; init; }
|
||||||
public string? Queue { get; init; }
|
public string? Queue { get; init; }
|
||||||
public required string Sid { get; init; }
|
public required string Sid { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Pre-encoded ASCII bytes of the SID, cached to avoid per-delivery encoding.
|
||||||
|
/// </summary>
|
||||||
|
public byte[] SidBytes => _sidBytes ??= Encoding.ASCII.GetBytes(Sid);
|
||||||
|
|
||||||
public long MessageCount; // Interlocked
|
public long MessageCount; // Interlocked
|
||||||
public long MaxMessages; // 0 = unlimited
|
public long MaxMessages; // 0 = unlimited
|
||||||
public INatsClient? Client { get; set; }
|
public INatsClient? Client { get; set; }
|
||||||
|
|||||||
@@ -5,3 +5,12 @@ public class BenchmarkCoreCollection : ICollectionFixture<CoreServerPairFixture>
|
|||||||
|
|
||||||
[CollectionDefinition("Benchmark-JetStream")]
|
[CollectionDefinition("Benchmark-JetStream")]
|
||||||
public class BenchmarkJetStreamCollection : ICollectionFixture<JetStreamServerPairFixture>;
|
public class BenchmarkJetStreamCollection : ICollectionFixture<JetStreamServerPairFixture>;
|
||||||
|
|
||||||
|
[CollectionDefinition("Benchmark-Mqtt")]
|
||||||
|
public class BenchmarkMqttCollection : ICollectionFixture<MqttServerFixture>;
|
||||||
|
|
||||||
|
[CollectionDefinition("Benchmark-Tls")]
|
||||||
|
public class BenchmarkTlsCollection : ICollectionFixture<TlsServerFixture>;
|
||||||
|
|
||||||
|
[CollectionDefinition("Benchmark-WebSocket")]
|
||||||
|
public class BenchmarkWebSocketCollection : ICollectionFixture<WebSocketServerFixture>;
|
||||||
|
|||||||
@@ -139,14 +139,14 @@ public sealed class DotNetServerProcess : IAsyncDisposable
|
|||||||
{
|
{
|
||||||
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
|
if (File.Exists(Path.Combine(dir.FullName, "NatsDotNet.slnx")))
|
||||||
{
|
{
|
||||||
var dll = Path.Combine(dir.FullName, "src", "NATS.Server.Host", "bin", "Debug", "net10.0", "NATS.Server.Host.dll");
|
var dll = Path.Combine(dir.FullName, "src", "NATS.Server.Host", "bin", "Release", "net10.0", "NATS.Server.Host.dll");
|
||||||
if (File.Exists(dll))
|
if (File.Exists(dll))
|
||||||
return dll;
|
return dll;
|
||||||
|
|
||||||
var build = Process.Start(new ProcessStartInfo
|
var build = Process.Start(new ProcessStartInfo
|
||||||
{
|
{
|
||||||
FileName = "dotnet",
|
FileName = "dotnet",
|
||||||
Arguments = "build src/NATS.Server.Host/NATS.Server.Host.csproj -c Debug",
|
Arguments = "build src/NATS.Server.Host/NATS.Server.Host.csproj -c Release",
|
||||||
WorkingDirectory = dir.FullName,
|
WorkingDirectory = dir.FullName,
|
||||||
UseShellExecute = false,
|
UseShellExecute = false,
|
||||||
RedirectStandardOutput = true,
|
RedirectStandardOutput = true,
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
using NATS.Client.Core;
|
||||||
|
|
||||||
|
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Starts both a Go and .NET NATS server with MQTT and JetStream enabled for MQTT benchmarks.
|
||||||
|
/// Shared across all tests in the "Benchmark-Mqtt" collection.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class MqttServerFixture : IAsyncLifetime
|
||||||
|
{
|
||||||
|
private GoServerProcess? _goServer;
|
||||||
|
private DotNetServerProcess? _dotNetServer;
|
||||||
|
private string? _goStoreDir;
|
||||||
|
private string? _dotNetStoreDir;
|
||||||
|
|
||||||
|
public int GoNatsPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||||
|
public int GoMqttPort { get; private set; }
|
||||||
|
public int DotNetNatsPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||||
|
public int DotNetMqttPort { get; private set; }
|
||||||
|
public bool GoAvailable => _goServer is not null;
|
||||||
|
|
||||||
|
public async Task InitializeAsync()
|
||||||
|
{
|
||||||
|
DotNetMqttPort = PortAllocator.AllocateFreePort();
|
||||||
|
_dotNetStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-dotnet-mqtt-" + Guid.NewGuid().ToString("N")[..8]);
|
||||||
|
Directory.CreateDirectory(_dotNetStoreDir);
|
||||||
|
|
||||||
|
var dotNetConfig = $$"""
|
||||||
|
jetstream {
|
||||||
|
store_dir: "{{_dotNetStoreDir}}"
|
||||||
|
max_mem_store: 64mb
|
||||||
|
max_file_store: 256mb
|
||||||
|
}
|
||||||
|
mqtt {
|
||||||
|
listen: 127.0.0.1:{{DotNetMqttPort}}
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
_dotNetServer = new DotNetServerProcess(dotNetConfig);
|
||||||
|
var dotNetTask = _dotNetServer.StartAsync();
|
||||||
|
|
||||||
|
if (GoServerProcess.IsAvailable())
|
||||||
|
{
|
||||||
|
GoMqttPort = PortAllocator.AllocateFreePort();
|
||||||
|
_goStoreDir = Path.Combine(Path.GetTempPath(), "nats-bench-go-mqtt-" + Guid.NewGuid().ToString("N")[..8]);
|
||||||
|
Directory.CreateDirectory(_goStoreDir);
|
||||||
|
|
||||||
|
var goConfig = $$"""
|
||||||
|
jetstream {
|
||||||
|
store_dir: "{{_goStoreDir}}"
|
||||||
|
max_mem_store: 64mb
|
||||||
|
max_file_store: 256mb
|
||||||
|
}
|
||||||
|
mqtt {
|
||||||
|
listen: 127.0.0.1:{{GoMqttPort}}
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
_goServer = new GoServerProcess(goConfig);
|
||||||
|
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
await dotNetTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task DisposeAsync()
|
||||||
|
{
|
||||||
|
if (_goServer is not null)
|
||||||
|
await _goServer.DisposeAsync();
|
||||||
|
if (_dotNetServer is not null)
|
||||||
|
await _dotNetServer.DisposeAsync();
|
||||||
|
|
||||||
|
CleanupDir(_goStoreDir);
|
||||||
|
CleanupDir(_dotNetStoreDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NatsConnection CreateGoNatsClient()
|
||||||
|
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoNatsPort}" });
|
||||||
|
|
||||||
|
public NatsConnection CreateDotNetNatsClient()
|
||||||
|
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetNatsPort}" });
|
||||||
|
|
||||||
|
private static void CleanupDir(string? dir)
|
||||||
|
{
|
||||||
|
if (dir is not null && Directory.Exists(dir))
|
||||||
|
{
|
||||||
|
try { Directory.Delete(dir, recursive: true); }
|
||||||
|
catch { /* best-effort cleanup */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
using System.Security.Cryptography;
|
||||||
|
using System.Security.Cryptography.X509Certificates;
|
||||||
|
using NATS.Client.Core;
|
||||||
|
|
||||||
|
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Starts both a Go and .NET NATS server with TLS enabled for transport overhead benchmarks.
|
||||||
|
/// Shared across all tests in the "Benchmark-Tls" collection.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class TlsServerFixture : IAsyncLifetime
|
||||||
|
{
|
||||||
|
private GoServerProcess? _goServer;
|
||||||
|
private DotNetServerProcess? _dotNetServer;
|
||||||
|
private string? _tempDir;
|
||||||
|
|
||||||
|
public int GoPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||||
|
public int DotNetPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||||
|
public bool GoAvailable => _goServer is not null;
|
||||||
|
|
||||||
|
public async Task InitializeAsync()
|
||||||
|
{
|
||||||
|
_tempDir = Path.Combine(Path.GetTempPath(), $"nats-bench-tls-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_tempDir);
|
||||||
|
|
||||||
|
var caCertPath = Path.Combine(_tempDir, "ca.pem");
|
||||||
|
var serverCertPath = Path.Combine(_tempDir, "server-cert.pem");
|
||||||
|
var serverKeyPath = Path.Combine(_tempDir, "server-key.pem");
|
||||||
|
|
||||||
|
GenerateCertificates(caCertPath, serverCertPath, serverKeyPath);
|
||||||
|
|
||||||
|
var config = $$"""
|
||||||
|
tls {
|
||||||
|
cert_file: "{{serverCertPath}}"
|
||||||
|
key_file: "{{serverKeyPath}}"
|
||||||
|
ca_file: "{{caCertPath}}"
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
_dotNetServer = new DotNetServerProcess(config);
|
||||||
|
var dotNetTask = _dotNetServer.StartAsync();
|
||||||
|
|
||||||
|
if (GoServerProcess.IsAvailable())
|
||||||
|
{
|
||||||
|
_goServer = new GoServerProcess(config);
|
||||||
|
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
await dotNetTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task DisposeAsync()
|
||||||
|
{
|
||||||
|
if (_goServer is not null)
|
||||||
|
await _goServer.DisposeAsync();
|
||||||
|
if (_dotNetServer is not null)
|
||||||
|
await _dotNetServer.DisposeAsync();
|
||||||
|
|
||||||
|
if (_tempDir is not null && Directory.Exists(_tempDir))
|
||||||
|
{
|
||||||
|
try { Directory.Delete(_tempDir, recursive: true); }
|
||||||
|
catch { /* best-effort cleanup */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public NatsConnection CreateGoTlsClient()
|
||||||
|
=> CreateTlsClient(GoPort);
|
||||||
|
|
||||||
|
public NatsConnection CreateDotNetTlsClient()
|
||||||
|
=> CreateTlsClient(DotNetPort);
|
||||||
|
|
||||||
|
private static NatsConnection CreateTlsClient(int port)
|
||||||
|
{
|
||||||
|
var opts = new NatsOpts
|
||||||
|
{
|
||||||
|
Url = $"nats://127.0.0.1:{port}",
|
||||||
|
TlsOpts = new NatsTlsOpts
|
||||||
|
{
|
||||||
|
Mode = TlsMode.Require,
|
||||||
|
InsecureSkipVerify = true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return new NatsConnection(opts);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void GenerateCertificates(string caCertPath, string serverCertPath, string serverKeyPath)
|
||||||
|
{
|
||||||
|
using var caKey = RSA.Create(2048);
|
||||||
|
var caReq = new CertificateRequest(
|
||||||
|
"CN=Benchmark Test CA",
|
||||||
|
caKey,
|
||||||
|
HashAlgorithmName.SHA256,
|
||||||
|
RSASignaturePadding.Pkcs1);
|
||||||
|
caReq.CertificateExtensions.Add(
|
||||||
|
new X509BasicConstraintsExtension(certificateAuthority: true, hasPathLengthConstraint: false, pathLengthConstraint: 0, critical: true));
|
||||||
|
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
using var caCert = caReq.CreateSelfSigned(now.AddMinutes(-5), now.AddDays(1));
|
||||||
|
|
||||||
|
using var serverKey = RSA.Create(2048);
|
||||||
|
var serverReq = new CertificateRequest(
|
||||||
|
"CN=localhost",
|
||||||
|
serverKey,
|
||||||
|
HashAlgorithmName.SHA256,
|
||||||
|
RSASignaturePadding.Pkcs1);
|
||||||
|
|
||||||
|
var sanBuilder = new SubjectAlternativeNameBuilder();
|
||||||
|
sanBuilder.AddIpAddress(System.Net.IPAddress.Loopback);
|
||||||
|
sanBuilder.AddDnsName("localhost");
|
||||||
|
serverReq.CertificateExtensions.Add(sanBuilder.Build());
|
||||||
|
serverReq.CertificateExtensions.Add(
|
||||||
|
new X509BasicConstraintsExtension(certificateAuthority: false, hasPathLengthConstraint: false, pathLengthConstraint: 0, critical: false));
|
||||||
|
|
||||||
|
using var serverCert = serverReq.Create(caCert, now.AddMinutes(-5), now.AddDays(1), [1, 2, 3, 4]);
|
||||||
|
|
||||||
|
File.WriteAllText(caCertPath, caCert.ExportCertificatePem());
|
||||||
|
File.WriteAllText(serverCertPath, serverCert.ExportCertificatePem());
|
||||||
|
File.WriteAllText(serverKeyPath, serverKey.ExportRSAPrivateKeyPem());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
using NATS.Client.Core;
|
||||||
|
|
||||||
|
namespace NATS.Server.Benchmark.Tests.Infrastructure;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Starts both a Go and .NET NATS server with WebSocket enabled for transport overhead benchmarks.
|
||||||
|
/// Shared across all tests in the "Benchmark-WebSocket" collection.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class WebSocketServerFixture : IAsyncLifetime
|
||||||
|
{
|
||||||
|
private GoServerProcess? _goServer;
|
||||||
|
private DotNetServerProcess? _dotNetServer;
|
||||||
|
|
||||||
|
public int GoNatsPort => _goServer?.Port ?? throw new InvalidOperationException("Go server not started");
|
||||||
|
public int GoWsPort { get; private set; }
|
||||||
|
public int DotNetNatsPort => _dotNetServer?.Port ?? throw new InvalidOperationException(".NET server not started");
|
||||||
|
public int DotNetWsPort { get; private set; }
|
||||||
|
public bool GoAvailable => _goServer is not null;
|
||||||
|
|
||||||
|
public async Task InitializeAsync()
|
||||||
|
{
|
||||||
|
DotNetWsPort = PortAllocator.AllocateFreePort();
|
||||||
|
|
||||||
|
var dotNetConfig = $$"""
|
||||||
|
websocket {
|
||||||
|
listen: 127.0.0.1:{{DotNetWsPort}}
|
||||||
|
no_tls: true
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
_dotNetServer = new DotNetServerProcess(dotNetConfig);
|
||||||
|
var dotNetTask = _dotNetServer.StartAsync();
|
||||||
|
|
||||||
|
if (GoServerProcess.IsAvailable())
|
||||||
|
{
|
||||||
|
GoWsPort = PortAllocator.AllocateFreePort();
|
||||||
|
|
||||||
|
var goConfig = $$"""
|
||||||
|
websocket {
|
||||||
|
listen: 127.0.0.1:{{GoWsPort}}
|
||||||
|
no_tls: true
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
_goServer = new GoServerProcess(goConfig);
|
||||||
|
await Task.WhenAll(dotNetTask, _goServer.StartAsync());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
await dotNetTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task DisposeAsync()
|
||||||
|
{
|
||||||
|
if (_goServer is not null)
|
||||||
|
await _goServer.DisposeAsync();
|
||||||
|
if (_dotNetServer is not null)
|
||||||
|
await _dotNetServer.DisposeAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
public NatsConnection CreateGoNatsClient()
|
||||||
|
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{GoNatsPort}" });
|
||||||
|
|
||||||
|
public NatsConnection CreateDotNetNatsClient()
|
||||||
|
=> new(new NatsOpts { Url = $"nats://127.0.0.1:{DotNetNatsPort}" });
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
using MQTTnet;
|
||||||
|
using MQTTnet.Client;
|
||||||
|
using NATS.Client.Core;
|
||||||
|
using NATS.Server.Benchmark.Tests.Harness;
|
||||||
|
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||||
|
using Xunit.Abstractions;
|
||||||
|
|
||||||
|
namespace NATS.Server.Benchmark.Tests.Mqtt;
|
||||||
|
|
||||||
|
[Collection("Benchmark-Mqtt")]
|
||||||
|
public class MqttThroughputTests(MqttServerFixture fixture, ITestOutputHelper output)
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
[Trait("Category", "Benchmark")]
|
||||||
|
public async Task MqttPubSub_128B()
|
||||||
|
{
|
||||||
|
const int payloadSize = 128;
|
||||||
|
const int messageCount = 5_000;
|
||||||
|
|
||||||
|
var dotnetResult = await RunMqttPubSub("MQTT PubSub (128B)", "DotNet", fixture.DotNetMqttPort, payloadSize, messageCount);
|
||||||
|
|
||||||
|
if (fixture.GoAvailable)
|
||||||
|
{
|
||||||
|
var goResult = await RunMqttPubSub("MQTT PubSub (128B)", "Go", fixture.GoMqttPort, payloadSize, messageCount);
|
||||||
|
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
[Trait("Category", "Benchmark")]
|
||||||
|
public async Task MqttCrossProtocol_NatsPub_MqttSub_128B()
|
||||||
|
{
|
||||||
|
const int payloadSize = 128;
|
||||||
|
const int messageCount = 5_000;
|
||||||
|
|
||||||
|
var dotnetResult = await RunCrossProtocol("Cross-Protocol NATS→MQTT (128B)", "DotNet", fixture.DotNetMqttPort, fixture.CreateDotNetNatsClient, payloadSize, messageCount);
|
||||||
|
|
||||||
|
if (fixture.GoAvailable)
|
||||||
|
{
|
||||||
|
var goResult = await RunCrossProtocol("Cross-Protocol NATS→MQTT (128B)", "Go", fixture.GoMqttPort, fixture.CreateGoNatsClient, payloadSize, messageCount);
|
||||||
|
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<BenchmarkResult> RunMqttPubSub(string name, string serverType, int mqttPort, int payloadSize, int messageCount)
|
||||||
|
{
|
||||||
|
var payload = new byte[payloadSize];
|
||||||
|
var topic = $"bench/mqtt/pubsub/{Guid.NewGuid():N}";
|
||||||
|
|
||||||
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||||
|
|
||||||
|
var factory = new MqttFactory();
|
||||||
|
using var subscriber = factory.CreateMqttClient();
|
||||||
|
using var publisher = factory.CreateMqttClient();
|
||||||
|
|
||||||
|
var subOpts = new MqttClientOptionsBuilder()
|
||||||
|
.WithTcpServer("127.0.0.1", mqttPort)
|
||||||
|
.WithClientId($"bench-sub-{Guid.NewGuid():N}")
|
||||||
|
.WithProtocolVersion(MQTTnet.Formatter.MqttProtocolVersion.V311)
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
var pubOpts = new MqttClientOptionsBuilder()
|
||||||
|
.WithTcpServer("127.0.0.1", mqttPort)
|
||||||
|
.WithClientId($"bench-pub-{Guid.NewGuid():N}")
|
||||||
|
.WithProtocolVersion(MQTTnet.Formatter.MqttProtocolVersion.V311)
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
await subscriber.ConnectAsync(subOpts, cts.Token);
|
||||||
|
await publisher.ConnectAsync(pubOpts, cts.Token);
|
||||||
|
|
||||||
|
var received = 0;
|
||||||
|
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
subscriber.ApplicationMessageReceivedAsync += _ =>
|
||||||
|
{
|
||||||
|
if (Interlocked.Increment(ref received) >= messageCount)
|
||||||
|
tcs.TrySetResult();
|
||||||
|
return Task.CompletedTask;
|
||||||
|
};
|
||||||
|
|
||||||
|
await subscriber.SubscribeAsync(
|
||||||
|
factory.CreateSubscribeOptionsBuilder()
|
||||||
|
.WithTopicFilter(topic)
|
||||||
|
.Build(),
|
||||||
|
cts.Token);
|
||||||
|
|
||||||
|
await Task.Delay(200, cts.Token);
|
||||||
|
|
||||||
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
for (var i = 0; i < messageCount; i++)
|
||||||
|
{
|
||||||
|
await publisher.PublishAsync(
|
||||||
|
new MqttApplicationMessageBuilder()
|
||||||
|
.WithTopic(topic)
|
||||||
|
.WithPayload(payload)
|
||||||
|
.Build(),
|
||||||
|
cts.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
await tcs.Task.WaitAsync(cts.Token);
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
await subscriber.DisconnectAsync(cancellationToken: cts.Token);
|
||||||
|
await publisher.DisconnectAsync(cancellationToken: cts.Token);
|
||||||
|
|
||||||
|
return new BenchmarkResult
|
||||||
|
{
|
||||||
|
Name = name,
|
||||||
|
ServerType = serverType,
|
||||||
|
TotalMessages = messageCount,
|
||||||
|
TotalBytes = (long)messageCount * payloadSize,
|
||||||
|
Duration = sw.Elapsed,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<BenchmarkResult> RunCrossProtocol(string name, string serverType, int mqttPort, Func<NatsConnection> createNatsClient, int payloadSize, int messageCount)
|
||||||
|
{
|
||||||
|
var payload = new byte[payloadSize];
|
||||||
|
var natsSubject = $"bench.mqtt.cross.{Guid.NewGuid():N}";
|
||||||
|
var mqttTopic = natsSubject.Replace('.', '/');
|
||||||
|
|
||||||
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||||
|
|
||||||
|
var factory = new MqttFactory();
|
||||||
|
using var mqttSub = factory.CreateMqttClient();
|
||||||
|
|
||||||
|
var subOpts = new MqttClientOptionsBuilder()
|
||||||
|
.WithTcpServer("127.0.0.1", mqttPort)
|
||||||
|
.WithClientId($"bench-cross-sub-{Guid.NewGuid():N}")
|
||||||
|
.WithProtocolVersion(MQTTnet.Formatter.MqttProtocolVersion.V311)
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
await mqttSub.ConnectAsync(subOpts, cts.Token);
|
||||||
|
|
||||||
|
var received = 0;
|
||||||
|
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
mqttSub.ApplicationMessageReceivedAsync += _ =>
|
||||||
|
{
|
||||||
|
if (Interlocked.Increment(ref received) >= messageCount)
|
||||||
|
tcs.TrySetResult();
|
||||||
|
return Task.CompletedTask;
|
||||||
|
};
|
||||||
|
|
||||||
|
await mqttSub.SubscribeAsync(
|
||||||
|
factory.CreateSubscribeOptionsBuilder()
|
||||||
|
.WithTopicFilter(mqttTopic)
|
||||||
|
.Build(),
|
||||||
|
cts.Token);
|
||||||
|
|
||||||
|
await Task.Delay(200, cts.Token);
|
||||||
|
|
||||||
|
await using var natsPub = createNatsClient();
|
||||||
|
await natsPub.ConnectAsync();
|
||||||
|
await natsPub.PingAsync(cts.Token);
|
||||||
|
|
||||||
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
for (var i = 0; i < messageCount; i++)
|
||||||
|
await natsPub.PublishAsync(natsSubject, payload, cancellationToken: cts.Token);
|
||||||
|
await natsPub.PingAsync(cts.Token);
|
||||||
|
|
||||||
|
await tcs.Task.WaitAsync(cts.Token);
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
await mqttSub.DisconnectAsync(cancellationToken: cts.Token);
|
||||||
|
|
||||||
|
return new BenchmarkResult
|
||||||
|
{
|
||||||
|
Name = name,
|
||||||
|
ServerType = serverType,
|
||||||
|
TotalMessages = messageCount,
|
||||||
|
TotalBytes = (long)messageCount * payloadSize,
|
||||||
|
Duration = sw.Elapsed,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
<PackageReference Include="coverlet.collector" />
|
<PackageReference Include="coverlet.collector" />
|
||||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||||
<PackageReference Include="NATS.Client.Core" />
|
<PackageReference Include="NATS.Client.Core" />
|
||||||
|
<PackageReference Include="MQTTnet" />
|
||||||
<PackageReference Include="NATS.Client.JetStream" />
|
<PackageReference Include="NATS.Client.JetStream" />
|
||||||
<PackageReference Include="Shouldly" />
|
<PackageReference Include="Shouldly" />
|
||||||
<PackageReference Include="xunit" />
|
<PackageReference Include="xunit" />
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&Fully
|
|||||||
# JetStream only
|
# JetStream only
|
||||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~JetStream" -v normal
|
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~JetStream" -v normal
|
||||||
|
|
||||||
|
# MQTT benchmarks
|
||||||
|
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~Mqtt" -v normal
|
||||||
|
|
||||||
|
# Transport benchmarks (TLS + WebSocket)
|
||||||
|
dotnet test tests/NATS.Server.Benchmark.Tests --filter "Category=Benchmark&FullyQualifiedName~Transport" -v normal
|
||||||
|
|
||||||
# A single benchmark by name
|
# A single benchmark by name
|
||||||
dotnet test tests/NATS.Server.Benchmark.Tests --filter "FullyQualifiedName=NATS.Server.Benchmark.Tests.CorePubSub.SinglePublisherThroughputTests.PubNoSub_16B" -v normal
|
dotnet test tests/NATS.Server.Benchmark.Tests --filter "FullyQualifiedName=NATS.Server.Benchmark.Tests.CorePubSub.SinglePublisherThroughputTests.PubNoSub_16B" -v normal
|
||||||
```
|
```
|
||||||
@@ -50,6 +56,12 @@ Use `-v normal` or `--logger "console;verbosity=detailed"` to see the comparison
|
|||||||
| `FileStoreAppendBenchmarks` | `FileStore_PurgeEx_Trim_Overhead` | FileStore purge/trim maintenance overhead under repeated updates |
|
| `FileStoreAppendBenchmarks` | `FileStore_PurgeEx_Trim_Overhead` | FileStore purge/trim maintenance overhead under repeated updates |
|
||||||
| `OrderedConsumerTests` | `JSOrderedConsumer_Throughput` | JetStream ordered ephemeral consumer read throughput |
|
| `OrderedConsumerTests` | `JSOrderedConsumer_Throughput` | JetStream ordered ephemeral consumer read throughput |
|
||||||
| `DurableConsumerFetchTests` | `JSDurableFetch_Throughput` | JetStream durable consumer fetch-in-batches throughput |
|
| `DurableConsumerFetchTests` | `JSDurableFetch_Throughput` | JetStream durable consumer fetch-in-batches throughput |
|
||||||
|
| `MqttThroughputTests` | `MqttPubSub_128B` | MQTT pub/sub throughput, 128-byte payload, QoS 0 |
|
||||||
|
| `MqttThroughputTests` | `MqttCrossProtocol_NatsPub_MqttSub_128B` | Cross-protocol NATS→MQTT routing throughput |
|
||||||
|
| `TlsPubSubTests` | `TlsPubSub1To1_128B` | TLS pub/sub 1:1 throughput, 128-byte payload |
|
||||||
|
| `TlsPubSubTests` | `TlsPubNoSub_128B` | TLS publish-only throughput, 128-byte payload |
|
||||||
|
| `WebSocketPubSubTests` | `WsPubSub1To1_128B` | WebSocket pub/sub 1:1 throughput, 128-byte payload |
|
||||||
|
| `WebSocketPubSubTests` | `WsPubNoSub_128B` | WebSocket publish-only throughput, 128-byte payload |
|
||||||
|
|
||||||
## Output Format
|
## Output Format
|
||||||
|
|
||||||
@@ -97,6 +109,9 @@ Infrastructure/
|
|||||||
GoServerProcess.cs # Builds + launches golang/nats-server
|
GoServerProcess.cs # Builds + launches golang/nats-server
|
||||||
CoreServerPairFixture.cs # IAsyncLifetime: Go + .NET servers for core tests
|
CoreServerPairFixture.cs # IAsyncLifetime: Go + .NET servers for core tests
|
||||||
JetStreamServerPairFixture # IAsyncLifetime: Go + .NET servers with JetStream
|
JetStreamServerPairFixture # IAsyncLifetime: Go + .NET servers with JetStream
|
||||||
|
MqttServerFixture.cs # IAsyncLifetime: .NET server with MQTT + JetStream
|
||||||
|
TlsServerFixture.cs # IAsyncLifetime: .NET server with TLS
|
||||||
|
WebSocketServerFixture.cs # IAsyncLifetime: .NET server with WebSocket
|
||||||
Collections.cs # xUnit collection definitions
|
Collections.cs # xUnit collection definitions
|
||||||
|
|
||||||
Harness/
|
Harness/
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
using NATS.Client.Core;
|
||||||
|
using NATS.Server.Benchmark.Tests.Harness;
|
||||||
|
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||||
|
using Xunit.Abstractions;
|
||||||
|
|
||||||
|
namespace NATS.Server.Benchmark.Tests.Transport;
|
||||||
|
|
||||||
|
[Collection("Benchmark-Tls")]
|
||||||
|
public class TlsPubSubTests(TlsServerFixture fixture, ITestOutputHelper output)
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
[Trait("Category", "Benchmark")]
|
||||||
|
public async Task TlsPubSub1To1_128B()
|
||||||
|
{
|
||||||
|
const int payloadSize = 128;
|
||||||
|
const int messageCount = 10_000;
|
||||||
|
|
||||||
|
var dotnetResult = await RunTlsPubSub("TLS PubSub 1:1 (128B)", "DotNet", fixture.CreateDotNetTlsClient, payloadSize, messageCount);
|
||||||
|
|
||||||
|
if (fixture.GoAvailable)
|
||||||
|
{
|
||||||
|
var goResult = await RunTlsPubSub("TLS PubSub 1:1 (128B)", "Go", fixture.CreateGoTlsClient, payloadSize, messageCount);
|
||||||
|
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
[Trait("Category", "Benchmark")]
|
||||||
|
public async Task TlsPubNoSub_128B()
|
||||||
|
{
|
||||||
|
const int payloadSize = 128;
|
||||||
|
|
||||||
|
var dotnetResult = await RunTlsPubOnly("TLS Pub-Only (128B)", "DotNet", fixture.CreateDotNetTlsClient, payloadSize);
|
||||||
|
|
||||||
|
if (fixture.GoAvailable)
|
||||||
|
{
|
||||||
|
var goResult = await RunTlsPubOnly("TLS Pub-Only (128B)", "Go", fixture.CreateGoTlsClient, payloadSize);
|
||||||
|
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<BenchmarkResult> RunTlsPubSub(string name, string serverType, Func<NatsConnection> createClient, int payloadSize, int messageCount)
|
||||||
|
{
|
||||||
|
var payload = new byte[payloadSize];
|
||||||
|
var subject = $"bench.tls.pubsub.{Guid.NewGuid():N}";
|
||||||
|
|
||||||
|
await using var pubClient = createClient();
|
||||||
|
await using var subClient = createClient();
|
||||||
|
await pubClient.ConnectAsync();
|
||||||
|
await subClient.ConnectAsync();
|
||||||
|
|
||||||
|
var received = 0;
|
||||||
|
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
var sub = await subClient.SubscribeCoreAsync<byte[]>(subject);
|
||||||
|
|
||||||
|
await subClient.PingAsync();
|
||||||
|
await pubClient.PingAsync();
|
||||||
|
|
||||||
|
var subTask = Task.Run(async () =>
|
||||||
|
{
|
||||||
|
await foreach (var msg in sub.Msgs.ReadAllAsync())
|
||||||
|
{
|
||||||
|
if (Interlocked.Increment(ref received) >= messageCount)
|
||||||
|
{
|
||||||
|
tcs.TrySetResult();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
for (var i = 0; i < messageCount; i++)
|
||||||
|
await pubClient.PublishAsync(subject, payload);
|
||||||
|
await pubClient.PingAsync();
|
||||||
|
|
||||||
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||||
|
await tcs.Task.WaitAsync(cts.Token);
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
await sub.UnsubscribeAsync();
|
||||||
|
|
||||||
|
return new BenchmarkResult
|
||||||
|
{
|
||||||
|
Name = name,
|
||||||
|
ServerType = serverType,
|
||||||
|
TotalMessages = messageCount,
|
||||||
|
TotalBytes = (long)messageCount * payloadSize,
|
||||||
|
Duration = sw.Elapsed,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<BenchmarkResult> RunTlsPubOnly(string name, string serverType, Func<NatsConnection> createClient, int payloadSize)
|
||||||
|
{
|
||||||
|
var subject = $"bench.tls.pubonly.{Guid.NewGuid():N}";
|
||||||
|
|
||||||
|
await using var client = createClient();
|
||||||
|
await client.ConnectAsync();
|
||||||
|
|
||||||
|
var runner = new BenchmarkRunner { WarmupCount = 1_000, MeasurementCount = 100_000 };
|
||||||
|
|
||||||
|
return await runner.MeasureThroughputAsync(
|
||||||
|
name,
|
||||||
|
serverType,
|
||||||
|
payloadSize,
|
||||||
|
async _ => await client.PublishAsync(subject, new byte[payloadSize]));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
using System.Net.WebSockets;
|
||||||
|
using System.Text;
|
||||||
|
using NATS.Client.Core;
|
||||||
|
using NATS.Server.Benchmark.Tests.Harness;
|
||||||
|
using NATS.Server.Benchmark.Tests.Infrastructure;
|
||||||
|
using Xunit.Abstractions;
|
||||||
|
|
||||||
|
namespace NATS.Server.Benchmark.Tests.Transport;
|
||||||
|
|
||||||
|
[Collection("Benchmark-WebSocket")]
|
||||||
|
public class WebSocketPubSubTests(WebSocketServerFixture fixture, ITestOutputHelper output)
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
[Trait("Category", "Benchmark")]
|
||||||
|
public async Task WsPubSub1To1_128B()
|
||||||
|
{
|
||||||
|
const int payloadSize = 128;
|
||||||
|
const int messageCount = 5_000;
|
||||||
|
|
||||||
|
var dotnetResult = await RunWsPubSub("WebSocket PubSub 1:1 (128B)", "DotNet", fixture.DotNetWsPort, fixture.CreateDotNetNatsClient, payloadSize, messageCount);
|
||||||
|
|
||||||
|
if (fixture.GoAvailable)
|
||||||
|
{
|
||||||
|
var goResult = await RunWsPubSub("WebSocket PubSub 1:1 (128B)", "Go", fixture.GoWsPort, fixture.CreateGoNatsClient, payloadSize, messageCount);
|
||||||
|
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
[Trait("Category", "Benchmark")]
|
||||||
|
public async Task WsPubNoSub_128B()
|
||||||
|
{
|
||||||
|
const int payloadSize = 128;
|
||||||
|
const int messageCount = 10_000;
|
||||||
|
|
||||||
|
var dotnetResult = await RunWsPubOnly("WebSocket Pub-Only (128B)", "DotNet", fixture.DotNetWsPort, payloadSize, messageCount);
|
||||||
|
|
||||||
|
if (fixture.GoAvailable)
|
||||||
|
{
|
||||||
|
var goResult = await RunWsPubOnly("WebSocket Pub-Only (128B)", "Go", fixture.GoWsPort, payloadSize, messageCount);
|
||||||
|
BenchmarkResultWriter.WriteComparison(output, goResult, dotnetResult);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
BenchmarkResultWriter.WriteSingle(output, dotnetResult);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<BenchmarkResult> RunWsPubSub(string name, string serverType, int wsPort, Func<NatsConnection> createNatsClient, int payloadSize, int messageCount)
|
||||||
|
{
|
||||||
|
var payload = new byte[payloadSize];
|
||||||
|
var subject = $"bench.ws.pubsub.{Guid.NewGuid():N}";
|
||||||
|
|
||||||
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||||
|
|
||||||
|
using var ws = new ClientWebSocket();
|
||||||
|
await ws.ConnectAsync(new Uri($"ws://127.0.0.1:{wsPort}"), cts.Token);
|
||||||
|
|
||||||
|
var reader = new WsLineReader(ws);
|
||||||
|
|
||||||
|
// Read INFO
|
||||||
|
await reader.ReadLineAsync(cts.Token);
|
||||||
|
|
||||||
|
// Send CONNECT + SUB + PING
|
||||||
|
await WsSend(ws, "CONNECT {\"verbose\":false,\"protocol\":1}\r\n", cts.Token);
|
||||||
|
await WsSend(ws, $"SUB {subject} 1\r\n", cts.Token);
|
||||||
|
await WsSend(ws, "PING\r\n", cts.Token);
|
||||||
|
await WaitForPong(reader, cts.Token);
|
||||||
|
|
||||||
|
// NATS publisher
|
||||||
|
await using var natsPub = createNatsClient();
|
||||||
|
await natsPub.ConnectAsync();
|
||||||
|
await natsPub.PingAsync(cts.Token);
|
||||||
|
|
||||||
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
|
||||||
|
for (var i = 0; i < messageCount; i++)
|
||||||
|
await natsPub.PublishAsync(subject, payload, cancellationToken: cts.Token);
|
||||||
|
await natsPub.PingAsync(cts.Token);
|
||||||
|
|
||||||
|
// Read all MSG responses from WebSocket
|
||||||
|
var received = 0;
|
||||||
|
while (received < messageCount)
|
||||||
|
{
|
||||||
|
var line = await reader.ReadLineAsync(cts.Token);
|
||||||
|
if (line.StartsWith("MSG ", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
await reader.ReadLineAsync(cts.Token);
|
||||||
|
received++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
await ws.CloseAsync(WebSocketCloseStatus.NormalClosure, null, cts.Token);
|
||||||
|
|
||||||
|
return new BenchmarkResult
|
||||||
|
{
|
||||||
|
Name = name,
|
||||||
|
ServerType = serverType,
|
||||||
|
TotalMessages = messageCount,
|
||||||
|
TotalBytes = (long)messageCount * payloadSize,
|
||||||
|
Duration = sw.Elapsed,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<BenchmarkResult> RunWsPubOnly(string name, string serverType, int wsPort, int payloadSize, int messageCount)
|
||||||
|
{
|
||||||
|
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
|
||||||
|
|
||||||
|
using var ws = new ClientWebSocket();
|
||||||
|
await ws.ConnectAsync(new Uri($"ws://127.0.0.1:{wsPort}"), cts.Token);
|
||||||
|
|
||||||
|
var reader = new WsLineReader(ws);
|
||||||
|
|
||||||
|
// Read INFO
|
||||||
|
await reader.ReadLineAsync(cts.Token);
|
||||||
|
|
||||||
|
// Send CONNECT
|
||||||
|
await WsSend(ws, "CONNECT {\"verbose\":false,\"protocol\":1}\r\n", cts.Token);
|
||||||
|
await WsSend(ws, "PING\r\n", cts.Token);
|
||||||
|
await WaitForPong(reader, cts.Token);
|
||||||
|
|
||||||
|
// Build a PUB command with raw binary payload
|
||||||
|
var subject = $"bench.ws.pubonly.{Guid.NewGuid():N}";
|
||||||
|
var pubLine = $"PUB {subject} {payloadSize}\r\n";
|
||||||
|
var pubPayload = new byte[payloadSize];
|
||||||
|
var pubCmd = Encoding.ASCII.GetBytes(pubLine)
|
||||||
|
.Concat(pubPayload)
|
||||||
|
.Concat(Encoding.ASCII.GetBytes("\r\n"))
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
for (var i = 0; i < messageCount; i++)
|
||||||
|
await ws.SendAsync(pubCmd, WebSocketMessageType.Binary, true, cts.Token);
|
||||||
|
|
||||||
|
// Flush with PING/PONG
|
||||||
|
await WsSend(ws, "PING\r\n", cts.Token);
|
||||||
|
await WaitForPong(reader, cts.Token);
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
await ws.CloseAsync(WebSocketCloseStatus.NormalClosure, null, cts.Token);
|
||||||
|
|
||||||
|
return new BenchmarkResult
|
||||||
|
{
|
||||||
|
Name = name,
|
||||||
|
ServerType = serverType,
|
||||||
|
TotalMessages = messageCount,
|
||||||
|
TotalBytes = (long)messageCount * payloadSize,
|
||||||
|
Duration = sw.Elapsed,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads lines until PONG is received, skipping any INFO lines
|
||||||
|
/// (Go server sends a second INFO after CONNECT with connect_info:true).
|
||||||
|
/// </summary>
|
||||||
|
private static async Task WaitForPong(WsLineReader reader, CancellationToken ct)
|
||||||
|
{
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
var line = await reader.ReadLineAsync(ct);
|
||||||
|
if (line == "PONG")
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task WsSend(ClientWebSocket ws, string data, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var bytes = Encoding.ASCII.GetBytes(data);
|
||||||
|
await ws.SendAsync(bytes, WebSocketMessageType.Binary, true, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Buffers incoming WebSocket frames and returns one NATS protocol line at a time.
|
||||||
|
/// </summary>
|
||||||
|
private sealed class WsLineReader(ClientWebSocket ws)
|
||||||
|
{
|
||||||
|
private readonly byte[] _recvBuffer = new byte[65536];
|
||||||
|
private readonly StringBuilder _pending = new();
|
||||||
|
|
||||||
|
public async Task<string> ReadLineAsync(CancellationToken ct)
|
||||||
|
{
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
var full = _pending.ToString();
|
||||||
|
var crlfIdx = full.IndexOf("\r\n", StringComparison.Ordinal);
|
||||||
|
if (crlfIdx >= 0)
|
||||||
|
{
|
||||||
|
var line = full[..crlfIdx];
|
||||||
|
_pending.Clear();
|
||||||
|
_pending.Append(full[(crlfIdx + 2)..]);
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
var result = await ws.ReceiveAsync(_recvBuffer, ct);
|
||||||
|
if (result.MessageType == WebSocketMessageType.Close)
|
||||||
|
throw new InvalidOperationException("WebSocket closed unexpectedly while reading");
|
||||||
|
|
||||||
|
var chunk = Encoding.ASCII.GetString(_recvBuffer, 0, result.Count);
|
||||||
|
_pending.Append(chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user