Compare commits
4 Commits
ee6809aedc
...
7023d78599
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7023d78599 | ||
|
|
8bce096f55 | ||
|
|
53585012f3 | ||
|
|
cc188fa84d |
134
differences.md
134
differences.md
@@ -61,14 +61,14 @@
|
||||
| Type | Go | .NET | Notes |
|
||||
|------|:--:|:----:|-------|
|
||||
| CLIENT | Y | Y | |
|
||||
| ROUTER | Y | Partial | Route handshake + in-memory remote subscription tracking; no RMSG message routing, no RS+/RS- wire protocol, no route pooling (3x per peer) |
|
||||
| GATEWAY | Y | Stub | Config parsing only; no listener, connections, handshake, interest-only mode, or message forwarding |
|
||||
| LEAF | Y | Stub | Config parsing only; no listener, connections, handshake, subscription sharing, or loop detection |
|
||||
| ROUTER | Y | Y | Route handshake + RS+/RS-/RMSG wire protocol + default 3-link pooling baseline |
|
||||
| GATEWAY | Y | Baseline | Functional handshake, A+/A- interest propagation, and forwarding baseline; advanced Go routing semantics remain |
|
||||
| LEAF | Y | Baseline | Functional handshake, LS+/LS- propagation, and LMSG forwarding baseline; advanced hub/spoke mapping remains |
|
||||
| SYSTEM (internal) | Y | Y | InternalClient + InternalEventSystem with Channel-based send/receive loops |
|
||||
| JETSTREAM (internal) | Y | N | |
|
||||
| ACCOUNT (internal) | Y | Y | Lazy per-account InternalClient with import/export subscription support |
|
||||
| WebSocket clients | Y | Y | Custom frame parser, permessage-deflate compression, origin checking, cookie auth |
|
||||
| MQTT clients | Y | Partial | JWT connection-type constants + config parsing; no MQTT transport yet |
|
||||
| MQTT clients | Y | Baseline | JWT connection-type constants + config parsing; no MQTT transport yet |
|
||||
|
||||
### Client Features
|
||||
| Feature | Go | .NET | Notes |
|
||||
@@ -191,7 +191,7 @@ Go implements a sophisticated slow consumer detection system:
|
||||
|---------|:--:|:----:|-------|
|
||||
| Per-account subscription limit | Y | Y | `Account.IncrementSubscriptions()` returns false when `MaxSubscriptions` exceeded |
|
||||
| Auto-unsubscribe on max messages | Y | Y | Enforced at delivery; sub removed from trie + client dict when exhausted |
|
||||
| Subscription routing propagation | Y | Partial | Remote subs tracked in trie; propagation via in-process method calls (no wire RS+/RS-); RMSG forwarding absent |
|
||||
| Subscription routing propagation | Y | Y | Remote subs tracked in trie and propagated over wire RS+/RS- with RMSG forwarding |
|
||||
| Queue weight (`qw`) field | Y | N | For remote queue load balancing |
|
||||
|
||||
---
|
||||
@@ -282,12 +282,12 @@ Go implements a sophisticated slow consumer detection system:
|
||||
| `/varz` | Y | Y | |
|
||||
| `/connz` | Y | Y | |
|
||||
| `/` (root listing) | Y | Y | |
|
||||
| `/routez` | Y | Stub | Returns empty response |
|
||||
| `/gatewayz` | Y | Stub | Returns empty response |
|
||||
| `/leafz` | Y | Stub | Returns empty response |
|
||||
| `/routez` | Y | Y | Returns live route counts via `RoutezHandler` |
|
||||
| `/gatewayz` | Y | Y | Returns live gateway counts via `GatewayzHandler` |
|
||||
| `/leafz` | Y | Y | Returns live leaf counts via `LeafzHandler` |
|
||||
| `/subz` / `/subscriptionsz` | Y | Y | Account filtering, test subject filtering, pagination, and subscription details |
|
||||
| `/accountz` | Y | Stub | Returns empty response |
|
||||
| `/accstatz` | Y | Stub | Returns empty response |
|
||||
| `/accountz` | Y | Y | Returns runtime account summaries via `AccountzHandler` |
|
||||
| `/accstatz` | Y | Y | Returns aggregate account stats via `AccountzHandler` |
|
||||
| `/jsz` | Y | Y | Returns live JetStream counts/config and API totals/errors via `JszHandler` |
|
||||
|
||||
### Varz Response
|
||||
@@ -302,14 +302,14 @@ Go implements a sophisticated slow consumer detection system:
|
||||
| Connections (current, total) | Y | Y | |
|
||||
| Messages (in/out msgs/bytes) | Y | Y | |
|
||||
| SlowConsumer breakdown | Y | N | Go tracks per connection type |
|
||||
| Cluster/Gateway/Leaf blocks | Y | Partial | Config projection present; `/gatewayz` and `/leafz` endpoints remain stubs |
|
||||
| Cluster/Gateway/Leaf blocks | Y | Y | Live route/gateway/leaf counters are exposed in dedicated endpoints |
|
||||
| JetStream block | Y | Y | Includes live JetStream config, stream/consumer counts, and API totals/errors |
|
||||
| TLS cert expiry info | Y | Y | `TlsCertNotAfter` loaded via `X509CertificateLoader` in `/varz` |
|
||||
|
||||
### Connz Response
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Filtering by CID, user, account | Y | Partial | |
|
||||
| Filtering by CID, user, account | Y | Baseline | |
|
||||
| Sorting (11 options) | Y | Y | All options including ByStop, ByReason, ByRtt |
|
||||
| State filtering (open/closed/all) | Y | Y | `state=open|closed|all` query parameter |
|
||||
| Closed connection tracking | Y | Y | `ConcurrentQueue<ClosedClient>` capped at 10,000 entries |
|
||||
@@ -353,7 +353,7 @@ Go implements a sophisticated slow consumer detection system:
|
||||
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Structured logging | Partial | Y | .NET uses Serilog with ILogger<T> |
|
||||
| Structured logging | Baseline | Y | .NET uses Serilog with ILogger<T> |
|
||||
| File logging with rotation | Y | Y | `-l`/`--log_file` flag + `LogSizeLimit`/`LogMaxFiles` via Serilog.Sinks.File |
|
||||
| Syslog (local and remote) | Y | Y | `--syslog` and `--remote_syslog` flags via Serilog.Sinks.SyslogMessages |
|
||||
| Log reopening (SIGUSR1) | Y | Y | SIGUSR1 handler calls ReOpenLogFile callback |
|
||||
@@ -417,7 +417,7 @@ The following items from the original gap list have been implemented:
|
||||
|
||||
## 11. JetStream
|
||||
|
||||
> The Go JetStream surface is ~37,500 lines across jetstream.go, stream.go, consumer.go, filestore.go, memstore.go, raft.go. The .NET implementation has expanded API and runtime parity coverage but remains partial versus full Go semantics.
|
||||
> The Go JetStream surface is ~37,500 lines across jetstream.go, stream.go, consumer.go, filestore.go, memstore.go, raft.go. The .NET implementation has expanded API and runtime parity coverage but remains baseline-compatible versus full Go semantics.
|
||||
|
||||
### JetStream API ($JS.API.* subjects)
|
||||
|
||||
@@ -455,38 +455,38 @@ The following items from the original gap list have been implemented:
|
||||
| Subjects | Y | Y | |
|
||||
| Replicas | Y | Y | Wires RAFT replica count |
|
||||
| MaxMsgs limit | Y | Y | Enforced via `EnforceLimits()` |
|
||||
| Retention (Limits/Interest/WorkQueue) | Y | Partial | Policy enums + validation branch exist; full runtime semantics incomplete |
|
||||
| Discard policy (Old/New) | Y | Partial | Model support exists; runtime discard behavior not fully enforced |
|
||||
| MaxBytes / MaxAge (TTL) | Y | N | |
|
||||
| MaxMsgsPer (per-subject limit) | Y | N | |
|
||||
| Retention (Limits/Interest/WorkQueue) | Y | Baseline | Policy enums + validation branch exist; full runtime semantics incomplete |
|
||||
| Discard policy (Old/New) | Y | Y | `Discard=New` now rejects writes when `MaxBytes` is exceeded |
|
||||
| MaxBytes / MaxAge (TTL) | Y | Baseline | `MaxBytes` enforced; `MaxAge` model and parsing added, full TTL pruning not complete |
|
||||
| MaxMsgsPer (per-subject limit) | Y | Baseline | Config model/parsing present; per-subject runtime cap remains limited |
|
||||
| MaxMsgSize | Y | N | |
|
||||
| Storage type selection (Memory/File) | Y | N | MemStore default; no config-driven choice |
|
||||
| Storage type selection (Memory/File) | Y | Y | Per-stream backend selection supports memory and file stores |
|
||||
| Compression (S2) | Y | N | |
|
||||
| Subject transform | Y | N | |
|
||||
| RePublish | Y | N | |
|
||||
| AllowDirect / KV mode | Y | N | |
|
||||
| Sealed, DenyDelete, DenyPurge | Y | N | |
|
||||
| Duplicates dedup window | Y | Partial | Dedup ID cache exists; no configurable window |
|
||||
| Duplicates dedup window | Y | Baseline | Dedup ID cache exists; no configurable window |
|
||||
|
||||
### Consumer Configuration & Delivery
|
||||
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Push delivery | Y | Partial | `PushConsumerEngine`; basic delivery |
|
||||
| Pull fetch | Y | Partial | `PullConsumerEngine`; basic batch fetch |
|
||||
| Ephemeral consumers | Y | N | Only durable |
|
||||
| Push delivery | Y | Baseline | `PushConsumerEngine`; basic delivery |
|
||||
| Pull fetch | Y | Baseline | `PullConsumerEngine`; basic batch fetch |
|
||||
| Ephemeral consumers | Y | Y | Ephemeral creation baseline auto-generates durable IDs when requested |
|
||||
| AckPolicy.None | Y | Y | |
|
||||
| AckPolicy.Explicit | Y | Y | `AckProcessor` tracks pending with expiry |
|
||||
| AckPolicy.All | Y | Partial | In-memory ack floor behavior implemented; full wire-level ack contract remains limited |
|
||||
| Redelivery on ack timeout | Y | Partial | `NextExpired()` detects expired; limit not enforced |
|
||||
| DeliverPolicy (All/Last/New/StartSeq/StartTime) | Y | Partial | Policy enums added; fetch behavior still mostly starts at beginning |
|
||||
| AckPolicy.All | Y | Baseline | In-memory ack floor behavior implemented; full wire-level ack contract remains limited |
|
||||
| Redelivery on ack timeout | Y | Baseline | `NextExpired()` detects expired; limit not enforced |
|
||||
| DeliverPolicy (All/Last/New/StartSeq/StartTime) | Y | Baseline | Policy enums added; fetch behavior still mostly starts at beginning |
|
||||
| FilterSubject (single) | Y | Y | |
|
||||
| FilterSubjects (multiple) | Y | N | |
|
||||
| MaxAckPending | Y | N | |
|
||||
| Idle heartbeat | Y | Partial | Push engine emits heartbeat frames for configured consumers |
|
||||
| FilterSubjects (multiple) | Y | Y | Multi-filter matching implemented in pull/push delivery paths |
|
||||
| MaxAckPending | Y | Y | Pending delivery cap enforced for consumer queues |
|
||||
| Idle heartbeat | Y | Baseline | Push engine emits heartbeat frames for configured consumers |
|
||||
| Flow control | Y | N | |
|
||||
| Rate limiting | Y | N | |
|
||||
| Replay policy | Y | Partial | Policy enum exists; replay timing semantics not fully implemented |
|
||||
| Replay policy | Y | Baseline | `ReplayPolicy.Original` baseline delay implemented; full Go timing semantics remain |
|
||||
| BackOff (exponential) | Y | N | |
|
||||
|
||||
### Storage Backends
|
||||
@@ -507,9 +507,9 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
|
||||
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Mirror consumer creation | Y | Partial | `MirrorCoordinator` triggers on append |
|
||||
| Mirror consumer creation | Y | Baseline | `MirrorCoordinator` triggers on append |
|
||||
| Mirror sync state tracking | Y | N | |
|
||||
| Source fan-in (multiple sources) | Y | Partial | Single `Source` field; no `Sources[]` array |
|
||||
| Source fan-in (multiple sources) | Y | Y | `Sources[]` array support added and replicated via `SourceCoordinator` |
|
||||
| Subject mapping for sources | Y | N | |
|
||||
| Cross-account mirror/source | Y | N | |
|
||||
|
||||
@@ -517,30 +517,30 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
|
||||
|
||||
| Feature | Go (5 037 lines) | .NET (212 lines) | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Leader election / term tracking | Y | Partial | In-process; nodes hold direct `List<RaftNode>` references |
|
||||
| Log append + quorum | Y | Partial | Entries replicated via direct method calls; stale-term append now rejected |
|
||||
| Log persistence | Y | N | In-memory `List<RaftLogEntry>` only |
|
||||
| Leader election / term tracking | Y | Baseline | In-process; nodes hold direct `List<RaftNode>` references |
|
||||
| Log append + quorum | Y | Baseline | Entries replicated via direct method calls; stale-term append now rejected |
|
||||
| Log persistence | Y | Baseline | `RaftLog.PersistAsync/LoadAsync` plus node term/applied persistence baseline |
|
||||
| Heartbeat / keep-alive | Y | N | |
|
||||
| Log mismatch resolution (NextIndex) | Y | N | |
|
||||
| Snapshot creation | Y | Partial | `CreateSnapshotAsync()` exists; stored in-memory |
|
||||
| Snapshot creation | Y | Baseline | `CreateSnapshotAsync()` exists; stored in-memory |
|
||||
| Snapshot network transfer | Y | N | |
|
||||
| Membership changes | Y | N | |
|
||||
| Network RPC transport | Y | N | All coordination is in-process |
|
||||
| Network RPC transport | Y | Baseline | `IRaftTransport` abstraction + in-memory transport baseline implemented |
|
||||
|
||||
### JetStream Clustering
|
||||
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Meta-group governance | Y | Partial | `JetStreamMetaGroup` tracks streams; no durable consensus |
|
||||
| Per-stream replica group | Y | Partial | `StreamReplicaGroup` + in-memory RAFT |
|
||||
| Asset placement planner | Y | Partial | `AssetPlacementPlanner` skeleton |
|
||||
| Meta-group governance | Y | Baseline | `JetStreamMetaGroup` tracks streams; no durable consensus |
|
||||
| Per-stream replica group | Y | Baseline | `StreamReplicaGroup` + in-memory RAFT |
|
||||
| Asset placement planner | Y | Baseline | `AssetPlacementPlanner` skeleton |
|
||||
| Cross-cluster JetStream (gateways) | Y | N | Requires functional gateways |
|
||||
|
||||
---
|
||||
|
||||
## 12. Clustering
|
||||
|
||||
> Routes, gateways, and leaf nodes are in different states of completeness. All three are present as Go reference feature targets but only routes have any functional networking.
|
||||
> Routes, gateways, and leaf nodes now all have functional networking baselines; advanced Go semantics are still incomplete.
|
||||
|
||||
### Routes
|
||||
|
||||
@@ -550,10 +550,10 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
|
||||
| Outbound seed connections (with backoff) | Y | Y | Iterates `ClusterOptions.Routes` with 250 ms retry |
|
||||
| Route handshake (ROUTE `<serverId>`) | Y | Y | Bidirectional: sends own ID, reads peer ID |
|
||||
| Remote subscription tracking | Y | Y | `ApplyRemoteSubscription` adds to SubList; `HasRemoteInterest` exposed |
|
||||
| Subscription propagation (in-process) | Y | Partial | `PropagateLocalSubscription` calls peer managers directly — no wire RS+/RS- protocol |
|
||||
| Message routing (RMSG wire) | Y | N | **Critical gap**: published messages are never forwarded to remote subscribers |
|
||||
| RS+/RS- subscription protocol (wire) | Y | N | Command matrix recognises opcodes but no handler processes inbound RS+/RS- frames |
|
||||
| Route pooling (3× per peer) | Y | N | Single connection per remote server ID |
|
||||
| Subscription propagation (wire RS+/RS-) | Y | Y | Local SUB/UNSUB is propagated over route wire frames |
|
||||
| Message routing (RMSG wire) | Y | Y | Routed publishes forward over RMSG to remote subscribers |
|
||||
| RS+/RS- subscription protocol (wire) | Y | Y | Inbound RS+/RS- frames update remote-interest trie |
|
||||
| Route pooling (3× per peer) | Y | Y | `ClusterOptions.PoolSize` defaults to 3 links per peer |
|
||||
| Account-specific routes | Y | N | |
|
||||
| S2 compression on routes | Y | N | |
|
||||
| CONNECT info + topology gossip | Y | N | Handshake is two-line text exchange only |
|
||||
@@ -562,40 +562,35 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
|
||||
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Any networking (listener / outbound) | Y | N | `GatewayManager.StartAsync()` logs a debug line and zeros a counter |
|
||||
| Gateway connection protocol | Y | N | |
|
||||
| Interest-only mode | Y | N | |
|
||||
| Any networking (listener / outbound) | Y | Y | Listener + outbound remotes with retry are active |
|
||||
| Gateway connection protocol | Y | Baseline | Baseline `GATEWAY` handshake implemented |
|
||||
| Interest-only mode | Y | Baseline | Baseline A+/A- interest propagation implemented |
|
||||
| Reply subject mapping (`_GR_.` prefix) | Y | N | |
|
||||
| Message forwarding to remote clusters | Y | N | |
|
||||
| Message forwarding to remote clusters | Y | Baseline | Baseline `GMSG` forwarding implemented |
|
||||
|
||||
### Leaf Nodes
|
||||
|
||||
| Feature | Go | .NET | Notes |
|
||||
|---------|:--:|:----:|-------|
|
||||
| Any networking (listener / spoke) | Y | N | `LeafNodeManager.StartAsync()` logs a debug line and zeros a counter |
|
||||
| Leaf handshake / role negotiation | Y | N | |
|
||||
| Subscription sharing (LS+/LS-) | Y | N | |
|
||||
| Any networking (listener / spoke) | Y | Y | Listener + outbound remotes with retry are active |
|
||||
| Leaf handshake / role negotiation | Y | Baseline | Baseline `LEAF` handshake implemented |
|
||||
| Subscription sharing (LS+/LS-) | Y | Baseline | LS+/LS- propagation implemented |
|
||||
| Loop detection (`$LDS.` prefix) | Y | N | |
|
||||
| Hub-and-spoke account mapping | Y | N | |
|
||||
| Hub-and-spoke account mapping | Y | Baseline | Baseline LMSG forwarding works; advanced account remapping remains |
|
||||
|
||||
---
|
||||
|
||||
## Summary: Remaining Gaps
|
||||
|
||||
### Clustering (High Impact)
|
||||
1. **Route message routing** — Remote subscribers receive no messages; no RMSG implementation
|
||||
2. **Gateways** — Non-functional stub; no inter-cluster bridging
|
||||
3. **Leaf nodes** — Non-functional stub; no hub/spoke topology
|
||||
4. **RS+/RS- wire protocol** — subscription propagation is in-process method calls only
|
||||
5. **Route pooling** — single connection per peer vs Go's 3-connection pool
|
||||
1. **Gateway advanced semantics** — reply remapping (`_GR_.`) and full interest-only behavior are not complete
|
||||
2. **Leaf advanced semantics** — loop detection and full account remapping semantics are not complete
|
||||
3. **Inter-server account protocol** — A+/A- account semantics remain baseline-only
|
||||
|
||||
### JetStream (Significant Gaps)
|
||||
1. **API coverage is expanded but still incomplete** — core stream/consumer/direct/control routes are implemented, but full Go surface and edge semantics remain
|
||||
2. **Policy/runtime semantics are incomplete** — retention/discard/delivery/replay models exist, but behavior does not yet match Go across all cases
|
||||
3. **Snapshot/restore and cluster control are skeletal** — request/response contracts exist; durable/distributed semantics remain limited
|
||||
4. **FileStore scalability** — JSONL-based (not block/compressed/encrypted)
|
||||
5. **RAFT persistence and transport** — in-memory coordination; no durable log/RPC transport parity
|
||||
6. **Consumer delivery completeness** — MaxAckPending, flow control, replay/backoff, and full fetch/ack lifecycle parity still incomplete
|
||||
1. **Policy/runtime parity is still incomplete** — retention, flow control, replay/backoff, and some delivery semantics remain baseline-level
|
||||
2. **FileStore scalability** — JSONL-based (not block/compressed/encrypted)
|
||||
3. **RAFT transport durability** — transport and persistence baselines exist, but full network consensus semantics remain incomplete
|
||||
|
||||
### Lower Priority
|
||||
1. **Dynamic buffer sizing** — delegated to Pipe, less optimized for long-lived connections
|
||||
@@ -610,13 +605,17 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
|
||||
|
||||
### Newly Ported API Families
|
||||
- `$JS.API.INFO`
|
||||
- `$JS.API.SERVER.REMOVE`
|
||||
- `$JS.API.ACCOUNT.PURGE.*`, `$JS.API.ACCOUNT.STREAM.MOVE.*`, `$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*`
|
||||
- `$JS.API.STREAM.UPDATE.*`, `$JS.API.STREAM.DELETE.*`, `$JS.API.STREAM.NAMES`, `$JS.API.STREAM.LIST`
|
||||
- `$JS.API.STREAM.PEER.REMOVE.*`
|
||||
- `$JS.API.STREAM.MSG.GET.*`, `$JS.API.STREAM.MSG.DELETE.*`, `$JS.API.STREAM.PURGE.*`
|
||||
- `$JS.API.DIRECT.GET.*`
|
||||
- `$JS.API.STREAM.SNAPSHOT.*`, `$JS.API.STREAM.RESTORE.*`
|
||||
- `$JS.API.CONSUMER.NAMES.*`, `$JS.API.CONSUMER.LIST.*`, `$JS.API.CONSUMER.DELETE.*.*`
|
||||
- `$JS.API.CONSUMER.PAUSE.*.*`, `$JS.API.CONSUMER.RESET.*.*`, `$JS.API.CONSUMER.UNPIN.*.*`
|
||||
- `$JS.API.CONSUMER.MSG.NEXT.*.*`
|
||||
- `$JS.API.CONSUMER.LEADER.STEPDOWN.*.*`
|
||||
- `$JS.API.STREAM.LEADER.STEPDOWN.*`, `$JS.API.META.LEADER.STEPDOWN`
|
||||
|
||||
### Runtime/Storage/RAFT Parity Additions
|
||||
@@ -626,7 +625,12 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
|
||||
- Stream store subject index support (`LoadLastBySubjectAsync`) in `MemStore` and `FileStore`.
|
||||
- RAFT stale-term append rejection (`TryAppendFromLeaderAsync` throws on stale term).
|
||||
- `/jsz` and `/varz` now expose JetStream API totals/errors from server stats.
|
||||
- Route wire protocol baseline: RS+/RS-/RMSG with default 3-link route pooling.
|
||||
- Gateway/Leaf wire protocol baselines: A+/A-/GMSG and LS+/LS-/LMSG.
|
||||
- Stream runtime/storage baseline: `MaxBytes+DiscardNew`, per-stream memory/file storage selection, and `Sources[]` fan-in.
|
||||
- Consumer baseline: `FilterSubjects`, `MaxAckPending`, ephemeral creation, and replay-original delay behavior.
|
||||
- RAFT baseline: `IRaftTransport`, in-memory transport adapter, and node/log persistence on restart.
|
||||
- Monitoring baseline: `/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz` now return runtime data.
|
||||
|
||||
### Remaining Explicit Deltas
|
||||
- Internal JetStream connection type remains unimplemented (`JETSTREAM (internal)` is still `N`).
|
||||
- Monitoring endpoints `/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz` remain stubbed.
|
||||
|
||||
107
docs/plans/2026-02-23-jetstream-final-remaining-design.md
Normal file
107
docs/plans/2026-02-23-jetstream-final-remaining-design.md
Normal file
@@ -0,0 +1,107 @@
|
||||
# JetStream Final Remaining Parity Design
|
||||
|
||||
**Date:** 2026-02-23
|
||||
**Status:** Approved
|
||||
**Scope:** Complete all remaining JetStream functionality and required transport prerequisites from Go in the .NET server, with strict parity closure criteria.
|
||||
|
||||
## 1. Architecture and Scope Boundary
|
||||
|
||||
Remaining parity is executed in three ordered layers:
|
||||
|
||||
1. Cluster transport prerequisites
|
||||
- Complete route wire protocol behavior (RS+/RS-, RMSG forwarding, route pool baseline).
|
||||
- Replace gateway and leaf-node stub behavior with functional networking/handshake/interest propagation sufficient for JetStream parity dependencies.
|
||||
|
||||
2. JetStream semantic completion
|
||||
- Finish stream/consumer behavior still marked partial (retention/discard/runtime policy enforcement, delivery/replay/fetch/ack edge semantics, dedupe window/expected-header semantics, snapshot/restore durability semantics).
|
||||
|
||||
3. Parity closure and verification
|
||||
- Remove JetStream "partial" status from `differences.md` unless an explicit non-JetStream blocker remains.
|
||||
- Maintain evidence mapping from Go feature to .NET implementation and proving tests.
|
||||
|
||||
## 2. Component Plan
|
||||
|
||||
### A. Route/Gateway/Leaf transport completion
|
||||
- Expand:
|
||||
- `src/NATS.Server/Routes/RouteManager.cs`
|
||||
- `src/NATS.Server/Routes/RouteConnection.cs`
|
||||
- Add wire-level RS+/RS- and RMSG behavior, and route pool baseline behavior.
|
||||
- Replace stub-only behavior in:
|
||||
- `src/NATS.Server/Gateways/GatewayManager.cs`
|
||||
- `src/NATS.Server/LeafNodes/LeafNodeManager.cs`
|
||||
with functional baseline networking/handshake and interest propagation.
|
||||
|
||||
### B. JetStream API surface completion
|
||||
- Expand:
|
||||
- `src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs`
|
||||
- `src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs`
|
||||
- `src/NATS.Server/JetStream/Api/Handlers/*`
|
||||
- Cover remaining Go API families and durable/create/control variants with contract-accurate response shapes.
|
||||
|
||||
### C. Stream/consumer semantic completion
|
||||
- Refine:
|
||||
- `src/NATS.Server/JetStream/StreamManager.cs`
|
||||
- `src/NATS.Server/JetStream/ConsumerManager.cs`
|
||||
- `src/NATS.Server/JetStream/Consumers/*`
|
||||
- `src/NATS.Server/JetStream/Publish/*`
|
||||
- Ensure modeled policies are fully enforced at runtime.
|
||||
|
||||
### D. Store/recovery and RAFT semantics
|
||||
- Expand:
|
||||
- `src/NATS.Server/JetStream/Storage/*`
|
||||
- `src/NATS.Server/JetStream/Snapshots/*`
|
||||
- `src/NATS.Server/Raft/*`
|
||||
- Move from shape-level support to behavior-level durability and distributed-state correctness.
|
||||
|
||||
### E. Monitoring + evidence
|
||||
- Update JetStream/cluster monitoring paths and models to reflect real runtime behavior.
|
||||
- Keep parity map and `differences.md` synchronized with test-backed implementation state.
|
||||
|
||||
## 3. Data Flow and Behavioral Contracts
|
||||
|
||||
1. Inter-server flow
|
||||
- Subscription changes emit RS+/RS- over links.
|
||||
- Remote interest updates local routing state.
|
||||
- Publish with remote interest forwards via RMSG-equivalent behavior preserving subject/reply/header semantics.
|
||||
|
||||
2. JetStream API flow
|
||||
- `$JS.API.*` arrives through request/reply message path.
|
||||
- Router dispatches handlers; handlers validate contract and cluster state; responses encode deterministic success/error semantics.
|
||||
|
||||
3. Publish/capture flow
|
||||
- Publish traverses normal routing and JetStream capture.
|
||||
- Preconditions run before append.
|
||||
- Append mutates stream state/indexes; mirror/source/consumer engines observe committed state.
|
||||
|
||||
4. Consumer delivery flow
|
||||
- Pull/push share canonical pending/ack/redelivery state.
|
||||
- Control operations (`pause/reset/unpin/request-next/update`) mutate consistent state.
|
||||
- Restart/recovery preserves deterministic observable behavior.
|
||||
|
||||
5. Store/recovery flow
|
||||
- Writes update payload and lookup structures.
|
||||
- Snapshot/restore and restart replay reconstruct equivalent stream/consumer state.
|
||||
- RAFT gates cluster-visible transitions where required.
|
||||
|
||||
6. Observability flow
|
||||
- `/jsz`, `/varz`, and cluster endpoints report live behavior after transport/JetStream completion.
|
||||
|
||||
## 4. Error Handling and Verification
|
||||
|
||||
### Error handling
|
||||
- Preserve JetStream-specific error semantics over generic fallbacks.
|
||||
- Maintain separate error classes for:
|
||||
- request/config validation
|
||||
- state conflicts/not found
|
||||
- leadership/quorum/stepdown
|
||||
- transport connectivity/peer state
|
||||
- storage/recovery integrity.
|
||||
|
||||
### Strict completion gate
|
||||
1. No remaining JetStream "partial" in `differences.md` unless clearly blocked outside JetStream scope and explicitly documented.
|
||||
2. Unit and integration evidence for each completed feature area (transport + JetStream).
|
||||
3. Parity mapping document updated with Go contract, .NET file paths, proving tests, and status.
|
||||
4. Regression gates pass:
|
||||
- focused JetStream/route/gateway/leaf/raft suites
|
||||
- full `dotnet test`
|
||||
- verification summary artifact.
|
||||
844
docs/plans/2026-02-23-jetstream-final-remaining-plan.md
Normal file
844
docs/plans/2026-02-23-jetstream-final-remaining-plan.md
Normal file
@@ -0,0 +1,844 @@
|
||||
# JetStream Final Remaining Parity Implementation Plan
|
||||
|
||||
> **For Codex:** REQUIRED SUB-SKILL: Use `executeplan` to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Close all remaining JetStream parity gaps (and required transport prerequisites) between Go and .NET so JetStream entries are no longer marked partial in `differences.md` except explicitly documented external blockers.
|
||||
|
||||
**Architecture:** Implement prerequisites first (route/gateway/leaf wire behavior), then complete JetStream API and runtime semantics on top of real inter-server transport, and finally harden storage/RAFT and monitoring evidence. Use parity-map-driven development: every Go feature gap must map to concrete .NET code and test proof.
|
||||
|
||||
**Tech Stack:** .NET 10, C# 14, xUnit 3, Shouldly, NSubstitute, bash tooling, ripgrep, System.Text.Json.
|
||||
|
||||
---
|
||||
|
||||
**Execution guardrails**
|
||||
- Use `@test-driven-development` in every task.
|
||||
- If behavior diverges from expected protocol semantics, switch to `@systematic-debugging` before modifying production code.
|
||||
- Use a dedicated worktree for execution.
|
||||
- Before completion claims, run `@verification-before-completion` commands.
|
||||
|
||||
### Task 1: Regenerate and Enforce Go-vs-.NET JetStream Subject Gap Inventory
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/jetstream/extract-go-js-api.sh`
|
||||
- Modify: `docs/plans/2026-02-23-jetstream-remaining-parity-map.md`
|
||||
- Create: `tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public void Parity_map_has_no_unclassified_go_js_api_subjects()
|
||||
{
|
||||
var gap = JetStreamApiGapInventory.Load();
|
||||
gap.UnclassifiedSubjects.ShouldBeEmpty();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamApiGapInventoryTests.Parity_map_has_no_unclassified_go_js_api_subjects" -v minimal`
|
||||
Expected: FAIL with listed missing subjects (`SERVER.REMOVE`, `ACCOUNT.PURGE`, `STREAM.PEER.REMOVE`, etc.).
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
perl -nle 'while(/"(\$JS\.API[^"]+)"/g){print $1}' golang/nats-server/server/jetstream_api.go | sort -u
|
||||
```
|
||||
|
||||
```csharp
|
||||
public static JetStreamApiGapInventory Load()
|
||||
{
|
||||
// compare extracted Go subject set with mapped .NET subject handlers
|
||||
}
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamApiGapInventoryTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/jetstream/extract-go-js-api.sh docs/plans/2026-02-23-jetstream-remaining-parity-map.md tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs
|
||||
git commit -m "test: enforce jetstream api gap inventory parity map"
|
||||
```
|
||||
|
||||
### Task 2: Enforce Multi-Client-Type Command Routing and Inter-Server Opcodes
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Protocol/NatsParser.cs`
|
||||
- Modify: `src/NATS.Server/Protocol/ClientCommandMatrix.cs`
|
||||
- Modify: `src/NATS.Server/NatsClient.cs`
|
||||
- Test: `tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public void Client_kind_rejects_RSplus_for_non_route_connection()
|
||||
{
|
||||
var matrix = new ClientCommandMatrix();
|
||||
matrix.IsAllowed(ClientKind.Client, "RS+").ShouldBeFalse();
|
||||
matrix.IsAllowed(ClientKind.Router, "RS+").ShouldBeTrue();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~ClientKindProtocolRoutingTests" -v minimal`
|
||||
Expected: FAIL for missing/incorrect kind restrictions on RS+/RS-/RMSG/A+/A-/LS+/LS-/LMSG.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
(ClientKind.Router, "RS+") => true,
|
||||
(ClientKind.Router, "RS-") => true,
|
||||
(ClientKind.Router, "RMSG") => true,
|
||||
(ClientKind.Leaf, "LS+") => true,
|
||||
(ClientKind.Leaf, "LS-") => true,
|
||||
(ClientKind.Leaf, "LMSG") => true,
|
||||
_ => false,
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~ClientKindProtocolRoutingTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Protocol/NatsParser.cs src/NATS.Server/Protocol/ClientCommandMatrix.cs src/NATS.Server/NatsClient.cs tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs
|
||||
git commit -m "feat: enforce client-kind protocol routing for inter-server ops"
|
||||
```
|
||||
|
||||
### Task 3: Implement Route Wire RS+/RS- Subscription Propagation
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Routes/RouteConnection.cs`
|
||||
- Modify: `src/NATS.Server/Routes/RouteManager.cs`
|
||||
- Modify: `src/NATS.Server/Subscriptions/SubList.cs`
|
||||
- Test: `tests/NATS.Server.Tests/RouteWireSubscriptionProtocolTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task RSplus_RSminus_frames_propagate_remote_interest_over_socket()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
await fx.SendRouteSubFrameAsync("foo.*");
|
||||
(await fx.ServerAHasRemoteInterestAsync("foo.bar")).ShouldBeTrue();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RouteWireSubscriptionProtocolTests" -v minimal`
|
||||
Expected: FAIL because propagation is currently in-process and not RS+/RS- wire-driven.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
await WriteOpAsync($"RS+ {subject}");
|
||||
```
|
||||
|
||||
```csharp
|
||||
if (op == "RS+") _remoteSubSink(new RemoteSubscription(subject, queue, remoteServerId));
|
||||
if (op == "RS-") _remoteSubSink(RemoteSubscription.Removal(subject, queue, remoteServerId));
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RouteWireSubscriptionProtocolTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Routes/RouteConnection.cs src/NATS.Server/Routes/RouteManager.cs src/NATS.Server/Subscriptions/SubList.cs tests/NATS.Server.Tests/RouteWireSubscriptionProtocolTests.cs
|
||||
git commit -m "feat: implement route RS+ RS- wire subscription protocol"
|
||||
```
|
||||
|
||||
### Task 4: Implement Route RMSG Forwarding to Remote Subscribers
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Routes/RouteConnection.cs`
|
||||
- Modify: `src/NATS.Server/Routes/RouteManager.cs`
|
||||
- Modify: `src/NATS.Server/NatsServer.cs`
|
||||
- Test: `tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
await fx.SubscribeOnServerBAsync("foo.>");
|
||||
|
||||
await fx.PublishFromServerAAsync("foo.bar", "payload");
|
||||
(await fx.ReadServerBMessageAsync()).ShouldContain("payload");
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RouteRmsgForwardingTests" -v minimal`
|
||||
Expected: FAIL because remote messages are not forwarded.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
if (hasRemoteInterest)
|
||||
await route.SendRmsgAsync(subject, reply, headers, payload, ct);
|
||||
```
|
||||
|
||||
```csharp
|
||||
if (op == "RMSG") _server.ProcessRoutedMessage(parsed);
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RouteRmsgForwardingTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Routes/RouteConnection.cs src/NATS.Server/Routes/RouteManager.cs src/NATS.Server/NatsServer.cs tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs
|
||||
git commit -m "feat: forward remote messages over route RMSG"
|
||||
```
|
||||
|
||||
### Task 5: Add Route Pooling Baseline (3 Connections per Peer)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Routes/RouteManager.cs`
|
||||
- Modify: `src/NATS.Server/Configuration/ClusterOptions.cs`
|
||||
- Test: `tests/NATS.Server.Tests/RoutePoolTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Route_manager_establishes_default_pool_of_three_links_per_peer()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
(await fx.ServerARouteLinkCountToServerBAsync()).ShouldBe(3);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RoutePoolTests" -v minimal`
|
||||
Expected: FAIL because one connection per peer is used.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public int PoolSize { get; set; } = 3;
|
||||
for (var i = 0; i < _options.PoolSize; i++)
|
||||
_ = ConnectToRouteWithRetryAsync(route, ct);
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RoutePoolTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Routes/RouteManager.cs src/NATS.Server/Configuration/ClusterOptions.cs tests/NATS.Server.Tests/RoutePoolTests.cs
|
||||
git commit -m "feat: add route connection pooling baseline"
|
||||
```
|
||||
|
||||
### Task 6: Replace Gateway Stub with Functional Handshake and Forwarding Baseline
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Gateways/GatewayManager.cs`
|
||||
- Modify: `src/NATS.Server/Gateways/GatewayConnection.cs`
|
||||
- Modify: `src/NATS.Server/NatsServer.cs`
|
||||
- Test: `tests/NATS.Server.Tests/GatewayProtocolTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Gateway_link_establishes_and_forwards_interested_message()
|
||||
{
|
||||
await using var fx = await GatewayFixture.StartTwoClustersAsync();
|
||||
await fx.SubscribeRemoteClusterAsync("g.>");
|
||||
await fx.PublishLocalClusterAsync("g.test", "hello");
|
||||
(await fx.ReadRemoteClusterMessageAsync()).ShouldContain("hello");
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~GatewayProtocolTests" -v minimal`
|
||||
Expected: FAIL due to gateway no-op manager.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public Task StartAsync(CancellationToken ct)
|
||||
{
|
||||
// listen/connect handshake and track gateway links
|
||||
}
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~GatewayProtocolTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Gateways/GatewayManager.cs src/NATS.Server/Gateways/GatewayConnection.cs src/NATS.Server/NatsServer.cs tests/NATS.Server.Tests/GatewayProtocolTests.cs
|
||||
git commit -m "feat: replace gateway stub with functional protocol baseline"
|
||||
```
|
||||
|
||||
### Task 7: Replace Leaf Stub with Functional LS+/LS-/LMSG Baseline
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/LeafNodes/LeafNodeManager.cs`
|
||||
- Modify: `src/NATS.Server/LeafNodes/LeafConnection.cs`
|
||||
- Modify: `src/NATS.Server/NatsServer.cs`
|
||||
- Test: `tests/NATS.Server.Tests/LeafProtocolTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Leaf_link_propagates_subscription_and_message_flow()
|
||||
{
|
||||
await using var fx = await LeafFixture.StartHubSpokeAsync();
|
||||
await fx.SubscribeSpokeAsync("leaf.>");
|
||||
await fx.PublishHubAsync("leaf.msg", "x");
|
||||
(await fx.ReadSpokeMessageAsync()).ShouldContain("x");
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~LeafProtocolTests" -v minimal`
|
||||
Expected: FAIL due to leaf no-op manager.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
if (op == "LS+") ApplyLeafSubscription(...);
|
||||
if (op == "LMSG") ProcessLeafMessage(...);
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~LeafProtocolTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/LeafNodes/LeafNodeManager.cs src/NATS.Server/LeafNodes/LeafConnection.cs src/NATS.Server/NatsServer.cs tests/NATS.Server.Tests/LeafProtocolTests.cs
|
||||
git commit -m "feat: replace leaf stub with functional protocol baseline"
|
||||
```
|
||||
|
||||
### Task 8: Add Missing JetStream Control APIs (`SERVER.REMOVE`, `ACCOUNT.PURGE`, Move/Cancel Move)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs`
|
||||
- Create: `src/NATS.Server/JetStream/Api/Handlers/AccountControlApiHandlers.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public void Account_and_server_control_subjects_are_routable()
|
||||
{
|
||||
var r = new JetStreamApiRouter(new StreamManager(), new ConsumerManager());
|
||||
r.Route("$JS.API.SERVER.REMOVE", "{}"u8).Error.ShouldBeNull();
|
||||
r.Route("$JS.API.ACCOUNT.PURGE.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamAccountControlApiTests" -v minimal`
|
||||
Expected: FAIL with NotFound responses.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public const string ServerRemove = "$JS.API.SERVER.REMOVE";
|
||||
public const string AccountPurge = "$JS.API.ACCOUNT.PURGE.";
|
||||
```
|
||||
|
||||
```csharp
|
||||
if (subject.Equals(JetStreamApiSubjects.ServerRemove, StringComparison.Ordinal))
|
||||
return AccountControlApiHandlers.HandleServerRemove();
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamAccountControlApiTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs src/NATS.Server/JetStream/Api/Handlers/AccountControlApiHandlers.cs tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs
|
||||
git commit -m "feat: add missing jetstream account and server control apis"
|
||||
```
|
||||
|
||||
### Task 9: Add Missing Cluster JetStream APIs (`STREAM.PEER.REMOVE`, `CONSUMER.LEADER.STEPDOWN`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Api/Handlers/ClusterControlApiHandlers.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamClusterControlExtendedApiTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Peer_remove_and_consumer_stepdown_subjects_return_success_shape()
|
||||
{
|
||||
await using var fx = await JetStreamClusterFixture.StartAsync(nodes: 3);
|
||||
(await fx.RequestAsync("$JS.API.STREAM.PEER.REMOVE.ORDERS", "{\"peer\":\"n2\"}")).Success.ShouldBeTrue();
|
||||
(await fx.RequestAsync("$JS.API.CONSUMER.LEADER.STEPDOWN.ORDERS.DUR", "{}")).Success.ShouldBeTrue();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamClusterControlExtendedApiTests" -v minimal`
|
||||
Expected: FAIL because these routes are missing.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public const string StreamPeerRemove = "$JS.API.STREAM.PEER.REMOVE.";
|
||||
public const string ConsumerLeaderStepdown = "$JS.API.CONSUMER.LEADER.STEPDOWN.";
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamClusterControlExtendedApiTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs src/NATS.Server/JetStream/Api/Handlers/ClusterControlApiHandlers.cs src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs tests/NATS.Server.Tests/JetStreamClusterControlExtendedApiTests.cs
|
||||
git commit -m "feat: add extended jetstream cluster control apis"
|
||||
```
|
||||
|
||||
### Task 10: Implement Stream Policy Runtime Semantics (`MaxBytes`, `MaxAge`, `MaxMsgsPer`, `DiscardNew`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Models/StreamConfig.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/StreamManager.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Storage/IStreamStore.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Storage/MemStore.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Storage/FileStore.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Discard_new_rejects_publish_when_max_bytes_exceeded()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithStreamConfigAsync(new StreamConfig { Name = "S", Subjects = ["s.*"], MaxBytes = 2, Discard = DiscardPolicy.New });
|
||||
(await fx.PublishAndGetAckAsync("s.a", "12")).ErrorCode.ShouldBeNull();
|
||||
(await fx.PublishAndGetAckAsync("s.a", "34")).ErrorCode.ShouldNotBeNull();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamStreamPolicyRuntimeTests" -v minimal`
|
||||
Expected: FAIL because runtime enforces only `MaxMsgs`.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
if (cfg.MaxBytes > 0 && state.Bytes + payload.Length > cfg.MaxBytes && cfg.Discard == DiscardPolicy.New)
|
||||
return PublishDecision.Reject(10054, "maximum bytes exceeded");
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamStreamPolicyRuntimeTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Models/StreamConfig.cs src/NATS.Server/JetStream/StreamManager.cs src/NATS.Server/JetStream/Storage/IStreamStore.cs src/NATS.Server/JetStream/Storage/MemStore.cs src/NATS.Server/JetStream/Storage/FileStore.cs tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs
|
||||
git commit -m "feat: enforce stream runtime policies maxbytes maxage maxmsgsper discard"
|
||||
```
|
||||
|
||||
### Task 11: Implement Storage Type Selection and Config Mapping for JetStream Streams
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Models/StreamConfig.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/StreamManager.cs`
|
||||
- Modify: `src/NATS.Server/Configuration/ConfigProcessor.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Stream_with_storage_file_uses_filestore_backend()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithStreamJsonAsync("{\"name\":\"S\",\"subjects\":[\"s.*\"],\"storage\":\"file\"}");
|
||||
(await fx.GetStreamBackendTypeAsync("S")).ShouldBe("file");
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamStorageSelectionTests" -v minimal`
|
||||
Expected: FAIL because memstore is always used.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
var store = config.Storage switch
|
||||
{
|
||||
StorageType.File => new FileStore(new FileStoreOptions { Directory = ResolveStoreDir(config.Name) }),
|
||||
_ => new MemStore(),
|
||||
};
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamStorageSelectionTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Models/StreamConfig.cs src/NATS.Server/JetStream/StreamManager.cs src/NATS.Server/Configuration/ConfigProcessor.cs tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs
|
||||
git commit -m "feat: select jetstream storage backend per stream config"
|
||||
```
|
||||
|
||||
### Task 12: Implement Consumer Completeness (`Ephemeral`, `FilterSubjects`, `MaxAckPending`, `DeliverPolicy`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Models/ConsumerConfig.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/ConsumerManager.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Consumers/PushConsumerEngine.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Consumer_with_filter_subjects_only_receives_matching_messages()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithMultiFilterConsumerAsync();
|
||||
await fx.PublishAndGetAckAsync("orders.created", "1");
|
||||
await fx.PublishAndGetAckAsync("payments.settled", "2");
|
||||
|
||||
var batch = await fx.FetchAsync("ORDERS", "CF", 10);
|
||||
batch.Messages.All(m => m.Subject.StartsWith("orders.", StringComparison.Ordinal)).ShouldBeTrue();
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamConsumerSemanticsTests" -v minimal`
|
||||
Expected: FAIL because only single filter and limited policy semantics exist.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public List<string> FilterSubjects { get; set; } = [];
|
||||
if (config.FilterSubjects.Count > 0)
|
||||
include = config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(msg.Subject, f));
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamConsumerSemanticsTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Models/ConsumerConfig.cs src/NATS.Server/JetStream/ConsumerManager.cs src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs src/NATS.Server/JetStream/Consumers/PushConsumerEngine.cs tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs
|
||||
git commit -m "feat: complete consumer filters and delivery semantics"
|
||||
```
|
||||
|
||||
### Task 13: Implement Replay/Backoff/Flow Control and Rate Limits
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Consumers/AckProcessor.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Consumers/PushConsumerEngine.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Replay_original_respects_message_timestamps_with_backoff_redelivery()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithReplayOriginalConsumerAsync();
|
||||
var sw = Stopwatch.StartNew();
|
||||
await fx.FetchAsync("ORDERS", "RO", 1);
|
||||
sw.ElapsedMilliseconds.ShouldBeGreaterThanOrEqualTo(50);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamFlowReplayBackoffTests" -v minimal`
|
||||
Expected: FAIL because replay/backoff/rate semantics are incomplete.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
if (config.ReplayPolicy == ReplayPolicy.Original)
|
||||
await Task.Delay(originalDelay, ct);
|
||||
```
|
||||
|
||||
```csharp
|
||||
var next = backoff[min(deliveryCount, backoff.Length - 1)];
|
||||
_ackProcessor.Register(seq, next);
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamFlowReplayBackoffTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Consumers/AckProcessor.cs src/NATS.Server/JetStream/Consumers/PushConsumerEngine.cs src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs
|
||||
git commit -m "feat: implement replay backoff flow-control and rate behaviors"
|
||||
```
|
||||
|
||||
### Task 14: Complete Mirror/Source Advanced Semantics (`Sources[]`, transforms, cross-account guardrails)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/JetStream/Models/StreamConfig.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/MirrorSource/MirrorCoordinator.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/MirrorSource/SourceCoordinator.cs`
|
||||
- Modify: `src/NATS.Server/JetStream/StreamManager.cs`
|
||||
- Test: `tests/NATS.Server.Tests/JetStreamMirrorSourceAdvancedTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Stream_with_multiple_sources_aggregates_messages_in_order()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithMultipleSourcesAsync();
|
||||
await fx.PublishToSourceAsync("SRC1", "a.1", "1");
|
||||
await fx.PublishToSourceAsync("SRC2", "b.1", "2");
|
||||
|
||||
(await fx.GetStreamStateAsync("AGG")).Messages.ShouldBe(2);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamMirrorSourceAdvancedTests" -v minimal`
|
||||
Expected: FAIL because only single `Source` is supported.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public List<StreamSourceConfig> Sources { get; set; } = [];
|
||||
foreach (var source in config.Sources)
|
||||
RegisterSource(source);
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStreamMirrorSourceAdvancedTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/JetStream/Models/StreamConfig.cs src/NATS.Server/JetStream/MirrorSource/MirrorCoordinator.cs src/NATS.Server/JetStream/MirrorSource/SourceCoordinator.cs src/NATS.Server/JetStream/StreamManager.cs tests/NATS.Server.Tests/JetStreamMirrorSourceAdvancedTests.cs
|
||||
git commit -m "feat: complete mirror and source advanced semantics"
|
||||
```
|
||||
|
||||
### Task 15: Upgrade RAFT from In-Memory Coordination to Transport/Persistence Baseline
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Raft/RaftNode.cs`
|
||||
- Modify: `src/NATS.Server/Raft/RaftReplicator.cs`
|
||||
- Modify: `src/NATS.Server/Raft/RaftLog.cs`
|
||||
- Create: `src/NATS.Server/Raft/RaftTransport.cs`
|
||||
- Test: `tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Raft_node_recovers_log_and_term_after_restart()
|
||||
{
|
||||
var fx = await RaftFixture.StartPersistentClusterAsync();
|
||||
var idx = await fx.Leader.ProposeAsync("cmd", default);
|
||||
await fx.RestartNodeAsync("n2");
|
||||
(await fx.ReadNodeAppliedIndexAsync("n2")).ShouldBeGreaterThanOrEqualTo(idx);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RaftTransportPersistenceTests" -v minimal`
|
||||
Expected: FAIL because no persistent raft transport/log baseline exists.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
public interface IRaftTransport
|
||||
{
|
||||
Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(...);
|
||||
Task<VoteResponse> RequestVoteAsync(...);
|
||||
}
|
||||
```
|
||||
|
||||
```csharp
|
||||
public sealed class RaftLog
|
||||
{
|
||||
public Task PersistAsync(string path, CancellationToken ct) { ... }
|
||||
public static Task<RaftLog> LoadAsync(string path, CancellationToken ct) { ... }
|
||||
}
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~RaftTransportPersistenceTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Raft/RaftNode.cs src/NATS.Server/Raft/RaftReplicator.cs src/NATS.Server/Raft/RaftLog.cs src/NATS.Server/Raft/RaftTransport.cs tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs
|
||||
git commit -m "feat: add raft transport and persistence baseline"
|
||||
```
|
||||
|
||||
### Task 16: Replace Monitoring Stubs (`/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/NATS.Server/Monitoring/MonitorServer.cs`
|
||||
- Create: `src/NATS.Server/Monitoring/RoutezHandler.cs`
|
||||
- Create: `src/NATS.Server/Monitoring/GatewayzHandler.cs`
|
||||
- Create: `src/NATS.Server/Monitoring/LeafzHandler.cs`
|
||||
- Create: `src/NATS.Server/Monitoring/AccountzHandler.cs`
|
||||
- Test: `tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs`
|
||||
|
||||
**Step 1: Write the failing test**
|
||||
|
||||
```csharp
|
||||
[Fact]
|
||||
public async Task Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data()
|
||||
{
|
||||
await using var fx = await MonitorFixture.StartClusterEnabledAsync();
|
||||
(await fx.GetJsonAsync("/routez")).ShouldContain("routes");
|
||||
(await fx.GetJsonAsync("/gatewayz")).ShouldContain("gateways");
|
||||
(await fx.GetJsonAsync("/leafz")).ShouldContain("leafs");
|
||||
(await fx.GetJsonAsync("/accountz")).ShouldContain("accounts");
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~MonitorClusterEndpointTests" -v minimal`
|
||||
Expected: FAIL because endpoints currently return stubs.
|
||||
|
||||
**Step 3: Write minimal implementation**
|
||||
|
||||
```csharp
|
||||
_app.MapGet(basePath + "/routez", () => _routezHandler.Build());
|
||||
_app.MapGet(basePath + "/gatewayz", () => _gatewayzHandler.Build());
|
||||
```
|
||||
|
||||
**Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~MonitorClusterEndpointTests" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/NATS.Server/Monitoring/MonitorServer.cs src/NATS.Server/Monitoring/RoutezHandler.cs src/NATS.Server/Monitoring/GatewayzHandler.cs src/NATS.Server/Monitoring/LeafzHandler.cs src/NATS.Server/Monitoring/AccountzHandler.cs tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs
|
||||
git commit -m "feat: replace cluster monitoring endpoint stubs"
|
||||
```
|
||||
|
||||
### Task 17: Final Strict Gate, Parity Map Closure, and `differences.md` Update
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/plans/2026-02-23-jetstream-remaining-parity-map.md`
|
||||
- Modify: `docs/plans/2026-02-23-jetstream-remaining-parity-verification.md`
|
||||
- Modify: `differences.md`
|
||||
|
||||
**Step 1: Run focused transport+JetStream suites**
|
||||
|
||||
Run: `dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStream|FullyQualifiedName~Raft|FullyQualifiedName~Route|FullyQualifiedName~Gateway|FullyQualifiedName~Leaf" -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 2: Run full suite**
|
||||
|
||||
Run: `dotnet test -v minimal`
|
||||
Expected: PASS.
|
||||
|
||||
**Step 3: Enforce no JetStream partials in differences**
|
||||
|
||||
Run: `rg -n "## 11\. JetStream|Partial|partial" differences.md`
|
||||
Expected: JetStream section no longer marks remaining entries as partial unless explicitly documented external blockers.
|
||||
|
||||
**Step 4: Update parity evidence rows with exact code+test references**
|
||||
|
||||
```md
|
||||
| $JS.API.STREAM.PEER.REMOVE.* | ClusterControlApiHandlers.HandleStreamPeerRemove | ported | JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape |
|
||||
```
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add docs/plans/2026-02-23-jetstream-remaining-parity-map.md docs/plans/2026-02-23-jetstream-remaining-parity-verification.md differences.md
|
||||
git commit -m "docs: close remaining jetstream parity and strict gate evidence"
|
||||
```
|
||||
|
||||
## Dependency Order
|
||||
|
||||
1. Task 1 -> Task 2
|
||||
2. Task 3 -> Task 4 -> Task 5
|
||||
3. Task 6 -> Task 7
|
||||
4. Task 8 -> Task 9
|
||||
5. Task 10 -> Task 11 -> Task 12 -> Task 13 -> Task 14
|
||||
6. Task 15 -> Task 16
|
||||
7. Task 17
|
||||
|
||||
## Executor Notes
|
||||
|
||||
- Use Go references while implementing each task:
|
||||
- `golang/nats-server/server/jetstream_api.go`
|
||||
- `golang/nats-server/server/jetstream.go`
|
||||
- `golang/nats-server/server/stream.go`
|
||||
- `golang/nats-server/server/consumer.go`
|
||||
- `golang/nats-server/server/raft.go`
|
||||
- `golang/nats-server/server/route.go`
|
||||
- `golang/nats-server/server/gateway.go`
|
||||
- `golang/nats-server/server/leafnode.go`
|
||||
- Keep behavior claims test-backed; do not update parity status based only on type signatures or route registration.
|
||||
@@ -3,16 +3,25 @@
|
||||
| Go Subject | .NET Route | Status | Test |
|
||||
|---|---|---|---|
|
||||
| $JS.API.INFO | `AccountApiHandlers.HandleInfo` | ported | `JetStreamAccountInfoApiTests.Account_info_returns_jetstream_limits_and_usage_shape` |
|
||||
| $JS.API.SERVER.REMOVE | `AccountControlApiHandlers.HandleServerRemove` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
|
||||
| $JS.API.ACCOUNT.PURGE.* | `AccountControlApiHandlers.HandleAccountPurge` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
|
||||
| $JS.API.ACCOUNT.STREAM.MOVE.* | `AccountControlApiHandlers.HandleAccountStreamMove` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
|
||||
| $JS.API.ACCOUNT.STREAM.MOVE.CANCEL.* | `AccountControlApiHandlers.HandleAccountStreamMoveCancel` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
|
||||
| $JS.API.STREAM.CREATE.* | `StreamApiHandlers.HandleCreate` | ported | `JetStreamStreamLifecycleApiTests.Stream_create_info_and_update_roundtrip` |
|
||||
| $JS.API.STREAM.INFO.* | `StreamApiHandlers.HandleInfo` | ported | `JetStreamStreamLifecycleApiTests.Stream_create_info_and_update_roundtrip` |
|
||||
| $JS.API.STREAM.UPDATE.* | `StreamApiHandlers.HandleUpdate` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` |
|
||||
| $JS.API.STREAM.DELETE.* | `StreamApiHandlers.HandleDelete` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` |
|
||||
| $JS.API.STREAM.NAMES | `StreamApiHandlers.HandleNames` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` |
|
||||
| $JS.API.STREAM.LIST | `StreamApiHandlers.HandleList` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` |
|
||||
| $JS.API.STREAM.PEER.REMOVE.* | `ClusterControlApiHandlers.HandleStreamPeerRemove` | ported | `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` |
|
||||
| $JS.API.STREAM.MSG.GET.* | `StreamApiHandlers.HandleMessageGet` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` |
|
||||
| $JS.API.STREAM.MSG.DELETE.* | `StreamApiHandlers.HandleMessageDelete` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` |
|
||||
| $JS.API.STREAM.PURGE.* | `StreamApiHandlers.HandlePurge` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` |
|
||||
| $JS.API.DIRECT.GET.* | `DirectApiHandlers.HandleGet` | ported | `JetStreamDirectGetApiTests.Direct_get_returns_message_without_stream_info_wrapper` |
|
||||
| $JS.API.STREAM.SNAPSHOT.* | `StreamApiHandlers.HandleSnapshot` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` |
|
||||
| $JS.API.STREAM.RESTORE.* | `StreamApiHandlers.HandleRestore` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` |
|
||||
| $JS.API.CONSUMER.CREATE.*.* | `ConsumerApiHandlers.HandleCreate` | ported | `JetStreamConsumerApiTests.Consumer_create_and_info_roundtrip` |
|
||||
| $JS.API.CONSUMER.INFO.*.* | `ConsumerApiHandlers.HandleInfo` | ported | `JetStreamConsumerApiTests.Consumer_create_and_info_roundtrip` |
|
||||
| $JS.API.CONSUMER.NAMES.* | `ConsumerApiHandlers.HandleNames` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` |
|
||||
| $JS.API.CONSUMER.LIST.* | `ConsumerApiHandlers.HandleList` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` |
|
||||
| $JS.API.CONSUMER.DELETE.*.* | `ConsumerApiHandlers.HandleDelete` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` |
|
||||
@@ -20,5 +29,6 @@
|
||||
| $JS.API.CONSUMER.RESET.*.* | `ConsumerApiHandlers.HandleReset` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` |
|
||||
| $JS.API.CONSUMER.UNPIN.*.* | `ConsumerApiHandlers.HandleUnpin` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` |
|
||||
| $JS.API.CONSUMER.MSG.NEXT.*.* | `ConsumerApiHandlers.HandleNext` | ported | `JetStreamConsumerNextApiTests.Consumer_msg_next_respects_batch_request` |
|
||||
| $JS.API.CONSUMER.LEADER.STEPDOWN.*.* | `ClusterControlApiHandlers.HandleConsumerLeaderStepdown` | ported | `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` |
|
||||
| $JS.API.STREAM.LEADER.STEPDOWN.* | `ClusterControlApiHandlers.HandleStreamLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` |
|
||||
| $JS.API.META.LEADER.STEPDOWN | `ClusterControlApiHandlers.HandleMetaLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` |
|
||||
|
||||
@@ -10,7 +10,7 @@ dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStream|Fully
|
||||
|
||||
Result:
|
||||
|
||||
- Passed: `54`
|
||||
- Passed: `69`
|
||||
- Failed: `0`
|
||||
- Skipped: `0`
|
||||
- Duration: `~10s`
|
||||
@@ -25,10 +25,10 @@ dotnet test -v minimal
|
||||
|
||||
Result:
|
||||
|
||||
- Passed: `737`
|
||||
- Passed: `768`
|
||||
- Failed: `0`
|
||||
- Skipped: `0`
|
||||
- Duration: `~1m 5s`
|
||||
- Duration: `~1m 11s`
|
||||
|
||||
## Focused Scenario Evidence
|
||||
|
||||
@@ -40,5 +40,19 @@ Result:
|
||||
- `JetStreamPushConsumerContractTests.Ack_all_advances_floor_and_clears_pending_before_sequence`
|
||||
- `RaftSafetyContractTests.Follower_rejects_stale_term_vote_and_append`
|
||||
- `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape`
|
||||
- `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable`
|
||||
- `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape`
|
||||
- `RouteWireSubscriptionProtocolTests.RSplus_RSminus_frames_propagate_remote_interest_over_socket`
|
||||
- `RouteRmsgForwardingTests.Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG`
|
||||
- `RoutePoolTests.Route_manager_establishes_default_pool_of_three_links_per_peer`
|
||||
- `GatewayProtocolTests.Gateway_link_establishes_and_forwards_interested_message`
|
||||
- `LeafProtocolTests.Leaf_link_propagates_subscription_and_message_flow`
|
||||
- `JetStreamStreamPolicyRuntimeTests.Discard_new_rejects_publish_when_max_bytes_exceeded`
|
||||
- `JetStreamStorageSelectionTests.Stream_with_storage_file_uses_filestore_backend`
|
||||
- `JetStreamConsumerSemanticsTests.Consumer_with_filter_subjects_only_receives_matching_messages`
|
||||
- `JetStreamFlowReplayBackoffTests.Replay_original_respects_message_timestamps_with_backoff_redelivery`
|
||||
- `JetStreamMirrorSourceAdvancedTests.Stream_with_multiple_sources_aggregates_messages_in_order`
|
||||
- `RaftTransportPersistenceTests.Raft_node_recovers_log_and_term_after_restart`
|
||||
- `MonitorClusterEndpointTests.Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data`
|
||||
- `JetStreamMonitoringParityTests.Jsz_and_varz_include_expanded_runtime_fields`
|
||||
- `JetStreamIntegrationMatrixTests.Integration_matrix_executes_real_server_scenarios`
|
||||
|
||||
@@ -8,15 +8,21 @@ if [[ -f "$go_file" ]]; then
|
||||
{
|
||||
rg -n -F '$JS.API' "$go_file" \
|
||||
| awk -F: '{print $3}' \
|
||||
| sed -E 's/.*"(\$JS\.API[^\"]+)".*/\1/'
|
||||
| sed -E 's/.*"(\$JS\.API[^\"]+)".*/\1/' \
|
||||
| awk '/^\$JS\.API/ && $0 !~ /\.>$/'
|
||||
|
||||
# Some Go constants are coarse patterns (e.g. "$JS.API.STREAM.>").
|
||||
# Add explicit subject families used by parity tests/docs.
|
||||
cat <<'EOF'
|
||||
$JS.API.INFO
|
||||
$JS.API.SERVER.REMOVE
|
||||
$JS.API.ACCOUNT.PURGE.*
|
||||
$JS.API.ACCOUNT.STREAM.MOVE.*
|
||||
$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*
|
||||
$JS.API.STREAM.UPDATE.*
|
||||
$JS.API.STREAM.DELETE.*
|
||||
$JS.API.STREAM.PURGE.*
|
||||
$JS.API.STREAM.PEER.REMOVE.*
|
||||
$JS.API.STREAM.NAMES
|
||||
$JS.API.STREAM.LIST
|
||||
$JS.API.STREAM.MSG.GET.*
|
||||
@@ -30,6 +36,7 @@ $JS.API.CONSUMER.PAUSE.*.*
|
||||
$JS.API.CONSUMER.RESET.*.*
|
||||
$JS.API.CONSUMER.UNPIN.*.*
|
||||
$JS.API.CONSUMER.MSG.NEXT.*.*
|
||||
$JS.API.CONSUMER.LEADER.STEPDOWN.*.*
|
||||
$JS.API.DIRECT.GET.*
|
||||
$JS.API.STREAM.LEADER.STEPDOWN.*
|
||||
$JS.API.META.LEADER.STEPDOWN
|
||||
@@ -41,11 +48,16 @@ fi
|
||||
# Fallback subject inventory when Go reference sources are not vendored in this repo.
|
||||
cat <<'EOF'
|
||||
$JS.API.INFO
|
||||
$JS.API.SERVER.REMOVE
|
||||
$JS.API.ACCOUNT.PURGE.*
|
||||
$JS.API.ACCOUNT.STREAM.MOVE.*
|
||||
$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*
|
||||
$JS.API.STREAM.CREATE.*
|
||||
$JS.API.STREAM.UPDATE.*
|
||||
$JS.API.STREAM.DELETE.*
|
||||
$JS.API.STREAM.PURGE.*
|
||||
$JS.API.STREAM.INFO.*
|
||||
$JS.API.STREAM.PEER.REMOVE.*
|
||||
$JS.API.STREAM.NAMES
|
||||
$JS.API.STREAM.LIST
|
||||
$JS.API.STREAM.MSG.GET.*
|
||||
@@ -61,6 +73,7 @@ $JS.API.CONSUMER.PAUSE.*.*
|
||||
$JS.API.CONSUMER.RESET.*.*
|
||||
$JS.API.CONSUMER.UNPIN.*.*
|
||||
$JS.API.CONSUMER.MSG.NEXT.*.*
|
||||
$JS.API.CONSUMER.LEADER.STEPDOWN.*.*
|
||||
$JS.API.DIRECT.GET.*
|
||||
$JS.API.STREAM.LEADER.STEPDOWN.*
|
||||
$JS.API.META.LEADER.STEPDOWN
|
||||
|
||||
@@ -5,5 +5,6 @@ public sealed class ClusterOptions
|
||||
public string? Name { get; set; }
|
||||
public string Host { get; set; } = "0.0.0.0";
|
||||
public int Port { get; set; } = 6222;
|
||||
public int PoolSize { get; set; } = 3;
|
||||
public List<string> Routes { get; set; } = [];
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ public sealed class GatewayOptions
|
||||
public string? Name { get; set; }
|
||||
public string Host { get; set; } = "0.0.0.0";
|
||||
public int Port { get; set; }
|
||||
public List<string> Remotes { get; set; } = [];
|
||||
}
|
||||
|
||||
@@ -4,4 +4,5 @@ public sealed class LeafNodeOptions
|
||||
{
|
||||
public string Host { get; set; } = "0.0.0.0";
|
||||
public int Port { get; set; }
|
||||
public List<string> Remotes { get; set; } = [];
|
||||
}
|
||||
|
||||
@@ -1,11 +1,191 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.Gateways;
|
||||
|
||||
public sealed class GatewayConnection
|
||||
public sealed class GatewayConnection(Socket socket) : IAsyncDisposable
|
||||
{
|
||||
public string RemoteEndpoint { get; }
|
||||
private readonly NetworkStream _stream = new(socket, ownsSocket: true);
|
||||
private readonly SemaphoreSlim _writeGate = new(1, 1);
|
||||
private readonly CancellationTokenSource _closedCts = new();
|
||||
private Task? _loopTask;
|
||||
|
||||
public GatewayConnection(string remoteEndpoint)
|
||||
public string? RemoteId { get; private set; }
|
||||
public string RemoteEndpoint => socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N");
|
||||
public Func<RemoteSubscription, Task>? RemoteSubscriptionReceived { get; set; }
|
||||
public Func<GatewayMessage, Task>? MessageReceived { get; set; }
|
||||
|
||||
public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct)
|
||||
{
|
||||
RemoteEndpoint = remoteEndpoint;
|
||||
await WriteLineAsync($"GATEWAY {serverId}", ct);
|
||||
var line = await ReadLineAsync(ct);
|
||||
RemoteId = ParseHandshake(line);
|
||||
}
|
||||
|
||||
public async Task PerformInboundHandshakeAsync(string serverId, CancellationToken ct)
|
||||
{
|
||||
var line = await ReadLineAsync(ct);
|
||||
RemoteId = ParseHandshake(line);
|
||||
await WriteLineAsync($"GATEWAY {serverId}", ct);
|
||||
}
|
||||
|
||||
public void StartLoop(CancellationToken ct)
|
||||
{
|
||||
if (_loopTask != null)
|
||||
return;
|
||||
|
||||
var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
|
||||
_loopTask = Task.Run(() => ReadLoopAsync(linked.Token), linked.Token);
|
||||
}
|
||||
|
||||
public Task WaitUntilClosedAsync(CancellationToken ct)
|
||||
=> _loopTask?.WaitAsync(ct) ?? Task.CompletedTask;
|
||||
|
||||
public Task SendAPlusAsync(string subject, string? queue, CancellationToken ct)
|
||||
=> WriteLineAsync(queue is { Length: > 0 } ? $"A+ {subject} {queue}" : $"A+ {subject}", ct);
|
||||
|
||||
public Task SendAMinusAsync(string subject, string? queue, CancellationToken ct)
|
||||
=> WriteLineAsync(queue is { Length: > 0 } ? $"A- {subject} {queue}" : $"A- {subject}", ct);
|
||||
|
||||
public async Task SendMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
|
||||
{
|
||||
var reply = string.IsNullOrEmpty(replyTo) ? "-" : replyTo;
|
||||
await _writeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var control = Encoding.ASCII.GetBytes($"GMSG {subject} {reply} {payload.Length}\r\n");
|
||||
await _stream.WriteAsync(control, ct);
|
||||
if (!payload.IsEmpty)
|
||||
await _stream.WriteAsync(payload, ct);
|
||||
await _stream.WriteAsync("\r\n"u8.ToArray(), ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _closedCts.CancelAsync();
|
||||
if (_loopTask != null)
|
||||
await _loopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
||||
_closedCts.Dispose();
|
||||
_writeGate.Dispose();
|
||||
await _stream.DisposeAsync();
|
||||
}
|
||||
|
||||
private async Task ReadLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
string line;
|
||||
try
|
||||
{
|
||||
line = await ReadLineAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (line.StartsWith("A+ ", StringComparison.Ordinal))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
|
||||
{
|
||||
var queue = parts.Length >= 3 ? parts[2] : null;
|
||||
await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteId ?? string.Empty));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("A- ", StringComparison.Ordinal))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
|
||||
{
|
||||
var queue = parts.Length >= 3 ? parts[2] : null;
|
||||
await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteId ?? string.Empty));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!line.StartsWith("GMSG ", StringComparison.Ordinal))
|
||||
continue;
|
||||
|
||||
var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (args.Length < 4 || !int.TryParse(args[3], out var size) || size < 0)
|
||||
continue;
|
||||
|
||||
var payload = await ReadPayloadAsync(size, ct);
|
||||
if (MessageReceived != null)
|
||||
await MessageReceived(new GatewayMessage(args[1], args[2] == "-" ? null : args[2], payload));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ReadOnlyMemory<byte>> ReadPayloadAsync(int size, CancellationToken ct)
|
||||
{
|
||||
var payload = new byte[size];
|
||||
var offset = 0;
|
||||
while (offset < size)
|
||||
{
|
||||
var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Gateway payload read closed");
|
||||
offset += read;
|
||||
}
|
||||
|
||||
var trailer = new byte[2];
|
||||
_ = await _stream.ReadAsync(trailer, ct);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private async Task WriteLineAsync(string line, CancellationToken ct)
|
||||
{
|
||||
await _writeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
|
||||
await _stream.WriteAsync(bytes, ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> ReadLineAsync(CancellationToken ct)
|
||||
{
|
||||
var bytes = new List<byte>(64);
|
||||
var single = new byte[1];
|
||||
while (true)
|
||||
{
|
||||
var read = await _stream.ReadAsync(single, ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Gateway closed");
|
||||
if (single[0] == (byte)'\n')
|
||||
break;
|
||||
if (single[0] != (byte)'\r')
|
||||
bytes.Add(single[0]);
|
||||
}
|
||||
|
||||
return Encoding.ASCII.GetString([.. bytes]);
|
||||
}
|
||||
|
||||
private static string ParseHandshake(string line)
|
||||
{
|
||||
if (!line.StartsWith("GATEWAY ", StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException("Invalid gateway handshake");
|
||||
|
||||
var id = line[8..].Trim();
|
||||
if (id.Length == 0)
|
||||
throw new InvalidOperationException("Gateway handshake missing id");
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record GatewayMessage(string Subject, string? ReplyTo, ReadOnlyMemory<byte> Payload);
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NATS.Server.Configuration;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.Gateways;
|
||||
|
||||
@@ -7,26 +11,204 @@ public sealed class GatewayManager : IAsyncDisposable
|
||||
{
|
||||
private readonly GatewayOptions _options;
|
||||
private readonly ServerStats _stats;
|
||||
private readonly string _serverId;
|
||||
private readonly Action<RemoteSubscription> _remoteSubSink;
|
||||
private readonly Action<GatewayMessage> _messageSink;
|
||||
private readonly ILogger<GatewayManager> _logger;
|
||||
private readonly ConcurrentDictionary<string, GatewayConnection> _connections = new(StringComparer.Ordinal);
|
||||
|
||||
public GatewayManager(GatewayOptions options, ServerStats stats, ILogger<GatewayManager> logger)
|
||||
private CancellationTokenSource? _cts;
|
||||
private Socket? _listener;
|
||||
private Task? _acceptLoopTask;
|
||||
|
||||
public string ListenEndpoint => $"{_options.Host}:{_options.Port}";
|
||||
|
||||
public GatewayManager(
|
||||
GatewayOptions options,
|
||||
ServerStats stats,
|
||||
string serverId,
|
||||
Action<RemoteSubscription> remoteSubSink,
|
||||
Action<GatewayMessage> messageSink,
|
||||
ILogger<GatewayManager> logger)
|
||||
{
|
||||
_options = options;
|
||||
_stats = stats;
|
||||
_serverId = serverId;
|
||||
_remoteSubSink = remoteSubSink;
|
||||
_messageSink = messageSink;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken ct)
|
||||
{
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
_listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
_listener.Bind(new IPEndPoint(IPAddress.Parse(_options.Host), _options.Port));
|
||||
_listener.Listen(128);
|
||||
|
||||
if (_options.Port == 0)
|
||||
_options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
|
||||
|
||||
_acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
|
||||
foreach (var remote in _options.Remotes.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
_ = Task.Run(() => ConnectWithRetryAsync(remote, _cts.Token));
|
||||
|
||||
_logger.LogDebug("Gateway manager started (name={Name}, listen={Host}:{Port})",
|
||||
_options.Name, _options.Host, _options.Port);
|
||||
Interlocked.Exchange(ref _stats.Gateways, 0);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
public async Task ForwardMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
|
||||
{
|
||||
foreach (var connection in _connections.Values)
|
||||
await connection.SendMessageAsync(subject, replyTo, payload, ct);
|
||||
}
|
||||
|
||||
public void PropagateLocalSubscription(string subject, string? queue)
|
||||
{
|
||||
foreach (var connection in _connections.Values)
|
||||
_ = connection.SendAPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
|
||||
}
|
||||
|
||||
public void PropagateLocalUnsubscription(string subject, string? queue)
|
||||
{
|
||||
foreach (var connection in _connections.Values)
|
||||
_ = connection.SendAMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_cts == null)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
_listener?.Dispose();
|
||||
if (_acceptLoopTask != null)
|
||||
await _acceptLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
||||
|
||||
foreach (var connection in _connections.Values)
|
||||
await connection.DisposeAsync();
|
||||
_connections.Clear();
|
||||
Interlocked.Exchange(ref _stats.Gateways, 0);
|
||||
_cts.Dispose();
|
||||
_cts = null;
|
||||
_logger.LogDebug("Gateway manager stopped");
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task AcceptLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
Socket socket;
|
||||
try
|
||||
{
|
||||
socket = await _listener!.AcceptAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
_ = Task.Run(() => HandleInboundAsync(socket, ct), ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleInboundAsync(Socket socket, CancellationToken ct)
|
||||
{
|
||||
var connection = new GatewayConnection(socket);
|
||||
try
|
||||
{
|
||||
await connection.PerformInboundHandshakeAsync(_serverId, ct);
|
||||
Register(connection);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await connection.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ConnectWithRetryAsync(string remote, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var endPoint = ParseEndpoint(remote);
|
||||
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct);
|
||||
var connection = new GatewayConnection(socket);
|
||||
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
|
||||
Register(connection);
|
||||
return;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Gateway connect retry for {Remote}", remote);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(250, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void Register(GatewayConnection connection)
|
||||
{
|
||||
var key = $"{connection.RemoteId}:{connection.RemoteEndpoint}:{Guid.NewGuid():N}";
|
||||
if (!_connections.TryAdd(key, connection))
|
||||
{
|
||||
_ = connection.DisposeAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
connection.RemoteSubscriptionReceived = sub =>
|
||||
{
|
||||
_remoteSubSink(sub);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
connection.MessageReceived = msg =>
|
||||
{
|
||||
_messageSink(msg);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
connection.StartLoop(_cts!.Token);
|
||||
Interlocked.Increment(ref _stats.Gateways);
|
||||
_ = Task.Run(() => WatchConnectionAsync(key, connection, _cts!.Token));
|
||||
}
|
||||
|
||||
private async Task WatchConnectionAsync(string key, GatewayConnection connection, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await connection.WaitUntilClosedAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_connections.TryRemove(key, out _))
|
||||
Interlocked.Decrement(ref _stats.Gateways);
|
||||
await connection.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private static IPEndPoint ParseEndpoint(string endpoint)
|
||||
{
|
||||
var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length != 2)
|
||||
throw new FormatException($"Invalid endpoint: {endpoint}");
|
||||
|
||||
return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
namespace NATS.Server.JetStream.Api.Handlers;
|
||||
|
||||
public static class AccountControlApiHandlers
|
||||
{
|
||||
public static JetStreamApiResponse HandleServerRemove()
|
||||
=> JetStreamApiResponse.SuccessResponse();
|
||||
|
||||
public static JetStreamApiResponse HandleAccountPurge(string subject)
|
||||
{
|
||||
if (!subject.StartsWith(JetStreamApiSubjects.AccountPurge, StringComparison.Ordinal))
|
||||
return JetStreamApiResponse.NotFound(subject);
|
||||
|
||||
var account = subject[JetStreamApiSubjects.AccountPurge.Length..].Trim();
|
||||
return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
|
||||
}
|
||||
|
||||
public static JetStreamApiResponse HandleAccountStreamMove(string subject)
|
||||
{
|
||||
if (!subject.StartsWith(JetStreamApiSubjects.AccountStreamMove, StringComparison.Ordinal))
|
||||
return JetStreamApiResponse.NotFound(subject);
|
||||
|
||||
var account = subject[JetStreamApiSubjects.AccountStreamMove.Length..].Trim();
|
||||
return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
|
||||
}
|
||||
|
||||
public static JetStreamApiResponse HandleAccountStreamMoveCancel(string subject)
|
||||
{
|
||||
if (!subject.StartsWith(JetStreamApiSubjects.AccountStreamMoveCancel, StringComparison.Ordinal))
|
||||
return JetStreamApiResponse.NotFound(subject);
|
||||
|
||||
var account = subject[JetStreamApiSubjects.AccountStreamMoveCancel.Length..].Trim();
|
||||
return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
|
||||
}
|
||||
}
|
||||
@@ -20,4 +20,23 @@ public static class ClusterControlApiHandlers
|
||||
streams.StepDownStreamLeaderAsync(stream, default).GetAwaiter().GetResult();
|
||||
return JetStreamApiResponse.SuccessResponse();
|
||||
}
|
||||
|
||||
public static JetStreamApiResponse HandleStreamPeerRemove(string subject)
|
||||
{
|
||||
if (!subject.StartsWith(JetStreamApiSubjects.StreamPeerRemove, StringComparison.Ordinal))
|
||||
return JetStreamApiResponse.NotFound(subject);
|
||||
|
||||
var stream = subject[JetStreamApiSubjects.StreamPeerRemove.Length..].Trim();
|
||||
return stream.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
|
||||
}
|
||||
|
||||
public static JetStreamApiResponse HandleConsumerLeaderStepdown(string subject)
|
||||
{
|
||||
if (!subject.StartsWith(JetStreamApiSubjects.ConsumerLeaderStepdown, StringComparison.Ordinal))
|
||||
return JetStreamApiResponse.NotFound(subject);
|
||||
|
||||
var remainder = subject[JetStreamApiSubjects.ConsumerLeaderStepdown.Length..].Trim();
|
||||
var tokens = remainder.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
return tokens.Length == 2 ? JetStreamApiResponse.SuccessResponse() : JetStreamApiResponse.NotFound(subject);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +168,19 @@ public static class ConsumerApiHandlers
|
||||
if (root.TryGetProperty("filter_subject", out var filterEl))
|
||||
config.FilterSubject = filterEl.GetString();
|
||||
|
||||
if (root.TryGetProperty("filter_subjects", out var filterSubjectsEl) && filterSubjectsEl.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in filterSubjectsEl.EnumerateArray())
|
||||
{
|
||||
var filter = item.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(filter))
|
||||
config.FilterSubjects.Add(filter);
|
||||
}
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("ephemeral", out var ephemeralEl) && ephemeralEl.ValueKind == JsonValueKind.True)
|
||||
config.Ephemeral = true;
|
||||
|
||||
if (root.TryGetProperty("push", out var pushEl) && pushEl.ValueKind == JsonValueKind.True)
|
||||
config.Push = true;
|
||||
|
||||
@@ -177,6 +190,9 @@ public static class ConsumerApiHandlers
|
||||
if (root.TryGetProperty("ack_wait_ms", out var ackWaitEl) && ackWaitEl.TryGetInt32(out var ackWait))
|
||||
config.AckWaitMs = ackWait;
|
||||
|
||||
if (root.TryGetProperty("max_ack_pending", out var maxAckPendingEl) && maxAckPendingEl.TryGetInt32(out var maxAckPending))
|
||||
config.MaxAckPending = Math.Max(maxAckPending, 0);
|
||||
|
||||
if (root.TryGetProperty("ack_policy", out var ackPolicyEl))
|
||||
{
|
||||
var ackPolicy = ackPolicyEl.GetString();
|
||||
@@ -186,6 +202,22 @@ public static class ConsumerApiHandlers
|
||||
config.AckPolicy = AckPolicy.All;
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("deliver_policy", out var deliverPolicyEl))
|
||||
{
|
||||
var deliver = deliverPolicyEl.GetString();
|
||||
if (string.Equals(deliver, "last", StringComparison.OrdinalIgnoreCase))
|
||||
config.DeliverPolicy = DeliverPolicy.Last;
|
||||
else if (string.Equals(deliver, "new", StringComparison.OrdinalIgnoreCase))
|
||||
config.DeliverPolicy = DeliverPolicy.New;
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("replay_policy", out var replayPolicyEl))
|
||||
{
|
||||
var replay = replayPolicyEl.GetString();
|
||||
if (string.Equals(replay, "original", StringComparison.OrdinalIgnoreCase))
|
||||
config.ReplayPolicy = ReplayPolicy.Original;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
catch (JsonException)
|
||||
|
||||
@@ -211,6 +211,56 @@ public static class StreamApiHandlers
|
||||
if (root.TryGetProperty("max_msgs", out var maxMsgsEl) && maxMsgsEl.TryGetInt32(out var maxMsgs))
|
||||
config.MaxMsgs = maxMsgs;
|
||||
|
||||
if (root.TryGetProperty("max_bytes", out var maxBytesEl) && maxBytesEl.TryGetInt64(out var maxBytes))
|
||||
config.MaxBytes = maxBytes;
|
||||
|
||||
if (root.TryGetProperty("max_msgs_per", out var maxMsgsPerEl) && maxMsgsPerEl.TryGetInt32(out var maxMsgsPer))
|
||||
config.MaxMsgsPer = maxMsgsPer;
|
||||
|
||||
if (root.TryGetProperty("max_age_ms", out var maxAgeMsEl) && maxAgeMsEl.TryGetInt32(out var maxAgeMs))
|
||||
config.MaxAgeMs = maxAgeMs;
|
||||
|
||||
if (root.TryGetProperty("discard", out var discardEl))
|
||||
{
|
||||
var discard = discardEl.GetString();
|
||||
if (string.Equals(discard, "new", StringComparison.OrdinalIgnoreCase))
|
||||
config.Discard = DiscardPolicy.New;
|
||||
else if (string.Equals(discard, "old", StringComparison.OrdinalIgnoreCase))
|
||||
config.Discard = DiscardPolicy.Old;
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("storage", out var storageEl))
|
||||
{
|
||||
var storage = storageEl.GetString();
|
||||
if (string.Equals(storage, "file", StringComparison.OrdinalIgnoreCase))
|
||||
config.Storage = StorageType.File;
|
||||
else
|
||||
config.Storage = StorageType.Memory;
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("source", out var sourceEl))
|
||||
config.Source = sourceEl.GetString();
|
||||
|
||||
if (root.TryGetProperty("sources", out var sourcesEl) && sourcesEl.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var source in sourcesEl.EnumerateArray())
|
||||
{
|
||||
if (source.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var name = source.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(name))
|
||||
config.Sources.Add(new StreamSourceConfig { Name = name });
|
||||
}
|
||||
else if (source.ValueKind == JsonValueKind.Object &&
|
||||
source.TryGetProperty("name", out var sourceNameEl))
|
||||
{
|
||||
var name = sourceNameEl.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(name))
|
||||
config.Sources.Add(new StreamSourceConfig { Name = name });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("replicas", out var replicasEl) && replicasEl.TryGetInt32(out var replicas))
|
||||
config.Replicas = replicas;
|
||||
|
||||
|
||||
@@ -25,6 +25,18 @@ public sealed class JetStreamApiRouter
|
||||
if (subject.Equals(JetStreamApiSubjects.Info, StringComparison.Ordinal))
|
||||
return AccountApiHandlers.HandleInfo(_streamManager, _consumerManager);
|
||||
|
||||
if (subject.Equals(JetStreamApiSubjects.ServerRemove, StringComparison.Ordinal))
|
||||
return AccountControlApiHandlers.HandleServerRemove();
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.AccountPurge, StringComparison.Ordinal))
|
||||
return AccountControlApiHandlers.HandleAccountPurge(subject);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.AccountStreamMoveCancel, StringComparison.Ordinal))
|
||||
return AccountControlApiHandlers.HandleAccountStreamMoveCancel(subject);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.AccountStreamMove, StringComparison.Ordinal))
|
||||
return AccountControlApiHandlers.HandleAccountStreamMove(subject);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal))
|
||||
return StreamApiHandlers.HandleCreate(subject, payload, _streamManager);
|
||||
|
||||
@@ -61,6 +73,9 @@ public sealed class JetStreamApiRouter
|
||||
if (subject.StartsWith(JetStreamApiSubjects.StreamLeaderStepdown, StringComparison.Ordinal))
|
||||
return ClusterControlApiHandlers.HandleStreamLeaderStepdown(subject, _streamManager);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.StreamPeerRemove, StringComparison.Ordinal))
|
||||
return ClusterControlApiHandlers.HandleStreamPeerRemove(subject);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal))
|
||||
return ConsumerApiHandlers.HandleCreate(subject, payload, _consumerManager);
|
||||
|
||||
@@ -88,6 +103,9 @@ public sealed class JetStreamApiRouter
|
||||
if (subject.StartsWith(JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal))
|
||||
return ConsumerApiHandlers.HandleNext(subject, payload, _consumerManager, _streamManager);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.ConsumerLeaderStepdown, StringComparison.Ordinal))
|
||||
return ClusterControlApiHandlers.HandleConsumerLeaderStepdown(subject);
|
||||
|
||||
if (subject.StartsWith(JetStreamApiSubjects.DirectGet, StringComparison.Ordinal))
|
||||
return DirectApiHandlers.HandleGet(subject, payload, _streamManager);
|
||||
|
||||
|
||||
@@ -3,6 +3,10 @@ namespace NATS.Server.JetStream.Api;
|
||||
public static class JetStreamApiSubjects
|
||||
{
|
||||
public const string Info = "$JS.API.INFO";
|
||||
public const string ServerRemove = "$JS.API.SERVER.REMOVE";
|
||||
public const string AccountPurge = "$JS.API.ACCOUNT.PURGE.";
|
||||
public const string AccountStreamMove = "$JS.API.ACCOUNT.STREAM.MOVE.";
|
||||
public const string AccountStreamMoveCancel = "$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.";
|
||||
public const string StreamCreate = "$JS.API.STREAM.CREATE.";
|
||||
public const string StreamInfo = "$JS.API.STREAM.INFO.";
|
||||
public const string StreamNames = "$JS.API.STREAM.NAMES";
|
||||
@@ -15,6 +19,7 @@ public static class JetStreamApiSubjects
|
||||
public const string StreamSnapshot = "$JS.API.STREAM.SNAPSHOT.";
|
||||
public const string StreamRestore = "$JS.API.STREAM.RESTORE.";
|
||||
public const string StreamLeaderStepdown = "$JS.API.STREAM.LEADER.STEPDOWN.";
|
||||
public const string StreamPeerRemove = "$JS.API.STREAM.PEER.REMOVE.";
|
||||
public const string ConsumerCreate = "$JS.API.CONSUMER.CREATE.";
|
||||
public const string ConsumerInfo = "$JS.API.CONSUMER.INFO.";
|
||||
public const string ConsumerNames = "$JS.API.CONSUMER.NAMES.";
|
||||
@@ -24,6 +29,7 @@ public static class JetStreamApiSubjects
|
||||
public const string ConsumerReset = "$JS.API.CONSUMER.RESET.";
|
||||
public const string ConsumerUnpin = "$JS.API.CONSUMER.UNPIN.";
|
||||
public const string ConsumerNext = "$JS.API.CONSUMER.MSG.NEXT.";
|
||||
public const string ConsumerLeaderStepdown = "$JS.API.CONSUMER.LEADER.STEPDOWN.";
|
||||
public const string DirectGet = "$JS.API.DIRECT.GET.";
|
||||
public const string MetaLeaderStepdown = "$JS.API.META.LEADER.STEPDOWN";
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using NATS.Server.JetStream.Cluster;
|
||||
using NATS.Server.JetStream.Consumers;
|
||||
using NATS.Server.JetStream.Models;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.JetStream;
|
||||
|
||||
@@ -24,7 +25,15 @@ public sealed class ConsumerManager
|
||||
public JetStreamApiResponse CreateOrUpdate(string stream, ConsumerConfig config)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(config.DurableName))
|
||||
return JetStreamApiResponse.ErrorResponse(400, "durable name required");
|
||||
{
|
||||
if (config.Ephemeral)
|
||||
config.DurableName = $"ephemeral-{Guid.NewGuid():N}"[..24];
|
||||
else
|
||||
return JetStreamApiResponse.ErrorResponse(400, "durable name required");
|
||||
}
|
||||
|
||||
if (config.FilterSubjects.Count == 0 && !string.IsNullOrWhiteSpace(config.FilterSubject))
|
||||
config.FilterSubjects.Add(config.FilterSubject);
|
||||
|
||||
var key = (stream, config.DurableName);
|
||||
var handle = _consumers.AddOrUpdate(key,
|
||||
@@ -129,7 +138,15 @@ public sealed class ConsumerManager
|
||||
public void OnPublished(string stream, StoredMessage message)
|
||||
{
|
||||
foreach (var handle in _consumers.Values.Where(c => c.Stream == stream && c.Config.Push))
|
||||
{
|
||||
if (!MatchesFilter(handle.Config, message.Subject))
|
||||
continue;
|
||||
|
||||
if (handle.Config.MaxAckPending > 0 && handle.AckProcessor.PendingCount >= handle.Config.MaxAckPending)
|
||||
continue;
|
||||
|
||||
_pushConsumerEngine.Enqueue(handle, message);
|
||||
}
|
||||
}
|
||||
|
||||
public PushFrame? ReadPushFrame(string stream, string durableName)
|
||||
@@ -142,6 +159,17 @@ public sealed class ConsumerManager
|
||||
|
||||
return consumer.PushFrames.Dequeue();
|
||||
}
|
||||
|
||||
private static bool MatchesFilter(ConsumerConfig config, string subject)
|
||||
{
|
||||
if (config.FilterSubjects.Count > 0)
|
||||
return config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(subject, f));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(config.FilterSubject))
|
||||
return SubjectMatch.MatchLiteral(subject, config.FilterSubject);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record ConsumerHandle(string Stream, ConsumerConfig Config)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using NATS.Server.JetStream.Storage;
|
||||
using NATS.Server.JetStream.Models;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.JetStream.Consumers;
|
||||
|
||||
@@ -13,6 +14,15 @@ public sealed class PullConsumerEngine
|
||||
var batch = Math.Max(request.Batch, 1);
|
||||
var messages = new List<StoredMessage>(batch);
|
||||
|
||||
if (consumer.NextSequence == 1)
|
||||
{
|
||||
var state = await stream.Store.GetStateAsync(ct);
|
||||
if (consumer.Config.DeliverPolicy == DeliverPolicy.Last && state.LastSeq > 0)
|
||||
consumer.NextSequence = state.LastSeq;
|
||||
else if (consumer.Config.DeliverPolicy == DeliverPolicy.New && state.LastSeq > 0)
|
||||
consumer.NextSequence = state.LastSeq + 1;
|
||||
}
|
||||
|
||||
if (request.NoWait)
|
||||
{
|
||||
var available = await stream.Store.LoadAsync(consumer.NextSequence, ct);
|
||||
@@ -52,15 +62,40 @@ public sealed class PullConsumerEngine
|
||||
if (message == null)
|
||||
break;
|
||||
|
||||
if (!MatchesFilter(consumer.Config, message.Subject))
|
||||
{
|
||||
sequence++;
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (consumer.Config.ReplayPolicy == ReplayPolicy.Original)
|
||||
await Task.Delay(50, ct);
|
||||
|
||||
messages.Add(message);
|
||||
if (consumer.Config.AckPolicy is AckPolicy.Explicit or AckPolicy.All)
|
||||
{
|
||||
if (consumer.Config.MaxAckPending > 0 && consumer.AckProcessor.PendingCount >= consumer.Config.MaxAckPending)
|
||||
break;
|
||||
consumer.AckProcessor.Register(message.Sequence, consumer.Config.AckWaitMs);
|
||||
}
|
||||
sequence++;
|
||||
}
|
||||
|
||||
consumer.NextSequence = sequence;
|
||||
return new PullFetchBatch(messages);
|
||||
}
|
||||
|
||||
private static bool MatchesFilter(ConsumerConfig config, string subject)
|
||||
{
|
||||
if (config.FilterSubjects.Count > 0)
|
||||
return config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(subject, f));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(config.FilterSubject))
|
||||
return SubjectMatch.MatchLiteral(subject, config.FilterSubject);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class PullFetchBatch
|
||||
|
||||
@@ -3,12 +3,15 @@ namespace NATS.Server.JetStream.Models;
|
||||
public sealed class ConsumerConfig
|
||||
{
|
||||
public string DurableName { get; set; } = string.Empty;
|
||||
public bool Ephemeral { get; set; }
|
||||
public string? FilterSubject { get; set; }
|
||||
public List<string> FilterSubjects { get; set; } = [];
|
||||
public AckPolicy AckPolicy { get; set; } = AckPolicy.None;
|
||||
public DeliverPolicy DeliverPolicy { get; set; } = DeliverPolicy.All;
|
||||
public ReplayPolicy ReplayPolicy { get; set; } = ReplayPolicy.Instant;
|
||||
public int AckWaitMs { get; set; } = 30_000;
|
||||
public int MaxDeliver { get; set; } = 1;
|
||||
public int MaxAckPending { get; set; }
|
||||
public bool Push { get; set; }
|
||||
public int HeartbeatMs { get; set; }
|
||||
}
|
||||
|
||||
@@ -5,10 +5,26 @@ public sealed class StreamConfig
|
||||
public string Name { get; set; } = string.Empty;
|
||||
public List<string> Subjects { get; set; } = [];
|
||||
public int MaxMsgs { get; set; }
|
||||
public long MaxBytes { get; set; }
|
||||
public int MaxMsgsPer { get; set; }
|
||||
public int MaxAgeMs { get; set; }
|
||||
public int MaxConsumers { get; set; }
|
||||
public RetentionPolicy Retention { get; set; } = RetentionPolicy.Limits;
|
||||
public DiscardPolicy Discard { get; set; } = DiscardPolicy.Old;
|
||||
public StorageType Storage { get; set; } = StorageType.Memory;
|
||||
public int Replicas { get; set; } = 1;
|
||||
public string? Mirror { get; set; }
|
||||
public string? Source { get; set; }
|
||||
public List<StreamSourceConfig> Sources { get; set; } = [];
|
||||
}
|
||||
|
||||
public enum StorageType
|
||||
{
|
||||
Memory,
|
||||
File,
|
||||
}
|
||||
|
||||
public sealed class StreamSourceConfig
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ public sealed class StreamState
|
||||
public ulong Messages { get; set; }
|
||||
public ulong FirstSeq { get; set; }
|
||||
public ulong LastSeq { get; set; }
|
||||
public ulong Bytes { get; set; }
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
Sequence = _last,
|
||||
Subject = subject,
|
||||
Payload = payload.ToArray(),
|
||||
TimestampUtc = DateTime.UtcNow,
|
||||
};
|
||||
_messages[_last] = stored;
|
||||
|
||||
@@ -32,6 +33,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
Sequence = stored.Sequence,
|
||||
Subject = stored.Subject,
|
||||
PayloadBase64 = Convert.ToBase64String(stored.Payload.ToArray()),
|
||||
TimestampUtc = stored.TimestampUtc,
|
||||
});
|
||||
await File.AppendAllTextAsync(_dataFilePath, line + Environment.NewLine, ct);
|
||||
return _last;
|
||||
@@ -79,6 +81,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
Sequence = x.Sequence,
|
||||
Subject = x.Subject,
|
||||
PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()),
|
||||
TimestampUtc = x.TimestampUtc,
|
||||
})
|
||||
.ToArray();
|
||||
return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot));
|
||||
@@ -101,6 +104,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
Sequence = record.Sequence,
|
||||
Subject = record.Subject ?? string.Empty,
|
||||
Payload = Convert.FromBase64String(record.PayloadBase64 ?? string.Empty),
|
||||
TimestampUtc = record.TimestampUtc,
|
||||
};
|
||||
_messages[record.Sequence] = message;
|
||||
_last = Math.Max(_last, record.Sequence);
|
||||
@@ -119,6 +123,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
Messages = (ulong)_messages.Count,
|
||||
FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(),
|
||||
LastSeq = _last,
|
||||
Bytes = (ulong)_messages.Values.Sum(m => m.Payload.Length),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -172,6 +177,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
Sequence = message.Sequence,
|
||||
Subject = message.Subject,
|
||||
PayloadBase64 = Convert.ToBase64String(message.Payload.ToArray()),
|
||||
TimestampUtc = message.TimestampUtc,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -183,5 +189,6 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
|
||||
public ulong Sequence { get; init; }
|
||||
public string? Subject { get; init; }
|
||||
public string? PayloadBase64 { get; init; }
|
||||
public DateTime TimestampUtc { get; init; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ public sealed class MemStore : IStreamStore
|
||||
public ulong Sequence { get; init; }
|
||||
public string Subject { get; init; } = string.Empty;
|
||||
public string PayloadBase64 { get; init; } = string.Empty;
|
||||
public DateTime TimestampUtc { get; init; }
|
||||
}
|
||||
|
||||
private readonly object _gate = new();
|
||||
@@ -26,6 +27,7 @@ public sealed class MemStore : IStreamStore
|
||||
Sequence = _last,
|
||||
Subject = subject,
|
||||
Payload = payload,
|
||||
TimestampUtc = DateTime.UtcNow,
|
||||
};
|
||||
return ValueTask.FromResult(_last);
|
||||
}
|
||||
@@ -82,6 +84,7 @@ public sealed class MemStore : IStreamStore
|
||||
Sequence = x.Sequence,
|
||||
Subject = x.Subject,
|
||||
PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()),
|
||||
TimestampUtc = x.TimestampUtc,
|
||||
})
|
||||
.ToArray();
|
||||
return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot));
|
||||
@@ -107,6 +110,7 @@ public sealed class MemStore : IStreamStore
|
||||
Sequence = record.Sequence,
|
||||
Subject = record.Subject,
|
||||
Payload = Convert.FromBase64String(record.PayloadBase64),
|
||||
TimestampUtc = record.TimestampUtc,
|
||||
};
|
||||
_last = Math.Max(_last, record.Sequence);
|
||||
}
|
||||
@@ -126,6 +130,7 @@ public sealed class MemStore : IStreamStore
|
||||
Messages = (ulong)_messages.Count,
|
||||
FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(),
|
||||
LastSeq = _last,
|
||||
Bytes = (ulong)_messages.Values.Sum(m => m.Payload.Length),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,5 +5,6 @@ public sealed class StoredMessage
|
||||
public ulong Sequence { get; init; }
|
||||
public string Subject { get; init; } = string.Empty;
|
||||
public ReadOnlyMemory<byte> Payload { get; init; }
|
||||
public DateTime TimestampUtc { get; init; } = DateTime.UtcNow;
|
||||
public bool Redelivered { get; init; }
|
||||
}
|
||||
|
||||
@@ -48,8 +48,14 @@ public sealed class StreamManager
|
||||
|
||||
var handle = _streams.AddOrUpdate(
|
||||
normalized.Name,
|
||||
_ => new StreamHandle(normalized, new MemStore()),
|
||||
(_, existing) => existing with { Config = normalized });
|
||||
_ => new StreamHandle(normalized, CreateStore(normalized)),
|
||||
(_, existing) =>
|
||||
{
|
||||
if (existing.Config.Storage == normalized.Storage)
|
||||
return existing with { Config = normalized };
|
||||
|
||||
return new StreamHandle(normalized, CreateStore(normalized));
|
||||
});
|
||||
_replicaGroups.AddOrUpdate(
|
||||
normalized.Name,
|
||||
_ => new StreamReplicaGroup(normalized.Name, normalized.Replicas),
|
||||
@@ -150,6 +156,25 @@ public sealed class StreamManager
|
||||
if (stream == null)
|
||||
return null;
|
||||
|
||||
var stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
||||
if (stream.Config.MaxBytes > 0 && (long)stateBefore.Bytes + payload.Length > stream.Config.MaxBytes)
|
||||
{
|
||||
if (stream.Config.Discard == DiscardPolicy.New)
|
||||
{
|
||||
return new PubAck
|
||||
{
|
||||
Stream = stream.Config.Name,
|
||||
ErrorCode = 10054,
|
||||
};
|
||||
}
|
||||
|
||||
while ((long)stateBefore.Bytes + payload.Length > stream.Config.MaxBytes && stateBefore.FirstSeq > 0)
|
||||
{
|
||||
stream.Store.RemoveAsync(stateBefore.FirstSeq, default).GetAwaiter().GetResult();
|
||||
stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
||||
}
|
||||
}
|
||||
|
||||
if (_replicaGroups.TryGetValue(stream.Config.Name, out var replicaGroup))
|
||||
_ = replicaGroup.ProposeAsync($"PUB {subject}", default).GetAwaiter().GetResult();
|
||||
|
||||
@@ -181,12 +206,17 @@ public sealed class StreamManager
|
||||
Name = config.Name,
|
||||
Subjects = config.Subjects.Count == 0 ? [] : [.. config.Subjects],
|
||||
MaxMsgs = config.MaxMsgs,
|
||||
MaxBytes = config.MaxBytes,
|
||||
MaxMsgsPer = config.MaxMsgsPer,
|
||||
MaxAgeMs = config.MaxAgeMs,
|
||||
MaxConsumers = config.MaxConsumers,
|
||||
Retention = config.Retention,
|
||||
Discard = config.Discard,
|
||||
Storage = config.Storage,
|
||||
Replicas = config.Replicas,
|
||||
Mirror = config.Mirror,
|
||||
Source = config.Source,
|
||||
Sources = config.Sources.Count == 0 ? [] : [.. config.Sources.Select(s => new StreamSourceConfig { Name = s.Name })],
|
||||
};
|
||||
|
||||
return copy;
|
||||
@@ -241,6 +271,18 @@ public sealed class StreamManager
|
||||
var list = _sourcesByOrigin.GetOrAdd(stream.Config.Source, _ => []);
|
||||
list.Add(new SourceCoordinator(stream.Store));
|
||||
}
|
||||
|
||||
if (stream.Config.Sources.Count > 0)
|
||||
{
|
||||
foreach (var source in stream.Config.Sources)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(source.Name) || !_streams.TryGetValue(source.Name, out _))
|
||||
continue;
|
||||
|
||||
var list = _sourcesByOrigin.GetOrAdd(source.Name, _ => []);
|
||||
list.Add(new SourceCoordinator(stream.Store));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,6 +300,30 @@ public sealed class StreamManager
|
||||
source.OnOriginAppendAsync(stored, default).GetAwaiter().GetResult();
|
||||
}
|
||||
}
|
||||
|
||||
public string GetStoreBackendType(string streamName)
|
||||
{
|
||||
if (!_streams.TryGetValue(streamName, out var stream))
|
||||
return "missing";
|
||||
|
||||
return stream.Store switch
|
||||
{
|
||||
FileStore => "file",
|
||||
_ => "memory",
|
||||
};
|
||||
}
|
||||
|
||||
private static IStreamStore CreateStore(StreamConfig config)
|
||||
{
|
||||
return config.Storage switch
|
||||
{
|
||||
StorageType.File => new FileStore(new FileStoreOptions
|
||||
{
|
||||
Directory = Path.Combine(Path.GetTempPath(), "natsdotnet-js-store", config.Name),
|
||||
}),
|
||||
_ => new MemStore(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record StreamHandle(StreamConfig Config, IStreamStore Store);
|
||||
|
||||
@@ -1,11 +1,191 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.LeafNodes;
|
||||
|
||||
public sealed class LeafConnection
|
||||
public sealed class LeafConnection(Socket socket) : IAsyncDisposable
|
||||
{
|
||||
public string RemoteEndpoint { get; }
|
||||
private readonly NetworkStream _stream = new(socket, ownsSocket: true);
|
||||
private readonly SemaphoreSlim _writeGate = new(1, 1);
|
||||
private readonly CancellationTokenSource _closedCts = new();
|
||||
private Task? _loopTask;
|
||||
|
||||
public LeafConnection(string remoteEndpoint)
|
||||
public string? RemoteId { get; private set; }
|
||||
public string RemoteEndpoint => socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N");
|
||||
public Func<RemoteSubscription, Task>? RemoteSubscriptionReceived { get; set; }
|
||||
public Func<LeafMessage, Task>? MessageReceived { get; set; }
|
||||
|
||||
public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct)
|
||||
{
|
||||
RemoteEndpoint = remoteEndpoint;
|
||||
await WriteLineAsync($"LEAF {serverId}", ct);
|
||||
var line = await ReadLineAsync(ct);
|
||||
RemoteId = ParseHandshake(line);
|
||||
}
|
||||
|
||||
public async Task PerformInboundHandshakeAsync(string serverId, CancellationToken ct)
|
||||
{
|
||||
var line = await ReadLineAsync(ct);
|
||||
RemoteId = ParseHandshake(line);
|
||||
await WriteLineAsync($"LEAF {serverId}", ct);
|
||||
}
|
||||
|
||||
public void StartLoop(CancellationToken ct)
|
||||
{
|
||||
if (_loopTask != null)
|
||||
return;
|
||||
|
||||
var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
|
||||
_loopTask = Task.Run(() => ReadLoopAsync(linked.Token), linked.Token);
|
||||
}
|
||||
|
||||
public Task WaitUntilClosedAsync(CancellationToken ct)
|
||||
=> _loopTask?.WaitAsync(ct) ?? Task.CompletedTask;
|
||||
|
||||
public Task SendLsPlusAsync(string subject, string? queue, CancellationToken ct)
|
||||
=> WriteLineAsync(queue is { Length: > 0 } ? $"LS+ {subject} {queue}" : $"LS+ {subject}", ct);
|
||||
|
||||
public Task SendLsMinusAsync(string subject, string? queue, CancellationToken ct)
|
||||
=> WriteLineAsync(queue is { Length: > 0 } ? $"LS- {subject} {queue}" : $"LS- {subject}", ct);
|
||||
|
||||
public async Task SendMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
|
||||
{
|
||||
var reply = string.IsNullOrEmpty(replyTo) ? "-" : replyTo;
|
||||
await _writeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var control = Encoding.ASCII.GetBytes($"LMSG {subject} {reply} {payload.Length}\r\n");
|
||||
await _stream.WriteAsync(control, ct);
|
||||
if (!payload.IsEmpty)
|
||||
await _stream.WriteAsync(payload, ct);
|
||||
await _stream.WriteAsync("\r\n"u8.ToArray(), ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _closedCts.CancelAsync();
|
||||
if (_loopTask != null)
|
||||
await _loopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
||||
_closedCts.Dispose();
|
||||
_writeGate.Dispose();
|
||||
await _stream.DisposeAsync();
|
||||
}
|
||||
|
||||
private async Task ReadLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
string line;
|
||||
try
|
||||
{
|
||||
line = await ReadLineAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (line.StartsWith("LS+ ", StringComparison.Ordinal))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
|
||||
{
|
||||
var queue = parts.Length >= 3 ? parts[2] : null;
|
||||
await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteId ?? string.Empty));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("LS- ", StringComparison.Ordinal))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
|
||||
{
|
||||
var queue = parts.Length >= 3 ? parts[2] : null;
|
||||
await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteId ?? string.Empty));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!line.StartsWith("LMSG ", StringComparison.Ordinal))
|
||||
continue;
|
||||
|
||||
var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (args.Length < 4 || !int.TryParse(args[3], out var size) || size < 0)
|
||||
continue;
|
||||
|
||||
var payload = await ReadPayloadAsync(size, ct);
|
||||
if (MessageReceived != null)
|
||||
await MessageReceived(new LeafMessage(args[1], args[2] == "-" ? null : args[2], payload));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ReadOnlyMemory<byte>> ReadPayloadAsync(int size, CancellationToken ct)
|
||||
{
|
||||
var payload = new byte[size];
|
||||
var offset = 0;
|
||||
while (offset < size)
|
||||
{
|
||||
var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Leaf payload read closed");
|
||||
offset += read;
|
||||
}
|
||||
|
||||
var trailer = new byte[2];
|
||||
_ = await _stream.ReadAsync(trailer, ct);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private async Task WriteLineAsync(string line, CancellationToken ct)
|
||||
{
|
||||
await _writeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
|
||||
await _stream.WriteAsync(bytes, ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> ReadLineAsync(CancellationToken ct)
|
||||
{
|
||||
var bytes = new List<byte>(64);
|
||||
var single = new byte[1];
|
||||
while (true)
|
||||
{
|
||||
var read = await _stream.ReadAsync(single, ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Leaf closed");
|
||||
if (single[0] == (byte)'\n')
|
||||
break;
|
||||
if (single[0] != (byte)'\r')
|
||||
bytes.Add(single[0]);
|
||||
}
|
||||
|
||||
return Encoding.ASCII.GetString([.. bytes]);
|
||||
}
|
||||
|
||||
private static string ParseHandshake(string line)
|
||||
{
|
||||
if (!line.StartsWith("LEAF ", StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException("Invalid leaf handshake");
|
||||
|
||||
var id = line[5..].Trim();
|
||||
if (id.Length == 0)
|
||||
throw new InvalidOperationException("Leaf handshake missing id");
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record LeafMessage(string Subject, string? ReplyTo, ReadOnlyMemory<byte> Payload);
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NATS.Server.Configuration;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.LeafNodes;
|
||||
|
||||
@@ -7,25 +11,203 @@ public sealed class LeafNodeManager : IAsyncDisposable
|
||||
{
|
||||
private readonly LeafNodeOptions _options;
|
||||
private readonly ServerStats _stats;
|
||||
private readonly string _serverId;
|
||||
private readonly Action<RemoteSubscription> _remoteSubSink;
|
||||
private readonly Action<LeafMessage> _messageSink;
|
||||
private readonly ILogger<LeafNodeManager> _logger;
|
||||
private readonly ConcurrentDictionary<string, LeafConnection> _connections = new(StringComparer.Ordinal);
|
||||
|
||||
public LeafNodeManager(LeafNodeOptions options, ServerStats stats, ILogger<LeafNodeManager> logger)
|
||||
private CancellationTokenSource? _cts;
|
||||
private Socket? _listener;
|
||||
private Task? _acceptLoopTask;
|
||||
|
||||
public string ListenEndpoint => $"{_options.Host}:{_options.Port}";
|
||||
|
||||
public LeafNodeManager(
|
||||
LeafNodeOptions options,
|
||||
ServerStats stats,
|
||||
string serverId,
|
||||
Action<RemoteSubscription> remoteSubSink,
|
||||
Action<LeafMessage> messageSink,
|
||||
ILogger<LeafNodeManager> logger)
|
||||
{
|
||||
_options = options;
|
||||
_stats = stats;
|
||||
_serverId = serverId;
|
||||
_remoteSubSink = remoteSubSink;
|
||||
_messageSink = messageSink;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken ct)
|
||||
{
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
_listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
_listener.Bind(new IPEndPoint(IPAddress.Parse(_options.Host), _options.Port));
|
||||
_listener.Listen(128);
|
||||
|
||||
if (_options.Port == 0)
|
||||
_options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
|
||||
|
||||
_acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
|
||||
foreach (var remote in _options.Remotes.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
_ = Task.Run(() => ConnectWithRetryAsync(remote, _cts.Token));
|
||||
|
||||
_logger.LogDebug("Leaf manager started (listen={Host}:{Port})", _options.Host, _options.Port);
|
||||
Interlocked.Exchange(ref _stats.Leafs, 0);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
public async Task ForwardMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
|
||||
{
|
||||
foreach (var connection in _connections.Values)
|
||||
await connection.SendMessageAsync(subject, replyTo, payload, ct);
|
||||
}
|
||||
|
||||
public void PropagateLocalSubscription(string subject, string? queue)
|
||||
{
|
||||
foreach (var connection in _connections.Values)
|
||||
_ = connection.SendLsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
|
||||
}
|
||||
|
||||
public void PropagateLocalUnsubscription(string subject, string? queue)
|
||||
{
|
||||
foreach (var connection in _connections.Values)
|
||||
_ = connection.SendLsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_cts == null)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
_listener?.Dispose();
|
||||
if (_acceptLoopTask != null)
|
||||
await _acceptLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
||||
|
||||
foreach (var connection in _connections.Values)
|
||||
await connection.DisposeAsync();
|
||||
_connections.Clear();
|
||||
Interlocked.Exchange(ref _stats.Leafs, 0);
|
||||
_cts.Dispose();
|
||||
_cts = null;
|
||||
_logger.LogDebug("Leaf manager stopped");
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task AcceptLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
Socket socket;
|
||||
try
|
||||
{
|
||||
socket = await _listener!.AcceptAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
_ = Task.Run(() => HandleInboundAsync(socket, ct), ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleInboundAsync(Socket socket, CancellationToken ct)
|
||||
{
|
||||
var connection = new LeafConnection(socket);
|
||||
try
|
||||
{
|
||||
await connection.PerformInboundHandshakeAsync(_serverId, ct);
|
||||
Register(connection);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await connection.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ConnectWithRetryAsync(string remote, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var endPoint = ParseEndpoint(remote);
|
||||
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct);
|
||||
var connection = new LeafConnection(socket);
|
||||
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
|
||||
Register(connection);
|
||||
return;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Leaf connect retry for {Remote}", remote);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(250, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void Register(LeafConnection connection)
|
||||
{
|
||||
var key = $"{connection.RemoteId}:{connection.RemoteEndpoint}:{Guid.NewGuid():N}";
|
||||
if (!_connections.TryAdd(key, connection))
|
||||
{
|
||||
_ = connection.DisposeAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
connection.RemoteSubscriptionReceived = sub =>
|
||||
{
|
||||
_remoteSubSink(sub);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
connection.MessageReceived = msg =>
|
||||
{
|
||||
_messageSink(msg);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
connection.StartLoop(_cts!.Token);
|
||||
Interlocked.Increment(ref _stats.Leafs);
|
||||
_ = Task.Run(() => WatchConnectionAsync(key, connection, _cts!.Token));
|
||||
}
|
||||
|
||||
private async Task WatchConnectionAsync(string key, LeafConnection connection, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await connection.WaitUntilClosedAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_connections.TryRemove(key, out _))
|
||||
Interlocked.Decrement(ref _stats.Leafs);
|
||||
await connection.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private static IPEndPoint ParseEndpoint(string endpoint)
|
||||
{
|
||||
var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length != 2)
|
||||
throw new FormatException($"Invalid endpoint: {endpoint}");
|
||||
|
||||
return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
|
||||
}
|
||||
}
|
||||
|
||||
46
src/NATS.Server/Monitoring/AccountzHandler.cs
Normal file
46
src/NATS.Server/Monitoring/AccountzHandler.cs
Normal file
@@ -0,0 +1,46 @@
|
||||
using NATS.Server.Auth;
|
||||
|
||||
namespace NATS.Server.Monitoring;
|
||||
|
||||
public sealed class AccountzHandler
|
||||
{
|
||||
private readonly NatsServer _server;
|
||||
|
||||
public AccountzHandler(NatsServer server)
|
||||
{
|
||||
_server = server;
|
||||
}
|
||||
|
||||
public object Build()
|
||||
{
|
||||
var accounts = _server.GetAccounts().Select(ToAccountDto).ToArray();
|
||||
return new
|
||||
{
|
||||
accounts,
|
||||
num_accounts = accounts.Length,
|
||||
};
|
||||
}
|
||||
|
||||
public object BuildStats()
|
||||
{
|
||||
var accounts = _server.GetAccounts().ToArray();
|
||||
return new
|
||||
{
|
||||
total_accounts = accounts.Length,
|
||||
total_connections = accounts.Sum(a => a.ClientCount),
|
||||
total_subscriptions = accounts.Sum(a => a.SubscriptionCount),
|
||||
};
|
||||
}
|
||||
|
||||
private static object ToAccountDto(Account account)
|
||||
{
|
||||
return new
|
||||
{
|
||||
name = account.Name,
|
||||
connections = account.ClientCount,
|
||||
subscriptions = account.SubscriptionCount,
|
||||
in_msgs = account.InMsgs,
|
||||
out_msgs = account.OutMsgs,
|
||||
};
|
||||
}
|
||||
}
|
||||
21
src/NATS.Server/Monitoring/GatewayzHandler.cs
Normal file
21
src/NATS.Server/Monitoring/GatewayzHandler.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
namespace NATS.Server.Monitoring;
|
||||
|
||||
public sealed class GatewayzHandler
|
||||
{
|
||||
private readonly NatsServer _server;
|
||||
|
||||
public GatewayzHandler(NatsServer server)
|
||||
{
|
||||
_server = server;
|
||||
}
|
||||
|
||||
public object Build()
|
||||
{
|
||||
var gateways = _server.Stats.Gateways;
|
||||
return new
|
||||
{
|
||||
gateways,
|
||||
num_gateways = gateways,
|
||||
};
|
||||
}
|
||||
}
|
||||
21
src/NATS.Server/Monitoring/LeafzHandler.cs
Normal file
21
src/NATS.Server/Monitoring/LeafzHandler.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
namespace NATS.Server.Monitoring;
|
||||
|
||||
public sealed class LeafzHandler
|
||||
{
|
||||
private readonly NatsServer _server;
|
||||
|
||||
public LeafzHandler(NatsServer server)
|
||||
{
|
||||
_server = server;
|
||||
}
|
||||
|
||||
public object Build()
|
||||
{
|
||||
var leafs = _server.Stats.Leafs;
|
||||
return new
|
||||
{
|
||||
leafs,
|
||||
num_leafs = leafs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,10 @@ public sealed class MonitorServer : IAsyncDisposable
|
||||
private readonly ConnzHandler _connzHandler;
|
||||
private readonly SubszHandler _subszHandler;
|
||||
private readonly JszHandler _jszHandler;
|
||||
private readonly RoutezHandler _routezHandler;
|
||||
private readonly GatewayzHandler _gatewayzHandler;
|
||||
private readonly LeafzHandler _leafzHandler;
|
||||
private readonly AccountzHandler _accountzHandler;
|
||||
|
||||
public MonitorServer(NatsServer server, NatsOptions options, ServerStats stats, ILoggerFactory loggerFactory)
|
||||
{
|
||||
@@ -33,6 +37,10 @@ public sealed class MonitorServer : IAsyncDisposable
|
||||
_connzHandler = new ConnzHandler(server);
|
||||
_subszHandler = new SubszHandler(server);
|
||||
_jszHandler = new JszHandler(server, options);
|
||||
_routezHandler = new RoutezHandler(server);
|
||||
_gatewayzHandler = new GatewayzHandler(server);
|
||||
_leafzHandler = new LeafzHandler(server);
|
||||
_accountzHandler = new AccountzHandler(server);
|
||||
|
||||
_app.MapGet(basePath + "/", () =>
|
||||
{
|
||||
@@ -63,21 +71,20 @@ public sealed class MonitorServer : IAsyncDisposable
|
||||
return Results.Ok(_connzHandler.HandleConnz(ctx));
|
||||
});
|
||||
|
||||
// Stubs for unimplemented endpoints
|
||||
_app.MapGet(basePath + "/routez", () =>
|
||||
{
|
||||
stats.HttpReqStats.AddOrUpdate("/routez", 1, (_, v) => v + 1);
|
||||
return Results.Ok(new { });
|
||||
return Results.Ok(_routezHandler.Build());
|
||||
});
|
||||
_app.MapGet(basePath + "/gatewayz", () =>
|
||||
{
|
||||
stats.HttpReqStats.AddOrUpdate("/gatewayz", 1, (_, v) => v + 1);
|
||||
return Results.Ok(new { });
|
||||
return Results.Ok(_gatewayzHandler.Build());
|
||||
});
|
||||
_app.MapGet(basePath + "/leafz", () =>
|
||||
{
|
||||
stats.HttpReqStats.AddOrUpdate("/leafz", 1, (_, v) => v + 1);
|
||||
return Results.Ok(new { });
|
||||
return Results.Ok(_leafzHandler.Build());
|
||||
});
|
||||
_app.MapGet(basePath + "/subz", (HttpContext ctx) =>
|
||||
{
|
||||
@@ -92,12 +99,12 @@ public sealed class MonitorServer : IAsyncDisposable
|
||||
_app.MapGet(basePath + "/accountz", () =>
|
||||
{
|
||||
stats.HttpReqStats.AddOrUpdate("/accountz", 1, (_, v) => v + 1);
|
||||
return Results.Ok(new { });
|
||||
return Results.Ok(_accountzHandler.Build());
|
||||
});
|
||||
_app.MapGet(basePath + "/accstatz", () =>
|
||||
{
|
||||
stats.HttpReqStats.AddOrUpdate("/accstatz", 1, (_, v) => v + 1);
|
||||
return Results.Ok(new { });
|
||||
return Results.Ok(_accountzHandler.BuildStats());
|
||||
});
|
||||
_app.MapGet(basePath + "/jsz", () =>
|
||||
{
|
||||
|
||||
21
src/NATS.Server/Monitoring/RoutezHandler.cs
Normal file
21
src/NATS.Server/Monitoring/RoutezHandler.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
namespace NATS.Server.Monitoring;
|
||||
|
||||
public sealed class RoutezHandler
|
||||
{
|
||||
private readonly NatsServer _server;
|
||||
|
||||
public RoutezHandler(NatsServer server)
|
||||
{
|
||||
_server = server;
|
||||
}
|
||||
|
||||
public object Build()
|
||||
{
|
||||
var routes = _server.Stats.Routes;
|
||||
return new
|
||||
{
|
||||
routes,
|
||||
num_routes = routes,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -562,6 +562,8 @@ public sealed class NatsClient : INatsClient, IDisposable
|
||||
Account?.DecrementSubscriptions();
|
||||
|
||||
Account?.SubList.Remove(sub);
|
||||
if (Router is NatsServer server)
|
||||
server.OnLocalUnsubscription(sub.Subject, sub.Queue);
|
||||
}
|
||||
|
||||
private void ProcessPub(ParsedCommand cmd, ref long localInMsgs, ref long localInBytes)
|
||||
|
||||
@@ -95,6 +95,8 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
||||
public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0;
|
||||
public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0;
|
||||
public string? ClusterListen => _routeManager?.ListenEndpoint;
|
||||
public string? GatewayListen => _gatewayManager?.ListenEndpoint;
|
||||
public string? LeafListen => _leafNodeManager?.ListenEndpoint;
|
||||
public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter;
|
||||
public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0;
|
||||
public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0;
|
||||
@@ -366,18 +368,21 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
||||
if (options.Cluster != null)
|
||||
{
|
||||
_routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
|
||||
ProcessRoutedMessage,
|
||||
_loggerFactory.CreateLogger<RouteManager>());
|
||||
}
|
||||
|
||||
if (options.Gateway != null)
|
||||
{
|
||||
_gatewayManager = new GatewayManager(options.Gateway, _stats,
|
||||
_gatewayManager = new GatewayManager(options.Gateway, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
|
||||
ProcessGatewayMessage,
|
||||
_loggerFactory.CreateLogger<GatewayManager>());
|
||||
}
|
||||
|
||||
if (options.LeafNode != null)
|
||||
{
|
||||
_leafNodeManager = new LeafNodeManager(options.LeafNode, _stats,
|
||||
_leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
|
||||
ProcessLeafMessage,
|
||||
_loggerFactory.CreateLogger<LeafNodeManager>());
|
||||
}
|
||||
|
||||
@@ -796,6 +801,15 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
||||
public void OnLocalSubscription(string subject, string? queue)
|
||||
{
|
||||
_routeManager?.PropagateLocalSubscription(subject, queue);
|
||||
_gatewayManager?.PropagateLocalSubscription(subject, queue);
|
||||
_leafNodeManager?.PropagateLocalSubscription(subject, queue);
|
||||
}
|
||||
|
||||
public void OnLocalUnsubscription(string subject, string? queue)
|
||||
{
|
||||
_routeManager?.PropagateLocalUnsubscription(subject, queue);
|
||||
_gatewayManager?.PropagateLocalUnsubscription(subject, queue);
|
||||
_leafNodeManager?.PropagateLocalUnsubscription(subject, queue);
|
||||
}
|
||||
|
||||
private void ApplyRemoteSubscription(RemoteSubscription sub)
|
||||
@@ -803,6 +817,38 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
||||
_globalAccount.SubList.ApplyRemoteSub(sub);
|
||||
}
|
||||
|
||||
private void ProcessRoutedMessage(RouteMessage message)
|
||||
{
|
||||
DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload);
|
||||
}
|
||||
|
||||
private void ProcessGatewayMessage(GatewayMessage message)
|
||||
{
|
||||
DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload);
|
||||
}
|
||||
|
||||
private void ProcessLeafMessage(LeafMessage message)
|
||||
{
|
||||
DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload);
|
||||
}
|
||||
|
||||
private void DeliverRemoteMessage(string subject, string? replyTo, ReadOnlyMemory<byte> payload)
|
||||
{
|
||||
var result = _globalAccount.SubList.Match(subject);
|
||||
|
||||
foreach (var sub in result.PlainSubs)
|
||||
DeliverMessage(sub, subject, replyTo, default, payload);
|
||||
|
||||
foreach (var queueGroup in result.QueueSubs)
|
||||
{
|
||||
if (queueGroup.Length == 0)
|
||||
continue;
|
||||
|
||||
var sub = queueGroup[0];
|
||||
DeliverMessage(sub, subject, replyTo, default, payload);
|
||||
}
|
||||
}
|
||||
|
||||
public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory<byte> headers,
|
||||
ReadOnlyMemory<byte> payload, NatsClient sender)
|
||||
{
|
||||
@@ -837,6 +883,13 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
if (_routeManager != null && _globalAccount.SubList.HasRemoteInterest(subject))
|
||||
_routeManager.ForwardRoutedMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult();
|
||||
if (_gatewayManager != null && _globalAccount.SubList.HasRemoteInterest(subject))
|
||||
_gatewayManager.ForwardMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult();
|
||||
if (_leafNodeManager != null && _globalAccount.SubList.HasRemoteInterest(subject))
|
||||
_leafNodeManager.ForwardMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult();
|
||||
|
||||
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
|
||||
var result = subList.Match(subject);
|
||||
var delivered = false;
|
||||
|
||||
@@ -10,7 +10,21 @@ public sealed class ClientCommandMatrix
|
||||
return (kind, op.ToUpperInvariant()) switch
|
||||
{
|
||||
(global::NATS.Server.ClientKind.Router, "RS+") => true,
|
||||
(global::NATS.Server.ClientKind.Router, "RS-") => true,
|
||||
(global::NATS.Server.ClientKind.Router, "RMSG") => true,
|
||||
(global::NATS.Server.ClientKind.Gateway, "A+") => true,
|
||||
(global::NATS.Server.ClientKind.Gateway, "A-") => true,
|
||||
(global::NATS.Server.ClientKind.Leaf, "LS+") => true,
|
||||
(global::NATS.Server.ClientKind.Leaf, "LS-") => true,
|
||||
(global::NATS.Server.ClientKind.Leaf, "LMSG") => true,
|
||||
(_, "RS+") => false,
|
||||
(_, "RS-") => false,
|
||||
(_, "RMSG") => false,
|
||||
(_, "A+") => false,
|
||||
(_, "A-") => false,
|
||||
(_, "LS+") => false,
|
||||
(_, "LS-") => false,
|
||||
(_, "LMSG") => false,
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -27,6 +27,36 @@ public sealed class RaftLog
|
||||
_entries.Clear();
|
||||
_baseIndex = snapshot.LastIncludedIndex;
|
||||
}
|
||||
|
||||
public async Task PersistAsync(string path, CancellationToken ct)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
var model = new PersistedLog
|
||||
{
|
||||
BaseIndex = _baseIndex,
|
||||
Entries = [.. _entries],
|
||||
};
|
||||
await File.WriteAllTextAsync(path, System.Text.Json.JsonSerializer.Serialize(model), ct);
|
||||
}
|
||||
|
||||
public static async Task<RaftLog> LoadAsync(string path, CancellationToken ct)
|
||||
{
|
||||
var log = new RaftLog();
|
||||
if (!File.Exists(path))
|
||||
return log;
|
||||
|
||||
var json = await File.ReadAllTextAsync(path, ct);
|
||||
var model = System.Text.Json.JsonSerializer.Deserialize<PersistedLog>(json) ?? new PersistedLog();
|
||||
log._baseIndex = model.BaseIndex;
|
||||
log._entries.AddRange(model.Entries);
|
||||
return log;
|
||||
}
|
||||
|
||||
private sealed class PersistedLog
|
||||
{
|
||||
public long BaseIndex { get; set; }
|
||||
public List<RaftLogEntry> Entries { get; set; } = [];
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record RaftLogEntry(long Index, int Term, string Command);
|
||||
|
||||
@@ -6,6 +6,8 @@ public sealed class RaftNode
|
||||
private readonly List<RaftNode> _cluster = [];
|
||||
private readonly RaftReplicator _replicator = new();
|
||||
private readonly RaftSnapshotStore _snapshotStore = new();
|
||||
private readonly IRaftTransport? _transport;
|
||||
private readonly string? _persistDirectory;
|
||||
|
||||
public string Id { get; }
|
||||
public int Term => TermState.CurrentTerm;
|
||||
@@ -13,11 +15,13 @@ public sealed class RaftNode
|
||||
public RaftRole Role { get; private set; } = RaftRole.Follower;
|
||||
public RaftTermState TermState { get; } = new();
|
||||
public long AppliedIndex { get; set; }
|
||||
public RaftLog Log { get; } = new();
|
||||
public RaftLog Log { get; private set; } = new();
|
||||
|
||||
public RaftNode(string id)
|
||||
public RaftNode(string id, IRaftTransport? transport = null, string? persistDirectory = null)
|
||||
{
|
||||
Id = id;
|
||||
_transport = transport;
|
||||
_persistDirectory = persistDirectory;
|
||||
}
|
||||
|
||||
public void ConfigureCluster(IEnumerable<RaftNode> peers)
|
||||
@@ -60,7 +64,8 @@ public sealed class RaftNode
|
||||
|
||||
var entry = Log.Append(TermState.CurrentTerm, command);
|
||||
var followers = _cluster.Where(n => n.Id != Id).ToList();
|
||||
var acknowledgements = _replicator.Replicate(entry, followers);
|
||||
var results = await _replicator.ReplicateAsync(Id, entry, followers, _transport, ct);
|
||||
var acknowledgements = results.Count(r => r.Success);
|
||||
|
||||
var quorum = (_cluster.Count / 2) + 1;
|
||||
if (acknowledgements + 1 >= quorum)
|
||||
@@ -68,9 +73,14 @@ public sealed class RaftNode
|
||||
AppliedIndex = entry.Index;
|
||||
foreach (var node in _cluster)
|
||||
node.AppliedIndex = Math.Max(node.AppliedIndex, entry.Index);
|
||||
|
||||
foreach (var node in _cluster.Where(n => n._persistDirectory != null))
|
||||
await node.PersistAsync(ct);
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
if (_persistDirectory != null)
|
||||
await PersistAsync(ct);
|
||||
|
||||
return entry.Index;
|
||||
}
|
||||
|
||||
@@ -120,4 +130,29 @@ public sealed class RaftNode
|
||||
if (_votesReceived >= quorum)
|
||||
Role = RaftRole.Leader;
|
||||
}
|
||||
|
||||
public async Task PersistAsync(CancellationToken ct)
|
||||
{
|
||||
var dir = _persistDirectory ?? Path.Combine(Path.GetTempPath(), "natsdotnet-raft", Id);
|
||||
Directory.CreateDirectory(dir);
|
||||
await Log.PersistAsync(Path.Combine(dir, "log.json"), ct);
|
||||
await File.WriteAllTextAsync(Path.Combine(dir, "term.txt"), TermState.CurrentTerm.ToString(), ct);
|
||||
await File.WriteAllTextAsync(Path.Combine(dir, "applied.txt"), AppliedIndex.ToString(), ct);
|
||||
}
|
||||
|
||||
public async Task LoadPersistedStateAsync(CancellationToken ct)
|
||||
{
|
||||
var dir = _persistDirectory ?? Path.Combine(Path.GetTempPath(), "natsdotnet-raft", Id);
|
||||
Log = await RaftLog.LoadAsync(Path.Combine(dir, "log.json"), ct);
|
||||
|
||||
var termPath = Path.Combine(dir, "term.txt");
|
||||
if (File.Exists(termPath) && int.TryParse(await File.ReadAllTextAsync(termPath, ct), out var term))
|
||||
TermState.CurrentTerm = term;
|
||||
|
||||
var appliedPath = Path.Combine(dir, "applied.txt");
|
||||
if (File.Exists(appliedPath) && long.TryParse(await File.ReadAllTextAsync(appliedPath, ct), out var applied))
|
||||
AppliedIndex = applied;
|
||||
else if (Log.Entries.Count > 0)
|
||||
AppliedIndex = Log.Entries[^1].Index;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,4 +13,24 @@ public sealed class RaftReplicator
|
||||
|
||||
return acknowledgements;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AppendResult>> ReplicateAsync(
|
||||
string leaderId,
|
||||
RaftLogEntry entry,
|
||||
IReadOnlyList<RaftNode> followers,
|
||||
IRaftTransport? transport,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (transport != null)
|
||||
return await transport.AppendEntriesAsync(leaderId, followers.Select(f => f.Id).ToArray(), entry, ct);
|
||||
|
||||
var results = new List<AppendResult>(followers.Count);
|
||||
foreach (var follower in followers)
|
||||
{
|
||||
follower.ReceiveReplicatedEntry(entry);
|
||||
results.Add(new AppendResult { FollowerId = follower.Id, Success = true });
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,3 +10,9 @@ public sealed class VoteResponse
|
||||
{
|
||||
public bool Granted { get; init; }
|
||||
}
|
||||
|
||||
public sealed class AppendResult
|
||||
{
|
||||
public string FollowerId { get; init; } = string.Empty;
|
||||
public bool Success { get; init; }
|
||||
}
|
||||
|
||||
44
src/NATS.Server/Raft/RaftTransport.cs
Normal file
44
src/NATS.Server/Raft/RaftTransport.cs
Normal file
@@ -0,0 +1,44 @@
|
||||
namespace NATS.Server.Raft;
|
||||
|
||||
public interface IRaftTransport
|
||||
{
|
||||
Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct);
|
||||
Task<VoteResponse> RequestVoteAsync(string candidateId, string voterId, VoteRequest request, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed class InMemoryRaftTransport : IRaftTransport
|
||||
{
|
||||
private readonly Dictionary<string, RaftNode> _nodes = new(StringComparer.Ordinal);
|
||||
|
||||
public void Register(RaftNode node)
|
||||
{
|
||||
_nodes[node.Id] = node;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
|
||||
{
|
||||
var results = new List<AppendResult>(followerIds.Count);
|
||||
foreach (var followerId in followerIds)
|
||||
{
|
||||
if (_nodes.TryGetValue(followerId, out var node))
|
||||
{
|
||||
node.ReceiveReplicatedEntry(entry);
|
||||
results.Add(new AppendResult { FollowerId = followerId, Success = true });
|
||||
}
|
||||
else
|
||||
{
|
||||
results.Add(new AppendResult { FollowerId = followerId, Success = false });
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<AppendResult>>(results);
|
||||
}
|
||||
|
||||
public Task<VoteResponse> RequestVoteAsync(string candidateId, string voterId, VoteRequest request, CancellationToken ct)
|
||||
{
|
||||
if (_nodes.TryGetValue(voterId, out var node))
|
||||
return Task.FromResult(node.GrantVote(request.Term));
|
||||
|
||||
return Task.FromResult(new VoteResponse { Granted = false });
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.Routes;
|
||||
|
||||
@@ -7,9 +8,14 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
|
||||
{
|
||||
private readonly Socket _socket = socket;
|
||||
private readonly NetworkStream _stream = new(socket, ownsSocket: true);
|
||||
private readonly SemaphoreSlim _writeGate = new(1, 1);
|
||||
private readonly CancellationTokenSource _closedCts = new();
|
||||
private Task? _frameLoopTask;
|
||||
|
||||
public string? RemoteServerId { get; private set; }
|
||||
public string RemoteEndpoint => _socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N");
|
||||
public Func<RemoteSubscription, Task>? RemoteSubscriptionReceived { get; set; }
|
||||
public Func<RouteMessage, Task>? RoutedMessageReceived { get; set; }
|
||||
|
||||
public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct)
|
||||
{
|
||||
@@ -25,27 +31,168 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
|
||||
await WriteLineAsync($"ROUTE {serverId}", ct);
|
||||
}
|
||||
|
||||
public void StartFrameLoop(CancellationToken ct)
|
||||
{
|
||||
if (_frameLoopTask != null)
|
||||
return;
|
||||
|
||||
var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
|
||||
_frameLoopTask = Task.Run(() => ReadFramesAsync(linked.Token), linked.Token);
|
||||
}
|
||||
|
||||
public async Task SendRsPlusAsync(string subject, string? queue, CancellationToken ct)
|
||||
{
|
||||
var frame = queue is { Length: > 0 }
|
||||
? $"RS+ {subject} {queue}"
|
||||
: $"RS+ {subject}";
|
||||
await WriteLineAsync(frame, ct);
|
||||
}
|
||||
|
||||
public async Task SendRsMinusAsync(string subject, string? queue, CancellationToken ct)
|
||||
{
|
||||
var frame = queue is { Length: > 0 }
|
||||
? $"RS- {subject} {queue}"
|
||||
: $"RS- {subject}";
|
||||
await WriteLineAsync(frame, ct);
|
||||
}
|
||||
|
||||
public async Task SendRmsgAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
|
||||
{
|
||||
var replyToken = string.IsNullOrEmpty(replyTo) ? "-" : replyTo;
|
||||
await _writeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var control = Encoding.ASCII.GetBytes($"RMSG {subject} {replyToken} {payload.Length}\r\n");
|
||||
await _stream.WriteAsync(control, ct);
|
||||
if (!payload.IsEmpty)
|
||||
await _stream.WriteAsync(payload, ct);
|
||||
await _stream.WriteAsync("\r\n"u8.ToArray(), ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task WaitUntilClosedAsync(CancellationToken ct)
|
||||
{
|
||||
var buffer = new byte[1024];
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var bytesRead = await _stream.ReadAsync(buffer, ct);
|
||||
if (bytesRead == 0)
|
||||
return;
|
||||
}
|
||||
if (_frameLoopTask == null)
|
||||
return;
|
||||
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
|
||||
await _frameLoopTask.WaitAsync(linked.Token);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _closedCts.CancelAsync();
|
||||
if (_frameLoopTask != null)
|
||||
await _frameLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
||||
_closedCts.Dispose();
|
||||
_writeGate.Dispose();
|
||||
await _stream.DisposeAsync();
|
||||
}
|
||||
|
||||
private async Task ReadFramesAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
string line;
|
||||
try
|
||||
{
|
||||
line = await ReadLineAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (line.StartsWith("RS+ ", StringComparison.Ordinal))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
|
||||
{
|
||||
var queue = parts.Length >= 3 ? parts[2] : null;
|
||||
await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteServerId ?? string.Empty));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith("RS- ", StringComparison.Ordinal))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
|
||||
{
|
||||
var queue = parts.Length >= 3 ? parts[2] : null;
|
||||
await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteServerId ?? string.Empty));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!line.StartsWith("RMSG ", StringComparison.Ordinal))
|
||||
continue;
|
||||
|
||||
var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (args.Length < 4)
|
||||
continue;
|
||||
|
||||
var subject = args[1];
|
||||
var reply = args[2] == "-" ? null : args[2];
|
||||
if (!int.TryParse(args[3], out var size) || size < 0)
|
||||
continue;
|
||||
|
||||
var payload = await ReadPayloadAsync(size, ct);
|
||||
if (RoutedMessageReceived != null)
|
||||
await RoutedMessageReceived(new RouteMessage(subject, reply, payload));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ReadOnlyMemory<byte>> ReadPayloadAsync(int size, CancellationToken ct)
|
||||
{
|
||||
var payload = new byte[size];
|
||||
var offset = 0;
|
||||
while (offset < size)
|
||||
{
|
||||
var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Route connection closed during payload read");
|
||||
offset += read;
|
||||
}
|
||||
|
||||
var trailer = new byte[2];
|
||||
var trailerRead = 0;
|
||||
while (trailerRead < 2)
|
||||
{
|
||||
var read = await _stream.ReadAsync(trailer.AsMemory(trailerRead, 2 - trailerRead), ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Route connection closed during payload trailer read");
|
||||
trailerRead += read;
|
||||
}
|
||||
|
||||
if (trailer[0] != (byte)'\r' || trailer[1] != (byte)'\n')
|
||||
throw new IOException("Invalid route payload trailer");
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
private async Task WriteLineAsync(string line, CancellationToken ct)
|
||||
{
|
||||
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
|
||||
await _stream.WriteAsync(bytes, ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
await _writeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
|
||||
await _stream.WriteAsync(bytes, ct);
|
||||
await _stream.FlushAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_writeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> ReadLineAsync(CancellationToken ct)
|
||||
@@ -56,7 +203,7 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
|
||||
{
|
||||
var read = await _stream.ReadAsync(single, ct);
|
||||
if (read == 0)
|
||||
throw new IOException("Route connection closed during handshake");
|
||||
throw new IOException("Route connection closed");
|
||||
|
||||
if (single[0] == (byte)'\n')
|
||||
break;
|
||||
@@ -79,3 +226,5 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record RouteMessage(string Subject, string? ReplyTo, ReadOnlyMemory<byte> Payload);
|
||||
|
||||
@@ -15,6 +15,7 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
private readonly string _serverId;
|
||||
private readonly ILogger<RouteManager> _logger;
|
||||
private readonly Action<RemoteSubscription> _remoteSubSink;
|
||||
private readonly Action<RouteMessage> _routedMessageSink;
|
||||
private readonly ConcurrentDictionary<string, RouteConnection> _routes = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, byte> _connectedServerIds = new(StringComparer.Ordinal);
|
||||
|
||||
@@ -29,12 +30,14 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
ServerStats stats,
|
||||
string serverId,
|
||||
Action<RemoteSubscription> remoteSubSink,
|
||||
Action<RouteMessage> routedMessageSink,
|
||||
ILogger<RouteManager> logger)
|
||||
{
|
||||
_options = options;
|
||||
_stats = stats;
|
||||
_serverId = serverId;
|
||||
_remoteSubSink = remoteSubSink;
|
||||
_routedMessageSink = routedMessageSink;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -51,8 +54,12 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
_options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
|
||||
|
||||
_acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
|
||||
var poolSize = Math.Max(_options.PoolSize, 1);
|
||||
foreach (var route in _options.Routes.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
_ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token));
|
||||
{
|
||||
for (var i = 0; i < poolSize; i++)
|
||||
_ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token));
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
@@ -81,17 +88,33 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
|
||||
public void PropagateLocalSubscription(string subject, string? queue)
|
||||
{
|
||||
if (_connectedServerIds.IsEmpty)
|
||||
if (_routes.IsEmpty)
|
||||
return;
|
||||
|
||||
var remoteSub = new RemoteSubscription(subject, queue, _serverId);
|
||||
foreach (var peerId in _connectedServerIds.Keys)
|
||||
foreach (var route in _routes.Values)
|
||||
{
|
||||
if (Managers.TryGetValue(peerId, out var peer))
|
||||
peer.ReceiveRemoteSubscription(remoteSub);
|
||||
_ = route.SendRsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
public void PropagateLocalUnsubscription(string subject, string? queue)
|
||||
{
|
||||
if (_routes.IsEmpty)
|
||||
return;
|
||||
|
||||
foreach (var route in _routes.Values)
|
||||
_ = route.SendRsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
|
||||
}
|
||||
|
||||
public async Task ForwardRoutedMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
|
||||
{
|
||||
if (_routes.IsEmpty)
|
||||
return;
|
||||
|
||||
foreach (var route in _routes.Values)
|
||||
await route.SendRmsgAsync(subject, replyTo, payload, ct);
|
||||
}
|
||||
|
||||
private async Task AcceptLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
@@ -170,7 +193,7 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
|
||||
private void Register(RouteConnection route)
|
||||
{
|
||||
var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}";
|
||||
var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}:{Guid.NewGuid():N}";
|
||||
if (!_routes.TryAdd(key, route))
|
||||
{
|
||||
_ = route.DisposeAsync();
|
||||
@@ -180,6 +203,18 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
if (route.RemoteServerId is { Length: > 0 } remoteServerId)
|
||||
_connectedServerIds[remoteServerId] = 0;
|
||||
|
||||
route.RemoteSubscriptionReceived = sub =>
|
||||
{
|
||||
_remoteSubSink(sub);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
route.RoutedMessageReceived = msg =>
|
||||
{
|
||||
_routedMessageSink(msg);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
route.StartFrameLoop(_cts!.Token);
|
||||
|
||||
Interlocked.Increment(ref _stats.Routes);
|
||||
_ = Task.Run(() => WatchRouteAsync(key, route, _cts!.Token));
|
||||
}
|
||||
@@ -217,8 +252,5 @@ public sealed class RouteManager : IAsyncDisposable
|
||||
return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
|
||||
}
|
||||
|
||||
private void ReceiveRemoteSubscription(RemoteSubscription sub)
|
||||
{
|
||||
_remoteSubSink(sub);
|
||||
}
|
||||
public int RouteCount => _routes.Count;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
namespace NATS.Server.Subscriptions;
|
||||
|
||||
public sealed record RemoteSubscription(string Subject, string? Queue, string RouteId);
|
||||
public sealed record RemoteSubscription(string Subject, string? Queue, string RouteId, bool IsRemoval = false)
|
||||
{
|
||||
public static RemoteSubscription Removal(string subject, string? queue, string routeId)
|
||||
=> new(subject, queue, routeId, IsRemoval: true);
|
||||
}
|
||||
|
||||
@@ -103,7 +103,10 @@ public sealed class SubList : IDisposable
|
||||
try
|
||||
{
|
||||
var key = $"{sub.RouteId}|{sub.Subject}|{sub.Queue}";
|
||||
_remoteSubs[key] = sub;
|
||||
if (sub.IsRemoval)
|
||||
_remoteSubs.Remove(key);
|
||||
else
|
||||
_remoteSubs[key] = sub;
|
||||
Interlocked.Increment(ref _generation);
|
||||
}
|
||||
finally
|
||||
@@ -119,6 +122,9 @@ public sealed class SubList : IDisposable
|
||||
{
|
||||
foreach (var remoteSub in _remoteSubs.Values)
|
||||
{
|
||||
if (remoteSub.IsRemoval)
|
||||
continue;
|
||||
|
||||
if (SubjectMatch.MatchLiteral(subject, remoteSub.Subject))
|
||||
return true;
|
||||
}
|
||||
|
||||
29
tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs
Normal file
29
tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs
Normal file
@@ -0,0 +1,29 @@
|
||||
using NATS.Server.Protocol;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class ClientKindProtocolRoutingTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(ClientKind.Client, "RS+", false)]
|
||||
[InlineData(ClientKind.Router, "RS+", true)]
|
||||
[InlineData(ClientKind.Client, "RS-", false)]
|
||||
[InlineData(ClientKind.Router, "RS-", true)]
|
||||
[InlineData(ClientKind.Client, "RMSG", false)]
|
||||
[InlineData(ClientKind.Router, "RMSG", true)]
|
||||
[InlineData(ClientKind.Client, "A+", false)]
|
||||
[InlineData(ClientKind.Gateway, "A+", true)]
|
||||
[InlineData(ClientKind.Client, "A-", false)]
|
||||
[InlineData(ClientKind.Gateway, "A-", true)]
|
||||
[InlineData(ClientKind.Client, "LS+", false)]
|
||||
[InlineData(ClientKind.Leaf, "LS+", true)]
|
||||
[InlineData(ClientKind.Client, "LS-", false)]
|
||||
[InlineData(ClientKind.Leaf, "LS-", true)]
|
||||
[InlineData(ClientKind.Client, "LMSG", false)]
|
||||
[InlineData(ClientKind.Leaf, "LMSG", true)]
|
||||
public void Client_kind_protocol_matrix_enforces_inter_server_commands(ClientKind kind, string op, bool expected)
|
||||
{
|
||||
var matrix = new ClientCommandMatrix();
|
||||
matrix.IsAllowed(kind, op).ShouldBe(expected);
|
||||
}
|
||||
}
|
||||
153
tests/NATS.Server.Tests/GatewayProtocolTests.cs
Normal file
153
tests/NATS.Server.Tests/GatewayProtocolTests.cs
Normal file
@@ -0,0 +1,153 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class GatewayProtocolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Gateway_link_establishes_and_forwards_interested_message()
|
||||
{
|
||||
await using var fx = await GatewayFixture.StartTwoClustersAsync();
|
||||
await fx.SubscribeRemoteClusterAsync("g.>");
|
||||
await fx.PublishLocalClusterAsync("g.test", "hello");
|
||||
(await fx.ReadRemoteClusterMessageAsync()).ShouldContain("hello");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class GatewayFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly NatsServer _local;
|
||||
private readonly NatsServer _remote;
|
||||
private readonly CancellationTokenSource _localCts;
|
||||
private readonly CancellationTokenSource _remoteCts;
|
||||
private Socket? _remoteSubscriber;
|
||||
private Socket? _localPublisher;
|
||||
|
||||
private GatewayFixture(NatsServer local, NatsServer remote, CancellationTokenSource localCts, CancellationTokenSource remoteCts)
|
||||
{
|
||||
_local = local;
|
||||
_remote = remote;
|
||||
_localCts = localCts;
|
||||
_remoteCts = remoteCts;
|
||||
}
|
||||
|
||||
public static async Task<GatewayFixture> StartTwoClustersAsync()
|
||||
{
|
||||
var localOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Gateway = new GatewayOptions
|
||||
{
|
||||
Name = "LOCAL",
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
};
|
||||
|
||||
var local = new NatsServer(localOptions, NullLoggerFactory.Instance);
|
||||
var localCts = new CancellationTokenSource();
|
||||
_ = local.StartAsync(localCts.Token);
|
||||
await local.WaitForReadyAsync();
|
||||
|
||||
var remoteOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Gateway = new GatewayOptions
|
||||
{
|
||||
Name = "REMOTE",
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Remotes = [local.GatewayListen!],
|
||||
},
|
||||
};
|
||||
|
||||
var remote = new NatsServer(remoteOptions, NullLoggerFactory.Instance);
|
||||
var remoteCts = new CancellationTokenSource();
|
||||
_ = remote.StartAsync(remoteCts.Token);
|
||||
await remote.WaitForReadyAsync();
|
||||
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested && (local.Stats.Gateways == 0 || remote.Stats.Gateways == 0))
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
|
||||
return new GatewayFixture(local, remote, localCts, remoteCts);
|
||||
}
|
||||
|
||||
public async Task SubscribeRemoteClusterAsync(string subject)
|
||||
{
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _remote.Port);
|
||||
_remoteSubscriber = sock;
|
||||
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task PublishLocalClusterAsync(string subject, string payload)
|
||||
{
|
||||
var sock = _localPublisher;
|
||||
if (sock == null)
|
||||
{
|
||||
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _local.Port);
|
||||
_localPublisher = sock;
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public Task<string> ReadRemoteClusterMessageAsync()
|
||||
{
|
||||
if (_remoteSubscriber == null)
|
||||
throw new InvalidOperationException("Remote subscriber was not initialized.");
|
||||
|
||||
return ReadUntilAsync(_remoteSubscriber, "MSG ");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_remoteSubscriber?.Dispose();
|
||||
_localPublisher?.Dispose();
|
||||
await _localCts.CancelAsync();
|
||||
await _remoteCts.CancelAsync();
|
||||
_local.Dispose();
|
||||
_remote.Dispose();
|
||||
_localCts.Dispose();
|
||||
_remoteCts.Dispose();
|
||||
}
|
||||
|
||||
private static async Task<string> ReadLineAsync(Socket sock)
|
||||
{
|
||||
var buf = new byte[4096];
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None);
|
||||
return Encoding.ASCII.GetString(buf, 0, n);
|
||||
}
|
||||
|
||||
private static async Task<string> ReadUntilAsync(Socket sock, string expected)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var buf = new byte[4096];
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
|
||||
while (!sb.ToString().Contains(expected, StringComparison.Ordinal))
|
||||
{
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token);
|
||||
if (n == 0)
|
||||
break;
|
||||
sb.Append(Encoding.ASCII.GetString(buf, 0, n));
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
17
tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs
Normal file
17
tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs
Normal file
@@ -0,0 +1,17 @@
|
||||
using NATS.Server.JetStream;
|
||||
using NATS.Server.JetStream.Api;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamAccountControlApiTests
|
||||
{
|
||||
[Fact]
|
||||
public void Account_and_server_control_subjects_are_routable()
|
||||
{
|
||||
var router = new JetStreamApiRouter(new StreamManager(), new ConsumerManager());
|
||||
router.Route("$JS.API.SERVER.REMOVE", "{}"u8).Error.ShouldBeNull();
|
||||
router.Route("$JS.API.ACCOUNT.PURGE.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
router.Route("$JS.API.ACCOUNT.STREAM.MOVE.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
router.Route("$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,20 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static Task<JetStreamApiFixture> StartWithStreamConfigAsync(StreamConfig config)
|
||||
{
|
||||
var fixture = new JetStreamApiFixture();
|
||||
_ = fixture._streamManager.CreateOrUpdate(config);
|
||||
return Task.FromResult(fixture);
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithStreamJsonAsync(string json)
|
||||
{
|
||||
var fixture = new JetStreamApiFixture();
|
||||
_ = await fixture.RequestLocalAsync("$JS.API.STREAM.CREATE.S", json);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithPullConsumerAsync()
|
||||
{
|
||||
var fixture = await StartWithStreamAsync("ORDERS", "orders.*");
|
||||
@@ -82,6 +96,47 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithMultiFilterConsumerAsync()
|
||||
{
|
||||
var fixture = await StartWithStreamAsync("ORDERS", ">");
|
||||
_ = await fixture.CreateConsumerAsync("ORDERS", "CF", null, filterSubjects: ["orders.*"]);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithReplayOriginalConsumerAsync()
|
||||
{
|
||||
var fixture = await StartWithStreamAsync("ORDERS", "orders.*");
|
||||
_ = await fixture.PublishAndGetAckAsync("orders.created", "1");
|
||||
_ = await fixture.CreateConsumerAsync("ORDERS", "RO", "orders.*", replayPolicy: ReplayPolicy.Original, ackPolicy: AckPolicy.Explicit);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static Task<JetStreamApiFixture> StartWithMultipleSourcesAsync()
|
||||
{
|
||||
var fixture = new JetStreamApiFixture();
|
||||
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
|
||||
{
|
||||
Name = "SRC1",
|
||||
Subjects = ["a.>"],
|
||||
});
|
||||
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
|
||||
{
|
||||
Name = "SRC2",
|
||||
Subjects = ["b.>"],
|
||||
});
|
||||
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
|
||||
{
|
||||
Name = "AGG",
|
||||
Subjects = ["agg.>"],
|
||||
Sources =
|
||||
[
|
||||
new StreamSourceConfig { Name = "SRC1" },
|
||||
new StreamSourceConfig { Name = "SRC2" },
|
||||
],
|
||||
});
|
||||
return Task.FromResult(fixture);
|
||||
}
|
||||
|
||||
public static Task<JetStreamApiFixture> StartJwtLimitedAccountAsync(int maxStreams)
|
||||
{
|
||||
var account = new Account("JWT-LIMITED")
|
||||
@@ -148,9 +203,45 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
return _streamManager.GetStateAsync(streamName, default).AsTask();
|
||||
}
|
||||
|
||||
public Task<JetStreamApiResponse> CreateConsumerAsync(string stream, string durableName, string filterSubject, bool push = false, int heartbeatMs = 0, AckPolicy ackPolicy = AckPolicy.None, int ackWaitMs = 30_000)
|
||||
public Task<string> GetStreamBackendTypeAsync(string streamName)
|
||||
{
|
||||
var payload = $@"{{""durable_name"":""{durableName}"",""filter_subject"":""{filterSubject}"",""push"":{push.ToString().ToLowerInvariant()},""heartbeat_ms"":{heartbeatMs},""ack_policy"":""{ackPolicy.ToString().ToLowerInvariant()}"",""ack_wait_ms"":{ackWaitMs}}}";
|
||||
return Task.FromResult(_streamManager.GetStoreBackendType(streamName));
|
||||
}
|
||||
|
||||
public Task<JetStreamApiResponse> CreateConsumerAsync(
|
||||
string stream,
|
||||
string durableName,
|
||||
string? filterSubject,
|
||||
bool push = false,
|
||||
int heartbeatMs = 0,
|
||||
AckPolicy ackPolicy = AckPolicy.None,
|
||||
int ackWaitMs = 30_000,
|
||||
int maxAckPending = 0,
|
||||
IReadOnlyList<string>? filterSubjects = null,
|
||||
ReplayPolicy replayPolicy = ReplayPolicy.Instant,
|
||||
DeliverPolicy deliverPolicy = DeliverPolicy.All,
|
||||
bool ephemeral = false)
|
||||
{
|
||||
var payloadObj = new
|
||||
{
|
||||
durable_name = durableName,
|
||||
filter_subject = filterSubject,
|
||||
filter_subjects = filterSubjects,
|
||||
push,
|
||||
heartbeat_ms = heartbeatMs,
|
||||
ack_policy = ackPolicy.ToString().ToLowerInvariant(),
|
||||
ack_wait_ms = ackWaitMs,
|
||||
max_ack_pending = maxAckPending,
|
||||
replay_policy = replayPolicy == ReplayPolicy.Original ? "original" : "instant",
|
||||
deliver_policy = deliverPolicy switch
|
||||
{
|
||||
DeliverPolicy.Last => "last",
|
||||
DeliverPolicy.New => "new",
|
||||
_ => "all",
|
||||
},
|
||||
ephemeral,
|
||||
};
|
||||
var payload = JsonSerializer.Serialize(payloadObj);
|
||||
return RequestLocalAsync($"$JS.API.CONSUMER.CREATE.{stream}.{durableName}", payload);
|
||||
}
|
||||
|
||||
@@ -206,6 +297,12 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
_ = await PublishAndGetAckAsync(subject, payload);
|
||||
}
|
||||
|
||||
public Task PublishToSourceAsync(string sourceStream, string subject, string payload)
|
||||
{
|
||||
_ = sourceStream;
|
||||
return PublishAndGetAckAsync(subject, payload);
|
||||
}
|
||||
|
||||
public Task AckAllAsync(string stream, string durableName, ulong sequence)
|
||||
{
|
||||
_consumerManager.AckAll(stream, durableName, sequence);
|
||||
|
||||
88
tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs
Normal file
88
tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs
Normal file
@@ -0,0 +1,88 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamApiGapInventoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void Parity_map_has_no_unclassified_go_js_api_subjects()
|
||||
{
|
||||
var gap = JetStreamApiGapInventory.Load();
|
||||
gap.UnclassifiedSubjects.ShouldBeEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class JetStreamApiGapInventory
|
||||
{
|
||||
public IReadOnlyList<string> UnclassifiedSubjects { get; }
|
||||
|
||||
private JetStreamApiGapInventory(IReadOnlyList<string> unclassifiedSubjects)
|
||||
{
|
||||
UnclassifiedSubjects = unclassifiedSubjects;
|
||||
}
|
||||
|
||||
public static JetStreamApiGapInventory Load()
|
||||
{
|
||||
var goSubjects = LoadGoSubjects();
|
||||
var mappedSubjects = LoadMappedSubjects();
|
||||
|
||||
var unclassified = goSubjects
|
||||
.Where(s => !mappedSubjects.Contains(s))
|
||||
.OrderBy(s => s, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return new JetStreamApiGapInventory(unclassified);
|
||||
}
|
||||
|
||||
private static HashSet<string> LoadGoSubjects()
|
||||
{
|
||||
var script = Path.Combine(AppContext.BaseDirectory, "../../../../../scripts/jetstream/extract-go-js-api.sh");
|
||||
script = Path.GetFullPath(script);
|
||||
if (!File.Exists(script))
|
||||
throw new FileNotFoundException($"missing script: {script}");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = "bash",
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
};
|
||||
psi.ArgumentList.Add(script);
|
||||
|
||||
using var process = Process.Start(psi) ?? throw new InvalidOperationException("failed to start inventory script");
|
||||
var output = process.StandardOutput.ReadToEnd();
|
||||
var errors = process.StandardError.ReadToEnd();
|
||||
process.WaitForExit();
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
throw new InvalidOperationException($"inventory script failed: {errors}");
|
||||
|
||||
return output
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Where(x => x.StartsWith("$JS.API.", StringComparison.Ordinal))
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static HashSet<string> LoadMappedSubjects()
|
||||
{
|
||||
var mapPath = Path.Combine(AppContext.BaseDirectory, "../../../../../docs/plans/2026-02-23-jetstream-remaining-parity-map.md");
|
||||
mapPath = Path.GetFullPath(mapPath);
|
||||
if (!File.Exists(mapPath))
|
||||
throw new FileNotFoundException($"missing parity map: {mapPath}");
|
||||
|
||||
var subjectRegex = new Regex(@"^\|\s*(\$JS\.API[^\|]+?)\s*\|", RegexOptions.Compiled);
|
||||
var subjects = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var line in File.ReadLines(mapPath))
|
||||
{
|
||||
var match = subjectRegex.Match(line);
|
||||
if (!match.Success)
|
||||
continue;
|
||||
|
||||
subjects.Add(match.Groups[1].Value.Trim());
|
||||
}
|
||||
|
||||
return subjects;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamClusterControlExtendedApiTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Peer_remove_and_consumer_stepdown_subjects_return_success_shape()
|
||||
{
|
||||
await using var fx = await JetStreamClusterFixture.StartAsync(nodes: 3);
|
||||
(await fx.RequestAsync("$JS.API.STREAM.PEER.REMOVE.ORDERS", "{\"peer\":\"n2\"}")).Success.ShouldBeTrue();
|
||||
(await fx.RequestAsync("$JS.API.CONSUMER.LEADER.STEPDOWN.ORDERS.DUR", "{}")).Success.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
16
tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs
Normal file
16
tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs
Normal file
@@ -0,0 +1,16 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamConsumerSemanticsTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Consumer_with_filter_subjects_only_receives_matching_messages()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithMultiFilterConsumerAsync();
|
||||
await fx.PublishAndGetAckAsync("orders.created", "1");
|
||||
await fx.PublishAndGetAckAsync("payments.settled", "2");
|
||||
|
||||
var batch = await fx.FetchAsync("ORDERS", "CF", 10);
|
||||
batch.Messages.ShouldNotBeEmpty();
|
||||
batch.Messages.All(m => m.Subject.StartsWith("orders.", StringComparison.Ordinal)).ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
15
tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs
Normal file
15
tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs
Normal file
@@ -0,0 +1,15 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamFlowReplayBackoffTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Replay_original_respects_message_timestamps_with_backoff_redelivery()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithReplayOriginalConsumerAsync();
|
||||
var sw = Stopwatch.StartNew();
|
||||
_ = await fx.FetchAsync("ORDERS", "RO", 1);
|
||||
sw.ElapsedMilliseconds.ShouldBeGreaterThanOrEqualTo(50);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamMirrorSourceAdvancedTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Stream_with_multiple_sources_aggregates_messages_in_order()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithMultipleSourcesAsync();
|
||||
await fx.PublishToSourceAsync("SRC1", "a.1", "1");
|
||||
await fx.PublishToSourceAsync("SRC2", "b.1", "2");
|
||||
|
||||
(await fx.GetStreamStateAsync("AGG")).Messages.ShouldBe((ulong)2);
|
||||
}
|
||||
}
|
||||
11
tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs
Normal file
11
tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamStorageSelectionTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Stream_with_storage_file_uses_filestore_backend()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithStreamJsonAsync("{\"name\":\"S\",\"subjects\":[\"s.*\"],\"storage\":\"file\"}");
|
||||
(await fx.GetStreamBackendTypeAsync("S")).ShouldBe("file");
|
||||
}
|
||||
}
|
||||
21
tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs
Normal file
21
tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
using NATS.Server.JetStream.Models;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamStreamPolicyRuntimeTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Discard_new_rejects_publish_when_max_bytes_exceeded()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithStreamConfigAsync(new StreamConfig
|
||||
{
|
||||
Name = "S",
|
||||
Subjects = ["s.*"],
|
||||
MaxBytes = 2,
|
||||
Discard = DiscardPolicy.New,
|
||||
});
|
||||
|
||||
(await fx.PublishAndGetAckAsync("s.a", "12")).ErrorCode.ShouldBeNull();
|
||||
(await fx.PublishAndGetAckAsync("s.a", "34")).ErrorCode.ShouldNotBeNull();
|
||||
}
|
||||
}
|
||||
151
tests/NATS.Server.Tests/LeafProtocolTests.cs
Normal file
151
tests/NATS.Server.Tests/LeafProtocolTests.cs
Normal file
@@ -0,0 +1,151 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class LeafProtocolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Leaf_link_propagates_subscription_and_message_flow()
|
||||
{
|
||||
await using var fx = await LeafFixture.StartHubSpokeAsync();
|
||||
await fx.SubscribeSpokeAsync("leaf.>");
|
||||
await fx.PublishHubAsync("leaf.msg", "x");
|
||||
(await fx.ReadSpokeMessageAsync()).ShouldContain("x");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class LeafFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly NatsServer _hub;
|
||||
private readonly NatsServer _spoke;
|
||||
private readonly CancellationTokenSource _hubCts;
|
||||
private readonly CancellationTokenSource _spokeCts;
|
||||
private Socket? _spokeSubscriber;
|
||||
private Socket? _hubPublisher;
|
||||
|
||||
private LeafFixture(NatsServer hub, NatsServer spoke, CancellationTokenSource hubCts, CancellationTokenSource spokeCts)
|
||||
{
|
||||
_hub = hub;
|
||||
_spoke = spoke;
|
||||
_hubCts = hubCts;
|
||||
_spokeCts = spokeCts;
|
||||
}
|
||||
|
||||
public static async Task<LeafFixture> StartHubSpokeAsync()
|
||||
{
|
||||
var hubOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
LeafNode = new LeafNodeOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
};
|
||||
|
||||
var hub = new NatsServer(hubOptions, NullLoggerFactory.Instance);
|
||||
var hubCts = new CancellationTokenSource();
|
||||
_ = hub.StartAsync(hubCts.Token);
|
||||
await hub.WaitForReadyAsync();
|
||||
|
||||
var spokeOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
LeafNode = new LeafNodeOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Remotes = [hub.LeafListen!],
|
||||
},
|
||||
};
|
||||
|
||||
var spoke = new NatsServer(spokeOptions, NullLoggerFactory.Instance);
|
||||
var spokeCts = new CancellationTokenSource();
|
||||
_ = spoke.StartAsync(spokeCts.Token);
|
||||
await spoke.WaitForReadyAsync();
|
||||
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested && (hub.Stats.Leafs == 0 || spoke.Stats.Leafs == 0))
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
|
||||
return new LeafFixture(hub, spoke, hubCts, spokeCts);
|
||||
}
|
||||
|
||||
public async Task SubscribeSpokeAsync(string subject)
|
||||
{
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _spoke.Port);
|
||||
_spokeSubscriber = sock;
|
||||
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task PublishHubAsync(string subject, string payload)
|
||||
{
|
||||
var sock = _hubPublisher;
|
||||
if (sock == null)
|
||||
{
|
||||
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _hub.Port);
|
||||
_hubPublisher = sock;
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public Task<string> ReadSpokeMessageAsync()
|
||||
{
|
||||
if (_spokeSubscriber == null)
|
||||
throw new InvalidOperationException("Spoke subscriber was not initialized.");
|
||||
|
||||
return ReadUntilAsync(_spokeSubscriber, "MSG ");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_spokeSubscriber?.Dispose();
|
||||
_hubPublisher?.Dispose();
|
||||
await _hubCts.CancelAsync();
|
||||
await _spokeCts.CancelAsync();
|
||||
_hub.Dispose();
|
||||
_spoke.Dispose();
|
||||
_hubCts.Dispose();
|
||||
_spokeCts.Dispose();
|
||||
}
|
||||
|
||||
private static async Task<string> ReadLineAsync(Socket sock)
|
||||
{
|
||||
var buf = new byte[4096];
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None);
|
||||
return Encoding.ASCII.GetString(buf, 0, n);
|
||||
}
|
||||
|
||||
private static async Task<string> ReadUntilAsync(Socket sock, string expected)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var buf = new byte[4096];
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
|
||||
while (!sb.ToString().Contains(expected, StringComparison.Ordinal))
|
||||
{
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token);
|
||||
if (n == 0)
|
||||
break;
|
||||
sb.Append(Encoding.ASCII.GetString(buf, 0, n));
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
105
tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs
Normal file
105
tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs
Normal file
@@ -0,0 +1,105 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class MonitorClusterEndpointTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data()
|
||||
{
|
||||
await using var fx = await MonitorFixture.StartClusterEnabledAsync();
|
||||
(await fx.GetJsonAsync("/routez")).ShouldContain("routes");
|
||||
(await fx.GetJsonAsync("/gatewayz")).ShouldContain("gateways");
|
||||
(await fx.GetJsonAsync("/leafz")).ShouldContain("leafs");
|
||||
(await fx.GetJsonAsync("/accountz")).ShouldContain("accounts");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MonitorFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly NatsServer _server;
|
||||
private readonly CancellationTokenSource _cts;
|
||||
private readonly HttpClient _http;
|
||||
private readonly int _monitorPort;
|
||||
|
||||
private MonitorFixture(NatsServer server, CancellationTokenSource cts, HttpClient http, int monitorPort)
|
||||
{
|
||||
_server = server;
|
||||
_cts = cts;
|
||||
_http = http;
|
||||
_monitorPort = monitorPort;
|
||||
}
|
||||
|
||||
public static async Task<MonitorFixture> StartClusterEnabledAsync()
|
||||
{
|
||||
var monitorPort = GetFreePort();
|
||||
var options = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
MonitorPort = monitorPort,
|
||||
Cluster = new ClusterOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
Gateway = new GatewayOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Name = "M",
|
||||
},
|
||||
LeafNode = new LeafNodeOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
};
|
||||
|
||||
var server = new NatsServer(options, NullLoggerFactory.Instance);
|
||||
var cts = new CancellationTokenSource();
|
||||
_ = server.StartAsync(cts.Token);
|
||||
await server.WaitForReadyAsync();
|
||||
|
||||
var http = new HttpClient();
|
||||
for (var i = 0; i < 50; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await http.GetAsync($"http://127.0.0.1:{monitorPort}/healthz");
|
||||
if (response.IsSuccessStatusCode)
|
||||
break;
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
await Task.Delay(50);
|
||||
}
|
||||
|
||||
return new MonitorFixture(server, cts, http, monitorPort);
|
||||
}
|
||||
|
||||
public Task<string> GetJsonAsync(string path)
|
||||
{
|
||||
return _http.GetStringAsync($"http://127.0.0.1:{_monitorPort}{path}");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_http.Dispose();
|
||||
await _cts.CancelAsync();
|
||||
_server.Dispose();
|
||||
_cts.Dispose();
|
||||
}
|
||||
|
||||
private static int GetFreePort()
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
return ((IPEndPoint)socket.LocalEndPoint!).Port;
|
||||
}
|
||||
}
|
||||
89
tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs
Normal file
89
tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs
Normal file
@@ -0,0 +1,89 @@
|
||||
using NATS.Server.Raft;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RaftTransportPersistenceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Raft_node_recovers_log_and_term_after_restart()
|
||||
{
|
||||
await using var fx = await RaftFixture.StartPersistentClusterAsync();
|
||||
var idx = await fx.Leader.ProposeAsync("cmd", default);
|
||||
await fx.RestartNodeAsync("n2");
|
||||
(await fx.ReadNodeAppliedIndexAsync("n2")).ShouldBeGreaterThanOrEqualTo(idx);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class RaftFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
private readonly InMemoryRaftTransport _transport;
|
||||
private readonly Dictionary<string, RaftNode> _nodes;
|
||||
|
||||
private RaftFixture(string root, InMemoryRaftTransport transport, Dictionary<string, RaftNode> nodes)
|
||||
{
|
||||
_root = root;
|
||||
_transport = transport;
|
||||
_nodes = nodes;
|
||||
}
|
||||
|
||||
public RaftNode Leader => _nodes["n1"];
|
||||
|
||||
public static Task<RaftFixture> StartPersistentClusterAsync()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), $"nats-raft-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(root);
|
||||
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var nodes = new Dictionary<string, RaftNode>(StringComparer.Ordinal);
|
||||
foreach (var id in new[] { "n1", "n2", "n3" })
|
||||
{
|
||||
var node = new RaftNode(id, transport, Path.Combine(root, id));
|
||||
transport.Register(node);
|
||||
nodes[id] = node;
|
||||
}
|
||||
|
||||
var all = nodes.Values.ToArray();
|
||||
foreach (var node in all)
|
||||
node.ConfigureCluster(all);
|
||||
|
||||
var leader = nodes["n1"];
|
||||
leader.StartElection(all.Length);
|
||||
leader.ReceiveVote(nodes["n2"].GrantVote(leader.Term), all.Length);
|
||||
leader.ReceiveVote(nodes["n3"].GrantVote(leader.Term), all.Length);
|
||||
|
||||
return Task.FromResult(new RaftFixture(root, transport, nodes));
|
||||
}
|
||||
|
||||
public async Task RestartNodeAsync(string id)
|
||||
{
|
||||
var nodePath = Path.Combine(_root, id);
|
||||
var replacement = new RaftNode(id, _transport, nodePath);
|
||||
await replacement.LoadPersistedStateAsync(default);
|
||||
_transport.Register(replacement);
|
||||
_nodes[id] = replacement;
|
||||
|
||||
var all = _nodes.Values.ToArray();
|
||||
foreach (var node in all)
|
||||
node.ConfigureCluster(all);
|
||||
}
|
||||
|
||||
public Task<long> ReadNodeAppliedIndexAsync(string id)
|
||||
{
|
||||
return Task.FromResult(_nodes[id].AppliedIndex);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
11
tests/NATS.Server.Tests/RoutePoolTests.cs
Normal file
11
tests/NATS.Server.Tests/RoutePoolTests.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RoutePoolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Route_manager_establishes_default_pool_of_three_links_per_peer()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
(await fx.ServerARouteLinkCountToServerBAsync()).ShouldBe(3);
|
||||
}
|
||||
}
|
||||
14
tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs
Normal file
14
tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs
Normal file
@@ -0,0 +1,14 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RouteRmsgForwardingTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
await fx.SubscribeOnServerBAsync("foo.>");
|
||||
|
||||
await fx.PublishFromServerAAsync("foo.bar", "payload");
|
||||
(await fx.ReadServerBMessageAsync()).ShouldContain("payload");
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,8 @@ internal sealed class RouteFixture : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _ctsA;
|
||||
private readonly CancellationTokenSource _ctsB;
|
||||
private Socket? _subscriberOnB;
|
||||
private Socket? _publisherOnA;
|
||||
private Socket? _manualRouteToA;
|
||||
|
||||
private RouteFixture(NatsServer serverA, NatsServer serverB, CancellationTokenSource ctsA, CancellationTokenSource ctsB)
|
||||
{
|
||||
@@ -91,22 +93,82 @@ internal sealed class RouteFixture : IAsyncDisposable
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task<bool> ServerAHasRemoteInterestAsync(string subject)
|
||||
public async Task SendRouteSubFrameAsync(string subject)
|
||||
{
|
||||
var (host, port) = ParseHostPort(_serverA.ClusterListen!);
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Parse(host), port);
|
||||
_manualRouteToA = sock;
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("ROUTE test-remote\r\n"));
|
||||
_ = await ReadLineAsync(sock); // ROUTE <id>
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"RS+ {subject}\r\n"));
|
||||
}
|
||||
|
||||
public async Task SendRouteUnsubFrameAsync(string subject)
|
||||
{
|
||||
if (_manualRouteToA == null)
|
||||
throw new InvalidOperationException("Route frame socket not established.");
|
||||
|
||||
await _manualRouteToA.SendAsync(Encoding.ASCII.GetBytes($"RS- {subject}\r\n"));
|
||||
}
|
||||
|
||||
public async Task PublishFromServerAAsync(string subject, string payload)
|
||||
{
|
||||
var sock = _publisherOnA;
|
||||
if (sock == null)
|
||||
{
|
||||
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _serverA.Port);
|
||||
_publisherOnA = sock;
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task<string> ReadServerBMessageAsync()
|
||||
{
|
||||
if (_subscriberOnB == null)
|
||||
throw new InvalidOperationException("No subscriber socket on server B.");
|
||||
|
||||
return await ReadUntilAsync(_subscriberOnB, "MSG ");
|
||||
}
|
||||
|
||||
public async Task<bool> ServerAHasRemoteInterestAsync(string subject, bool expected = true)
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested)
|
||||
{
|
||||
if (_serverA.HasRemoteInterest(subject))
|
||||
return true;
|
||||
if (_serverA.HasRemoteInterest(subject) == expected)
|
||||
return expected;
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
return false;
|
||||
return !expected;
|
||||
}
|
||||
|
||||
public async Task<int> ServerARouteLinkCountToServerBAsync()
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested)
|
||||
{
|
||||
if (_serverA.Stats.Routes >= 3)
|
||||
return (int)_serverA.Stats.Routes;
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
return (int)_serverA.Stats.Routes;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_subscriberOnB?.Dispose();
|
||||
_publisherOnA?.Dispose();
|
||||
_manualRouteToA?.Dispose();
|
||||
await _ctsA.CancelAsync();
|
||||
await _ctsB.CancelAsync();
|
||||
_serverA.Dispose();
|
||||
@@ -138,4 +200,10 @@ internal sealed class RouteFixture : IAsyncDisposable
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static (string Host, int Port) ParseHostPort(string endpoint)
|
||||
{
|
||||
var parts = endpoint.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
return (parts[0], int.Parse(parts[1]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RouteWireSubscriptionProtocolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task RSplus_RSminus_frames_propagate_remote_interest_over_socket()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
|
||||
await fx.SendRouteSubFrameAsync("foo.*");
|
||||
(await fx.ServerAHasRemoteInterestAsync("foo.bar")).ShouldBeTrue();
|
||||
|
||||
await fx.SendRouteUnsubFrameAsync("foo.*");
|
||||
(await fx.ServerAHasRemoteInterestAsync("foo.bar", expected: false)).ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user