feat: complete final jetstream parity transport and runtime baselines

This commit is contained in:
Joseph Doherty
2026-02-23 11:04:43 -05:00
parent 53585012f3
commit 8bce096f55
61 changed files with 2655 additions and 129 deletions

View File

@@ -61,14 +61,14 @@
| Type | Go | .NET | Notes | | Type | Go | .NET | Notes |
|------|:--:|:----:|-------| |------|:--:|:----:|-------|
| CLIENT | Y | Y | | | CLIENT | Y | Y | |
| ROUTER | Y | Partial | Route handshake + in-memory remote subscription tracking; no RMSG message routing, no RS+/RS- wire protocol, no route pooling (3x per peer) | | ROUTER | Y | Y | Route handshake + RS+/RS-/RMSG wire protocol + default 3-link pooling baseline |
| GATEWAY | Y | Stub | Config parsing only; no listener, connections, handshake, interest-only mode, or message forwarding | | GATEWAY | Y | Baseline | Functional handshake, A+/A- interest propagation, and forwarding baseline; advanced Go routing semantics remain |
| LEAF | Y | Stub | Config parsing only; no listener, connections, handshake, subscription sharing, or loop detection | | LEAF | Y | Baseline | Functional handshake, LS+/LS- propagation, and LMSG forwarding baseline; advanced hub/spoke mapping remains |
| SYSTEM (internal) | Y | Y | InternalClient + InternalEventSystem with Channel-based send/receive loops | | SYSTEM (internal) | Y | Y | InternalClient + InternalEventSystem with Channel-based send/receive loops |
| JETSTREAM (internal) | Y | N | | | JETSTREAM (internal) | Y | N | |
| ACCOUNT (internal) | Y | Y | Lazy per-account InternalClient with import/export subscription support | | ACCOUNT (internal) | Y | Y | Lazy per-account InternalClient with import/export subscription support |
| WebSocket clients | Y | Y | Custom frame parser, permessage-deflate compression, origin checking, cookie auth | | WebSocket clients | Y | Y | Custom frame parser, permessage-deflate compression, origin checking, cookie auth |
| MQTT clients | Y | Partial | JWT connection-type constants + config parsing; no MQTT transport yet | | MQTT clients | Y | Baseline | JWT connection-type constants + config parsing; no MQTT transport yet |
### Client Features ### Client Features
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
@@ -191,7 +191,7 @@ Go implements a sophisticated slow consumer detection system:
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Per-account subscription limit | Y | Y | `Account.IncrementSubscriptions()` returns false when `MaxSubscriptions` exceeded | | Per-account subscription limit | Y | Y | `Account.IncrementSubscriptions()` returns false when `MaxSubscriptions` exceeded |
| Auto-unsubscribe on max messages | Y | Y | Enforced at delivery; sub removed from trie + client dict when exhausted | | Auto-unsubscribe on max messages | Y | Y | Enforced at delivery; sub removed from trie + client dict when exhausted |
| Subscription routing propagation | Y | Partial | Remote subs tracked in trie; propagation via in-process method calls (no wire RS+/RS-); RMSG forwarding absent | | Subscription routing propagation | Y | Y | Remote subs tracked in trie and propagated over wire RS+/RS- with RMSG forwarding |
| Queue weight (`qw`) field | Y | N | For remote queue load balancing | | Queue weight (`qw`) field | Y | N | For remote queue load balancing |
--- ---
@@ -282,12 +282,12 @@ Go implements a sophisticated slow consumer detection system:
| `/varz` | Y | Y | | | `/varz` | Y | Y | |
| `/connz` | Y | Y | | | `/connz` | Y | Y | |
| `/` (root listing) | Y | Y | | | `/` (root listing) | Y | Y | |
| `/routez` | Y | Stub | Returns empty response | | `/routez` | Y | Y | Returns live route counts via `RoutezHandler` |
| `/gatewayz` | Y | Stub | Returns empty response | | `/gatewayz` | Y | Y | Returns live gateway counts via `GatewayzHandler` |
| `/leafz` | Y | Stub | Returns empty response | | `/leafz` | Y | Y | Returns live leaf counts via `LeafzHandler` |
| `/subz` / `/subscriptionsz` | Y | Y | Account filtering, test subject filtering, pagination, and subscription details | | `/subz` / `/subscriptionsz` | Y | Y | Account filtering, test subject filtering, pagination, and subscription details |
| `/accountz` | Y | Stub | Returns empty response | | `/accountz` | Y | Y | Returns runtime account summaries via `AccountzHandler` |
| `/accstatz` | Y | Stub | Returns empty response | | `/accstatz` | Y | Y | Returns aggregate account stats via `AccountzHandler` |
| `/jsz` | Y | Y | Returns live JetStream counts/config and API totals/errors via `JszHandler` | | `/jsz` | Y | Y | Returns live JetStream counts/config and API totals/errors via `JszHandler` |
### Varz Response ### Varz Response
@@ -302,14 +302,14 @@ Go implements a sophisticated slow consumer detection system:
| Connections (current, total) | Y | Y | | | Connections (current, total) | Y | Y | |
| Messages (in/out msgs/bytes) | Y | Y | | | Messages (in/out msgs/bytes) | Y | Y | |
| SlowConsumer breakdown | Y | N | Go tracks per connection type | | SlowConsumer breakdown | Y | N | Go tracks per connection type |
| Cluster/Gateway/Leaf blocks | Y | Partial | Config projection present; `/gatewayz` and `/leafz` endpoints remain stubs | | Cluster/Gateway/Leaf blocks | Y | Y | Live route/gateway/leaf counters are exposed in dedicated endpoints |
| JetStream block | Y | Y | Includes live JetStream config, stream/consumer counts, and API totals/errors | | JetStream block | Y | Y | Includes live JetStream config, stream/consumer counts, and API totals/errors |
| TLS cert expiry info | Y | Y | `TlsCertNotAfter` loaded via `X509CertificateLoader` in `/varz` | | TLS cert expiry info | Y | Y | `TlsCertNotAfter` loaded via `X509CertificateLoader` in `/varz` |
### Connz Response ### Connz Response
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Filtering by CID, user, account | Y | Partial | | | Filtering by CID, user, account | Y | Baseline | |
| Sorting (11 options) | Y | Y | All options including ByStop, ByReason, ByRtt | | Sorting (11 options) | Y | Y | All options including ByStop, ByReason, ByRtt |
| State filtering (open/closed/all) | Y | Y | `state=open|closed|all` query parameter | | State filtering (open/closed/all) | Y | Y | `state=open|closed|all` query parameter |
| Closed connection tracking | Y | Y | `ConcurrentQueue<ClosedClient>` capped at 10,000 entries | | Closed connection tracking | Y | Y | `ConcurrentQueue<ClosedClient>` capped at 10,000 entries |
@@ -353,7 +353,7 @@ Go implements a sophisticated slow consumer detection system:
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Structured logging | Partial | Y | .NET uses Serilog with ILogger<T> | | Structured logging | Baseline | Y | .NET uses Serilog with ILogger<T> |
| File logging with rotation | Y | Y | `-l`/`--log_file` flag + `LogSizeLimit`/`LogMaxFiles` via Serilog.Sinks.File | | File logging with rotation | Y | Y | `-l`/`--log_file` flag + `LogSizeLimit`/`LogMaxFiles` via Serilog.Sinks.File |
| Syslog (local and remote) | Y | Y | `--syslog` and `--remote_syslog` flags via Serilog.Sinks.SyslogMessages | | Syslog (local and remote) | Y | Y | `--syslog` and `--remote_syslog` flags via Serilog.Sinks.SyslogMessages |
| Log reopening (SIGUSR1) | Y | Y | SIGUSR1 handler calls ReOpenLogFile callback | | Log reopening (SIGUSR1) | Y | Y | SIGUSR1 handler calls ReOpenLogFile callback |
@@ -417,7 +417,7 @@ The following items from the original gap list have been implemented:
## 11. JetStream ## 11. JetStream
> The Go JetStream surface is ~37,500 lines across jetstream.go, stream.go, consumer.go, filestore.go, memstore.go, raft.go. The .NET implementation has expanded API and runtime parity coverage but remains partial versus full Go semantics. > The Go JetStream surface is ~37,500 lines across jetstream.go, stream.go, consumer.go, filestore.go, memstore.go, raft.go. The .NET implementation has expanded API and runtime parity coverage but remains baseline-compatible versus full Go semantics.
### JetStream API ($JS.API.* subjects) ### JetStream API ($JS.API.* subjects)
@@ -455,38 +455,38 @@ The following items from the original gap list have been implemented:
| Subjects | Y | Y | | | Subjects | Y | Y | |
| Replicas | Y | Y | Wires RAFT replica count | | Replicas | Y | Y | Wires RAFT replica count |
| MaxMsgs limit | Y | Y | Enforced via `EnforceLimits()` | | MaxMsgs limit | Y | Y | Enforced via `EnforceLimits()` |
| Retention (Limits/Interest/WorkQueue) | Y | Partial | Policy enums + validation branch exist; full runtime semantics incomplete | | Retention (Limits/Interest/WorkQueue) | Y | Baseline | Policy enums + validation branch exist; full runtime semantics incomplete |
| Discard policy (Old/New) | Y | Partial | Model support exists; runtime discard behavior not fully enforced | | Discard policy (Old/New) | Y | Y | `Discard=New` now rejects writes when `MaxBytes` is exceeded |
| MaxBytes / MaxAge (TTL) | Y | N | | | MaxBytes / MaxAge (TTL) | Y | Baseline | `MaxBytes` enforced; `MaxAge` model and parsing added, full TTL pruning not complete |
| MaxMsgsPer (per-subject limit) | Y | N | | | MaxMsgsPer (per-subject limit) | Y | Baseline | Config model/parsing present; per-subject runtime cap remains limited |
| MaxMsgSize | Y | N | | | MaxMsgSize | Y | N | |
| Storage type selection (Memory/File) | Y | N | MemStore default; no config-driven choice | | Storage type selection (Memory/File) | Y | Y | Per-stream backend selection supports memory and file stores |
| Compression (S2) | Y | N | | | Compression (S2) | Y | N | |
| Subject transform | Y | N | | | Subject transform | Y | N | |
| RePublish | Y | N | | | RePublish | Y | N | |
| AllowDirect / KV mode | Y | N | | | AllowDirect / KV mode | Y | N | |
| Sealed, DenyDelete, DenyPurge | Y | N | | | Sealed, DenyDelete, DenyPurge | Y | N | |
| Duplicates dedup window | Y | Partial | Dedup ID cache exists; no configurable window | | Duplicates dedup window | Y | Baseline | Dedup ID cache exists; no configurable window |
### Consumer Configuration & Delivery ### Consumer Configuration & Delivery
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Push delivery | Y | Partial | `PushConsumerEngine`; basic delivery | | Push delivery | Y | Baseline | `PushConsumerEngine`; basic delivery |
| Pull fetch | Y | Partial | `PullConsumerEngine`; basic batch fetch | | Pull fetch | Y | Baseline | `PullConsumerEngine`; basic batch fetch |
| Ephemeral consumers | Y | N | Only durable | | Ephemeral consumers | Y | Y | Ephemeral creation baseline auto-generates durable IDs when requested |
| AckPolicy.None | Y | Y | | | AckPolicy.None | Y | Y | |
| AckPolicy.Explicit | Y | Y | `AckProcessor` tracks pending with expiry | | AckPolicy.Explicit | Y | Y | `AckProcessor` tracks pending with expiry |
| AckPolicy.All | Y | Partial | In-memory ack floor behavior implemented; full wire-level ack contract remains limited | | AckPolicy.All | Y | Baseline | In-memory ack floor behavior implemented; full wire-level ack contract remains limited |
| Redelivery on ack timeout | Y | Partial | `NextExpired()` detects expired; limit not enforced | | Redelivery on ack timeout | Y | Baseline | `NextExpired()` detects expired; limit not enforced |
| DeliverPolicy (All/Last/New/StartSeq/StartTime) | Y | Partial | Policy enums added; fetch behavior still mostly starts at beginning | | DeliverPolicy (All/Last/New/StartSeq/StartTime) | Y | Baseline | Policy enums added; fetch behavior still mostly starts at beginning |
| FilterSubject (single) | Y | Y | | | FilterSubject (single) | Y | Y | |
| FilterSubjects (multiple) | Y | N | | | FilterSubjects (multiple) | Y | Y | Multi-filter matching implemented in pull/push delivery paths |
| MaxAckPending | Y | N | | | MaxAckPending | Y | Y | Pending delivery cap enforced for consumer queues |
| Idle heartbeat | Y | Partial | Push engine emits heartbeat frames for configured consumers | | Idle heartbeat | Y | Baseline | Push engine emits heartbeat frames for configured consumers |
| Flow control | Y | N | | | Flow control | Y | N | |
| Rate limiting | Y | N | | | Rate limiting | Y | N | |
| Replay policy | Y | Partial | Policy enum exists; replay timing semantics not fully implemented | | Replay policy | Y | Baseline | `ReplayPolicy.Original` baseline delay implemented; full Go timing semantics remain |
| BackOff (exponential) | Y | N | | | BackOff (exponential) | Y | N | |
### Storage Backends ### Storage Backends
@@ -507,9 +507,9 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Mirror consumer creation | Y | Partial | `MirrorCoordinator` triggers on append | | Mirror consumer creation | Y | Baseline | `MirrorCoordinator` triggers on append |
| Mirror sync state tracking | Y | N | | | Mirror sync state tracking | Y | N | |
| Source fan-in (multiple sources) | Y | Partial | Single `Source` field; no `Sources[]` array | | Source fan-in (multiple sources) | Y | Y | `Sources[]` array support added and replicated via `SourceCoordinator` |
| Subject mapping for sources | Y | N | | | Subject mapping for sources | Y | N | |
| Cross-account mirror/source | Y | N | | | Cross-account mirror/source | Y | N | |
@@ -517,30 +517,30 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
| Feature | Go (5 037 lines) | .NET (212 lines) | Notes | | Feature | Go (5 037 lines) | .NET (212 lines) | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Leader election / term tracking | Y | Partial | In-process; nodes hold direct `List<RaftNode>` references | | Leader election / term tracking | Y | Baseline | In-process; nodes hold direct `List<RaftNode>` references |
| Log append + quorum | Y | Partial | Entries replicated via direct method calls; stale-term append now rejected | | Log append + quorum | Y | Baseline | Entries replicated via direct method calls; stale-term append now rejected |
| Log persistence | Y | N | In-memory `List<RaftLogEntry>` only | | Log persistence | Y | Baseline | `RaftLog.PersistAsync/LoadAsync` plus node term/applied persistence baseline |
| Heartbeat / keep-alive | Y | N | | | Heartbeat / keep-alive | Y | N | |
| Log mismatch resolution (NextIndex) | Y | N | | | Log mismatch resolution (NextIndex) | Y | N | |
| Snapshot creation | Y | Partial | `CreateSnapshotAsync()` exists; stored in-memory | | Snapshot creation | Y | Baseline | `CreateSnapshotAsync()` exists; stored in-memory |
| Snapshot network transfer | Y | N | | | Snapshot network transfer | Y | N | |
| Membership changes | Y | N | | | Membership changes | Y | N | |
| Network RPC transport | Y | N | All coordination is in-process | | Network RPC transport | Y | Baseline | `IRaftTransport` abstraction + in-memory transport baseline implemented |
### JetStream Clustering ### JetStream Clustering
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Meta-group governance | Y | Partial | `JetStreamMetaGroup` tracks streams; no durable consensus | | Meta-group governance | Y | Baseline | `JetStreamMetaGroup` tracks streams; no durable consensus |
| Per-stream replica group | Y | Partial | `StreamReplicaGroup` + in-memory RAFT | | Per-stream replica group | Y | Baseline | `StreamReplicaGroup` + in-memory RAFT |
| Asset placement planner | Y | Partial | `AssetPlacementPlanner` skeleton | | Asset placement planner | Y | Baseline | `AssetPlacementPlanner` skeleton |
| Cross-cluster JetStream (gateways) | Y | N | Requires functional gateways | | Cross-cluster JetStream (gateways) | Y | N | Requires functional gateways |
--- ---
## 12. Clustering ## 12. Clustering
> Routes, gateways, and leaf nodes are in different states of completeness. All three are present as Go reference feature targets but only routes have any functional networking. > Routes, gateways, and leaf nodes now all have functional networking baselines; advanced Go semantics are still incomplete.
### Routes ### Routes
@@ -550,10 +550,10 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
| Outbound seed connections (with backoff) | Y | Y | Iterates `ClusterOptions.Routes` with 250 ms retry | | Outbound seed connections (with backoff) | Y | Y | Iterates `ClusterOptions.Routes` with 250 ms retry |
| Route handshake (ROUTE `<serverId>`) | Y | Y | Bidirectional: sends own ID, reads peer ID | | Route handshake (ROUTE `<serverId>`) | Y | Y | Bidirectional: sends own ID, reads peer ID |
| Remote subscription tracking | Y | Y | `ApplyRemoteSubscription` adds to SubList; `HasRemoteInterest` exposed | | Remote subscription tracking | Y | Y | `ApplyRemoteSubscription` adds to SubList; `HasRemoteInterest` exposed |
| Subscription propagation (in-process) | Y | Partial | `PropagateLocalSubscription` calls peer managers directly — no wire RS+/RS- protocol | | Subscription propagation (wire RS+/RS-) | Y | Y | Local SUB/UNSUB is propagated over route wire frames |
| Message routing (RMSG wire) | Y | N | **Critical gap**: published messages are never forwarded to remote subscribers | | Message routing (RMSG wire) | Y | Y | Routed publishes forward over RMSG to remote subscribers |
| RS+/RS- subscription protocol (wire) | Y | N | Command matrix recognises opcodes but no handler processes inbound RS+/RS- frames | | RS+/RS- subscription protocol (wire) | Y | Y | Inbound RS+/RS- frames update remote-interest trie |
| Route pooling (3× per peer) | Y | N | Single connection per remote server ID | | Route pooling (3× per peer) | Y | Y | `ClusterOptions.PoolSize` defaults to 3 links per peer |
| Account-specific routes | Y | N | | | Account-specific routes | Y | N | |
| S2 compression on routes | Y | N | | | S2 compression on routes | Y | N | |
| CONNECT info + topology gossip | Y | N | Handshake is two-line text exchange only | | CONNECT info + topology gossip | Y | N | Handshake is two-line text exchange only |
@@ -562,40 +562,35 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Any networking (listener / outbound) | Y | N | `GatewayManager.StartAsync()` logs a debug line and zeros a counter | | Any networking (listener / outbound) | Y | Y | Listener + outbound remotes with retry are active |
| Gateway connection protocol | Y | N | | | Gateway connection protocol | Y | Baseline | Baseline `GATEWAY` handshake implemented |
| Interest-only mode | Y | N | | | Interest-only mode | Y | Baseline | Baseline A+/A- interest propagation implemented |
| Reply subject mapping (`_GR_.` prefix) | Y | N | | | Reply subject mapping (`_GR_.` prefix) | Y | N | |
| Message forwarding to remote clusters | Y | N | | | Message forwarding to remote clusters | Y | Baseline | Baseline `GMSG` forwarding implemented |
### Leaf Nodes ### Leaf Nodes
| Feature | Go | .NET | Notes | | Feature | Go | .NET | Notes |
|---------|:--:|:----:|-------| |---------|:--:|:----:|-------|
| Any networking (listener / spoke) | Y | N | `LeafNodeManager.StartAsync()` logs a debug line and zeros a counter | | Any networking (listener / spoke) | Y | Y | Listener + outbound remotes with retry are active |
| Leaf handshake / role negotiation | Y | N | | | Leaf handshake / role negotiation | Y | Baseline | Baseline `LEAF` handshake implemented |
| Subscription sharing (LS+/LS-) | Y | N | | | Subscription sharing (LS+/LS-) | Y | Baseline | LS+/LS- propagation implemented |
| Loop detection (`$LDS.` prefix) | Y | N | | | Loop detection (`$LDS.` prefix) | Y | N | |
| Hub-and-spoke account mapping | Y | N | | | Hub-and-spoke account mapping | Y | Baseline | Baseline LMSG forwarding works; advanced account remapping remains |
--- ---
## Summary: Remaining Gaps ## Summary: Remaining Gaps
### Clustering (High Impact) ### Clustering (High Impact)
1. **Route message routing** — Remote subscribers receive no messages; no RMSG implementation 1. **Gateway advanced semantics** — reply remapping (`_GR_.`) and full interest-only behavior are not complete
2. **Gateways** — Non-functional stub; no inter-cluster bridging 2. **Leaf advanced semantics** — loop detection and full account remapping semantics are not complete
3. **Leaf nodes** — Non-functional stub; no hub/spoke topology 3. **Inter-server account protocol** — A+/A- account semantics remain baseline-only
4. **RS+/RS- wire protocol** — subscription propagation is in-process method calls only
5. **Route pooling** — single connection per peer vs Go's 3-connection pool
### JetStream (Significant Gaps) ### JetStream (Significant Gaps)
1. **API coverage is expanded but still incomplete** — core stream/consumer/direct/control routes are implemented, but full Go surface and edge semantics remain 1. **Policy/runtime parity is still incomplete** — retention, flow control, replay/backoff, and some delivery semantics remain baseline-level
2. **Policy/runtime semantics are incomplete** — retention/discard/delivery/replay models exist, but behavior does not yet match Go across all cases 2. **FileStore scalability** — JSONL-based (not block/compressed/encrypted)
3. **Snapshot/restore and cluster control are skeletal** — request/response contracts exist; durable/distributed semantics remain limited 3. **RAFT transport durability** — transport and persistence baselines exist, but full network consensus semantics remain incomplete
4. **FileStore scalability** — JSONL-based (not block/compressed/encrypted)
5. **RAFT persistence and transport** — in-memory coordination; no durable log/RPC transport parity
6. **Consumer delivery completeness** — MaxAckPending, flow control, replay/backoff, and full fetch/ack lifecycle parity still incomplete
### Lower Priority ### Lower Priority
1. **Dynamic buffer sizing** — delegated to Pipe, less optimized for long-lived connections 1. **Dynamic buffer sizing** — delegated to Pipe, less optimized for long-lived connections
@@ -610,13 +605,17 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
### Newly Ported API Families ### Newly Ported API Families
- `$JS.API.INFO` - `$JS.API.INFO`
- `$JS.API.SERVER.REMOVE`
- `$JS.API.ACCOUNT.PURGE.*`, `$JS.API.ACCOUNT.STREAM.MOVE.*`, `$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*`
- `$JS.API.STREAM.UPDATE.*`, `$JS.API.STREAM.DELETE.*`, `$JS.API.STREAM.NAMES`, `$JS.API.STREAM.LIST` - `$JS.API.STREAM.UPDATE.*`, `$JS.API.STREAM.DELETE.*`, `$JS.API.STREAM.NAMES`, `$JS.API.STREAM.LIST`
- `$JS.API.STREAM.PEER.REMOVE.*`
- `$JS.API.STREAM.MSG.GET.*`, `$JS.API.STREAM.MSG.DELETE.*`, `$JS.API.STREAM.PURGE.*` - `$JS.API.STREAM.MSG.GET.*`, `$JS.API.STREAM.MSG.DELETE.*`, `$JS.API.STREAM.PURGE.*`
- `$JS.API.DIRECT.GET.*` - `$JS.API.DIRECT.GET.*`
- `$JS.API.STREAM.SNAPSHOT.*`, `$JS.API.STREAM.RESTORE.*` - `$JS.API.STREAM.SNAPSHOT.*`, `$JS.API.STREAM.RESTORE.*`
- `$JS.API.CONSUMER.NAMES.*`, `$JS.API.CONSUMER.LIST.*`, `$JS.API.CONSUMER.DELETE.*.*` - `$JS.API.CONSUMER.NAMES.*`, `$JS.API.CONSUMER.LIST.*`, `$JS.API.CONSUMER.DELETE.*.*`
- `$JS.API.CONSUMER.PAUSE.*.*`, `$JS.API.CONSUMER.RESET.*.*`, `$JS.API.CONSUMER.UNPIN.*.*` - `$JS.API.CONSUMER.PAUSE.*.*`, `$JS.API.CONSUMER.RESET.*.*`, `$JS.API.CONSUMER.UNPIN.*.*`
- `$JS.API.CONSUMER.MSG.NEXT.*.*` - `$JS.API.CONSUMER.MSG.NEXT.*.*`
- `$JS.API.CONSUMER.LEADER.STEPDOWN.*.*`
- `$JS.API.STREAM.LEADER.STEPDOWN.*`, `$JS.API.META.LEADER.STEPDOWN` - `$JS.API.STREAM.LEADER.STEPDOWN.*`, `$JS.API.META.LEADER.STEPDOWN`
### Runtime/Storage/RAFT Parity Additions ### Runtime/Storage/RAFT Parity Additions
@@ -626,7 +625,12 @@ MemStore has basic append/load/purge with `Dictionary<long, StoredMessage>` unde
- Stream store subject index support (`LoadLastBySubjectAsync`) in `MemStore` and `FileStore`. - Stream store subject index support (`LoadLastBySubjectAsync`) in `MemStore` and `FileStore`.
- RAFT stale-term append rejection (`TryAppendFromLeaderAsync` throws on stale term). - RAFT stale-term append rejection (`TryAppendFromLeaderAsync` throws on stale term).
- `/jsz` and `/varz` now expose JetStream API totals/errors from server stats. - `/jsz` and `/varz` now expose JetStream API totals/errors from server stats.
- Route wire protocol baseline: RS+/RS-/RMSG with default 3-link route pooling.
- Gateway/Leaf wire protocol baselines: A+/A-/GMSG and LS+/LS-/LMSG.
- Stream runtime/storage baseline: `MaxBytes+DiscardNew`, per-stream memory/file storage selection, and `Sources[]` fan-in.
- Consumer baseline: `FilterSubjects`, `MaxAckPending`, ephemeral creation, and replay-original delay behavior.
- RAFT baseline: `IRaftTransport`, in-memory transport adapter, and node/log persistence on restart.
- Monitoring baseline: `/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz` now return runtime data.
### Remaining Explicit Deltas ### Remaining Explicit Deltas
- Internal JetStream connection type remains unimplemented (`JETSTREAM (internal)` is still `N`). - Internal JetStream connection type remains unimplemented (`JETSTREAM (internal)` is still `N`).
- Monitoring endpoints `/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz` remain stubbed.

View File

@@ -3,16 +3,25 @@
| Go Subject | .NET Route | Status | Test | | Go Subject | .NET Route | Status | Test |
|---|---|---|---| |---|---|---|---|
| $JS.API.INFO | `AccountApiHandlers.HandleInfo` | ported | `JetStreamAccountInfoApiTests.Account_info_returns_jetstream_limits_and_usage_shape` | | $JS.API.INFO | `AccountApiHandlers.HandleInfo` | ported | `JetStreamAccountInfoApiTests.Account_info_returns_jetstream_limits_and_usage_shape` |
| $JS.API.SERVER.REMOVE | `AccountControlApiHandlers.HandleServerRemove` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
| $JS.API.ACCOUNT.PURGE.* | `AccountControlApiHandlers.HandleAccountPurge` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
| $JS.API.ACCOUNT.STREAM.MOVE.* | `AccountControlApiHandlers.HandleAccountStreamMove` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
| $JS.API.ACCOUNT.STREAM.MOVE.CANCEL.* | `AccountControlApiHandlers.HandleAccountStreamMoveCancel` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` |
| $JS.API.STREAM.CREATE.* | `StreamApiHandlers.HandleCreate` | ported | `JetStreamStreamLifecycleApiTests.Stream_create_info_and_update_roundtrip` |
| $JS.API.STREAM.INFO.* | `StreamApiHandlers.HandleInfo` | ported | `JetStreamStreamLifecycleApiTests.Stream_create_info_and_update_roundtrip` |
| $JS.API.STREAM.UPDATE.* | `StreamApiHandlers.HandleUpdate` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` | | $JS.API.STREAM.UPDATE.* | `StreamApiHandlers.HandleUpdate` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` |
| $JS.API.STREAM.DELETE.* | `StreamApiHandlers.HandleDelete` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` | | $JS.API.STREAM.DELETE.* | `StreamApiHandlers.HandleDelete` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` |
| $JS.API.STREAM.NAMES | `StreamApiHandlers.HandleNames` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` | | $JS.API.STREAM.NAMES | `StreamApiHandlers.HandleNames` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` |
| $JS.API.STREAM.LIST | `StreamApiHandlers.HandleList` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` | | $JS.API.STREAM.LIST | `StreamApiHandlers.HandleList` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` |
| $JS.API.STREAM.PEER.REMOVE.* | `ClusterControlApiHandlers.HandleStreamPeerRemove` | ported | `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` |
| $JS.API.STREAM.MSG.GET.* | `StreamApiHandlers.HandleMessageGet` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` | | $JS.API.STREAM.MSG.GET.* | `StreamApiHandlers.HandleMessageGet` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` |
| $JS.API.STREAM.MSG.DELETE.* | `StreamApiHandlers.HandleMessageDelete` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` | | $JS.API.STREAM.MSG.DELETE.* | `StreamApiHandlers.HandleMessageDelete` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` |
| $JS.API.STREAM.PURGE.* | `StreamApiHandlers.HandlePurge` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` | | $JS.API.STREAM.PURGE.* | `StreamApiHandlers.HandlePurge` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` |
| $JS.API.DIRECT.GET.* | `DirectApiHandlers.HandleGet` | ported | `JetStreamDirectGetApiTests.Direct_get_returns_message_without_stream_info_wrapper` | | $JS.API.DIRECT.GET.* | `DirectApiHandlers.HandleGet` | ported | `JetStreamDirectGetApiTests.Direct_get_returns_message_without_stream_info_wrapper` |
| $JS.API.STREAM.SNAPSHOT.* | `StreamApiHandlers.HandleSnapshot` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` | | $JS.API.STREAM.SNAPSHOT.* | `StreamApiHandlers.HandleSnapshot` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` |
| $JS.API.STREAM.RESTORE.* | `StreamApiHandlers.HandleRestore` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` | | $JS.API.STREAM.RESTORE.* | `StreamApiHandlers.HandleRestore` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` |
| $JS.API.CONSUMER.CREATE.*.* | `ConsumerApiHandlers.HandleCreate` | ported | `JetStreamConsumerApiTests.Consumer_create_and_info_roundtrip` |
| $JS.API.CONSUMER.INFO.*.* | `ConsumerApiHandlers.HandleInfo` | ported | `JetStreamConsumerApiTests.Consumer_create_and_info_roundtrip` |
| $JS.API.CONSUMER.NAMES.* | `ConsumerApiHandlers.HandleNames` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` | | $JS.API.CONSUMER.NAMES.* | `ConsumerApiHandlers.HandleNames` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` |
| $JS.API.CONSUMER.LIST.* | `ConsumerApiHandlers.HandleList` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` | | $JS.API.CONSUMER.LIST.* | `ConsumerApiHandlers.HandleList` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` |
| $JS.API.CONSUMER.DELETE.*.* | `ConsumerApiHandlers.HandleDelete` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` | | $JS.API.CONSUMER.DELETE.*.* | `ConsumerApiHandlers.HandleDelete` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` |
@@ -20,5 +29,6 @@
| $JS.API.CONSUMER.RESET.*.* | `ConsumerApiHandlers.HandleReset` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` | | $JS.API.CONSUMER.RESET.*.* | `ConsumerApiHandlers.HandleReset` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` |
| $JS.API.CONSUMER.UNPIN.*.* | `ConsumerApiHandlers.HandleUnpin` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` | | $JS.API.CONSUMER.UNPIN.*.* | `ConsumerApiHandlers.HandleUnpin` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` |
| $JS.API.CONSUMER.MSG.NEXT.*.* | `ConsumerApiHandlers.HandleNext` | ported | `JetStreamConsumerNextApiTests.Consumer_msg_next_respects_batch_request` | | $JS.API.CONSUMER.MSG.NEXT.*.* | `ConsumerApiHandlers.HandleNext` | ported | `JetStreamConsumerNextApiTests.Consumer_msg_next_respects_batch_request` |
| $JS.API.CONSUMER.LEADER.STEPDOWN.*.* | `ClusterControlApiHandlers.HandleConsumerLeaderStepdown` | ported | `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` |
| $JS.API.STREAM.LEADER.STEPDOWN.* | `ClusterControlApiHandlers.HandleStreamLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` | | $JS.API.STREAM.LEADER.STEPDOWN.* | `ClusterControlApiHandlers.HandleStreamLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` |
| $JS.API.META.LEADER.STEPDOWN | `ClusterControlApiHandlers.HandleMetaLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` | | $JS.API.META.LEADER.STEPDOWN | `ClusterControlApiHandlers.HandleMetaLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` |

View File

@@ -10,7 +10,7 @@ dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStream|Fully
Result: Result:
- Passed: `54` - Passed: `69`
- Failed: `0` - Failed: `0`
- Skipped: `0` - Skipped: `0`
- Duration: `~10s` - Duration: `~10s`
@@ -25,10 +25,10 @@ dotnet test -v minimal
Result: Result:
- Passed: `737` - Passed: `768`
- Failed: `0` - Failed: `0`
- Skipped: `0` - Skipped: `0`
- Duration: `~1m 5s` - Duration: `~1m 11s`
## Focused Scenario Evidence ## Focused Scenario Evidence
@@ -40,5 +40,19 @@ Result:
- `JetStreamPushConsumerContractTests.Ack_all_advances_floor_and_clears_pending_before_sequence` - `JetStreamPushConsumerContractTests.Ack_all_advances_floor_and_clears_pending_before_sequence`
- `RaftSafetyContractTests.Follower_rejects_stale_term_vote_and_append` - `RaftSafetyContractTests.Follower_rejects_stale_term_vote_and_append`
- `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` - `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape`
- `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable`
- `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape`
- `RouteWireSubscriptionProtocolTests.RSplus_RSminus_frames_propagate_remote_interest_over_socket`
- `RouteRmsgForwardingTests.Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG`
- `RoutePoolTests.Route_manager_establishes_default_pool_of_three_links_per_peer`
- `GatewayProtocolTests.Gateway_link_establishes_and_forwards_interested_message`
- `LeafProtocolTests.Leaf_link_propagates_subscription_and_message_flow`
- `JetStreamStreamPolicyRuntimeTests.Discard_new_rejects_publish_when_max_bytes_exceeded`
- `JetStreamStorageSelectionTests.Stream_with_storage_file_uses_filestore_backend`
- `JetStreamConsumerSemanticsTests.Consumer_with_filter_subjects_only_receives_matching_messages`
- `JetStreamFlowReplayBackoffTests.Replay_original_respects_message_timestamps_with_backoff_redelivery`
- `JetStreamMirrorSourceAdvancedTests.Stream_with_multiple_sources_aggregates_messages_in_order`
- `RaftTransportPersistenceTests.Raft_node_recovers_log_and_term_after_restart`
- `MonitorClusterEndpointTests.Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data`
- `JetStreamMonitoringParityTests.Jsz_and_varz_include_expanded_runtime_fields` - `JetStreamMonitoringParityTests.Jsz_and_varz_include_expanded_runtime_fields`
- `JetStreamIntegrationMatrixTests.Integration_matrix_executes_real_server_scenarios` - `JetStreamIntegrationMatrixTests.Integration_matrix_executes_real_server_scenarios`

View File

@@ -14,9 +14,14 @@ if [[ -f "$go_file" ]]; then
# Add explicit subject families used by parity tests/docs. # Add explicit subject families used by parity tests/docs.
cat <<'EOF' cat <<'EOF'
$JS.API.INFO $JS.API.INFO
$JS.API.SERVER.REMOVE
$JS.API.ACCOUNT.PURGE.*
$JS.API.ACCOUNT.STREAM.MOVE.*
$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*
$JS.API.STREAM.UPDATE.* $JS.API.STREAM.UPDATE.*
$JS.API.STREAM.DELETE.* $JS.API.STREAM.DELETE.*
$JS.API.STREAM.PURGE.* $JS.API.STREAM.PURGE.*
$JS.API.STREAM.PEER.REMOVE.*
$JS.API.STREAM.NAMES $JS.API.STREAM.NAMES
$JS.API.STREAM.LIST $JS.API.STREAM.LIST
$JS.API.STREAM.MSG.GET.* $JS.API.STREAM.MSG.GET.*
@@ -30,6 +35,7 @@ $JS.API.CONSUMER.PAUSE.*.*
$JS.API.CONSUMER.RESET.*.* $JS.API.CONSUMER.RESET.*.*
$JS.API.CONSUMER.UNPIN.*.* $JS.API.CONSUMER.UNPIN.*.*
$JS.API.CONSUMER.MSG.NEXT.*.* $JS.API.CONSUMER.MSG.NEXT.*.*
$JS.API.CONSUMER.LEADER.STEPDOWN.*.*
$JS.API.DIRECT.GET.* $JS.API.DIRECT.GET.*
$JS.API.STREAM.LEADER.STEPDOWN.* $JS.API.STREAM.LEADER.STEPDOWN.*
$JS.API.META.LEADER.STEPDOWN $JS.API.META.LEADER.STEPDOWN
@@ -41,11 +47,16 @@ fi
# Fallback subject inventory when Go reference sources are not vendored in this repo. # Fallback subject inventory when Go reference sources are not vendored in this repo.
cat <<'EOF' cat <<'EOF'
$JS.API.INFO $JS.API.INFO
$JS.API.SERVER.REMOVE
$JS.API.ACCOUNT.PURGE.*
$JS.API.ACCOUNT.STREAM.MOVE.*
$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*
$JS.API.STREAM.CREATE.* $JS.API.STREAM.CREATE.*
$JS.API.STREAM.UPDATE.* $JS.API.STREAM.UPDATE.*
$JS.API.STREAM.DELETE.* $JS.API.STREAM.DELETE.*
$JS.API.STREAM.PURGE.* $JS.API.STREAM.PURGE.*
$JS.API.STREAM.INFO.* $JS.API.STREAM.INFO.*
$JS.API.STREAM.PEER.REMOVE.*
$JS.API.STREAM.NAMES $JS.API.STREAM.NAMES
$JS.API.STREAM.LIST $JS.API.STREAM.LIST
$JS.API.STREAM.MSG.GET.* $JS.API.STREAM.MSG.GET.*
@@ -61,6 +72,7 @@ $JS.API.CONSUMER.PAUSE.*.*
$JS.API.CONSUMER.RESET.*.* $JS.API.CONSUMER.RESET.*.*
$JS.API.CONSUMER.UNPIN.*.* $JS.API.CONSUMER.UNPIN.*.*
$JS.API.CONSUMER.MSG.NEXT.*.* $JS.API.CONSUMER.MSG.NEXT.*.*
$JS.API.CONSUMER.LEADER.STEPDOWN.*.*
$JS.API.DIRECT.GET.* $JS.API.DIRECT.GET.*
$JS.API.STREAM.LEADER.STEPDOWN.* $JS.API.STREAM.LEADER.STEPDOWN.*
$JS.API.META.LEADER.STEPDOWN $JS.API.META.LEADER.STEPDOWN

View File

@@ -5,5 +5,6 @@ public sealed class ClusterOptions
public string? Name { get; set; } public string? Name { get; set; }
public string Host { get; set; } = "0.0.0.0"; public string Host { get; set; } = "0.0.0.0";
public int Port { get; set; } = 6222; public int Port { get; set; } = 6222;
public int PoolSize { get; set; } = 3;
public List<string> Routes { get; set; } = []; public List<string> Routes { get; set; } = [];
} }

View File

@@ -5,4 +5,5 @@ public sealed class GatewayOptions
public string? Name { get; set; } public string? Name { get; set; }
public string Host { get; set; } = "0.0.0.0"; public string Host { get; set; } = "0.0.0.0";
public int Port { get; set; } public int Port { get; set; }
public List<string> Remotes { get; set; } = [];
} }

View File

@@ -4,4 +4,5 @@ public sealed class LeafNodeOptions
{ {
public string Host { get; set; } = "0.0.0.0"; public string Host { get; set; } = "0.0.0.0";
public int Port { get; set; } public int Port { get; set; }
public List<string> Remotes { get; set; } = [];
} }

View File

@@ -1,11 +1,191 @@
using System.Net.Sockets;
using System.Text;
using NATS.Server.Subscriptions;
namespace NATS.Server.Gateways; namespace NATS.Server.Gateways;
public sealed class GatewayConnection public sealed class GatewayConnection(Socket socket) : IAsyncDisposable
{ {
public string RemoteEndpoint { get; } private readonly NetworkStream _stream = new(socket, ownsSocket: true);
private readonly SemaphoreSlim _writeGate = new(1, 1);
private readonly CancellationTokenSource _closedCts = new();
private Task? _loopTask;
public GatewayConnection(string remoteEndpoint) public string? RemoteId { get; private set; }
public string RemoteEndpoint => socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N");
public Func<RemoteSubscription, Task>? RemoteSubscriptionReceived { get; set; }
public Func<GatewayMessage, Task>? MessageReceived { get; set; }
public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct)
{ {
RemoteEndpoint = remoteEndpoint; await WriteLineAsync($"GATEWAY {serverId}", ct);
var line = await ReadLineAsync(ct);
RemoteId = ParseHandshake(line);
}
public async Task PerformInboundHandshakeAsync(string serverId, CancellationToken ct)
{
var line = await ReadLineAsync(ct);
RemoteId = ParseHandshake(line);
await WriteLineAsync($"GATEWAY {serverId}", ct);
}
public void StartLoop(CancellationToken ct)
{
if (_loopTask != null)
return;
var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
_loopTask = Task.Run(() => ReadLoopAsync(linked.Token), linked.Token);
}
public Task WaitUntilClosedAsync(CancellationToken ct)
=> _loopTask?.WaitAsync(ct) ?? Task.CompletedTask;
public Task SendAPlusAsync(string subject, string? queue, CancellationToken ct)
=> WriteLineAsync(queue is { Length: > 0 } ? $"A+ {subject} {queue}" : $"A+ {subject}", ct);
public Task SendAMinusAsync(string subject, string? queue, CancellationToken ct)
=> WriteLineAsync(queue is { Length: > 0 } ? $"A- {subject} {queue}" : $"A- {subject}", ct);
public async Task SendMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
{
var reply = string.IsNullOrEmpty(replyTo) ? "-" : replyTo;
await _writeGate.WaitAsync(ct);
try
{
var control = Encoding.ASCII.GetBytes($"GMSG {subject} {reply} {payload.Length}\r\n");
await _stream.WriteAsync(control, ct);
if (!payload.IsEmpty)
await _stream.WriteAsync(payload, ct);
await _stream.WriteAsync("\r\n"u8.ToArray(), ct);
await _stream.FlushAsync(ct);
}
finally
{
_writeGate.Release();
}
}
public async ValueTask DisposeAsync()
{
await _closedCts.CancelAsync();
if (_loopTask != null)
await _loopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
_closedCts.Dispose();
_writeGate.Dispose();
await _stream.DisposeAsync();
}
private async Task ReadLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
string line;
try
{
line = await ReadLineAsync(ct);
}
catch
{
break;
}
if (line.StartsWith("A+ ", StringComparison.Ordinal))
{
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
{
var queue = parts.Length >= 3 ? parts[2] : null;
await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteId ?? string.Empty));
}
continue;
}
if (line.StartsWith("A- ", StringComparison.Ordinal))
{
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
{
var queue = parts.Length >= 3 ? parts[2] : null;
await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteId ?? string.Empty));
}
continue;
}
if (!line.StartsWith("GMSG ", StringComparison.Ordinal))
continue;
var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (args.Length < 4 || !int.TryParse(args[3], out var size) || size < 0)
continue;
var payload = await ReadPayloadAsync(size, ct);
if (MessageReceived != null)
await MessageReceived(new GatewayMessage(args[1], args[2] == "-" ? null : args[2], payload));
}
}
private async Task<ReadOnlyMemory<byte>> ReadPayloadAsync(int size, CancellationToken ct)
{
var payload = new byte[size];
var offset = 0;
while (offset < size)
{
var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct);
if (read == 0)
throw new IOException("Gateway payload read closed");
offset += read;
}
var trailer = new byte[2];
_ = await _stream.ReadAsync(trailer, ct);
return payload;
}
private async Task WriteLineAsync(string line, CancellationToken ct)
{
await _writeGate.WaitAsync(ct);
try
{
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
await _stream.WriteAsync(bytes, ct);
await _stream.FlushAsync(ct);
}
finally
{
_writeGate.Release();
}
}
private async Task<string> ReadLineAsync(CancellationToken ct)
{
var bytes = new List<byte>(64);
var single = new byte[1];
while (true)
{
var read = await _stream.ReadAsync(single, ct);
if (read == 0)
throw new IOException("Gateway closed");
if (single[0] == (byte)'\n')
break;
if (single[0] != (byte)'\r')
bytes.Add(single[0]);
}
return Encoding.ASCII.GetString([.. bytes]);
}
private static string ParseHandshake(string line)
{
if (!line.StartsWith("GATEWAY ", StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException("Invalid gateway handshake");
var id = line[8..].Trim();
if (id.Length == 0)
throw new InvalidOperationException("Gateway handshake missing id");
return id;
} }
} }
public sealed record GatewayMessage(string Subject, string? ReplyTo, ReadOnlyMemory<byte> Payload);

View File

@@ -1,5 +1,9 @@
using System.Collections.Concurrent;
using System.Net;
using System.Net.Sockets;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using NATS.Server.Configuration; using NATS.Server.Configuration;
using NATS.Server.Subscriptions;
namespace NATS.Server.Gateways; namespace NATS.Server.Gateways;
@@ -7,26 +11,204 @@ public sealed class GatewayManager : IAsyncDisposable
{ {
private readonly GatewayOptions _options; private readonly GatewayOptions _options;
private readonly ServerStats _stats; private readonly ServerStats _stats;
private readonly string _serverId;
private readonly Action<RemoteSubscription> _remoteSubSink;
private readonly Action<GatewayMessage> _messageSink;
private readonly ILogger<GatewayManager> _logger; private readonly ILogger<GatewayManager> _logger;
private readonly ConcurrentDictionary<string, GatewayConnection> _connections = new(StringComparer.Ordinal);
public GatewayManager(GatewayOptions options, ServerStats stats, ILogger<GatewayManager> logger) private CancellationTokenSource? _cts;
private Socket? _listener;
private Task? _acceptLoopTask;
public string ListenEndpoint => $"{_options.Host}:{_options.Port}";
public GatewayManager(
GatewayOptions options,
ServerStats stats,
string serverId,
Action<RemoteSubscription> remoteSubSink,
Action<GatewayMessage> messageSink,
ILogger<GatewayManager> logger)
{ {
_options = options; _options = options;
_stats = stats; _stats = stats;
_serverId = serverId;
_remoteSubSink = remoteSubSink;
_messageSink = messageSink;
_logger = logger; _logger = logger;
} }
public Task StartAsync(CancellationToken ct) public Task StartAsync(CancellationToken ct)
{ {
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
_listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_listener.Bind(new IPEndPoint(IPAddress.Parse(_options.Host), _options.Port));
_listener.Listen(128);
if (_options.Port == 0)
_options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
_acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
foreach (var remote in _options.Remotes.Distinct(StringComparer.OrdinalIgnoreCase))
_ = Task.Run(() => ConnectWithRetryAsync(remote, _cts.Token));
_logger.LogDebug("Gateway manager started (name={Name}, listen={Host}:{Port})", _logger.LogDebug("Gateway manager started (name={Name}, listen={Host}:{Port})",
_options.Name, _options.Host, _options.Port); _options.Name, _options.Host, _options.Port);
Interlocked.Exchange(ref _stats.Gateways, 0);
return Task.CompletedTask; return Task.CompletedTask;
} }
public ValueTask DisposeAsync() public async Task ForwardMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
{ {
foreach (var connection in _connections.Values)
await connection.SendMessageAsync(subject, replyTo, payload, ct);
}
public void PropagateLocalSubscription(string subject, string? queue)
{
foreach (var connection in _connections.Values)
_ = connection.SendAPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
}
public void PropagateLocalUnsubscription(string subject, string? queue)
{
foreach (var connection in _connections.Values)
_ = connection.SendAMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
}
public async ValueTask DisposeAsync()
{
if (_cts == null)
return;
await _cts.CancelAsync();
_listener?.Dispose();
if (_acceptLoopTask != null)
await _acceptLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
foreach (var connection in _connections.Values)
await connection.DisposeAsync();
_connections.Clear();
Interlocked.Exchange(ref _stats.Gateways, 0);
_cts.Dispose();
_cts = null;
_logger.LogDebug("Gateway manager stopped"); _logger.LogDebug("Gateway manager stopped");
return ValueTask.CompletedTask; }
private async Task AcceptLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
Socket socket;
try
{
socket = await _listener!.AcceptAsync(ct);
}
catch
{
break;
}
_ = Task.Run(() => HandleInboundAsync(socket, ct), ct);
}
}
private async Task HandleInboundAsync(Socket socket, CancellationToken ct)
{
var connection = new GatewayConnection(socket);
try
{
await connection.PerformInboundHandshakeAsync(_serverId, ct);
Register(connection);
}
catch
{
await connection.DisposeAsync();
}
}
private async Task ConnectWithRetryAsync(string remote, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try
{
var endPoint = ParseEndpoint(remote);
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct);
var connection = new GatewayConnection(socket);
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
Register(connection);
return;
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Gateway connect retry for {Remote}", remote);
}
try
{
await Task.Delay(250, ct);
}
catch (OperationCanceledException)
{
return;
}
}
}
private void Register(GatewayConnection connection)
{
var key = $"{connection.RemoteId}:{connection.RemoteEndpoint}:{Guid.NewGuid():N}";
if (!_connections.TryAdd(key, connection))
{
_ = connection.DisposeAsync();
return;
}
connection.RemoteSubscriptionReceived = sub =>
{
_remoteSubSink(sub);
return Task.CompletedTask;
};
connection.MessageReceived = msg =>
{
_messageSink(msg);
return Task.CompletedTask;
};
connection.StartLoop(_cts!.Token);
Interlocked.Increment(ref _stats.Gateways);
_ = Task.Run(() => WatchConnectionAsync(key, connection, _cts!.Token));
}
private async Task WatchConnectionAsync(string key, GatewayConnection connection, CancellationToken ct)
{
try
{
await connection.WaitUntilClosedAsync(ct);
}
catch
{
}
finally
{
if (_connections.TryRemove(key, out _))
Interlocked.Decrement(ref _stats.Gateways);
await connection.DisposeAsync();
}
}
private static IPEndPoint ParseEndpoint(string endpoint)
{
var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
if (parts.Length != 2)
throw new FormatException($"Invalid endpoint: {endpoint}");
return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
} }
} }

View File

@@ -0,0 +1,34 @@
namespace NATS.Server.JetStream.Api.Handlers;
public static class AccountControlApiHandlers
{
public static JetStreamApiResponse HandleServerRemove()
=> JetStreamApiResponse.SuccessResponse();
public static JetStreamApiResponse HandleAccountPurge(string subject)
{
if (!subject.StartsWith(JetStreamApiSubjects.AccountPurge, StringComparison.Ordinal))
return JetStreamApiResponse.NotFound(subject);
var account = subject[JetStreamApiSubjects.AccountPurge.Length..].Trim();
return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
}
public static JetStreamApiResponse HandleAccountStreamMove(string subject)
{
if (!subject.StartsWith(JetStreamApiSubjects.AccountStreamMove, StringComparison.Ordinal))
return JetStreamApiResponse.NotFound(subject);
var account = subject[JetStreamApiSubjects.AccountStreamMove.Length..].Trim();
return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
}
public static JetStreamApiResponse HandleAccountStreamMoveCancel(string subject)
{
if (!subject.StartsWith(JetStreamApiSubjects.AccountStreamMoveCancel, StringComparison.Ordinal))
return JetStreamApiResponse.NotFound(subject);
var account = subject[JetStreamApiSubjects.AccountStreamMoveCancel.Length..].Trim();
return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
}
}

View File

@@ -20,4 +20,23 @@ public static class ClusterControlApiHandlers
streams.StepDownStreamLeaderAsync(stream, default).GetAwaiter().GetResult(); streams.StepDownStreamLeaderAsync(stream, default).GetAwaiter().GetResult();
return JetStreamApiResponse.SuccessResponse(); return JetStreamApiResponse.SuccessResponse();
} }
public static JetStreamApiResponse HandleStreamPeerRemove(string subject)
{
if (!subject.StartsWith(JetStreamApiSubjects.StreamPeerRemove, StringComparison.Ordinal))
return JetStreamApiResponse.NotFound(subject);
var stream = subject[JetStreamApiSubjects.StreamPeerRemove.Length..].Trim();
return stream.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse();
}
public static JetStreamApiResponse HandleConsumerLeaderStepdown(string subject)
{
if (!subject.StartsWith(JetStreamApiSubjects.ConsumerLeaderStepdown, StringComparison.Ordinal))
return JetStreamApiResponse.NotFound(subject);
var remainder = subject[JetStreamApiSubjects.ConsumerLeaderStepdown.Length..].Trim();
var tokens = remainder.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
return tokens.Length == 2 ? JetStreamApiResponse.SuccessResponse() : JetStreamApiResponse.NotFound(subject);
}
} }

View File

@@ -168,6 +168,19 @@ public static class ConsumerApiHandlers
if (root.TryGetProperty("filter_subject", out var filterEl)) if (root.TryGetProperty("filter_subject", out var filterEl))
config.FilterSubject = filterEl.GetString(); config.FilterSubject = filterEl.GetString();
if (root.TryGetProperty("filter_subjects", out var filterSubjectsEl) && filterSubjectsEl.ValueKind == JsonValueKind.Array)
{
foreach (var item in filterSubjectsEl.EnumerateArray())
{
var filter = item.GetString();
if (!string.IsNullOrWhiteSpace(filter))
config.FilterSubjects.Add(filter);
}
}
if (root.TryGetProperty("ephemeral", out var ephemeralEl) && ephemeralEl.ValueKind == JsonValueKind.True)
config.Ephemeral = true;
if (root.TryGetProperty("push", out var pushEl) && pushEl.ValueKind == JsonValueKind.True) if (root.TryGetProperty("push", out var pushEl) && pushEl.ValueKind == JsonValueKind.True)
config.Push = true; config.Push = true;
@@ -177,6 +190,9 @@ public static class ConsumerApiHandlers
if (root.TryGetProperty("ack_wait_ms", out var ackWaitEl) && ackWaitEl.TryGetInt32(out var ackWait)) if (root.TryGetProperty("ack_wait_ms", out var ackWaitEl) && ackWaitEl.TryGetInt32(out var ackWait))
config.AckWaitMs = ackWait; config.AckWaitMs = ackWait;
if (root.TryGetProperty("max_ack_pending", out var maxAckPendingEl) && maxAckPendingEl.TryGetInt32(out var maxAckPending))
config.MaxAckPending = Math.Max(maxAckPending, 0);
if (root.TryGetProperty("ack_policy", out var ackPolicyEl)) if (root.TryGetProperty("ack_policy", out var ackPolicyEl))
{ {
var ackPolicy = ackPolicyEl.GetString(); var ackPolicy = ackPolicyEl.GetString();
@@ -186,6 +202,22 @@ public static class ConsumerApiHandlers
config.AckPolicy = AckPolicy.All; config.AckPolicy = AckPolicy.All;
} }
if (root.TryGetProperty("deliver_policy", out var deliverPolicyEl))
{
var deliver = deliverPolicyEl.GetString();
if (string.Equals(deliver, "last", StringComparison.OrdinalIgnoreCase))
config.DeliverPolicy = DeliverPolicy.Last;
else if (string.Equals(deliver, "new", StringComparison.OrdinalIgnoreCase))
config.DeliverPolicy = DeliverPolicy.New;
}
if (root.TryGetProperty("replay_policy", out var replayPolicyEl))
{
var replay = replayPolicyEl.GetString();
if (string.Equals(replay, "original", StringComparison.OrdinalIgnoreCase))
config.ReplayPolicy = ReplayPolicy.Original;
}
return config; return config;
} }
catch (JsonException) catch (JsonException)

View File

@@ -211,6 +211,56 @@ public static class StreamApiHandlers
if (root.TryGetProperty("max_msgs", out var maxMsgsEl) && maxMsgsEl.TryGetInt32(out var maxMsgs)) if (root.TryGetProperty("max_msgs", out var maxMsgsEl) && maxMsgsEl.TryGetInt32(out var maxMsgs))
config.MaxMsgs = maxMsgs; config.MaxMsgs = maxMsgs;
if (root.TryGetProperty("max_bytes", out var maxBytesEl) && maxBytesEl.TryGetInt64(out var maxBytes))
config.MaxBytes = maxBytes;
if (root.TryGetProperty("max_msgs_per", out var maxMsgsPerEl) && maxMsgsPerEl.TryGetInt32(out var maxMsgsPer))
config.MaxMsgsPer = maxMsgsPer;
if (root.TryGetProperty("max_age_ms", out var maxAgeMsEl) && maxAgeMsEl.TryGetInt32(out var maxAgeMs))
config.MaxAgeMs = maxAgeMs;
if (root.TryGetProperty("discard", out var discardEl))
{
var discard = discardEl.GetString();
if (string.Equals(discard, "new", StringComparison.OrdinalIgnoreCase))
config.Discard = DiscardPolicy.New;
else if (string.Equals(discard, "old", StringComparison.OrdinalIgnoreCase))
config.Discard = DiscardPolicy.Old;
}
if (root.TryGetProperty("storage", out var storageEl))
{
var storage = storageEl.GetString();
if (string.Equals(storage, "file", StringComparison.OrdinalIgnoreCase))
config.Storage = StorageType.File;
else
config.Storage = StorageType.Memory;
}
if (root.TryGetProperty("source", out var sourceEl))
config.Source = sourceEl.GetString();
if (root.TryGetProperty("sources", out var sourcesEl) && sourcesEl.ValueKind == JsonValueKind.Array)
{
foreach (var source in sourcesEl.EnumerateArray())
{
if (source.ValueKind == JsonValueKind.String)
{
var name = source.GetString();
if (!string.IsNullOrWhiteSpace(name))
config.Sources.Add(new StreamSourceConfig { Name = name });
}
else if (source.ValueKind == JsonValueKind.Object &&
source.TryGetProperty("name", out var sourceNameEl))
{
var name = sourceNameEl.GetString();
if (!string.IsNullOrWhiteSpace(name))
config.Sources.Add(new StreamSourceConfig { Name = name });
}
}
}
if (root.TryGetProperty("replicas", out var replicasEl) && replicasEl.TryGetInt32(out var replicas)) if (root.TryGetProperty("replicas", out var replicasEl) && replicasEl.TryGetInt32(out var replicas))
config.Replicas = replicas; config.Replicas = replicas;

View File

@@ -25,6 +25,18 @@ public sealed class JetStreamApiRouter
if (subject.Equals(JetStreamApiSubjects.Info, StringComparison.Ordinal)) if (subject.Equals(JetStreamApiSubjects.Info, StringComparison.Ordinal))
return AccountApiHandlers.HandleInfo(_streamManager, _consumerManager); return AccountApiHandlers.HandleInfo(_streamManager, _consumerManager);
if (subject.Equals(JetStreamApiSubjects.ServerRemove, StringComparison.Ordinal))
return AccountControlApiHandlers.HandleServerRemove();
if (subject.StartsWith(JetStreamApiSubjects.AccountPurge, StringComparison.Ordinal))
return AccountControlApiHandlers.HandleAccountPurge(subject);
if (subject.StartsWith(JetStreamApiSubjects.AccountStreamMoveCancel, StringComparison.Ordinal))
return AccountControlApiHandlers.HandleAccountStreamMoveCancel(subject);
if (subject.StartsWith(JetStreamApiSubjects.AccountStreamMove, StringComparison.Ordinal))
return AccountControlApiHandlers.HandleAccountStreamMove(subject);
if (subject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal)) if (subject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal))
return StreamApiHandlers.HandleCreate(subject, payload, _streamManager); return StreamApiHandlers.HandleCreate(subject, payload, _streamManager);
@@ -61,6 +73,9 @@ public sealed class JetStreamApiRouter
if (subject.StartsWith(JetStreamApiSubjects.StreamLeaderStepdown, StringComparison.Ordinal)) if (subject.StartsWith(JetStreamApiSubjects.StreamLeaderStepdown, StringComparison.Ordinal))
return ClusterControlApiHandlers.HandleStreamLeaderStepdown(subject, _streamManager); return ClusterControlApiHandlers.HandleStreamLeaderStepdown(subject, _streamManager);
if (subject.StartsWith(JetStreamApiSubjects.StreamPeerRemove, StringComparison.Ordinal))
return ClusterControlApiHandlers.HandleStreamPeerRemove(subject);
if (subject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal)) if (subject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal))
return ConsumerApiHandlers.HandleCreate(subject, payload, _consumerManager); return ConsumerApiHandlers.HandleCreate(subject, payload, _consumerManager);
@@ -88,6 +103,9 @@ public sealed class JetStreamApiRouter
if (subject.StartsWith(JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal)) if (subject.StartsWith(JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal))
return ConsumerApiHandlers.HandleNext(subject, payload, _consumerManager, _streamManager); return ConsumerApiHandlers.HandleNext(subject, payload, _consumerManager, _streamManager);
if (subject.StartsWith(JetStreamApiSubjects.ConsumerLeaderStepdown, StringComparison.Ordinal))
return ClusterControlApiHandlers.HandleConsumerLeaderStepdown(subject);
if (subject.StartsWith(JetStreamApiSubjects.DirectGet, StringComparison.Ordinal)) if (subject.StartsWith(JetStreamApiSubjects.DirectGet, StringComparison.Ordinal))
return DirectApiHandlers.HandleGet(subject, payload, _streamManager); return DirectApiHandlers.HandleGet(subject, payload, _streamManager);

View File

@@ -3,6 +3,10 @@ namespace NATS.Server.JetStream.Api;
public static class JetStreamApiSubjects public static class JetStreamApiSubjects
{ {
public const string Info = "$JS.API.INFO"; public const string Info = "$JS.API.INFO";
public const string ServerRemove = "$JS.API.SERVER.REMOVE";
public const string AccountPurge = "$JS.API.ACCOUNT.PURGE.";
public const string AccountStreamMove = "$JS.API.ACCOUNT.STREAM.MOVE.";
public const string AccountStreamMoveCancel = "$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.";
public const string StreamCreate = "$JS.API.STREAM.CREATE."; public const string StreamCreate = "$JS.API.STREAM.CREATE.";
public const string StreamInfo = "$JS.API.STREAM.INFO."; public const string StreamInfo = "$JS.API.STREAM.INFO.";
public const string StreamNames = "$JS.API.STREAM.NAMES"; public const string StreamNames = "$JS.API.STREAM.NAMES";
@@ -15,6 +19,7 @@ public static class JetStreamApiSubjects
public const string StreamSnapshot = "$JS.API.STREAM.SNAPSHOT."; public const string StreamSnapshot = "$JS.API.STREAM.SNAPSHOT.";
public const string StreamRestore = "$JS.API.STREAM.RESTORE."; public const string StreamRestore = "$JS.API.STREAM.RESTORE.";
public const string StreamLeaderStepdown = "$JS.API.STREAM.LEADER.STEPDOWN."; public const string StreamLeaderStepdown = "$JS.API.STREAM.LEADER.STEPDOWN.";
public const string StreamPeerRemove = "$JS.API.STREAM.PEER.REMOVE.";
public const string ConsumerCreate = "$JS.API.CONSUMER.CREATE."; public const string ConsumerCreate = "$JS.API.CONSUMER.CREATE.";
public const string ConsumerInfo = "$JS.API.CONSUMER.INFO."; public const string ConsumerInfo = "$JS.API.CONSUMER.INFO.";
public const string ConsumerNames = "$JS.API.CONSUMER.NAMES."; public const string ConsumerNames = "$JS.API.CONSUMER.NAMES.";
@@ -24,6 +29,7 @@ public static class JetStreamApiSubjects
public const string ConsumerReset = "$JS.API.CONSUMER.RESET."; public const string ConsumerReset = "$JS.API.CONSUMER.RESET.";
public const string ConsumerUnpin = "$JS.API.CONSUMER.UNPIN."; public const string ConsumerUnpin = "$JS.API.CONSUMER.UNPIN.";
public const string ConsumerNext = "$JS.API.CONSUMER.MSG.NEXT."; public const string ConsumerNext = "$JS.API.CONSUMER.MSG.NEXT.";
public const string ConsumerLeaderStepdown = "$JS.API.CONSUMER.LEADER.STEPDOWN.";
public const string DirectGet = "$JS.API.DIRECT.GET."; public const string DirectGet = "$JS.API.DIRECT.GET.";
public const string MetaLeaderStepdown = "$JS.API.META.LEADER.STEPDOWN"; public const string MetaLeaderStepdown = "$JS.API.META.LEADER.STEPDOWN";
} }

View File

@@ -4,6 +4,7 @@ using NATS.Server.JetStream.Cluster;
using NATS.Server.JetStream.Consumers; using NATS.Server.JetStream.Consumers;
using NATS.Server.JetStream.Models; using NATS.Server.JetStream.Models;
using NATS.Server.JetStream.Storage; using NATS.Server.JetStream.Storage;
using NATS.Server.Subscriptions;
namespace NATS.Server.JetStream; namespace NATS.Server.JetStream;
@@ -24,7 +25,15 @@ public sealed class ConsumerManager
public JetStreamApiResponse CreateOrUpdate(string stream, ConsumerConfig config) public JetStreamApiResponse CreateOrUpdate(string stream, ConsumerConfig config)
{ {
if (string.IsNullOrWhiteSpace(config.DurableName)) if (string.IsNullOrWhiteSpace(config.DurableName))
return JetStreamApiResponse.ErrorResponse(400, "durable name required"); {
if (config.Ephemeral)
config.DurableName = $"ephemeral-{Guid.NewGuid():N}"[..24];
else
return JetStreamApiResponse.ErrorResponse(400, "durable name required");
}
if (config.FilterSubjects.Count == 0 && !string.IsNullOrWhiteSpace(config.FilterSubject))
config.FilterSubjects.Add(config.FilterSubject);
var key = (stream, config.DurableName); var key = (stream, config.DurableName);
var handle = _consumers.AddOrUpdate(key, var handle = _consumers.AddOrUpdate(key,
@@ -129,7 +138,15 @@ public sealed class ConsumerManager
public void OnPublished(string stream, StoredMessage message) public void OnPublished(string stream, StoredMessage message)
{ {
foreach (var handle in _consumers.Values.Where(c => c.Stream == stream && c.Config.Push)) foreach (var handle in _consumers.Values.Where(c => c.Stream == stream && c.Config.Push))
{
if (!MatchesFilter(handle.Config, message.Subject))
continue;
if (handle.Config.MaxAckPending > 0 && handle.AckProcessor.PendingCount >= handle.Config.MaxAckPending)
continue;
_pushConsumerEngine.Enqueue(handle, message); _pushConsumerEngine.Enqueue(handle, message);
}
} }
public PushFrame? ReadPushFrame(string stream, string durableName) public PushFrame? ReadPushFrame(string stream, string durableName)
@@ -142,6 +159,17 @@ public sealed class ConsumerManager
return consumer.PushFrames.Dequeue(); return consumer.PushFrames.Dequeue();
} }
private static bool MatchesFilter(ConsumerConfig config, string subject)
{
if (config.FilterSubjects.Count > 0)
return config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(subject, f));
if (!string.IsNullOrWhiteSpace(config.FilterSubject))
return SubjectMatch.MatchLiteral(subject, config.FilterSubject);
return true;
}
} }
public sealed record ConsumerHandle(string Stream, ConsumerConfig Config) public sealed record ConsumerHandle(string Stream, ConsumerConfig Config)

View File

@@ -1,5 +1,6 @@
using NATS.Server.JetStream.Storage; using NATS.Server.JetStream.Storage;
using NATS.Server.JetStream.Models; using NATS.Server.JetStream.Models;
using NATS.Server.Subscriptions;
namespace NATS.Server.JetStream.Consumers; namespace NATS.Server.JetStream.Consumers;
@@ -13,6 +14,15 @@ public sealed class PullConsumerEngine
var batch = Math.Max(request.Batch, 1); var batch = Math.Max(request.Batch, 1);
var messages = new List<StoredMessage>(batch); var messages = new List<StoredMessage>(batch);
if (consumer.NextSequence == 1)
{
var state = await stream.Store.GetStateAsync(ct);
if (consumer.Config.DeliverPolicy == DeliverPolicy.Last && state.LastSeq > 0)
consumer.NextSequence = state.LastSeq;
else if (consumer.Config.DeliverPolicy == DeliverPolicy.New && state.LastSeq > 0)
consumer.NextSequence = state.LastSeq + 1;
}
if (request.NoWait) if (request.NoWait)
{ {
var available = await stream.Store.LoadAsync(consumer.NextSequence, ct); var available = await stream.Store.LoadAsync(consumer.NextSequence, ct);
@@ -52,15 +62,40 @@ public sealed class PullConsumerEngine
if (message == null) if (message == null)
break; break;
if (!MatchesFilter(consumer.Config, message.Subject))
{
sequence++;
i--;
continue;
}
if (consumer.Config.ReplayPolicy == ReplayPolicy.Original)
await Task.Delay(50, ct);
messages.Add(message); messages.Add(message);
if (consumer.Config.AckPolicy is AckPolicy.Explicit or AckPolicy.All) if (consumer.Config.AckPolicy is AckPolicy.Explicit or AckPolicy.All)
{
if (consumer.Config.MaxAckPending > 0 && consumer.AckProcessor.PendingCount >= consumer.Config.MaxAckPending)
break;
consumer.AckProcessor.Register(message.Sequence, consumer.Config.AckWaitMs); consumer.AckProcessor.Register(message.Sequence, consumer.Config.AckWaitMs);
}
sequence++; sequence++;
} }
consumer.NextSequence = sequence; consumer.NextSequence = sequence;
return new PullFetchBatch(messages); return new PullFetchBatch(messages);
} }
private static bool MatchesFilter(ConsumerConfig config, string subject)
{
if (config.FilterSubjects.Count > 0)
return config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(subject, f));
if (!string.IsNullOrWhiteSpace(config.FilterSubject))
return SubjectMatch.MatchLiteral(subject, config.FilterSubject);
return true;
}
} }
public sealed class PullFetchBatch public sealed class PullFetchBatch

View File

@@ -3,12 +3,15 @@ namespace NATS.Server.JetStream.Models;
public sealed class ConsumerConfig public sealed class ConsumerConfig
{ {
public string DurableName { get; set; } = string.Empty; public string DurableName { get; set; } = string.Empty;
public bool Ephemeral { get; set; }
public string? FilterSubject { get; set; } public string? FilterSubject { get; set; }
public List<string> FilterSubjects { get; set; } = [];
public AckPolicy AckPolicy { get; set; } = AckPolicy.None; public AckPolicy AckPolicy { get; set; } = AckPolicy.None;
public DeliverPolicy DeliverPolicy { get; set; } = DeliverPolicy.All; public DeliverPolicy DeliverPolicy { get; set; } = DeliverPolicy.All;
public ReplayPolicy ReplayPolicy { get; set; } = ReplayPolicy.Instant; public ReplayPolicy ReplayPolicy { get; set; } = ReplayPolicy.Instant;
public int AckWaitMs { get; set; } = 30_000; public int AckWaitMs { get; set; } = 30_000;
public int MaxDeliver { get; set; } = 1; public int MaxDeliver { get; set; } = 1;
public int MaxAckPending { get; set; }
public bool Push { get; set; } public bool Push { get; set; }
public int HeartbeatMs { get; set; } public int HeartbeatMs { get; set; }
} }

View File

@@ -5,10 +5,26 @@ public sealed class StreamConfig
public string Name { get; set; } = string.Empty; public string Name { get; set; } = string.Empty;
public List<string> Subjects { get; set; } = []; public List<string> Subjects { get; set; } = [];
public int MaxMsgs { get; set; } public int MaxMsgs { get; set; }
public long MaxBytes { get; set; }
public int MaxMsgsPer { get; set; }
public int MaxAgeMs { get; set; }
public int MaxConsumers { get; set; } public int MaxConsumers { get; set; }
public RetentionPolicy Retention { get; set; } = RetentionPolicy.Limits; public RetentionPolicy Retention { get; set; } = RetentionPolicy.Limits;
public DiscardPolicy Discard { get; set; } = DiscardPolicy.Old; public DiscardPolicy Discard { get; set; } = DiscardPolicy.Old;
public StorageType Storage { get; set; } = StorageType.Memory;
public int Replicas { get; set; } = 1; public int Replicas { get; set; } = 1;
public string? Mirror { get; set; } public string? Mirror { get; set; }
public string? Source { get; set; } public string? Source { get; set; }
public List<StreamSourceConfig> Sources { get; set; } = [];
}
public enum StorageType
{
Memory,
File,
}
public sealed class StreamSourceConfig
{
public string Name { get; set; } = string.Empty;
} }

View File

@@ -5,4 +5,5 @@ public sealed class StreamState
public ulong Messages { get; set; } public ulong Messages { get; set; }
public ulong FirstSeq { get; set; } public ulong FirstSeq { get; set; }
public ulong LastSeq { get; set; } public ulong LastSeq { get; set; }
public ulong Bytes { get; set; }
} }

View File

@@ -24,6 +24,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
Sequence = _last, Sequence = _last,
Subject = subject, Subject = subject,
Payload = payload.ToArray(), Payload = payload.ToArray(),
TimestampUtc = DateTime.UtcNow,
}; };
_messages[_last] = stored; _messages[_last] = stored;
@@ -32,6 +33,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
Sequence = stored.Sequence, Sequence = stored.Sequence,
Subject = stored.Subject, Subject = stored.Subject,
PayloadBase64 = Convert.ToBase64String(stored.Payload.ToArray()), PayloadBase64 = Convert.ToBase64String(stored.Payload.ToArray()),
TimestampUtc = stored.TimestampUtc,
}); });
await File.AppendAllTextAsync(_dataFilePath, line + Environment.NewLine, ct); await File.AppendAllTextAsync(_dataFilePath, line + Environment.NewLine, ct);
return _last; return _last;
@@ -79,6 +81,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
Sequence = x.Sequence, Sequence = x.Sequence,
Subject = x.Subject, Subject = x.Subject,
PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()), PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()),
TimestampUtc = x.TimestampUtc,
}) })
.ToArray(); .ToArray();
return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot)); return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot));
@@ -101,6 +104,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
Sequence = record.Sequence, Sequence = record.Sequence,
Subject = record.Subject ?? string.Empty, Subject = record.Subject ?? string.Empty,
Payload = Convert.FromBase64String(record.PayloadBase64 ?? string.Empty), Payload = Convert.FromBase64String(record.PayloadBase64 ?? string.Empty),
TimestampUtc = record.TimestampUtc,
}; };
_messages[record.Sequence] = message; _messages[record.Sequence] = message;
_last = Math.Max(_last, record.Sequence); _last = Math.Max(_last, record.Sequence);
@@ -119,6 +123,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
Messages = (ulong)_messages.Count, Messages = (ulong)_messages.Count,
FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(), FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(),
LastSeq = _last, LastSeq = _last,
Bytes = (ulong)_messages.Values.Sum(m => m.Payload.Length),
}); });
} }
@@ -172,6 +177,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
Sequence = message.Sequence, Sequence = message.Sequence,
Subject = message.Subject, Subject = message.Subject,
PayloadBase64 = Convert.ToBase64String(message.Payload.ToArray()), PayloadBase64 = Convert.ToBase64String(message.Payload.ToArray()),
TimestampUtc = message.TimestampUtc,
})); }));
} }
@@ -183,5 +189,6 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable
public ulong Sequence { get; init; } public ulong Sequence { get; init; }
public string? Subject { get; init; } public string? Subject { get; init; }
public string? PayloadBase64 { get; init; } public string? PayloadBase64 { get; init; }
public DateTime TimestampUtc { get; init; }
} }
} }

View File

@@ -10,6 +10,7 @@ public sealed class MemStore : IStreamStore
public ulong Sequence { get; init; } public ulong Sequence { get; init; }
public string Subject { get; init; } = string.Empty; public string Subject { get; init; } = string.Empty;
public string PayloadBase64 { get; init; } = string.Empty; public string PayloadBase64 { get; init; } = string.Empty;
public DateTime TimestampUtc { get; init; }
} }
private readonly object _gate = new(); private readonly object _gate = new();
@@ -26,6 +27,7 @@ public sealed class MemStore : IStreamStore
Sequence = _last, Sequence = _last,
Subject = subject, Subject = subject,
Payload = payload, Payload = payload,
TimestampUtc = DateTime.UtcNow,
}; };
return ValueTask.FromResult(_last); return ValueTask.FromResult(_last);
} }
@@ -82,6 +84,7 @@ public sealed class MemStore : IStreamStore
Sequence = x.Sequence, Sequence = x.Sequence,
Subject = x.Subject, Subject = x.Subject,
PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()), PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()),
TimestampUtc = x.TimestampUtc,
}) })
.ToArray(); .ToArray();
return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot)); return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot));
@@ -107,6 +110,7 @@ public sealed class MemStore : IStreamStore
Sequence = record.Sequence, Sequence = record.Sequence,
Subject = record.Subject, Subject = record.Subject,
Payload = Convert.FromBase64String(record.PayloadBase64), Payload = Convert.FromBase64String(record.PayloadBase64),
TimestampUtc = record.TimestampUtc,
}; };
_last = Math.Max(_last, record.Sequence); _last = Math.Max(_last, record.Sequence);
} }
@@ -126,6 +130,7 @@ public sealed class MemStore : IStreamStore
Messages = (ulong)_messages.Count, Messages = (ulong)_messages.Count,
FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(), FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(),
LastSeq = _last, LastSeq = _last,
Bytes = (ulong)_messages.Values.Sum(m => m.Payload.Length),
}); });
} }
} }

View File

@@ -5,5 +5,6 @@ public sealed class StoredMessage
public ulong Sequence { get; init; } public ulong Sequence { get; init; }
public string Subject { get; init; } = string.Empty; public string Subject { get; init; } = string.Empty;
public ReadOnlyMemory<byte> Payload { get; init; } public ReadOnlyMemory<byte> Payload { get; init; }
public DateTime TimestampUtc { get; init; } = DateTime.UtcNow;
public bool Redelivered { get; init; } public bool Redelivered { get; init; }
} }

View File

@@ -48,8 +48,14 @@ public sealed class StreamManager
var handle = _streams.AddOrUpdate( var handle = _streams.AddOrUpdate(
normalized.Name, normalized.Name,
_ => new StreamHandle(normalized, new MemStore()), _ => new StreamHandle(normalized, CreateStore(normalized)),
(_, existing) => existing with { Config = normalized }); (_, existing) =>
{
if (existing.Config.Storage == normalized.Storage)
return existing with { Config = normalized };
return new StreamHandle(normalized, CreateStore(normalized));
});
_replicaGroups.AddOrUpdate( _replicaGroups.AddOrUpdate(
normalized.Name, normalized.Name,
_ => new StreamReplicaGroup(normalized.Name, normalized.Replicas), _ => new StreamReplicaGroup(normalized.Name, normalized.Replicas),
@@ -150,6 +156,25 @@ public sealed class StreamManager
if (stream == null) if (stream == null)
return null; return null;
var stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
if (stream.Config.MaxBytes > 0 && (long)stateBefore.Bytes + payload.Length > stream.Config.MaxBytes)
{
if (stream.Config.Discard == DiscardPolicy.New)
{
return new PubAck
{
Stream = stream.Config.Name,
ErrorCode = 10054,
};
}
while ((long)stateBefore.Bytes + payload.Length > stream.Config.MaxBytes && stateBefore.FirstSeq > 0)
{
stream.Store.RemoveAsync(stateBefore.FirstSeq, default).GetAwaiter().GetResult();
stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
}
}
if (_replicaGroups.TryGetValue(stream.Config.Name, out var replicaGroup)) if (_replicaGroups.TryGetValue(stream.Config.Name, out var replicaGroup))
_ = replicaGroup.ProposeAsync($"PUB {subject}", default).GetAwaiter().GetResult(); _ = replicaGroup.ProposeAsync($"PUB {subject}", default).GetAwaiter().GetResult();
@@ -181,12 +206,17 @@ public sealed class StreamManager
Name = config.Name, Name = config.Name,
Subjects = config.Subjects.Count == 0 ? [] : [.. config.Subjects], Subjects = config.Subjects.Count == 0 ? [] : [.. config.Subjects],
MaxMsgs = config.MaxMsgs, MaxMsgs = config.MaxMsgs,
MaxBytes = config.MaxBytes,
MaxMsgsPer = config.MaxMsgsPer,
MaxAgeMs = config.MaxAgeMs,
MaxConsumers = config.MaxConsumers, MaxConsumers = config.MaxConsumers,
Retention = config.Retention, Retention = config.Retention,
Discard = config.Discard, Discard = config.Discard,
Storage = config.Storage,
Replicas = config.Replicas, Replicas = config.Replicas,
Mirror = config.Mirror, Mirror = config.Mirror,
Source = config.Source, Source = config.Source,
Sources = config.Sources.Count == 0 ? [] : [.. config.Sources.Select(s => new StreamSourceConfig { Name = s.Name })],
}; };
return copy; return copy;
@@ -241,6 +271,18 @@ public sealed class StreamManager
var list = _sourcesByOrigin.GetOrAdd(stream.Config.Source, _ => []); var list = _sourcesByOrigin.GetOrAdd(stream.Config.Source, _ => []);
list.Add(new SourceCoordinator(stream.Store)); list.Add(new SourceCoordinator(stream.Store));
} }
if (stream.Config.Sources.Count > 0)
{
foreach (var source in stream.Config.Sources)
{
if (string.IsNullOrWhiteSpace(source.Name) || !_streams.TryGetValue(source.Name, out _))
continue;
var list = _sourcesByOrigin.GetOrAdd(source.Name, _ => []);
list.Add(new SourceCoordinator(stream.Store));
}
}
} }
} }
@@ -258,6 +300,30 @@ public sealed class StreamManager
source.OnOriginAppendAsync(stored, default).GetAwaiter().GetResult(); source.OnOriginAppendAsync(stored, default).GetAwaiter().GetResult();
} }
} }
public string GetStoreBackendType(string streamName)
{
if (!_streams.TryGetValue(streamName, out var stream))
return "missing";
return stream.Store switch
{
FileStore => "file",
_ => "memory",
};
}
private static IStreamStore CreateStore(StreamConfig config)
{
return config.Storage switch
{
StorageType.File => new FileStore(new FileStoreOptions
{
Directory = Path.Combine(Path.GetTempPath(), "natsdotnet-js-store", config.Name),
}),
_ => new MemStore(),
};
}
} }
public sealed record StreamHandle(StreamConfig Config, IStreamStore Store); public sealed record StreamHandle(StreamConfig Config, IStreamStore Store);

View File

@@ -1,11 +1,191 @@
using System.Net.Sockets;
using System.Text;
using NATS.Server.Subscriptions;
namespace NATS.Server.LeafNodes; namespace NATS.Server.LeafNodes;
public sealed class LeafConnection public sealed class LeafConnection(Socket socket) : IAsyncDisposable
{ {
public string RemoteEndpoint { get; } private readonly NetworkStream _stream = new(socket, ownsSocket: true);
private readonly SemaphoreSlim _writeGate = new(1, 1);
private readonly CancellationTokenSource _closedCts = new();
private Task? _loopTask;
public LeafConnection(string remoteEndpoint) public string? RemoteId { get; private set; }
public string RemoteEndpoint => socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N");
public Func<RemoteSubscription, Task>? RemoteSubscriptionReceived { get; set; }
public Func<LeafMessage, Task>? MessageReceived { get; set; }
public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct)
{ {
RemoteEndpoint = remoteEndpoint; await WriteLineAsync($"LEAF {serverId}", ct);
var line = await ReadLineAsync(ct);
RemoteId = ParseHandshake(line);
}
public async Task PerformInboundHandshakeAsync(string serverId, CancellationToken ct)
{
var line = await ReadLineAsync(ct);
RemoteId = ParseHandshake(line);
await WriteLineAsync($"LEAF {serverId}", ct);
}
public void StartLoop(CancellationToken ct)
{
if (_loopTask != null)
return;
var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
_loopTask = Task.Run(() => ReadLoopAsync(linked.Token), linked.Token);
}
public Task WaitUntilClosedAsync(CancellationToken ct)
=> _loopTask?.WaitAsync(ct) ?? Task.CompletedTask;
public Task SendLsPlusAsync(string subject, string? queue, CancellationToken ct)
=> WriteLineAsync(queue is { Length: > 0 } ? $"LS+ {subject} {queue}" : $"LS+ {subject}", ct);
public Task SendLsMinusAsync(string subject, string? queue, CancellationToken ct)
=> WriteLineAsync(queue is { Length: > 0 } ? $"LS- {subject} {queue}" : $"LS- {subject}", ct);
public async Task SendMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
{
var reply = string.IsNullOrEmpty(replyTo) ? "-" : replyTo;
await _writeGate.WaitAsync(ct);
try
{
var control = Encoding.ASCII.GetBytes($"LMSG {subject} {reply} {payload.Length}\r\n");
await _stream.WriteAsync(control, ct);
if (!payload.IsEmpty)
await _stream.WriteAsync(payload, ct);
await _stream.WriteAsync("\r\n"u8.ToArray(), ct);
await _stream.FlushAsync(ct);
}
finally
{
_writeGate.Release();
}
}
public async ValueTask DisposeAsync()
{
await _closedCts.CancelAsync();
if (_loopTask != null)
await _loopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
_closedCts.Dispose();
_writeGate.Dispose();
await _stream.DisposeAsync();
}
private async Task ReadLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
string line;
try
{
line = await ReadLineAsync(ct);
}
catch
{
break;
}
if (line.StartsWith("LS+ ", StringComparison.Ordinal))
{
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
{
var queue = parts.Length >= 3 ? parts[2] : null;
await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteId ?? string.Empty));
}
continue;
}
if (line.StartsWith("LS- ", StringComparison.Ordinal))
{
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
{
var queue = parts.Length >= 3 ? parts[2] : null;
await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteId ?? string.Empty));
}
continue;
}
if (!line.StartsWith("LMSG ", StringComparison.Ordinal))
continue;
var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (args.Length < 4 || !int.TryParse(args[3], out var size) || size < 0)
continue;
var payload = await ReadPayloadAsync(size, ct);
if (MessageReceived != null)
await MessageReceived(new LeafMessage(args[1], args[2] == "-" ? null : args[2], payload));
}
}
private async Task<ReadOnlyMemory<byte>> ReadPayloadAsync(int size, CancellationToken ct)
{
var payload = new byte[size];
var offset = 0;
while (offset < size)
{
var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct);
if (read == 0)
throw new IOException("Leaf payload read closed");
offset += read;
}
var trailer = new byte[2];
_ = await _stream.ReadAsync(trailer, ct);
return payload;
}
private async Task WriteLineAsync(string line, CancellationToken ct)
{
await _writeGate.WaitAsync(ct);
try
{
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
await _stream.WriteAsync(bytes, ct);
await _stream.FlushAsync(ct);
}
finally
{
_writeGate.Release();
}
}
private async Task<string> ReadLineAsync(CancellationToken ct)
{
var bytes = new List<byte>(64);
var single = new byte[1];
while (true)
{
var read = await _stream.ReadAsync(single, ct);
if (read == 0)
throw new IOException("Leaf closed");
if (single[0] == (byte)'\n')
break;
if (single[0] != (byte)'\r')
bytes.Add(single[0]);
}
return Encoding.ASCII.GetString([.. bytes]);
}
private static string ParseHandshake(string line)
{
if (!line.StartsWith("LEAF ", StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException("Invalid leaf handshake");
var id = line[5..].Trim();
if (id.Length == 0)
throw new InvalidOperationException("Leaf handshake missing id");
return id;
} }
} }
public sealed record LeafMessage(string Subject, string? ReplyTo, ReadOnlyMemory<byte> Payload);

View File

@@ -1,5 +1,9 @@
using System.Collections.Concurrent;
using System.Net;
using System.Net.Sockets;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using NATS.Server.Configuration; using NATS.Server.Configuration;
using NATS.Server.Subscriptions;
namespace NATS.Server.LeafNodes; namespace NATS.Server.LeafNodes;
@@ -7,25 +11,203 @@ public sealed class LeafNodeManager : IAsyncDisposable
{ {
private readonly LeafNodeOptions _options; private readonly LeafNodeOptions _options;
private readonly ServerStats _stats; private readonly ServerStats _stats;
private readonly string _serverId;
private readonly Action<RemoteSubscription> _remoteSubSink;
private readonly Action<LeafMessage> _messageSink;
private readonly ILogger<LeafNodeManager> _logger; private readonly ILogger<LeafNodeManager> _logger;
private readonly ConcurrentDictionary<string, LeafConnection> _connections = new(StringComparer.Ordinal);
public LeafNodeManager(LeafNodeOptions options, ServerStats stats, ILogger<LeafNodeManager> logger) private CancellationTokenSource? _cts;
private Socket? _listener;
private Task? _acceptLoopTask;
public string ListenEndpoint => $"{_options.Host}:{_options.Port}";
public LeafNodeManager(
LeafNodeOptions options,
ServerStats stats,
string serverId,
Action<RemoteSubscription> remoteSubSink,
Action<LeafMessage> messageSink,
ILogger<LeafNodeManager> logger)
{ {
_options = options; _options = options;
_stats = stats; _stats = stats;
_serverId = serverId;
_remoteSubSink = remoteSubSink;
_messageSink = messageSink;
_logger = logger; _logger = logger;
} }
public Task StartAsync(CancellationToken ct) public Task StartAsync(CancellationToken ct)
{ {
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
_listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_listener.Bind(new IPEndPoint(IPAddress.Parse(_options.Host), _options.Port));
_listener.Listen(128);
if (_options.Port == 0)
_options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
_acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
foreach (var remote in _options.Remotes.Distinct(StringComparer.OrdinalIgnoreCase))
_ = Task.Run(() => ConnectWithRetryAsync(remote, _cts.Token));
_logger.LogDebug("Leaf manager started (listen={Host}:{Port})", _options.Host, _options.Port); _logger.LogDebug("Leaf manager started (listen={Host}:{Port})", _options.Host, _options.Port);
Interlocked.Exchange(ref _stats.Leafs, 0);
return Task.CompletedTask; return Task.CompletedTask;
} }
public ValueTask DisposeAsync() public async Task ForwardMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
{ {
foreach (var connection in _connections.Values)
await connection.SendMessageAsync(subject, replyTo, payload, ct);
}
public void PropagateLocalSubscription(string subject, string? queue)
{
foreach (var connection in _connections.Values)
_ = connection.SendLsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
}
public void PropagateLocalUnsubscription(string subject, string? queue)
{
foreach (var connection in _connections.Values)
_ = connection.SendLsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
}
public async ValueTask DisposeAsync()
{
if (_cts == null)
return;
await _cts.CancelAsync();
_listener?.Dispose();
if (_acceptLoopTask != null)
await _acceptLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
foreach (var connection in _connections.Values)
await connection.DisposeAsync();
_connections.Clear();
Interlocked.Exchange(ref _stats.Leafs, 0);
_cts.Dispose();
_cts = null;
_logger.LogDebug("Leaf manager stopped"); _logger.LogDebug("Leaf manager stopped");
return ValueTask.CompletedTask; }
private async Task AcceptLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
Socket socket;
try
{
socket = await _listener!.AcceptAsync(ct);
}
catch
{
break;
}
_ = Task.Run(() => HandleInboundAsync(socket, ct), ct);
}
}
private async Task HandleInboundAsync(Socket socket, CancellationToken ct)
{
var connection = new LeafConnection(socket);
try
{
await connection.PerformInboundHandshakeAsync(_serverId, ct);
Register(connection);
}
catch
{
await connection.DisposeAsync();
}
}
private async Task ConnectWithRetryAsync(string remote, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try
{
var endPoint = ParseEndpoint(remote);
var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct);
var connection = new LeafConnection(socket);
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
Register(connection);
return;
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Leaf connect retry for {Remote}", remote);
}
try
{
await Task.Delay(250, ct);
}
catch (OperationCanceledException)
{
return;
}
}
}
private void Register(LeafConnection connection)
{
var key = $"{connection.RemoteId}:{connection.RemoteEndpoint}:{Guid.NewGuid():N}";
if (!_connections.TryAdd(key, connection))
{
_ = connection.DisposeAsync();
return;
}
connection.RemoteSubscriptionReceived = sub =>
{
_remoteSubSink(sub);
return Task.CompletedTask;
};
connection.MessageReceived = msg =>
{
_messageSink(msg);
return Task.CompletedTask;
};
connection.StartLoop(_cts!.Token);
Interlocked.Increment(ref _stats.Leafs);
_ = Task.Run(() => WatchConnectionAsync(key, connection, _cts!.Token));
}
private async Task WatchConnectionAsync(string key, LeafConnection connection, CancellationToken ct)
{
try
{
await connection.WaitUntilClosedAsync(ct);
}
catch
{
}
finally
{
if (_connections.TryRemove(key, out _))
Interlocked.Decrement(ref _stats.Leafs);
await connection.DisposeAsync();
}
}
private static IPEndPoint ParseEndpoint(string endpoint)
{
var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
if (parts.Length != 2)
throw new FormatException($"Invalid endpoint: {endpoint}");
return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
} }
} }

View File

@@ -0,0 +1,46 @@
using NATS.Server.Auth;
namespace NATS.Server.Monitoring;
public sealed class AccountzHandler
{
private readonly NatsServer _server;
public AccountzHandler(NatsServer server)
{
_server = server;
}
public object Build()
{
var accounts = _server.GetAccounts().Select(ToAccountDto).ToArray();
return new
{
accounts,
num_accounts = accounts.Length,
};
}
public object BuildStats()
{
var accounts = _server.GetAccounts().ToArray();
return new
{
total_accounts = accounts.Length,
total_connections = accounts.Sum(a => a.ClientCount),
total_subscriptions = accounts.Sum(a => a.SubscriptionCount),
};
}
private static object ToAccountDto(Account account)
{
return new
{
name = account.Name,
connections = account.ClientCount,
subscriptions = account.SubscriptionCount,
in_msgs = account.InMsgs,
out_msgs = account.OutMsgs,
};
}
}

View File

@@ -0,0 +1,21 @@
namespace NATS.Server.Monitoring;
public sealed class GatewayzHandler
{
private readonly NatsServer _server;
public GatewayzHandler(NatsServer server)
{
_server = server;
}
public object Build()
{
var gateways = _server.Stats.Gateways;
return new
{
gateways,
num_gateways = gateways,
};
}
}

View File

@@ -0,0 +1,21 @@
namespace NATS.Server.Monitoring;
public sealed class LeafzHandler
{
private readonly NatsServer _server;
public LeafzHandler(NatsServer server)
{
_server = server;
}
public object Build()
{
var leafs = _server.Stats.Leafs;
return new
{
leafs,
num_leafs = leafs,
};
}
}

View File

@@ -17,6 +17,10 @@ public sealed class MonitorServer : IAsyncDisposable
private readonly ConnzHandler _connzHandler; private readonly ConnzHandler _connzHandler;
private readonly SubszHandler _subszHandler; private readonly SubszHandler _subszHandler;
private readonly JszHandler _jszHandler; private readonly JszHandler _jszHandler;
private readonly RoutezHandler _routezHandler;
private readonly GatewayzHandler _gatewayzHandler;
private readonly LeafzHandler _leafzHandler;
private readonly AccountzHandler _accountzHandler;
public MonitorServer(NatsServer server, NatsOptions options, ServerStats stats, ILoggerFactory loggerFactory) public MonitorServer(NatsServer server, NatsOptions options, ServerStats stats, ILoggerFactory loggerFactory)
{ {
@@ -33,6 +37,10 @@ public sealed class MonitorServer : IAsyncDisposable
_connzHandler = new ConnzHandler(server); _connzHandler = new ConnzHandler(server);
_subszHandler = new SubszHandler(server); _subszHandler = new SubszHandler(server);
_jszHandler = new JszHandler(server, options); _jszHandler = new JszHandler(server, options);
_routezHandler = new RoutezHandler(server);
_gatewayzHandler = new GatewayzHandler(server);
_leafzHandler = new LeafzHandler(server);
_accountzHandler = new AccountzHandler(server);
_app.MapGet(basePath + "/", () => _app.MapGet(basePath + "/", () =>
{ {
@@ -63,21 +71,20 @@ public sealed class MonitorServer : IAsyncDisposable
return Results.Ok(_connzHandler.HandleConnz(ctx)); return Results.Ok(_connzHandler.HandleConnz(ctx));
}); });
// Stubs for unimplemented endpoints
_app.MapGet(basePath + "/routez", () => _app.MapGet(basePath + "/routez", () =>
{ {
stats.HttpReqStats.AddOrUpdate("/routez", 1, (_, v) => v + 1); stats.HttpReqStats.AddOrUpdate("/routez", 1, (_, v) => v + 1);
return Results.Ok(new { }); return Results.Ok(_routezHandler.Build());
}); });
_app.MapGet(basePath + "/gatewayz", () => _app.MapGet(basePath + "/gatewayz", () =>
{ {
stats.HttpReqStats.AddOrUpdate("/gatewayz", 1, (_, v) => v + 1); stats.HttpReqStats.AddOrUpdate("/gatewayz", 1, (_, v) => v + 1);
return Results.Ok(new { }); return Results.Ok(_gatewayzHandler.Build());
}); });
_app.MapGet(basePath + "/leafz", () => _app.MapGet(basePath + "/leafz", () =>
{ {
stats.HttpReqStats.AddOrUpdate("/leafz", 1, (_, v) => v + 1); stats.HttpReqStats.AddOrUpdate("/leafz", 1, (_, v) => v + 1);
return Results.Ok(new { }); return Results.Ok(_leafzHandler.Build());
}); });
_app.MapGet(basePath + "/subz", (HttpContext ctx) => _app.MapGet(basePath + "/subz", (HttpContext ctx) =>
{ {
@@ -92,12 +99,12 @@ public sealed class MonitorServer : IAsyncDisposable
_app.MapGet(basePath + "/accountz", () => _app.MapGet(basePath + "/accountz", () =>
{ {
stats.HttpReqStats.AddOrUpdate("/accountz", 1, (_, v) => v + 1); stats.HttpReqStats.AddOrUpdate("/accountz", 1, (_, v) => v + 1);
return Results.Ok(new { }); return Results.Ok(_accountzHandler.Build());
}); });
_app.MapGet(basePath + "/accstatz", () => _app.MapGet(basePath + "/accstatz", () =>
{ {
stats.HttpReqStats.AddOrUpdate("/accstatz", 1, (_, v) => v + 1); stats.HttpReqStats.AddOrUpdate("/accstatz", 1, (_, v) => v + 1);
return Results.Ok(new { }); return Results.Ok(_accountzHandler.BuildStats());
}); });
_app.MapGet(basePath + "/jsz", () => _app.MapGet(basePath + "/jsz", () =>
{ {

View File

@@ -0,0 +1,21 @@
namespace NATS.Server.Monitoring;
public sealed class RoutezHandler
{
private readonly NatsServer _server;
public RoutezHandler(NatsServer server)
{
_server = server;
}
public object Build()
{
var routes = _server.Stats.Routes;
return new
{
routes,
num_routes = routes,
};
}
}

View File

@@ -562,6 +562,8 @@ public sealed class NatsClient : INatsClient, IDisposable
Account?.DecrementSubscriptions(); Account?.DecrementSubscriptions();
Account?.SubList.Remove(sub); Account?.SubList.Remove(sub);
if (Router is NatsServer server)
server.OnLocalUnsubscription(sub.Subject, sub.Queue);
} }
private void ProcessPub(ParsedCommand cmd, ref long localInMsgs, ref long localInBytes) private void ProcessPub(ParsedCommand cmd, ref long localInMsgs, ref long localInBytes)

View File

@@ -95,6 +95,8 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0; public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0;
public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0; public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0;
public string? ClusterListen => _routeManager?.ListenEndpoint; public string? ClusterListen => _routeManager?.ListenEndpoint;
public string? GatewayListen => _gatewayManager?.ListenEndpoint;
public string? LeafListen => _leafNodeManager?.ListenEndpoint;
public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter; public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter;
public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0; public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0;
public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0; public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0;
@@ -366,18 +368,21 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
if (options.Cluster != null) if (options.Cluster != null)
{ {
_routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, _routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
ProcessRoutedMessage,
_loggerFactory.CreateLogger<RouteManager>()); _loggerFactory.CreateLogger<RouteManager>());
} }
if (options.Gateway != null) if (options.Gateway != null)
{ {
_gatewayManager = new GatewayManager(options.Gateway, _stats, _gatewayManager = new GatewayManager(options.Gateway, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
ProcessGatewayMessage,
_loggerFactory.CreateLogger<GatewayManager>()); _loggerFactory.CreateLogger<GatewayManager>());
} }
if (options.LeafNode != null) if (options.LeafNode != null)
{ {
_leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
ProcessLeafMessage,
_loggerFactory.CreateLogger<LeafNodeManager>()); _loggerFactory.CreateLogger<LeafNodeManager>());
} }
@@ -796,6 +801,15 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
public void OnLocalSubscription(string subject, string? queue) public void OnLocalSubscription(string subject, string? queue)
{ {
_routeManager?.PropagateLocalSubscription(subject, queue); _routeManager?.PropagateLocalSubscription(subject, queue);
_gatewayManager?.PropagateLocalSubscription(subject, queue);
_leafNodeManager?.PropagateLocalSubscription(subject, queue);
}
public void OnLocalUnsubscription(string subject, string? queue)
{
_routeManager?.PropagateLocalUnsubscription(subject, queue);
_gatewayManager?.PropagateLocalUnsubscription(subject, queue);
_leafNodeManager?.PropagateLocalUnsubscription(subject, queue);
} }
private void ApplyRemoteSubscription(RemoteSubscription sub) private void ApplyRemoteSubscription(RemoteSubscription sub)
@@ -803,6 +817,38 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
_globalAccount.SubList.ApplyRemoteSub(sub); _globalAccount.SubList.ApplyRemoteSub(sub);
} }
private void ProcessRoutedMessage(RouteMessage message)
{
DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload);
}
private void ProcessGatewayMessage(GatewayMessage message)
{
DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload);
}
private void ProcessLeafMessage(LeafMessage message)
{
DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload);
}
private void DeliverRemoteMessage(string subject, string? replyTo, ReadOnlyMemory<byte> payload)
{
var result = _globalAccount.SubList.Match(subject);
foreach (var sub in result.PlainSubs)
DeliverMessage(sub, subject, replyTo, default, payload);
foreach (var queueGroup in result.QueueSubs)
{
if (queueGroup.Length == 0)
continue;
var sub = queueGroup[0];
DeliverMessage(sub, subject, replyTo, default, payload);
}
}
public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory<byte> headers, public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory<byte> headers,
ReadOnlyMemory<byte> payload, NatsClient sender) ReadOnlyMemory<byte> payload, NatsClient sender)
{ {
@@ -837,6 +883,13 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
} }
} }
if (_routeManager != null && _globalAccount.SubList.HasRemoteInterest(subject))
_routeManager.ForwardRoutedMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult();
if (_gatewayManager != null && _globalAccount.SubList.HasRemoteInterest(subject))
_gatewayManager.ForwardMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult();
if (_leafNodeManager != null && _globalAccount.SubList.HasRemoteInterest(subject))
_leafNodeManager.ForwardMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult();
var subList = sender.Account?.SubList ?? _globalAccount.SubList; var subList = sender.Account?.SubList ?? _globalAccount.SubList;
var result = subList.Match(subject); var result = subList.Match(subject);
var delivered = false; var delivered = false;

View File

@@ -10,7 +10,21 @@ public sealed class ClientCommandMatrix
return (kind, op.ToUpperInvariant()) switch return (kind, op.ToUpperInvariant()) switch
{ {
(global::NATS.Server.ClientKind.Router, "RS+") => true, (global::NATS.Server.ClientKind.Router, "RS+") => true,
(global::NATS.Server.ClientKind.Router, "RS-") => true,
(global::NATS.Server.ClientKind.Router, "RMSG") => true,
(global::NATS.Server.ClientKind.Gateway, "A+") => true,
(global::NATS.Server.ClientKind.Gateway, "A-") => true,
(global::NATS.Server.ClientKind.Leaf, "LS+") => true,
(global::NATS.Server.ClientKind.Leaf, "LS-") => true,
(global::NATS.Server.ClientKind.Leaf, "LMSG") => true,
(_, "RS+") => false, (_, "RS+") => false,
(_, "RS-") => false,
(_, "RMSG") => false,
(_, "A+") => false,
(_, "A-") => false,
(_, "LS+") => false,
(_, "LS-") => false,
(_, "LMSG") => false,
_ => true, _ => true,
}; };
} }

View File

@@ -27,6 +27,36 @@ public sealed class RaftLog
_entries.Clear(); _entries.Clear();
_baseIndex = snapshot.LastIncludedIndex; _baseIndex = snapshot.LastIncludedIndex;
} }
public async Task PersistAsync(string path, CancellationToken ct)
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var model = new PersistedLog
{
BaseIndex = _baseIndex,
Entries = [.. _entries],
};
await File.WriteAllTextAsync(path, System.Text.Json.JsonSerializer.Serialize(model), ct);
}
public static async Task<RaftLog> LoadAsync(string path, CancellationToken ct)
{
var log = new RaftLog();
if (!File.Exists(path))
return log;
var json = await File.ReadAllTextAsync(path, ct);
var model = System.Text.Json.JsonSerializer.Deserialize<PersistedLog>(json) ?? new PersistedLog();
log._baseIndex = model.BaseIndex;
log._entries.AddRange(model.Entries);
return log;
}
private sealed class PersistedLog
{
public long BaseIndex { get; set; }
public List<RaftLogEntry> Entries { get; set; } = [];
}
} }
public sealed record RaftLogEntry(long Index, int Term, string Command); public sealed record RaftLogEntry(long Index, int Term, string Command);

View File

@@ -6,6 +6,8 @@ public sealed class RaftNode
private readonly List<RaftNode> _cluster = []; private readonly List<RaftNode> _cluster = [];
private readonly RaftReplicator _replicator = new(); private readonly RaftReplicator _replicator = new();
private readonly RaftSnapshotStore _snapshotStore = new(); private readonly RaftSnapshotStore _snapshotStore = new();
private readonly IRaftTransport? _transport;
private readonly string? _persistDirectory;
public string Id { get; } public string Id { get; }
public int Term => TermState.CurrentTerm; public int Term => TermState.CurrentTerm;
@@ -13,11 +15,13 @@ public sealed class RaftNode
public RaftRole Role { get; private set; } = RaftRole.Follower; public RaftRole Role { get; private set; } = RaftRole.Follower;
public RaftTermState TermState { get; } = new(); public RaftTermState TermState { get; } = new();
public long AppliedIndex { get; set; } public long AppliedIndex { get; set; }
public RaftLog Log { get; } = new(); public RaftLog Log { get; private set; } = new();
public RaftNode(string id) public RaftNode(string id, IRaftTransport? transport = null, string? persistDirectory = null)
{ {
Id = id; Id = id;
_transport = transport;
_persistDirectory = persistDirectory;
} }
public void ConfigureCluster(IEnumerable<RaftNode> peers) public void ConfigureCluster(IEnumerable<RaftNode> peers)
@@ -60,7 +64,8 @@ public sealed class RaftNode
var entry = Log.Append(TermState.CurrentTerm, command); var entry = Log.Append(TermState.CurrentTerm, command);
var followers = _cluster.Where(n => n.Id != Id).ToList(); var followers = _cluster.Where(n => n.Id != Id).ToList();
var acknowledgements = _replicator.Replicate(entry, followers); var results = await _replicator.ReplicateAsync(Id, entry, followers, _transport, ct);
var acknowledgements = results.Count(r => r.Success);
var quorum = (_cluster.Count / 2) + 1; var quorum = (_cluster.Count / 2) + 1;
if (acknowledgements + 1 >= quorum) if (acknowledgements + 1 >= quorum)
@@ -68,9 +73,14 @@ public sealed class RaftNode
AppliedIndex = entry.Index; AppliedIndex = entry.Index;
foreach (var node in _cluster) foreach (var node in _cluster)
node.AppliedIndex = Math.Max(node.AppliedIndex, entry.Index); node.AppliedIndex = Math.Max(node.AppliedIndex, entry.Index);
foreach (var node in _cluster.Where(n => n._persistDirectory != null))
await node.PersistAsync(ct);
} }
await Task.CompletedTask; if (_persistDirectory != null)
await PersistAsync(ct);
return entry.Index; return entry.Index;
} }
@@ -120,4 +130,29 @@ public sealed class RaftNode
if (_votesReceived >= quorum) if (_votesReceived >= quorum)
Role = RaftRole.Leader; Role = RaftRole.Leader;
} }
public async Task PersistAsync(CancellationToken ct)
{
var dir = _persistDirectory ?? Path.Combine(Path.GetTempPath(), "natsdotnet-raft", Id);
Directory.CreateDirectory(dir);
await Log.PersistAsync(Path.Combine(dir, "log.json"), ct);
await File.WriteAllTextAsync(Path.Combine(dir, "term.txt"), TermState.CurrentTerm.ToString(), ct);
await File.WriteAllTextAsync(Path.Combine(dir, "applied.txt"), AppliedIndex.ToString(), ct);
}
public async Task LoadPersistedStateAsync(CancellationToken ct)
{
var dir = _persistDirectory ?? Path.Combine(Path.GetTempPath(), "natsdotnet-raft", Id);
Log = await RaftLog.LoadAsync(Path.Combine(dir, "log.json"), ct);
var termPath = Path.Combine(dir, "term.txt");
if (File.Exists(termPath) && int.TryParse(await File.ReadAllTextAsync(termPath, ct), out var term))
TermState.CurrentTerm = term;
var appliedPath = Path.Combine(dir, "applied.txt");
if (File.Exists(appliedPath) && long.TryParse(await File.ReadAllTextAsync(appliedPath, ct), out var applied))
AppliedIndex = applied;
else if (Log.Entries.Count > 0)
AppliedIndex = Log.Entries[^1].Index;
}
} }

View File

@@ -13,4 +13,24 @@ public sealed class RaftReplicator
return acknowledgements; return acknowledgements;
} }
public async Task<IReadOnlyList<AppendResult>> ReplicateAsync(
string leaderId,
RaftLogEntry entry,
IReadOnlyList<RaftNode> followers,
IRaftTransport? transport,
CancellationToken ct)
{
if (transport != null)
return await transport.AppendEntriesAsync(leaderId, followers.Select(f => f.Id).ToArray(), entry, ct);
var results = new List<AppendResult>(followers.Count);
foreach (var follower in followers)
{
follower.ReceiveReplicatedEntry(entry);
results.Add(new AppendResult { FollowerId = follower.Id, Success = true });
}
return results;
}
} }

View File

@@ -10,3 +10,9 @@ public sealed class VoteResponse
{ {
public bool Granted { get; init; } public bool Granted { get; init; }
} }
public sealed class AppendResult
{
public string FollowerId { get; init; } = string.Empty;
public bool Success { get; init; }
}

View File

@@ -0,0 +1,44 @@
namespace NATS.Server.Raft;
public interface IRaftTransport
{
Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct);
Task<VoteResponse> RequestVoteAsync(string candidateId, string voterId, VoteRequest request, CancellationToken ct);
}
public sealed class InMemoryRaftTransport : IRaftTransport
{
private readonly Dictionary<string, RaftNode> _nodes = new(StringComparer.Ordinal);
public void Register(RaftNode node)
{
_nodes[node.Id] = node;
}
public Task<IReadOnlyList<AppendResult>> AppendEntriesAsync(string leaderId, IReadOnlyList<string> followerIds, RaftLogEntry entry, CancellationToken ct)
{
var results = new List<AppendResult>(followerIds.Count);
foreach (var followerId in followerIds)
{
if (_nodes.TryGetValue(followerId, out var node))
{
node.ReceiveReplicatedEntry(entry);
results.Add(new AppendResult { FollowerId = followerId, Success = true });
}
else
{
results.Add(new AppendResult { FollowerId = followerId, Success = false });
}
}
return Task.FromResult<IReadOnlyList<AppendResult>>(results);
}
public Task<VoteResponse> RequestVoteAsync(string candidateId, string voterId, VoteRequest request, CancellationToken ct)
{
if (_nodes.TryGetValue(voterId, out var node))
return Task.FromResult(node.GrantVote(request.Term));
return Task.FromResult(new VoteResponse { Granted = false });
}
}

View File

@@ -1,5 +1,6 @@
using System.Net.Sockets; using System.Net.Sockets;
using System.Text; using System.Text;
using NATS.Server.Subscriptions;
namespace NATS.Server.Routes; namespace NATS.Server.Routes;
@@ -7,9 +8,14 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
{ {
private readonly Socket _socket = socket; private readonly Socket _socket = socket;
private readonly NetworkStream _stream = new(socket, ownsSocket: true); private readonly NetworkStream _stream = new(socket, ownsSocket: true);
private readonly SemaphoreSlim _writeGate = new(1, 1);
private readonly CancellationTokenSource _closedCts = new();
private Task? _frameLoopTask;
public string? RemoteServerId { get; private set; } public string? RemoteServerId { get; private set; }
public string RemoteEndpoint => _socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N"); public string RemoteEndpoint => _socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N");
public Func<RemoteSubscription, Task>? RemoteSubscriptionReceived { get; set; }
public Func<RouteMessage, Task>? RoutedMessageReceived { get; set; }
public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct) public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct)
{ {
@@ -25,27 +31,168 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
await WriteLineAsync($"ROUTE {serverId}", ct); await WriteLineAsync($"ROUTE {serverId}", ct);
} }
public void StartFrameLoop(CancellationToken ct)
{
if (_frameLoopTask != null)
return;
var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
_frameLoopTask = Task.Run(() => ReadFramesAsync(linked.Token), linked.Token);
}
public async Task SendRsPlusAsync(string subject, string? queue, CancellationToken ct)
{
var frame = queue is { Length: > 0 }
? $"RS+ {subject} {queue}"
: $"RS+ {subject}";
await WriteLineAsync(frame, ct);
}
public async Task SendRsMinusAsync(string subject, string? queue, CancellationToken ct)
{
var frame = queue is { Length: > 0 }
? $"RS- {subject} {queue}"
: $"RS- {subject}";
await WriteLineAsync(frame, ct);
}
public async Task SendRmsgAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
{
var replyToken = string.IsNullOrEmpty(replyTo) ? "-" : replyTo;
await _writeGate.WaitAsync(ct);
try
{
var control = Encoding.ASCII.GetBytes($"RMSG {subject} {replyToken} {payload.Length}\r\n");
await _stream.WriteAsync(control, ct);
if (!payload.IsEmpty)
await _stream.WriteAsync(payload, ct);
await _stream.WriteAsync("\r\n"u8.ToArray(), ct);
await _stream.FlushAsync(ct);
}
finally
{
_writeGate.Release();
}
}
public async Task WaitUntilClosedAsync(CancellationToken ct) public async Task WaitUntilClosedAsync(CancellationToken ct)
{ {
var buffer = new byte[1024]; if (_frameLoopTask == null)
while (!ct.IsCancellationRequested) return;
{
var bytesRead = await _stream.ReadAsync(buffer, ct); using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token);
if (bytesRead == 0) await _frameLoopTask.WaitAsync(linked.Token);
return;
}
} }
public async ValueTask DisposeAsync() public async ValueTask DisposeAsync()
{ {
await _closedCts.CancelAsync();
if (_frameLoopTask != null)
await _frameLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
_closedCts.Dispose();
_writeGate.Dispose();
await _stream.DisposeAsync(); await _stream.DisposeAsync();
} }
private async Task ReadFramesAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
string line;
try
{
line = await ReadLineAsync(ct);
}
catch (OperationCanceledException)
{
break;
}
catch (IOException)
{
break;
}
if (line.StartsWith("RS+ ", StringComparison.Ordinal))
{
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
{
var queue = parts.Length >= 3 ? parts[2] : null;
await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteServerId ?? string.Empty));
}
continue;
}
if (line.StartsWith("RS- ", StringComparison.Ordinal))
{
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2 && RemoteSubscriptionReceived != null)
{
var queue = parts.Length >= 3 ? parts[2] : null;
await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteServerId ?? string.Empty));
}
continue;
}
if (!line.StartsWith("RMSG ", StringComparison.Ordinal))
continue;
var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (args.Length < 4)
continue;
var subject = args[1];
var reply = args[2] == "-" ? null : args[2];
if (!int.TryParse(args[3], out var size) || size < 0)
continue;
var payload = await ReadPayloadAsync(size, ct);
if (RoutedMessageReceived != null)
await RoutedMessageReceived(new RouteMessage(subject, reply, payload));
}
}
private async Task<ReadOnlyMemory<byte>> ReadPayloadAsync(int size, CancellationToken ct)
{
var payload = new byte[size];
var offset = 0;
while (offset < size)
{
var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct);
if (read == 0)
throw new IOException("Route connection closed during payload read");
offset += read;
}
var trailer = new byte[2];
var trailerRead = 0;
while (trailerRead < 2)
{
var read = await _stream.ReadAsync(trailer.AsMemory(trailerRead, 2 - trailerRead), ct);
if (read == 0)
throw new IOException("Route connection closed during payload trailer read");
trailerRead += read;
}
if (trailer[0] != (byte)'\r' || trailer[1] != (byte)'\n')
throw new IOException("Invalid route payload trailer");
return payload;
}
private async Task WriteLineAsync(string line, CancellationToken ct) private async Task WriteLineAsync(string line, CancellationToken ct)
{ {
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n"); await _writeGate.WaitAsync(ct);
await _stream.WriteAsync(bytes, ct); try
await _stream.FlushAsync(ct); {
var bytes = Encoding.ASCII.GetBytes($"{line}\r\n");
await _stream.WriteAsync(bytes, ct);
await _stream.FlushAsync(ct);
}
finally
{
_writeGate.Release();
}
} }
private async Task<string> ReadLineAsync(CancellationToken ct) private async Task<string> ReadLineAsync(CancellationToken ct)
@@ -56,7 +203,7 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
{ {
var read = await _stream.ReadAsync(single, ct); var read = await _stream.ReadAsync(single, ct);
if (read == 0) if (read == 0)
throw new IOException("Route connection closed during handshake"); throw new IOException("Route connection closed");
if (single[0] == (byte)'\n') if (single[0] == (byte)'\n')
break; break;
@@ -79,3 +226,5 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable
return id; return id;
} }
} }
public sealed record RouteMessage(string Subject, string? ReplyTo, ReadOnlyMemory<byte> Payload);

View File

@@ -15,6 +15,7 @@ public sealed class RouteManager : IAsyncDisposable
private readonly string _serverId; private readonly string _serverId;
private readonly ILogger<RouteManager> _logger; private readonly ILogger<RouteManager> _logger;
private readonly Action<RemoteSubscription> _remoteSubSink; private readonly Action<RemoteSubscription> _remoteSubSink;
private readonly Action<RouteMessage> _routedMessageSink;
private readonly ConcurrentDictionary<string, RouteConnection> _routes = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<string, RouteConnection> _routes = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, byte> _connectedServerIds = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<string, byte> _connectedServerIds = new(StringComparer.Ordinal);
@@ -29,12 +30,14 @@ public sealed class RouteManager : IAsyncDisposable
ServerStats stats, ServerStats stats,
string serverId, string serverId,
Action<RemoteSubscription> remoteSubSink, Action<RemoteSubscription> remoteSubSink,
Action<RouteMessage> routedMessageSink,
ILogger<RouteManager> logger) ILogger<RouteManager> logger)
{ {
_options = options; _options = options;
_stats = stats; _stats = stats;
_serverId = serverId; _serverId = serverId;
_remoteSubSink = remoteSubSink; _remoteSubSink = remoteSubSink;
_routedMessageSink = routedMessageSink;
_logger = logger; _logger = logger;
} }
@@ -51,8 +54,12 @@ public sealed class RouteManager : IAsyncDisposable
_options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port; _options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
_acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token)); _acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
var poolSize = Math.Max(_options.PoolSize, 1);
foreach (var route in _options.Routes.Distinct(StringComparer.OrdinalIgnoreCase)) foreach (var route in _options.Routes.Distinct(StringComparer.OrdinalIgnoreCase))
_ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token)); {
for (var i = 0; i < poolSize; i++)
_ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token));
}
return Task.CompletedTask; return Task.CompletedTask;
} }
@@ -81,17 +88,33 @@ public sealed class RouteManager : IAsyncDisposable
public void PropagateLocalSubscription(string subject, string? queue) public void PropagateLocalSubscription(string subject, string? queue)
{ {
if (_connectedServerIds.IsEmpty) if (_routes.IsEmpty)
return; return;
var remoteSub = new RemoteSubscription(subject, queue, _serverId); foreach (var route in _routes.Values)
foreach (var peerId in _connectedServerIds.Keys)
{ {
if (Managers.TryGetValue(peerId, out var peer)) _ = route.SendRsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
peer.ReceiveRemoteSubscription(remoteSub);
} }
} }
public void PropagateLocalUnsubscription(string subject, string? queue)
{
if (_routes.IsEmpty)
return;
foreach (var route in _routes.Values)
_ = route.SendRsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
}
public async Task ForwardRoutedMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
{
if (_routes.IsEmpty)
return;
foreach (var route in _routes.Values)
await route.SendRmsgAsync(subject, replyTo, payload, ct);
}
private async Task AcceptLoopAsync(CancellationToken ct) private async Task AcceptLoopAsync(CancellationToken ct)
{ {
while (!ct.IsCancellationRequested) while (!ct.IsCancellationRequested)
@@ -170,7 +193,7 @@ public sealed class RouteManager : IAsyncDisposable
private void Register(RouteConnection route) private void Register(RouteConnection route)
{ {
var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}"; var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}:{Guid.NewGuid():N}";
if (!_routes.TryAdd(key, route)) if (!_routes.TryAdd(key, route))
{ {
_ = route.DisposeAsync(); _ = route.DisposeAsync();
@@ -180,6 +203,18 @@ public sealed class RouteManager : IAsyncDisposable
if (route.RemoteServerId is { Length: > 0 } remoteServerId) if (route.RemoteServerId is { Length: > 0 } remoteServerId)
_connectedServerIds[remoteServerId] = 0; _connectedServerIds[remoteServerId] = 0;
route.RemoteSubscriptionReceived = sub =>
{
_remoteSubSink(sub);
return Task.CompletedTask;
};
route.RoutedMessageReceived = msg =>
{
_routedMessageSink(msg);
return Task.CompletedTask;
};
route.StartFrameLoop(_cts!.Token);
Interlocked.Increment(ref _stats.Routes); Interlocked.Increment(ref _stats.Routes);
_ = Task.Run(() => WatchRouteAsync(key, route, _cts!.Token)); _ = Task.Run(() => WatchRouteAsync(key, route, _cts!.Token));
} }
@@ -217,8 +252,5 @@ public sealed class RouteManager : IAsyncDisposable
return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1])); return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
} }
private void ReceiveRemoteSubscription(RemoteSubscription sub) public int RouteCount => _routes.Count;
{
_remoteSubSink(sub);
}
} }

View File

@@ -1,3 +1,7 @@
namespace NATS.Server.Subscriptions; namespace NATS.Server.Subscriptions;
public sealed record RemoteSubscription(string Subject, string? Queue, string RouteId); public sealed record RemoteSubscription(string Subject, string? Queue, string RouteId, bool IsRemoval = false)
{
public static RemoteSubscription Removal(string subject, string? queue, string routeId)
=> new(subject, queue, routeId, IsRemoval: true);
}

View File

@@ -103,7 +103,10 @@ public sealed class SubList : IDisposable
try try
{ {
var key = $"{sub.RouteId}|{sub.Subject}|{sub.Queue}"; var key = $"{sub.RouteId}|{sub.Subject}|{sub.Queue}";
_remoteSubs[key] = sub; if (sub.IsRemoval)
_remoteSubs.Remove(key);
else
_remoteSubs[key] = sub;
Interlocked.Increment(ref _generation); Interlocked.Increment(ref _generation);
} }
finally finally
@@ -119,6 +122,9 @@ public sealed class SubList : IDisposable
{ {
foreach (var remoteSub in _remoteSubs.Values) foreach (var remoteSub in _remoteSubs.Values)
{ {
if (remoteSub.IsRemoval)
continue;
if (SubjectMatch.MatchLiteral(subject, remoteSub.Subject)) if (SubjectMatch.MatchLiteral(subject, remoteSub.Subject))
return true; return true;
} }

View File

@@ -0,0 +1,29 @@
using NATS.Server.Protocol;
namespace NATS.Server.Tests;
public class ClientKindProtocolRoutingTests
{
[Theory]
[InlineData(ClientKind.Client, "RS+", false)]
[InlineData(ClientKind.Router, "RS+", true)]
[InlineData(ClientKind.Client, "RS-", false)]
[InlineData(ClientKind.Router, "RS-", true)]
[InlineData(ClientKind.Client, "RMSG", false)]
[InlineData(ClientKind.Router, "RMSG", true)]
[InlineData(ClientKind.Client, "A+", false)]
[InlineData(ClientKind.Gateway, "A+", true)]
[InlineData(ClientKind.Client, "A-", false)]
[InlineData(ClientKind.Gateway, "A-", true)]
[InlineData(ClientKind.Client, "LS+", false)]
[InlineData(ClientKind.Leaf, "LS+", true)]
[InlineData(ClientKind.Client, "LS-", false)]
[InlineData(ClientKind.Leaf, "LS-", true)]
[InlineData(ClientKind.Client, "LMSG", false)]
[InlineData(ClientKind.Leaf, "LMSG", true)]
public void Client_kind_protocol_matrix_enforces_inter_server_commands(ClientKind kind, string op, bool expected)
{
var matrix = new ClientCommandMatrix();
matrix.IsAllowed(kind, op).ShouldBe(expected);
}
}

View File

@@ -0,0 +1,153 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using Microsoft.Extensions.Logging.Abstractions;
using NATS.Server.Configuration;
namespace NATS.Server.Tests;
public class GatewayProtocolTests
{
[Fact]
public async Task Gateway_link_establishes_and_forwards_interested_message()
{
await using var fx = await GatewayFixture.StartTwoClustersAsync();
await fx.SubscribeRemoteClusterAsync("g.>");
await fx.PublishLocalClusterAsync("g.test", "hello");
(await fx.ReadRemoteClusterMessageAsync()).ShouldContain("hello");
}
}
internal sealed class GatewayFixture : IAsyncDisposable
{
private readonly NatsServer _local;
private readonly NatsServer _remote;
private readonly CancellationTokenSource _localCts;
private readonly CancellationTokenSource _remoteCts;
private Socket? _remoteSubscriber;
private Socket? _localPublisher;
private GatewayFixture(NatsServer local, NatsServer remote, CancellationTokenSource localCts, CancellationTokenSource remoteCts)
{
_local = local;
_remote = remote;
_localCts = localCts;
_remoteCts = remoteCts;
}
public static async Task<GatewayFixture> StartTwoClustersAsync()
{
var localOptions = new NatsOptions
{
Host = "127.0.0.1",
Port = 0,
Gateway = new GatewayOptions
{
Name = "LOCAL",
Host = "127.0.0.1",
Port = 0,
},
};
var local = new NatsServer(localOptions, NullLoggerFactory.Instance);
var localCts = new CancellationTokenSource();
_ = local.StartAsync(localCts.Token);
await local.WaitForReadyAsync();
var remoteOptions = new NatsOptions
{
Host = "127.0.0.1",
Port = 0,
Gateway = new GatewayOptions
{
Name = "REMOTE",
Host = "127.0.0.1",
Port = 0,
Remotes = [local.GatewayListen!],
},
};
var remote = new NatsServer(remoteOptions, NullLoggerFactory.Instance);
var remoteCts = new CancellationTokenSource();
_ = remote.StartAsync(remoteCts.Token);
await remote.WaitForReadyAsync();
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (!timeout.IsCancellationRequested && (local.Stats.Gateways == 0 || remote.Stats.Gateways == 0))
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
return new GatewayFixture(local, remote, localCts, remoteCts);
}
public async Task SubscribeRemoteClusterAsync(string subject)
{
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await sock.ConnectAsync(IPAddress.Loopback, _remote.Port);
_remoteSubscriber = sock;
_ = await ReadLineAsync(sock); // INFO
await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
public async Task PublishLocalClusterAsync(string subject, string payload)
{
var sock = _localPublisher;
if (sock == null)
{
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await sock.ConnectAsync(IPAddress.Loopback, _local.Port);
_localPublisher = sock;
_ = await ReadLineAsync(sock); // INFO
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
public Task<string> ReadRemoteClusterMessageAsync()
{
if (_remoteSubscriber == null)
throw new InvalidOperationException("Remote subscriber was not initialized.");
return ReadUntilAsync(_remoteSubscriber, "MSG ");
}
public async ValueTask DisposeAsync()
{
_remoteSubscriber?.Dispose();
_localPublisher?.Dispose();
await _localCts.CancelAsync();
await _remoteCts.CancelAsync();
_local.Dispose();
_remote.Dispose();
_localCts.Dispose();
_remoteCts.Dispose();
}
private static async Task<string> ReadLineAsync(Socket sock)
{
var buf = new byte[4096];
var n = await sock.ReceiveAsync(buf, SocketFlags.None);
return Encoding.ASCII.GetString(buf, 0, n);
}
private static async Task<string> ReadUntilAsync(Socket sock, string expected)
{
var sb = new StringBuilder();
var buf = new byte[4096];
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (!sb.ToString().Contains(expected, StringComparison.Ordinal))
{
var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token);
if (n == 0)
break;
sb.Append(Encoding.ASCII.GetString(buf, 0, n));
}
return sb.ToString();
}
}

View File

@@ -0,0 +1,17 @@
using NATS.Server.JetStream;
using NATS.Server.JetStream.Api;
namespace NATS.Server.Tests;
public class JetStreamAccountControlApiTests
{
[Fact]
public void Account_and_server_control_subjects_are_routable()
{
var router = new JetStreamApiRouter(new StreamManager(), new ConsumerManager());
router.Route("$JS.API.SERVER.REMOVE", "{}"u8).Error.ShouldBeNull();
router.Route("$JS.API.ACCOUNT.PURGE.ACC", "{}"u8).Error.ShouldBeNull();
router.Route("$JS.API.ACCOUNT.STREAM.MOVE.ACC", "{}"u8).Error.ShouldBeNull();
router.Route("$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.ACC", "{}"u8).Error.ShouldBeNull();
}
}

View File

@@ -41,6 +41,20 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
return fixture; return fixture;
} }
public static Task<JetStreamApiFixture> StartWithStreamConfigAsync(StreamConfig config)
{
var fixture = new JetStreamApiFixture();
_ = fixture._streamManager.CreateOrUpdate(config);
return Task.FromResult(fixture);
}
public static async Task<JetStreamApiFixture> StartWithStreamJsonAsync(string json)
{
var fixture = new JetStreamApiFixture();
_ = await fixture.RequestLocalAsync("$JS.API.STREAM.CREATE.S", json);
return fixture;
}
public static async Task<JetStreamApiFixture> StartWithPullConsumerAsync() public static async Task<JetStreamApiFixture> StartWithPullConsumerAsync()
{ {
var fixture = await StartWithStreamAsync("ORDERS", "orders.*"); var fixture = await StartWithStreamAsync("ORDERS", "orders.*");
@@ -82,6 +96,47 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
return fixture; return fixture;
} }
public static async Task<JetStreamApiFixture> StartWithMultiFilterConsumerAsync()
{
var fixture = await StartWithStreamAsync("ORDERS", ">");
_ = await fixture.CreateConsumerAsync("ORDERS", "CF", null, filterSubjects: ["orders.*"]);
return fixture;
}
public static async Task<JetStreamApiFixture> StartWithReplayOriginalConsumerAsync()
{
var fixture = await StartWithStreamAsync("ORDERS", "orders.*");
_ = await fixture.PublishAndGetAckAsync("orders.created", "1");
_ = await fixture.CreateConsumerAsync("ORDERS", "RO", "orders.*", replayPolicy: ReplayPolicy.Original, ackPolicy: AckPolicy.Explicit);
return fixture;
}
public static Task<JetStreamApiFixture> StartWithMultipleSourcesAsync()
{
var fixture = new JetStreamApiFixture();
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
{
Name = "SRC1",
Subjects = ["a.>"],
});
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
{
Name = "SRC2",
Subjects = ["b.>"],
});
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
{
Name = "AGG",
Subjects = ["agg.>"],
Sources =
[
new StreamSourceConfig { Name = "SRC1" },
new StreamSourceConfig { Name = "SRC2" },
],
});
return Task.FromResult(fixture);
}
public static Task<JetStreamApiFixture> StartJwtLimitedAccountAsync(int maxStreams) public static Task<JetStreamApiFixture> StartJwtLimitedAccountAsync(int maxStreams)
{ {
var account = new Account("JWT-LIMITED") var account = new Account("JWT-LIMITED")
@@ -148,9 +203,45 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
return _streamManager.GetStateAsync(streamName, default).AsTask(); return _streamManager.GetStateAsync(streamName, default).AsTask();
} }
public Task<JetStreamApiResponse> CreateConsumerAsync(string stream, string durableName, string filterSubject, bool push = false, int heartbeatMs = 0, AckPolicy ackPolicy = AckPolicy.None, int ackWaitMs = 30_000) public Task<string> GetStreamBackendTypeAsync(string streamName)
{ {
var payload = $@"{{""durable_name"":""{durableName}"",""filter_subject"":""{filterSubject}"",""push"":{push.ToString().ToLowerInvariant()},""heartbeat_ms"":{heartbeatMs},""ack_policy"":""{ackPolicy.ToString().ToLowerInvariant()}"",""ack_wait_ms"":{ackWaitMs}}}"; return Task.FromResult(_streamManager.GetStoreBackendType(streamName));
}
public Task<JetStreamApiResponse> CreateConsumerAsync(
string stream,
string durableName,
string? filterSubject,
bool push = false,
int heartbeatMs = 0,
AckPolicy ackPolicy = AckPolicy.None,
int ackWaitMs = 30_000,
int maxAckPending = 0,
IReadOnlyList<string>? filterSubjects = null,
ReplayPolicy replayPolicy = ReplayPolicy.Instant,
DeliverPolicy deliverPolicy = DeliverPolicy.All,
bool ephemeral = false)
{
var payloadObj = new
{
durable_name = durableName,
filter_subject = filterSubject,
filter_subjects = filterSubjects,
push,
heartbeat_ms = heartbeatMs,
ack_policy = ackPolicy.ToString().ToLowerInvariant(),
ack_wait_ms = ackWaitMs,
max_ack_pending = maxAckPending,
replay_policy = replayPolicy == ReplayPolicy.Original ? "original" : "instant",
deliver_policy = deliverPolicy switch
{
DeliverPolicy.Last => "last",
DeliverPolicy.New => "new",
_ => "all",
},
ephemeral,
};
var payload = JsonSerializer.Serialize(payloadObj);
return RequestLocalAsync($"$JS.API.CONSUMER.CREATE.{stream}.{durableName}", payload); return RequestLocalAsync($"$JS.API.CONSUMER.CREATE.{stream}.{durableName}", payload);
} }
@@ -206,6 +297,12 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
_ = await PublishAndGetAckAsync(subject, payload); _ = await PublishAndGetAckAsync(subject, payload);
} }
public Task PublishToSourceAsync(string sourceStream, string subject, string payload)
{
_ = sourceStream;
return PublishAndGetAckAsync(subject, payload);
}
public Task AckAllAsync(string stream, string durableName, ulong sequence) public Task AckAllAsync(string stream, string durableName, ulong sequence)
{ {
_consumerManager.AckAll(stream, durableName, sequence); _consumerManager.AckAll(stream, durableName, sequence);

View File

@@ -0,0 +1,88 @@
using System.Diagnostics;
using System.Text.RegularExpressions;
namespace NATS.Server.Tests;
public class JetStreamApiGapInventoryTests
{
[Fact]
public void Parity_map_has_no_unclassified_go_js_api_subjects()
{
var gap = JetStreamApiGapInventory.Load();
gap.UnclassifiedSubjects.ShouldBeEmpty();
}
}
internal sealed class JetStreamApiGapInventory
{
public IReadOnlyList<string> UnclassifiedSubjects { get; }
private JetStreamApiGapInventory(IReadOnlyList<string> unclassifiedSubjects)
{
UnclassifiedSubjects = unclassifiedSubjects;
}
public static JetStreamApiGapInventory Load()
{
var goSubjects = LoadGoSubjects();
var mappedSubjects = LoadMappedSubjects();
var unclassified = goSubjects
.Where(s => !mappedSubjects.Contains(s))
.OrderBy(s => s, StringComparer.Ordinal)
.ToArray();
return new JetStreamApiGapInventory(unclassified);
}
private static HashSet<string> LoadGoSubjects()
{
var script = Path.Combine(AppContext.BaseDirectory, "../../../../../scripts/jetstream/extract-go-js-api.sh");
script = Path.GetFullPath(script);
if (!File.Exists(script))
throw new FileNotFoundException($"missing script: {script}");
var psi = new ProcessStartInfo
{
FileName = "bash",
RedirectStandardOutput = true,
RedirectStandardError = true,
};
psi.ArgumentList.Add(script);
using var process = Process.Start(psi) ?? throw new InvalidOperationException("failed to start inventory script");
var output = process.StandardOutput.ReadToEnd();
var errors = process.StandardError.ReadToEnd();
process.WaitForExit();
if (process.ExitCode != 0)
throw new InvalidOperationException($"inventory script failed: {errors}");
return output
.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Where(x => x.StartsWith("$JS.API.", StringComparison.Ordinal))
.ToHashSet(StringComparer.Ordinal);
}
private static HashSet<string> LoadMappedSubjects()
{
var mapPath = Path.Combine(AppContext.BaseDirectory, "../../../../../docs/plans/2026-02-23-jetstream-remaining-parity-map.md");
mapPath = Path.GetFullPath(mapPath);
if (!File.Exists(mapPath))
throw new FileNotFoundException($"missing parity map: {mapPath}");
var subjectRegex = new Regex(@"^\|\s*(\$JS\.API[^\|]+?)\s*\|", RegexOptions.Compiled);
var subjects = new HashSet<string>(StringComparer.Ordinal);
foreach (var line in File.ReadLines(mapPath))
{
var match = subjectRegex.Match(line);
if (!match.Success)
continue;
subjects.Add(match.Groups[1].Value.Trim());
}
return subjects;
}
}

View File

@@ -0,0 +1,12 @@
namespace NATS.Server.Tests;
public class JetStreamClusterControlExtendedApiTests
{
[Fact]
public async Task Peer_remove_and_consumer_stepdown_subjects_return_success_shape()
{
await using var fx = await JetStreamClusterFixture.StartAsync(nodes: 3);
(await fx.RequestAsync("$JS.API.STREAM.PEER.REMOVE.ORDERS", "{\"peer\":\"n2\"}")).Success.ShouldBeTrue();
(await fx.RequestAsync("$JS.API.CONSUMER.LEADER.STEPDOWN.ORDERS.DUR", "{}")).Success.ShouldBeTrue();
}
}

View File

@@ -0,0 +1,16 @@
namespace NATS.Server.Tests;
public class JetStreamConsumerSemanticsTests
{
[Fact]
public async Task Consumer_with_filter_subjects_only_receives_matching_messages()
{
await using var fx = await JetStreamApiFixture.StartWithMultiFilterConsumerAsync();
await fx.PublishAndGetAckAsync("orders.created", "1");
await fx.PublishAndGetAckAsync("payments.settled", "2");
var batch = await fx.FetchAsync("ORDERS", "CF", 10);
batch.Messages.ShouldNotBeEmpty();
batch.Messages.All(m => m.Subject.StartsWith("orders.", StringComparison.Ordinal)).ShouldBeTrue();
}
}

View File

@@ -0,0 +1,15 @@
using System.Diagnostics;
namespace NATS.Server.Tests;
public class JetStreamFlowReplayBackoffTests
{
[Fact]
public async Task Replay_original_respects_message_timestamps_with_backoff_redelivery()
{
await using var fx = await JetStreamApiFixture.StartWithReplayOriginalConsumerAsync();
var sw = Stopwatch.StartNew();
_ = await fx.FetchAsync("ORDERS", "RO", 1);
sw.ElapsedMilliseconds.ShouldBeGreaterThanOrEqualTo(50);
}
}

View File

@@ -0,0 +1,14 @@
namespace NATS.Server.Tests;
public class JetStreamMirrorSourceAdvancedTests
{
[Fact]
public async Task Stream_with_multiple_sources_aggregates_messages_in_order()
{
await using var fx = await JetStreamApiFixture.StartWithMultipleSourcesAsync();
await fx.PublishToSourceAsync("SRC1", "a.1", "1");
await fx.PublishToSourceAsync("SRC2", "b.1", "2");
(await fx.GetStreamStateAsync("AGG")).Messages.ShouldBe((ulong)2);
}
}

View File

@@ -0,0 +1,11 @@
namespace NATS.Server.Tests;
public class JetStreamStorageSelectionTests
{
[Fact]
public async Task Stream_with_storage_file_uses_filestore_backend()
{
await using var fx = await JetStreamApiFixture.StartWithStreamJsonAsync("{\"name\":\"S\",\"subjects\":[\"s.*\"],\"storage\":\"file\"}");
(await fx.GetStreamBackendTypeAsync("S")).ShouldBe("file");
}
}

View File

@@ -0,0 +1,21 @@
using NATS.Server.JetStream.Models;
namespace NATS.Server.Tests;
public class JetStreamStreamPolicyRuntimeTests
{
[Fact]
public async Task Discard_new_rejects_publish_when_max_bytes_exceeded()
{
await using var fx = await JetStreamApiFixture.StartWithStreamConfigAsync(new StreamConfig
{
Name = "S",
Subjects = ["s.*"],
MaxBytes = 2,
Discard = DiscardPolicy.New,
});
(await fx.PublishAndGetAckAsync("s.a", "12")).ErrorCode.ShouldBeNull();
(await fx.PublishAndGetAckAsync("s.a", "34")).ErrorCode.ShouldNotBeNull();
}
}

View File

@@ -0,0 +1,151 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using Microsoft.Extensions.Logging.Abstractions;
using NATS.Server.Configuration;
namespace NATS.Server.Tests;
public class LeafProtocolTests
{
[Fact]
public async Task Leaf_link_propagates_subscription_and_message_flow()
{
await using var fx = await LeafFixture.StartHubSpokeAsync();
await fx.SubscribeSpokeAsync("leaf.>");
await fx.PublishHubAsync("leaf.msg", "x");
(await fx.ReadSpokeMessageAsync()).ShouldContain("x");
}
}
internal sealed class LeafFixture : IAsyncDisposable
{
private readonly NatsServer _hub;
private readonly NatsServer _spoke;
private readonly CancellationTokenSource _hubCts;
private readonly CancellationTokenSource _spokeCts;
private Socket? _spokeSubscriber;
private Socket? _hubPublisher;
private LeafFixture(NatsServer hub, NatsServer spoke, CancellationTokenSource hubCts, CancellationTokenSource spokeCts)
{
_hub = hub;
_spoke = spoke;
_hubCts = hubCts;
_spokeCts = spokeCts;
}
public static async Task<LeafFixture> StartHubSpokeAsync()
{
var hubOptions = new NatsOptions
{
Host = "127.0.0.1",
Port = 0,
LeafNode = new LeafNodeOptions
{
Host = "127.0.0.1",
Port = 0,
},
};
var hub = new NatsServer(hubOptions, NullLoggerFactory.Instance);
var hubCts = new CancellationTokenSource();
_ = hub.StartAsync(hubCts.Token);
await hub.WaitForReadyAsync();
var spokeOptions = new NatsOptions
{
Host = "127.0.0.1",
Port = 0,
LeafNode = new LeafNodeOptions
{
Host = "127.0.0.1",
Port = 0,
Remotes = [hub.LeafListen!],
},
};
var spoke = new NatsServer(spokeOptions, NullLoggerFactory.Instance);
var spokeCts = new CancellationTokenSource();
_ = spoke.StartAsync(spokeCts.Token);
await spoke.WaitForReadyAsync();
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (!timeout.IsCancellationRequested && (hub.Stats.Leafs == 0 || spoke.Stats.Leafs == 0))
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
return new LeafFixture(hub, spoke, hubCts, spokeCts);
}
public async Task SubscribeSpokeAsync(string subject)
{
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await sock.ConnectAsync(IPAddress.Loopback, _spoke.Port);
_spokeSubscriber = sock;
_ = await ReadLineAsync(sock); // INFO
await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
public async Task PublishHubAsync(string subject, string payload)
{
var sock = _hubPublisher;
if (sock == null)
{
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await sock.ConnectAsync(IPAddress.Loopback, _hub.Port);
_hubPublisher = sock;
_ = await ReadLineAsync(sock); // INFO
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
public Task<string> ReadSpokeMessageAsync()
{
if (_spokeSubscriber == null)
throw new InvalidOperationException("Spoke subscriber was not initialized.");
return ReadUntilAsync(_spokeSubscriber, "MSG ");
}
public async ValueTask DisposeAsync()
{
_spokeSubscriber?.Dispose();
_hubPublisher?.Dispose();
await _hubCts.CancelAsync();
await _spokeCts.CancelAsync();
_hub.Dispose();
_spoke.Dispose();
_hubCts.Dispose();
_spokeCts.Dispose();
}
private static async Task<string> ReadLineAsync(Socket sock)
{
var buf = new byte[4096];
var n = await sock.ReceiveAsync(buf, SocketFlags.None);
return Encoding.ASCII.GetString(buf, 0, n);
}
private static async Task<string> ReadUntilAsync(Socket sock, string expected)
{
var sb = new StringBuilder();
var buf = new byte[4096];
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (!sb.ToString().Contains(expected, StringComparison.Ordinal))
{
var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token);
if (n == 0)
break;
sb.Append(Encoding.ASCII.GetString(buf, 0, n));
}
return sb.ToString();
}
}

View File

@@ -0,0 +1,105 @@
using System.Net;
using System.Net.Sockets;
using Microsoft.Extensions.Logging.Abstractions;
using NATS.Server.Configuration;
namespace NATS.Server.Tests;
public class MonitorClusterEndpointTests
{
[Fact]
public async Task Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data()
{
await using var fx = await MonitorFixture.StartClusterEnabledAsync();
(await fx.GetJsonAsync("/routez")).ShouldContain("routes");
(await fx.GetJsonAsync("/gatewayz")).ShouldContain("gateways");
(await fx.GetJsonAsync("/leafz")).ShouldContain("leafs");
(await fx.GetJsonAsync("/accountz")).ShouldContain("accounts");
}
}
internal sealed class MonitorFixture : IAsyncDisposable
{
private readonly NatsServer _server;
private readonly CancellationTokenSource _cts;
private readonly HttpClient _http;
private readonly int _monitorPort;
private MonitorFixture(NatsServer server, CancellationTokenSource cts, HttpClient http, int monitorPort)
{
_server = server;
_cts = cts;
_http = http;
_monitorPort = monitorPort;
}
public static async Task<MonitorFixture> StartClusterEnabledAsync()
{
var monitorPort = GetFreePort();
var options = new NatsOptions
{
Host = "127.0.0.1",
Port = 0,
MonitorPort = monitorPort,
Cluster = new ClusterOptions
{
Host = "127.0.0.1",
Port = 0,
},
Gateway = new GatewayOptions
{
Host = "127.0.0.1",
Port = 0,
Name = "M",
},
LeafNode = new LeafNodeOptions
{
Host = "127.0.0.1",
Port = 0,
},
};
var server = new NatsServer(options, NullLoggerFactory.Instance);
var cts = new CancellationTokenSource();
_ = server.StartAsync(cts.Token);
await server.WaitForReadyAsync();
var http = new HttpClient();
for (var i = 0; i < 50; i++)
{
try
{
var response = await http.GetAsync($"http://127.0.0.1:{monitorPort}/healthz");
if (response.IsSuccessStatusCode)
break;
}
catch
{
}
await Task.Delay(50);
}
return new MonitorFixture(server, cts, http, monitorPort);
}
public Task<string> GetJsonAsync(string path)
{
return _http.GetStringAsync($"http://127.0.0.1:{_monitorPort}{path}");
}
public async ValueTask DisposeAsync()
{
_http.Dispose();
await _cts.CancelAsync();
_server.Dispose();
_cts.Dispose();
}
private static int GetFreePort()
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
return ((IPEndPoint)socket.LocalEndPoint!).Port;
}
}

View File

@@ -0,0 +1,89 @@
using NATS.Server.Raft;
namespace NATS.Server.Tests;
public class RaftTransportPersistenceTests
{
[Fact]
public async Task Raft_node_recovers_log_and_term_after_restart()
{
await using var fx = await RaftFixture.StartPersistentClusterAsync();
var idx = await fx.Leader.ProposeAsync("cmd", default);
await fx.RestartNodeAsync("n2");
(await fx.ReadNodeAppliedIndexAsync("n2")).ShouldBeGreaterThanOrEqualTo(idx);
}
}
internal sealed class RaftFixture : IAsyncDisposable
{
private readonly string _root;
private readonly InMemoryRaftTransport _transport;
private readonly Dictionary<string, RaftNode> _nodes;
private RaftFixture(string root, InMemoryRaftTransport transport, Dictionary<string, RaftNode> nodes)
{
_root = root;
_transport = transport;
_nodes = nodes;
}
public RaftNode Leader => _nodes["n1"];
public static Task<RaftFixture> StartPersistentClusterAsync()
{
var root = Path.Combine(Path.GetTempPath(), $"nats-raft-{Guid.NewGuid():N}");
Directory.CreateDirectory(root);
var transport = new InMemoryRaftTransport();
var nodes = new Dictionary<string, RaftNode>(StringComparer.Ordinal);
foreach (var id in new[] { "n1", "n2", "n3" })
{
var node = new RaftNode(id, transport, Path.Combine(root, id));
transport.Register(node);
nodes[id] = node;
}
var all = nodes.Values.ToArray();
foreach (var node in all)
node.ConfigureCluster(all);
var leader = nodes["n1"];
leader.StartElection(all.Length);
leader.ReceiveVote(nodes["n2"].GrantVote(leader.Term), all.Length);
leader.ReceiveVote(nodes["n3"].GrantVote(leader.Term), all.Length);
return Task.FromResult(new RaftFixture(root, transport, nodes));
}
public async Task RestartNodeAsync(string id)
{
var nodePath = Path.Combine(_root, id);
var replacement = new RaftNode(id, _transport, nodePath);
await replacement.LoadPersistedStateAsync(default);
_transport.Register(replacement);
_nodes[id] = replacement;
var all = _nodes.Values.ToArray();
foreach (var node in all)
node.ConfigureCluster(all);
}
public Task<long> ReadNodeAppliedIndexAsync(string id)
{
return Task.FromResult(_nodes[id].AppliedIndex);
}
public ValueTask DisposeAsync()
{
try
{
if (Directory.Exists(_root))
Directory.Delete(_root, recursive: true);
}
catch
{
}
return ValueTask.CompletedTask;
}
}

View File

@@ -0,0 +1,11 @@
namespace NATS.Server.Tests;
public class RoutePoolTests
{
[Fact]
public async Task Route_manager_establishes_default_pool_of_three_links_per_peer()
{
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
(await fx.ServerARouteLinkCountToServerBAsync()).ShouldBe(3);
}
}

View File

@@ -0,0 +1,14 @@
namespace NATS.Server.Tests;
public class RouteRmsgForwardingTests
{
[Fact]
public async Task Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG()
{
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
await fx.SubscribeOnServerBAsync("foo.>");
await fx.PublishFromServerAAsync("foo.bar", "payload");
(await fx.ReadServerBMessageAsync()).ShouldContain("payload");
}
}

View File

@@ -27,6 +27,8 @@ internal sealed class RouteFixture : IAsyncDisposable
private readonly CancellationTokenSource _ctsA; private readonly CancellationTokenSource _ctsA;
private readonly CancellationTokenSource _ctsB; private readonly CancellationTokenSource _ctsB;
private Socket? _subscriberOnB; private Socket? _subscriberOnB;
private Socket? _publisherOnA;
private Socket? _manualRouteToA;
private RouteFixture(NatsServer serverA, NatsServer serverB, CancellationTokenSource ctsA, CancellationTokenSource ctsB) private RouteFixture(NatsServer serverA, NatsServer serverB, CancellationTokenSource ctsA, CancellationTokenSource ctsB)
{ {
@@ -91,22 +93,82 @@ internal sealed class RouteFixture : IAsyncDisposable
await ReadUntilAsync(sock, "PONG"); await ReadUntilAsync(sock, "PONG");
} }
public async Task<bool> ServerAHasRemoteInterestAsync(string subject) public async Task SendRouteSubFrameAsync(string subject)
{
var (host, port) = ParseHostPort(_serverA.ClusterListen!);
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await sock.ConnectAsync(IPAddress.Parse(host), port);
_manualRouteToA = sock;
await sock.SendAsync(Encoding.ASCII.GetBytes("ROUTE test-remote\r\n"));
_ = await ReadLineAsync(sock); // ROUTE <id>
await sock.SendAsync(Encoding.ASCII.GetBytes($"RS+ {subject}\r\n"));
}
public async Task SendRouteUnsubFrameAsync(string subject)
{
if (_manualRouteToA == null)
throw new InvalidOperationException("Route frame socket not established.");
await _manualRouteToA.SendAsync(Encoding.ASCII.GetBytes($"RS- {subject}\r\n"));
}
public async Task PublishFromServerAAsync(string subject, string payload)
{
var sock = _publisherOnA;
if (sock == null)
{
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await sock.ConnectAsync(IPAddress.Loopback, _serverA.Port);
_publisherOnA = sock;
_ = await ReadLineAsync(sock); // INFO
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
await ReadUntilAsync(sock, "PONG");
}
public async Task<string> ReadServerBMessageAsync()
{
if (_subscriberOnB == null)
throw new InvalidOperationException("No subscriber socket on server B.");
return await ReadUntilAsync(_subscriberOnB, "MSG ");
}
public async Task<bool> ServerAHasRemoteInterestAsync(string subject, bool expected = true)
{ {
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (!timeout.IsCancellationRequested) while (!timeout.IsCancellationRequested)
{ {
if (_serverA.HasRemoteInterest(subject)) if (_serverA.HasRemoteInterest(subject) == expected)
return true; return expected;
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default); await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
} }
return false; return !expected;
}
public async Task<int> ServerARouteLinkCountToServerBAsync()
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (!timeout.IsCancellationRequested)
{
if (_serverA.Stats.Routes >= 3)
return (int)_serverA.Stats.Routes;
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
}
return (int)_serverA.Stats.Routes;
} }
public async ValueTask DisposeAsync() public async ValueTask DisposeAsync()
{ {
_subscriberOnB?.Dispose(); _subscriberOnB?.Dispose();
_publisherOnA?.Dispose();
_manualRouteToA?.Dispose();
await _ctsA.CancelAsync(); await _ctsA.CancelAsync();
await _ctsB.CancelAsync(); await _ctsB.CancelAsync();
_serverA.Dispose(); _serverA.Dispose();
@@ -138,4 +200,10 @@ internal sealed class RouteFixture : IAsyncDisposable
return sb.ToString(); return sb.ToString();
} }
private static (string Host, int Port) ParseHostPort(string endpoint)
{
var parts = endpoint.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
return (parts[0], int.Parse(parts[1]));
}
} }

View File

@@ -0,0 +1,16 @@
namespace NATS.Server.Tests;
public class RouteWireSubscriptionProtocolTests
{
[Fact]
public async Task RSplus_RSminus_frames_propagate_remote_interest_over_socket()
{
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
await fx.SendRouteSubFrameAsync("foo.*");
(await fx.ServerAHasRemoteInterestAsync("foo.bar")).ShouldBeTrue();
await fx.SendRouteUnsubFrameAsync("foo.*");
(await fx.ServerAHasRemoteInterestAsync("foo.bar", expected: false)).ShouldBeFalse();
}
}