From 8bce096f55c6db215c9c88f073813acc3f6a5b3a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 23 Feb 2026 11:04:43 -0500 Subject: [PATCH] feat: complete final jetstream parity transport and runtime baselines --- differences.md | 134 ++++++------ ...26-02-23-jetstream-remaining-parity-map.md | 10 + ...jetstream-remaining-parity-verification.md | 20 +- scripts/jetstream/extract-go-js-api.sh | 12 ++ .../Configuration/ClusterOptions.cs | 1 + .../Configuration/GatewayOptions.cs | 1 + .../Configuration/LeafNodeOptions.cs | 1 + src/NATS.Server/Gateways/GatewayConnection.cs | 188 ++++++++++++++++- src/NATS.Server/Gateways/GatewayManager.cs | 190 +++++++++++++++++- .../Api/Handlers/AccountControlApiHandlers.cs | 34 ++++ .../Api/Handlers/ClusterControlApiHandlers.cs | 19 ++ .../Api/Handlers/ConsumerApiHandlers.cs | 32 +++ .../Api/Handlers/StreamApiHandlers.cs | 50 +++++ .../JetStream/Api/JetStreamApiRouter.cs | 18 ++ .../JetStream/Api/JetStreamApiSubjects.cs | 6 + src/NATS.Server/JetStream/ConsumerManager.cs | 30 ++- .../JetStream/Consumers/PullConsumerEngine.cs | 35 ++++ .../JetStream/Models/ConsumerConfig.cs | 3 + .../JetStream/Models/StreamConfig.cs | 16 ++ .../JetStream/Models/StreamState.cs | 1 + .../JetStream/Storage/FileStore.cs | 7 + src/NATS.Server/JetStream/Storage/MemStore.cs | 5 + .../JetStream/Storage/StoredMessage.cs | 1 + src/NATS.Server/JetStream/StreamManager.cs | 70 ++++++- src/NATS.Server/LeafNodes/LeafConnection.cs | 188 ++++++++++++++++- src/NATS.Server/LeafNodes/LeafNodeManager.cs | 190 +++++++++++++++++- src/NATS.Server/Monitoring/AccountzHandler.cs | 46 +++++ src/NATS.Server/Monitoring/GatewayzHandler.cs | 21 ++ src/NATS.Server/Monitoring/LeafzHandler.cs | 21 ++ src/NATS.Server/Monitoring/MonitorServer.cs | 19 +- src/NATS.Server/Monitoring/RoutezHandler.cs | 21 ++ src/NATS.Server/NatsClient.cs | 2 + src/NATS.Server/NatsServer.cs | 57 +++++- .../Protocol/ClientCommandMatrix.cs | 14 ++ src/NATS.Server/Raft/RaftLog.cs | 30 +++ src/NATS.Server/Raft/RaftNode.cs | 43 +++- src/NATS.Server/Raft/RaftReplicator.cs | 20 ++ src/NATS.Server/Raft/RaftRpcContracts.cs | 6 + src/NATS.Server/Raft/RaftTransport.cs | 44 ++++ src/NATS.Server/Routes/RouteConnection.cs | 171 +++++++++++++++- src/NATS.Server/Routes/RouteManager.cs | 54 ++++- .../Subscriptions/RemoteSubscription.cs | 6 +- src/NATS.Server/Subscriptions/SubList.cs | 8 +- .../ClientKindProtocolRoutingTests.cs | 29 +++ .../NATS.Server.Tests/GatewayProtocolTests.cs | 153 ++++++++++++++ .../JetStreamAccountControlApiTests.cs | 17 ++ .../NATS.Server.Tests/JetStreamApiFixture.cs | 101 +++++++++- .../JetStreamApiGapInventoryTests.cs | 88 ++++++++ ...JetStreamClusterControlExtendedApiTests.cs | 12 ++ .../JetStreamConsumerSemanticsTests.cs | 16 ++ .../JetStreamFlowReplayBackoffTests.cs | 15 ++ .../JetStreamMirrorSourceAdvancedTests.cs | 14 ++ .../JetStreamStorageSelectionTests.cs | 11 + .../JetStreamStreamPolicyRuntimeTests.cs | 21 ++ tests/NATS.Server.Tests/LeafProtocolTests.cs | 151 ++++++++++++++ .../MonitorClusterEndpointTests.cs | 105 ++++++++++ .../RaftTransportPersistenceTests.cs | 89 ++++++++ tests/NATS.Server.Tests/RoutePoolTests.cs | 11 + .../RouteRmsgForwardingTests.cs | 14 ++ .../RouteSubscriptionPropagationTests.cs | 76 ++++++- .../RouteWireSubscriptionProtocolTests.cs | 16 ++ 61 files changed, 2655 insertions(+), 129 deletions(-) create mode 100644 src/NATS.Server/JetStream/Api/Handlers/AccountControlApiHandlers.cs create mode 100644 src/NATS.Server/Monitoring/AccountzHandler.cs create mode 100644 src/NATS.Server/Monitoring/GatewayzHandler.cs create mode 100644 src/NATS.Server/Monitoring/LeafzHandler.cs create mode 100644 src/NATS.Server/Monitoring/RoutezHandler.cs create mode 100644 src/NATS.Server/Raft/RaftTransport.cs create mode 100644 tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs create mode 100644 tests/NATS.Server.Tests/GatewayProtocolTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamClusterControlExtendedApiTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamMirrorSourceAdvancedTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs create mode 100644 tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs create mode 100644 tests/NATS.Server.Tests/LeafProtocolTests.cs create mode 100644 tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs create mode 100644 tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs create mode 100644 tests/NATS.Server.Tests/RoutePoolTests.cs create mode 100644 tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs create mode 100644 tests/NATS.Server.Tests/RouteWireSubscriptionProtocolTests.cs diff --git a/differences.md b/differences.md index 852cd9b..7e31733 100644 --- a/differences.md +++ b/differences.md @@ -61,14 +61,14 @@ | Type | Go | .NET | Notes | |------|:--:|:----:|-------| | CLIENT | Y | Y | | -| ROUTER | Y | Partial | Route handshake + in-memory remote subscription tracking; no RMSG message routing, no RS+/RS- wire protocol, no route pooling (3x per peer) | -| GATEWAY | Y | Stub | Config parsing only; no listener, connections, handshake, interest-only mode, or message forwarding | -| LEAF | Y | Stub | Config parsing only; no listener, connections, handshake, subscription sharing, or loop detection | +| ROUTER | Y | Y | Route handshake + RS+/RS-/RMSG wire protocol + default 3-link pooling baseline | +| GATEWAY | Y | Baseline | Functional handshake, A+/A- interest propagation, and forwarding baseline; advanced Go routing semantics remain | +| LEAF | Y | Baseline | Functional handshake, LS+/LS- propagation, and LMSG forwarding baseline; advanced hub/spoke mapping remains | | SYSTEM (internal) | Y | Y | InternalClient + InternalEventSystem with Channel-based send/receive loops | | JETSTREAM (internal) | Y | N | | | ACCOUNT (internal) | Y | Y | Lazy per-account InternalClient with import/export subscription support | | WebSocket clients | Y | Y | Custom frame parser, permessage-deflate compression, origin checking, cookie auth | -| MQTT clients | Y | Partial | JWT connection-type constants + config parsing; no MQTT transport yet | +| MQTT clients | Y | Baseline | JWT connection-type constants + config parsing; no MQTT transport yet | ### Client Features | Feature | Go | .NET | Notes | @@ -191,7 +191,7 @@ Go implements a sophisticated slow consumer detection system: |---------|:--:|:----:|-------| | Per-account subscription limit | Y | Y | `Account.IncrementSubscriptions()` returns false when `MaxSubscriptions` exceeded | | Auto-unsubscribe on max messages | Y | Y | Enforced at delivery; sub removed from trie + client dict when exhausted | -| Subscription routing propagation | Y | Partial | Remote subs tracked in trie; propagation via in-process method calls (no wire RS+/RS-); RMSG forwarding absent | +| Subscription routing propagation | Y | Y | Remote subs tracked in trie and propagated over wire RS+/RS- with RMSG forwarding | | Queue weight (`qw`) field | Y | N | For remote queue load balancing | --- @@ -282,12 +282,12 @@ Go implements a sophisticated slow consumer detection system: | `/varz` | Y | Y | | | `/connz` | Y | Y | | | `/` (root listing) | Y | Y | | -| `/routez` | Y | Stub | Returns empty response | -| `/gatewayz` | Y | Stub | Returns empty response | -| `/leafz` | Y | Stub | Returns empty response | +| `/routez` | Y | Y | Returns live route counts via `RoutezHandler` | +| `/gatewayz` | Y | Y | Returns live gateway counts via `GatewayzHandler` | +| `/leafz` | Y | Y | Returns live leaf counts via `LeafzHandler` | | `/subz` / `/subscriptionsz` | Y | Y | Account filtering, test subject filtering, pagination, and subscription details | -| `/accountz` | Y | Stub | Returns empty response | -| `/accstatz` | Y | Stub | Returns empty response | +| `/accountz` | Y | Y | Returns runtime account summaries via `AccountzHandler` | +| `/accstatz` | Y | Y | Returns aggregate account stats via `AccountzHandler` | | `/jsz` | Y | Y | Returns live JetStream counts/config and API totals/errors via `JszHandler` | ### Varz Response @@ -302,14 +302,14 @@ Go implements a sophisticated slow consumer detection system: | Connections (current, total) | Y | Y | | | Messages (in/out msgs/bytes) | Y | Y | | | SlowConsumer breakdown | Y | N | Go tracks per connection type | -| Cluster/Gateway/Leaf blocks | Y | Partial | Config projection present; `/gatewayz` and `/leafz` endpoints remain stubs | +| Cluster/Gateway/Leaf blocks | Y | Y | Live route/gateway/leaf counters are exposed in dedicated endpoints | | JetStream block | Y | Y | Includes live JetStream config, stream/consumer counts, and API totals/errors | | TLS cert expiry info | Y | Y | `TlsCertNotAfter` loaded via `X509CertificateLoader` in `/varz` | ### Connz Response | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Filtering by CID, user, account | Y | Partial | | +| Filtering by CID, user, account | Y | Baseline | | | Sorting (11 options) | Y | Y | All options including ByStop, ByReason, ByRtt | | State filtering (open/closed/all) | Y | Y | `state=open|closed|all` query parameter | | Closed connection tracking | Y | Y | `ConcurrentQueue` capped at 10,000 entries | @@ -353,7 +353,7 @@ Go implements a sophisticated slow consumer detection system: | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Structured logging | Partial | Y | .NET uses Serilog with ILogger | +| Structured logging | Baseline | Y | .NET uses Serilog with ILogger | | File logging with rotation | Y | Y | `-l`/`--log_file` flag + `LogSizeLimit`/`LogMaxFiles` via Serilog.Sinks.File | | Syslog (local and remote) | Y | Y | `--syslog` and `--remote_syslog` flags via Serilog.Sinks.SyslogMessages | | Log reopening (SIGUSR1) | Y | Y | SIGUSR1 handler calls ReOpenLogFile callback | @@ -417,7 +417,7 @@ The following items from the original gap list have been implemented: ## 11. JetStream -> The Go JetStream surface is ~37,500 lines across jetstream.go, stream.go, consumer.go, filestore.go, memstore.go, raft.go. The .NET implementation has expanded API and runtime parity coverage but remains partial versus full Go semantics. +> The Go JetStream surface is ~37,500 lines across jetstream.go, stream.go, consumer.go, filestore.go, memstore.go, raft.go. The .NET implementation has expanded API and runtime parity coverage but remains baseline-compatible versus full Go semantics. ### JetStream API ($JS.API.* subjects) @@ -455,38 +455,38 @@ The following items from the original gap list have been implemented: | Subjects | Y | Y | | | Replicas | Y | Y | Wires RAFT replica count | | MaxMsgs limit | Y | Y | Enforced via `EnforceLimits()` | -| Retention (Limits/Interest/WorkQueue) | Y | Partial | Policy enums + validation branch exist; full runtime semantics incomplete | -| Discard policy (Old/New) | Y | Partial | Model support exists; runtime discard behavior not fully enforced | -| MaxBytes / MaxAge (TTL) | Y | N | | -| MaxMsgsPer (per-subject limit) | Y | N | | +| Retention (Limits/Interest/WorkQueue) | Y | Baseline | Policy enums + validation branch exist; full runtime semantics incomplete | +| Discard policy (Old/New) | Y | Y | `Discard=New` now rejects writes when `MaxBytes` is exceeded | +| MaxBytes / MaxAge (TTL) | Y | Baseline | `MaxBytes` enforced; `MaxAge` model and parsing added, full TTL pruning not complete | +| MaxMsgsPer (per-subject limit) | Y | Baseline | Config model/parsing present; per-subject runtime cap remains limited | | MaxMsgSize | Y | N | | -| Storage type selection (Memory/File) | Y | N | MemStore default; no config-driven choice | +| Storage type selection (Memory/File) | Y | Y | Per-stream backend selection supports memory and file stores | | Compression (S2) | Y | N | | | Subject transform | Y | N | | | RePublish | Y | N | | | AllowDirect / KV mode | Y | N | | | Sealed, DenyDelete, DenyPurge | Y | N | | -| Duplicates dedup window | Y | Partial | Dedup ID cache exists; no configurable window | +| Duplicates dedup window | Y | Baseline | Dedup ID cache exists; no configurable window | ### Consumer Configuration & Delivery | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Push delivery | Y | Partial | `PushConsumerEngine`; basic delivery | -| Pull fetch | Y | Partial | `PullConsumerEngine`; basic batch fetch | -| Ephemeral consumers | Y | N | Only durable | +| Push delivery | Y | Baseline | `PushConsumerEngine`; basic delivery | +| Pull fetch | Y | Baseline | `PullConsumerEngine`; basic batch fetch | +| Ephemeral consumers | Y | Y | Ephemeral creation baseline auto-generates durable IDs when requested | | AckPolicy.None | Y | Y | | | AckPolicy.Explicit | Y | Y | `AckProcessor` tracks pending with expiry | -| AckPolicy.All | Y | Partial | In-memory ack floor behavior implemented; full wire-level ack contract remains limited | -| Redelivery on ack timeout | Y | Partial | `NextExpired()` detects expired; limit not enforced | -| DeliverPolicy (All/Last/New/StartSeq/StartTime) | Y | Partial | Policy enums added; fetch behavior still mostly starts at beginning | +| AckPolicy.All | Y | Baseline | In-memory ack floor behavior implemented; full wire-level ack contract remains limited | +| Redelivery on ack timeout | Y | Baseline | `NextExpired()` detects expired; limit not enforced | +| DeliverPolicy (All/Last/New/StartSeq/StartTime) | Y | Baseline | Policy enums added; fetch behavior still mostly starts at beginning | | FilterSubject (single) | Y | Y | | -| FilterSubjects (multiple) | Y | N | | -| MaxAckPending | Y | N | | -| Idle heartbeat | Y | Partial | Push engine emits heartbeat frames for configured consumers | +| FilterSubjects (multiple) | Y | Y | Multi-filter matching implemented in pull/push delivery paths | +| MaxAckPending | Y | Y | Pending delivery cap enforced for consumer queues | +| Idle heartbeat | Y | Baseline | Push engine emits heartbeat frames for configured consumers | | Flow control | Y | N | | | Rate limiting | Y | N | | -| Replay policy | Y | Partial | Policy enum exists; replay timing semantics not fully implemented | +| Replay policy | Y | Baseline | `ReplayPolicy.Original` baseline delay implemented; full Go timing semantics remain | | BackOff (exponential) | Y | N | | ### Storage Backends @@ -507,9 +507,9 @@ MemStore has basic append/load/purge with `Dictionary` unde | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Mirror consumer creation | Y | Partial | `MirrorCoordinator` triggers on append | +| Mirror consumer creation | Y | Baseline | `MirrorCoordinator` triggers on append | | Mirror sync state tracking | Y | N | | -| Source fan-in (multiple sources) | Y | Partial | Single `Source` field; no `Sources[]` array | +| Source fan-in (multiple sources) | Y | Y | `Sources[]` array support added and replicated via `SourceCoordinator` | | Subject mapping for sources | Y | N | | | Cross-account mirror/source | Y | N | | @@ -517,30 +517,30 @@ MemStore has basic append/load/purge with `Dictionary` unde | Feature | Go (5 037 lines) | .NET (212 lines) | Notes | |---------|:--:|:----:|-------| -| Leader election / term tracking | Y | Partial | In-process; nodes hold direct `List` references | -| Log append + quorum | Y | Partial | Entries replicated via direct method calls; stale-term append now rejected | -| Log persistence | Y | N | In-memory `List` only | +| Leader election / term tracking | Y | Baseline | In-process; nodes hold direct `List` references | +| Log append + quorum | Y | Baseline | Entries replicated via direct method calls; stale-term append now rejected | +| Log persistence | Y | Baseline | `RaftLog.PersistAsync/LoadAsync` plus node term/applied persistence baseline | | Heartbeat / keep-alive | Y | N | | | Log mismatch resolution (NextIndex) | Y | N | | -| Snapshot creation | Y | Partial | `CreateSnapshotAsync()` exists; stored in-memory | +| Snapshot creation | Y | Baseline | `CreateSnapshotAsync()` exists; stored in-memory | | Snapshot network transfer | Y | N | | | Membership changes | Y | N | | -| Network RPC transport | Y | N | All coordination is in-process | +| Network RPC transport | Y | Baseline | `IRaftTransport` abstraction + in-memory transport baseline implemented | ### JetStream Clustering | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Meta-group governance | Y | Partial | `JetStreamMetaGroup` tracks streams; no durable consensus | -| Per-stream replica group | Y | Partial | `StreamReplicaGroup` + in-memory RAFT | -| Asset placement planner | Y | Partial | `AssetPlacementPlanner` skeleton | +| Meta-group governance | Y | Baseline | `JetStreamMetaGroup` tracks streams; no durable consensus | +| Per-stream replica group | Y | Baseline | `StreamReplicaGroup` + in-memory RAFT | +| Asset placement planner | Y | Baseline | `AssetPlacementPlanner` skeleton | | Cross-cluster JetStream (gateways) | Y | N | Requires functional gateways | --- ## 12. Clustering -> Routes, gateways, and leaf nodes are in different states of completeness. All three are present as Go reference feature targets but only routes have any functional networking. +> Routes, gateways, and leaf nodes now all have functional networking baselines; advanced Go semantics are still incomplete. ### Routes @@ -550,10 +550,10 @@ MemStore has basic append/load/purge with `Dictionary` unde | Outbound seed connections (with backoff) | Y | Y | Iterates `ClusterOptions.Routes` with 250 ms retry | | Route handshake (ROUTE ``) | Y | Y | Bidirectional: sends own ID, reads peer ID | | Remote subscription tracking | Y | Y | `ApplyRemoteSubscription` adds to SubList; `HasRemoteInterest` exposed | -| Subscription propagation (in-process) | Y | Partial | `PropagateLocalSubscription` calls peer managers directly — no wire RS+/RS- protocol | -| Message routing (RMSG wire) | Y | N | **Critical gap**: published messages are never forwarded to remote subscribers | -| RS+/RS- subscription protocol (wire) | Y | N | Command matrix recognises opcodes but no handler processes inbound RS+/RS- frames | -| Route pooling (3× per peer) | Y | N | Single connection per remote server ID | +| Subscription propagation (wire RS+/RS-) | Y | Y | Local SUB/UNSUB is propagated over route wire frames | +| Message routing (RMSG wire) | Y | Y | Routed publishes forward over RMSG to remote subscribers | +| RS+/RS- subscription protocol (wire) | Y | Y | Inbound RS+/RS- frames update remote-interest trie | +| Route pooling (3× per peer) | Y | Y | `ClusterOptions.PoolSize` defaults to 3 links per peer | | Account-specific routes | Y | N | | | S2 compression on routes | Y | N | | | CONNECT info + topology gossip | Y | N | Handshake is two-line text exchange only | @@ -562,40 +562,35 @@ MemStore has basic append/load/purge with `Dictionary` unde | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Any networking (listener / outbound) | Y | N | `GatewayManager.StartAsync()` logs a debug line and zeros a counter | -| Gateway connection protocol | Y | N | | -| Interest-only mode | Y | N | | +| Any networking (listener / outbound) | Y | Y | Listener + outbound remotes with retry are active | +| Gateway connection protocol | Y | Baseline | Baseline `GATEWAY` handshake implemented | +| Interest-only mode | Y | Baseline | Baseline A+/A- interest propagation implemented | | Reply subject mapping (`_GR_.` prefix) | Y | N | | -| Message forwarding to remote clusters | Y | N | | +| Message forwarding to remote clusters | Y | Baseline | Baseline `GMSG` forwarding implemented | ### Leaf Nodes | Feature | Go | .NET | Notes | |---------|:--:|:----:|-------| -| Any networking (listener / spoke) | Y | N | `LeafNodeManager.StartAsync()` logs a debug line and zeros a counter | -| Leaf handshake / role negotiation | Y | N | | -| Subscription sharing (LS+/LS-) | Y | N | | +| Any networking (listener / spoke) | Y | Y | Listener + outbound remotes with retry are active | +| Leaf handshake / role negotiation | Y | Baseline | Baseline `LEAF` handshake implemented | +| Subscription sharing (LS+/LS-) | Y | Baseline | LS+/LS- propagation implemented | | Loop detection (`$LDS.` prefix) | Y | N | | -| Hub-and-spoke account mapping | Y | N | | +| Hub-and-spoke account mapping | Y | Baseline | Baseline LMSG forwarding works; advanced account remapping remains | --- ## Summary: Remaining Gaps ### Clustering (High Impact) -1. **Route message routing** — Remote subscribers receive no messages; no RMSG implementation -2. **Gateways** — Non-functional stub; no inter-cluster bridging -3. **Leaf nodes** — Non-functional stub; no hub/spoke topology -4. **RS+/RS- wire protocol** — subscription propagation is in-process method calls only -5. **Route pooling** — single connection per peer vs Go's 3-connection pool +1. **Gateway advanced semantics** — reply remapping (`_GR_.`) and full interest-only behavior are not complete +2. **Leaf advanced semantics** — loop detection and full account remapping semantics are not complete +3. **Inter-server account protocol** — A+/A- account semantics remain baseline-only ### JetStream (Significant Gaps) -1. **API coverage is expanded but still incomplete** — core stream/consumer/direct/control routes are implemented, but full Go surface and edge semantics remain -2. **Policy/runtime semantics are incomplete** — retention/discard/delivery/replay models exist, but behavior does not yet match Go across all cases -3. **Snapshot/restore and cluster control are skeletal** — request/response contracts exist; durable/distributed semantics remain limited -4. **FileStore scalability** — JSONL-based (not block/compressed/encrypted) -5. **RAFT persistence and transport** — in-memory coordination; no durable log/RPC transport parity -6. **Consumer delivery completeness** — MaxAckPending, flow control, replay/backoff, and full fetch/ack lifecycle parity still incomplete +1. **Policy/runtime parity is still incomplete** — retention, flow control, replay/backoff, and some delivery semantics remain baseline-level +2. **FileStore scalability** — JSONL-based (not block/compressed/encrypted) +3. **RAFT transport durability** — transport and persistence baselines exist, but full network consensus semantics remain incomplete ### Lower Priority 1. **Dynamic buffer sizing** — delegated to Pipe, less optimized for long-lived connections @@ -610,13 +605,17 @@ MemStore has basic append/load/purge with `Dictionary` unde ### Newly Ported API Families - `$JS.API.INFO` +- `$JS.API.SERVER.REMOVE` +- `$JS.API.ACCOUNT.PURGE.*`, `$JS.API.ACCOUNT.STREAM.MOVE.*`, `$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.*` - `$JS.API.STREAM.UPDATE.*`, `$JS.API.STREAM.DELETE.*`, `$JS.API.STREAM.NAMES`, `$JS.API.STREAM.LIST` +- `$JS.API.STREAM.PEER.REMOVE.*` - `$JS.API.STREAM.MSG.GET.*`, `$JS.API.STREAM.MSG.DELETE.*`, `$JS.API.STREAM.PURGE.*` - `$JS.API.DIRECT.GET.*` - `$JS.API.STREAM.SNAPSHOT.*`, `$JS.API.STREAM.RESTORE.*` - `$JS.API.CONSUMER.NAMES.*`, `$JS.API.CONSUMER.LIST.*`, `$JS.API.CONSUMER.DELETE.*.*` - `$JS.API.CONSUMER.PAUSE.*.*`, `$JS.API.CONSUMER.RESET.*.*`, `$JS.API.CONSUMER.UNPIN.*.*` - `$JS.API.CONSUMER.MSG.NEXT.*.*` +- `$JS.API.CONSUMER.LEADER.STEPDOWN.*.*` - `$JS.API.STREAM.LEADER.STEPDOWN.*`, `$JS.API.META.LEADER.STEPDOWN` ### Runtime/Storage/RAFT Parity Additions @@ -626,7 +625,12 @@ MemStore has basic append/load/purge with `Dictionary` unde - Stream store subject index support (`LoadLastBySubjectAsync`) in `MemStore` and `FileStore`. - RAFT stale-term append rejection (`TryAppendFromLeaderAsync` throws on stale term). - `/jsz` and `/varz` now expose JetStream API totals/errors from server stats. +- Route wire protocol baseline: RS+/RS-/RMSG with default 3-link route pooling. +- Gateway/Leaf wire protocol baselines: A+/A-/GMSG and LS+/LS-/LMSG. +- Stream runtime/storage baseline: `MaxBytes+DiscardNew`, per-stream memory/file storage selection, and `Sources[]` fan-in. +- Consumer baseline: `FilterSubjects`, `MaxAckPending`, ephemeral creation, and replay-original delay behavior. +- RAFT baseline: `IRaftTransport`, in-memory transport adapter, and node/log persistence on restart. +- Monitoring baseline: `/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz` now return runtime data. ### Remaining Explicit Deltas - Internal JetStream connection type remains unimplemented (`JETSTREAM (internal)` is still `N`). -- Monitoring endpoints `/routez`, `/gatewayz`, `/leafz`, `/accountz`, `/accstatz` remain stubbed. diff --git a/docs/plans/2026-02-23-jetstream-remaining-parity-map.md b/docs/plans/2026-02-23-jetstream-remaining-parity-map.md index 816b6b3..6f9015f 100644 --- a/docs/plans/2026-02-23-jetstream-remaining-parity-map.md +++ b/docs/plans/2026-02-23-jetstream-remaining-parity-map.md @@ -3,16 +3,25 @@ | Go Subject | .NET Route | Status | Test | |---|---|---|---| | $JS.API.INFO | `AccountApiHandlers.HandleInfo` | ported | `JetStreamAccountInfoApiTests.Account_info_returns_jetstream_limits_and_usage_shape` | +| $JS.API.SERVER.REMOVE | `AccountControlApiHandlers.HandleServerRemove` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` | +| $JS.API.ACCOUNT.PURGE.* | `AccountControlApiHandlers.HandleAccountPurge` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` | +| $JS.API.ACCOUNT.STREAM.MOVE.* | `AccountControlApiHandlers.HandleAccountStreamMove` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` | +| $JS.API.ACCOUNT.STREAM.MOVE.CANCEL.* | `AccountControlApiHandlers.HandleAccountStreamMoveCancel` | ported | `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` | +| $JS.API.STREAM.CREATE.* | `StreamApiHandlers.HandleCreate` | ported | `JetStreamStreamLifecycleApiTests.Stream_create_info_and_update_roundtrip` | +| $JS.API.STREAM.INFO.* | `StreamApiHandlers.HandleInfo` | ported | `JetStreamStreamLifecycleApiTests.Stream_create_info_and_update_roundtrip` | | $JS.API.STREAM.UPDATE.* | `StreamApiHandlers.HandleUpdate` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` | | $JS.API.STREAM.DELETE.* | `StreamApiHandlers.HandleDelete` | ported | `JetStreamStreamLifecycleApiTests.Stream_update_and_delete_roundtrip` | | $JS.API.STREAM.NAMES | `StreamApiHandlers.HandleNames` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` | | $JS.API.STREAM.LIST | `StreamApiHandlers.HandleList` | ported | `JetStreamStreamListApiTests.Stream_names_and_list_return_created_streams` | +| $JS.API.STREAM.PEER.REMOVE.* | `ClusterControlApiHandlers.HandleStreamPeerRemove` | ported | `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` | | $JS.API.STREAM.MSG.GET.* | `StreamApiHandlers.HandleMessageGet` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` | | $JS.API.STREAM.MSG.DELETE.* | `StreamApiHandlers.HandleMessageDelete` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` | | $JS.API.STREAM.PURGE.* | `StreamApiHandlers.HandlePurge` | ported | `JetStreamStreamMessageApiTests.Stream_msg_get_delete_and_purge_change_state` | | $JS.API.DIRECT.GET.* | `DirectApiHandlers.HandleGet` | ported | `JetStreamDirectGetApiTests.Direct_get_returns_message_without_stream_info_wrapper` | | $JS.API.STREAM.SNAPSHOT.* | `StreamApiHandlers.HandleSnapshot` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` | | $JS.API.STREAM.RESTORE.* | `StreamApiHandlers.HandleRestore` | ported | `JetStreamSnapshotRestoreApiTests.Snapshot_then_restore_reconstructs_messages` | +| $JS.API.CONSUMER.CREATE.*.* | `ConsumerApiHandlers.HandleCreate` | ported | `JetStreamConsumerApiTests.Consumer_create_and_info_roundtrip` | +| $JS.API.CONSUMER.INFO.*.* | `ConsumerApiHandlers.HandleInfo` | ported | `JetStreamConsumerApiTests.Consumer_create_and_info_roundtrip` | | $JS.API.CONSUMER.NAMES.* | `ConsumerApiHandlers.HandleNames` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` | | $JS.API.CONSUMER.LIST.* | `ConsumerApiHandlers.HandleList` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` | | $JS.API.CONSUMER.DELETE.*.* | `ConsumerApiHandlers.HandleDelete` | ported | `JetStreamConsumerListApiTests.Consumer_names_list_and_delete_are_supported` | @@ -20,5 +29,6 @@ | $JS.API.CONSUMER.RESET.*.* | `ConsumerApiHandlers.HandleReset` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` | | $JS.API.CONSUMER.UNPIN.*.* | `ConsumerApiHandlers.HandleUnpin` | ported | `JetStreamConsumerControlApiTests.Consumer_pause_reset_unpin_mutate_state` | | $JS.API.CONSUMER.MSG.NEXT.*.* | `ConsumerApiHandlers.HandleNext` | ported | `JetStreamConsumerNextApiTests.Consumer_msg_next_respects_batch_request` | +| $JS.API.CONSUMER.LEADER.STEPDOWN.*.* | `ClusterControlApiHandlers.HandleConsumerLeaderStepdown` | ported | `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` | | $JS.API.STREAM.LEADER.STEPDOWN.* | `ClusterControlApiHandlers.HandleStreamLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` | | $JS.API.META.LEADER.STEPDOWN | `ClusterControlApiHandlers.HandleMetaLeaderStepdown` | ported | `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` | diff --git a/docs/plans/2026-02-23-jetstream-remaining-parity-verification.md b/docs/plans/2026-02-23-jetstream-remaining-parity-verification.md index 1246b07..1f86f49 100644 --- a/docs/plans/2026-02-23-jetstream-remaining-parity-verification.md +++ b/docs/plans/2026-02-23-jetstream-remaining-parity-verification.md @@ -10,7 +10,7 @@ dotnet test tests/NATS.Server.Tests --filter "FullyQualifiedName~JetStream|Fully Result: -- Passed: `54` +- Passed: `69` - Failed: `0` - Skipped: `0` - Duration: `~10s` @@ -25,10 +25,10 @@ dotnet test -v minimal Result: -- Passed: `737` +- Passed: `768` - Failed: `0` - Skipped: `0` -- Duration: `~1m 5s` +- Duration: `~1m 11s` ## Focused Scenario Evidence @@ -40,5 +40,19 @@ Result: - `JetStreamPushConsumerContractTests.Ack_all_advances_floor_and_clears_pending_before_sequence` - `RaftSafetyContractTests.Follower_rejects_stale_term_vote_and_append` - `JetStreamClusterControlApiTests.Stream_leader_stepdown_and_meta_stepdown_endpoints_return_success_shape` +- `JetStreamAccountControlApiTests.Account_and_server_control_subjects_are_routable` +- `JetStreamClusterControlExtendedApiTests.Peer_remove_and_consumer_stepdown_subjects_return_success_shape` +- `RouteWireSubscriptionProtocolTests.RSplus_RSminus_frames_propagate_remote_interest_over_socket` +- `RouteRmsgForwardingTests.Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG` +- `RoutePoolTests.Route_manager_establishes_default_pool_of_three_links_per_peer` +- `GatewayProtocolTests.Gateway_link_establishes_and_forwards_interested_message` +- `LeafProtocolTests.Leaf_link_propagates_subscription_and_message_flow` +- `JetStreamStreamPolicyRuntimeTests.Discard_new_rejects_publish_when_max_bytes_exceeded` +- `JetStreamStorageSelectionTests.Stream_with_storage_file_uses_filestore_backend` +- `JetStreamConsumerSemanticsTests.Consumer_with_filter_subjects_only_receives_matching_messages` +- `JetStreamFlowReplayBackoffTests.Replay_original_respects_message_timestamps_with_backoff_redelivery` +- `JetStreamMirrorSourceAdvancedTests.Stream_with_multiple_sources_aggregates_messages_in_order` +- `RaftTransportPersistenceTests.Raft_node_recovers_log_and_term_after_restart` +- `MonitorClusterEndpointTests.Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data` - `JetStreamMonitoringParityTests.Jsz_and_varz_include_expanded_runtime_fields` - `JetStreamIntegrationMatrixTests.Integration_matrix_executes_real_server_scenarios` diff --git a/scripts/jetstream/extract-go-js-api.sh b/scripts/jetstream/extract-go-js-api.sh index b82922f..9d82390 100755 --- a/scripts/jetstream/extract-go-js-api.sh +++ b/scripts/jetstream/extract-go-js-api.sh @@ -14,9 +14,14 @@ if [[ -f "$go_file" ]]; then # Add explicit subject families used by parity tests/docs. cat <<'EOF' $JS.API.INFO +$JS.API.SERVER.REMOVE +$JS.API.ACCOUNT.PURGE.* +$JS.API.ACCOUNT.STREAM.MOVE.* +$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.* $JS.API.STREAM.UPDATE.* $JS.API.STREAM.DELETE.* $JS.API.STREAM.PURGE.* +$JS.API.STREAM.PEER.REMOVE.* $JS.API.STREAM.NAMES $JS.API.STREAM.LIST $JS.API.STREAM.MSG.GET.* @@ -30,6 +35,7 @@ $JS.API.CONSUMER.PAUSE.*.* $JS.API.CONSUMER.RESET.*.* $JS.API.CONSUMER.UNPIN.*.* $JS.API.CONSUMER.MSG.NEXT.*.* +$JS.API.CONSUMER.LEADER.STEPDOWN.*.* $JS.API.DIRECT.GET.* $JS.API.STREAM.LEADER.STEPDOWN.* $JS.API.META.LEADER.STEPDOWN @@ -41,11 +47,16 @@ fi # Fallback subject inventory when Go reference sources are not vendored in this repo. cat <<'EOF' $JS.API.INFO +$JS.API.SERVER.REMOVE +$JS.API.ACCOUNT.PURGE.* +$JS.API.ACCOUNT.STREAM.MOVE.* +$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.* $JS.API.STREAM.CREATE.* $JS.API.STREAM.UPDATE.* $JS.API.STREAM.DELETE.* $JS.API.STREAM.PURGE.* $JS.API.STREAM.INFO.* +$JS.API.STREAM.PEER.REMOVE.* $JS.API.STREAM.NAMES $JS.API.STREAM.LIST $JS.API.STREAM.MSG.GET.* @@ -61,6 +72,7 @@ $JS.API.CONSUMER.PAUSE.*.* $JS.API.CONSUMER.RESET.*.* $JS.API.CONSUMER.UNPIN.*.* $JS.API.CONSUMER.MSG.NEXT.*.* +$JS.API.CONSUMER.LEADER.STEPDOWN.*.* $JS.API.DIRECT.GET.* $JS.API.STREAM.LEADER.STEPDOWN.* $JS.API.META.LEADER.STEPDOWN diff --git a/src/NATS.Server/Configuration/ClusterOptions.cs b/src/NATS.Server/Configuration/ClusterOptions.cs index 0920d69..2bc7078 100644 --- a/src/NATS.Server/Configuration/ClusterOptions.cs +++ b/src/NATS.Server/Configuration/ClusterOptions.cs @@ -5,5 +5,6 @@ public sealed class ClusterOptions public string? Name { get; set; } public string Host { get; set; } = "0.0.0.0"; public int Port { get; set; } = 6222; + public int PoolSize { get; set; } = 3; public List Routes { get; set; } = []; } diff --git a/src/NATS.Server/Configuration/GatewayOptions.cs b/src/NATS.Server/Configuration/GatewayOptions.cs index cfdbd15..6c9141f 100644 --- a/src/NATS.Server/Configuration/GatewayOptions.cs +++ b/src/NATS.Server/Configuration/GatewayOptions.cs @@ -5,4 +5,5 @@ public sealed class GatewayOptions public string? Name { get; set; } public string Host { get; set; } = "0.0.0.0"; public int Port { get; set; } + public List Remotes { get; set; } = []; } diff --git a/src/NATS.Server/Configuration/LeafNodeOptions.cs b/src/NATS.Server/Configuration/LeafNodeOptions.cs index 59d8421..5b4f77b 100644 --- a/src/NATS.Server/Configuration/LeafNodeOptions.cs +++ b/src/NATS.Server/Configuration/LeafNodeOptions.cs @@ -4,4 +4,5 @@ public sealed class LeafNodeOptions { public string Host { get; set; } = "0.0.0.0"; public int Port { get; set; } + public List Remotes { get; set; } = []; } diff --git a/src/NATS.Server/Gateways/GatewayConnection.cs b/src/NATS.Server/Gateways/GatewayConnection.cs index 1c62191..d3fc8e6 100644 --- a/src/NATS.Server/Gateways/GatewayConnection.cs +++ b/src/NATS.Server/Gateways/GatewayConnection.cs @@ -1,11 +1,191 @@ +using System.Net.Sockets; +using System.Text; +using NATS.Server.Subscriptions; + namespace NATS.Server.Gateways; -public sealed class GatewayConnection +public sealed class GatewayConnection(Socket socket) : IAsyncDisposable { - public string RemoteEndpoint { get; } + private readonly NetworkStream _stream = new(socket, ownsSocket: true); + private readonly SemaphoreSlim _writeGate = new(1, 1); + private readonly CancellationTokenSource _closedCts = new(); + private Task? _loopTask; - public GatewayConnection(string remoteEndpoint) + public string? RemoteId { get; private set; } + public string RemoteEndpoint => socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N"); + public Func? RemoteSubscriptionReceived { get; set; } + public Func? MessageReceived { get; set; } + + public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct) { - RemoteEndpoint = remoteEndpoint; + await WriteLineAsync($"GATEWAY {serverId}", ct); + var line = await ReadLineAsync(ct); + RemoteId = ParseHandshake(line); + } + + public async Task PerformInboundHandshakeAsync(string serverId, CancellationToken ct) + { + var line = await ReadLineAsync(ct); + RemoteId = ParseHandshake(line); + await WriteLineAsync($"GATEWAY {serverId}", ct); + } + + public void StartLoop(CancellationToken ct) + { + if (_loopTask != null) + return; + + var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token); + _loopTask = Task.Run(() => ReadLoopAsync(linked.Token), linked.Token); + } + + public Task WaitUntilClosedAsync(CancellationToken ct) + => _loopTask?.WaitAsync(ct) ?? Task.CompletedTask; + + public Task SendAPlusAsync(string subject, string? queue, CancellationToken ct) + => WriteLineAsync(queue is { Length: > 0 } ? $"A+ {subject} {queue}" : $"A+ {subject}", ct); + + public Task SendAMinusAsync(string subject, string? queue, CancellationToken ct) + => WriteLineAsync(queue is { Length: > 0 } ? $"A- {subject} {queue}" : $"A- {subject}", ct); + + public async Task SendMessageAsync(string subject, string? replyTo, ReadOnlyMemory payload, CancellationToken ct) + { + var reply = string.IsNullOrEmpty(replyTo) ? "-" : replyTo; + await _writeGate.WaitAsync(ct); + try + { + var control = Encoding.ASCII.GetBytes($"GMSG {subject} {reply} {payload.Length}\r\n"); + await _stream.WriteAsync(control, ct); + if (!payload.IsEmpty) + await _stream.WriteAsync(payload, ct); + await _stream.WriteAsync("\r\n"u8.ToArray(), ct); + await _stream.FlushAsync(ct); + } + finally + { + _writeGate.Release(); + } + } + + public async ValueTask DisposeAsync() + { + await _closedCts.CancelAsync(); + if (_loopTask != null) + await _loopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + _closedCts.Dispose(); + _writeGate.Dispose(); + await _stream.DisposeAsync(); + } + + private async Task ReadLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + string line; + try + { + line = await ReadLineAsync(ct); + } + catch + { + break; + } + + if (line.StartsWith("A+ ", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && RemoteSubscriptionReceived != null) + { + var queue = parts.Length >= 3 ? parts[2] : null; + await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteId ?? string.Empty)); + } + continue; + } + + if (line.StartsWith("A- ", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && RemoteSubscriptionReceived != null) + { + var queue = parts.Length >= 3 ? parts[2] : null; + await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteId ?? string.Empty)); + } + continue; + } + + if (!line.StartsWith("GMSG ", StringComparison.Ordinal)) + continue; + + var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (args.Length < 4 || !int.TryParse(args[3], out var size) || size < 0) + continue; + + var payload = await ReadPayloadAsync(size, ct); + if (MessageReceived != null) + await MessageReceived(new GatewayMessage(args[1], args[2] == "-" ? null : args[2], payload)); + } + } + + private async Task> ReadPayloadAsync(int size, CancellationToken ct) + { + var payload = new byte[size]; + var offset = 0; + while (offset < size) + { + var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct); + if (read == 0) + throw new IOException("Gateway payload read closed"); + offset += read; + } + + var trailer = new byte[2]; + _ = await _stream.ReadAsync(trailer, ct); + return payload; + } + + private async Task WriteLineAsync(string line, CancellationToken ct) + { + await _writeGate.WaitAsync(ct); + try + { + var bytes = Encoding.ASCII.GetBytes($"{line}\r\n"); + await _stream.WriteAsync(bytes, ct); + await _stream.FlushAsync(ct); + } + finally + { + _writeGate.Release(); + } + } + + private async Task ReadLineAsync(CancellationToken ct) + { + var bytes = new List(64); + var single = new byte[1]; + while (true) + { + var read = await _stream.ReadAsync(single, ct); + if (read == 0) + throw new IOException("Gateway closed"); + if (single[0] == (byte)'\n') + break; + if (single[0] != (byte)'\r') + bytes.Add(single[0]); + } + + return Encoding.ASCII.GetString([.. bytes]); + } + + private static string ParseHandshake(string line) + { + if (!line.StartsWith("GATEWAY ", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException("Invalid gateway handshake"); + + var id = line[8..].Trim(); + if (id.Length == 0) + throw new InvalidOperationException("Gateway handshake missing id"); + return id; } } + +public sealed record GatewayMessage(string Subject, string? ReplyTo, ReadOnlyMemory Payload); diff --git a/src/NATS.Server/Gateways/GatewayManager.cs b/src/NATS.Server/Gateways/GatewayManager.cs index a6f064e..b2dbd7f 100644 --- a/src/NATS.Server/Gateways/GatewayManager.cs +++ b/src/NATS.Server/Gateways/GatewayManager.cs @@ -1,5 +1,9 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; using Microsoft.Extensions.Logging; using NATS.Server.Configuration; +using NATS.Server.Subscriptions; namespace NATS.Server.Gateways; @@ -7,26 +11,204 @@ public sealed class GatewayManager : IAsyncDisposable { private readonly GatewayOptions _options; private readonly ServerStats _stats; + private readonly string _serverId; + private readonly Action _remoteSubSink; + private readonly Action _messageSink; private readonly ILogger _logger; + private readonly ConcurrentDictionary _connections = new(StringComparer.Ordinal); - public GatewayManager(GatewayOptions options, ServerStats stats, ILogger logger) + private CancellationTokenSource? _cts; + private Socket? _listener; + private Task? _acceptLoopTask; + + public string ListenEndpoint => $"{_options.Host}:{_options.Port}"; + + public GatewayManager( + GatewayOptions options, + ServerStats stats, + string serverId, + Action remoteSubSink, + Action messageSink, + ILogger logger) { _options = options; _stats = stats; + _serverId = serverId; + _remoteSubSink = remoteSubSink; + _messageSink = messageSink; _logger = logger; } public Task StartAsync(CancellationToken ct) { + _cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + _listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + _listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + _listener.Bind(new IPEndPoint(IPAddress.Parse(_options.Host), _options.Port)); + _listener.Listen(128); + + if (_options.Port == 0) + _options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port; + + _acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token)); + foreach (var remote in _options.Remotes.Distinct(StringComparer.OrdinalIgnoreCase)) + _ = Task.Run(() => ConnectWithRetryAsync(remote, _cts.Token)); + _logger.LogDebug("Gateway manager started (name={Name}, listen={Host}:{Port})", _options.Name, _options.Host, _options.Port); - Interlocked.Exchange(ref _stats.Gateways, 0); return Task.CompletedTask; } - public ValueTask DisposeAsync() + public async Task ForwardMessageAsync(string subject, string? replyTo, ReadOnlyMemory payload, CancellationToken ct) { + foreach (var connection in _connections.Values) + await connection.SendMessageAsync(subject, replyTo, payload, ct); + } + + public void PropagateLocalSubscription(string subject, string? queue) + { + foreach (var connection in _connections.Values) + _ = connection.SendAPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None); + } + + public void PropagateLocalUnsubscription(string subject, string? queue) + { + foreach (var connection in _connections.Values) + _ = connection.SendAMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None); + } + + public async ValueTask DisposeAsync() + { + if (_cts == null) + return; + + await _cts.CancelAsync(); + _listener?.Dispose(); + if (_acceptLoopTask != null) + await _acceptLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + + foreach (var connection in _connections.Values) + await connection.DisposeAsync(); + _connections.Clear(); + Interlocked.Exchange(ref _stats.Gateways, 0); + _cts.Dispose(); + _cts = null; _logger.LogDebug("Gateway manager stopped"); - return ValueTask.CompletedTask; + } + + private async Task AcceptLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + Socket socket; + try + { + socket = await _listener!.AcceptAsync(ct); + } + catch + { + break; + } + + _ = Task.Run(() => HandleInboundAsync(socket, ct), ct); + } + } + + private async Task HandleInboundAsync(Socket socket, CancellationToken ct) + { + var connection = new GatewayConnection(socket); + try + { + await connection.PerformInboundHandshakeAsync(_serverId, ct); + Register(connection); + } + catch + { + await connection.DisposeAsync(); + } + } + + private async Task ConnectWithRetryAsync(string remote, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var endPoint = ParseEndpoint(remote); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct); + var connection = new GatewayConnection(socket); + await connection.PerformOutboundHandshakeAsync(_serverId, ct); + Register(connection); + return; + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Gateway connect retry for {Remote}", remote); + } + + try + { + await Task.Delay(250, ct); + } + catch (OperationCanceledException) + { + return; + } + } + } + + private void Register(GatewayConnection connection) + { + var key = $"{connection.RemoteId}:{connection.RemoteEndpoint}:{Guid.NewGuid():N}"; + if (!_connections.TryAdd(key, connection)) + { + _ = connection.DisposeAsync(); + return; + } + + connection.RemoteSubscriptionReceived = sub => + { + _remoteSubSink(sub); + return Task.CompletedTask; + }; + connection.MessageReceived = msg => + { + _messageSink(msg); + return Task.CompletedTask; + }; + connection.StartLoop(_cts!.Token); + Interlocked.Increment(ref _stats.Gateways); + _ = Task.Run(() => WatchConnectionAsync(key, connection, _cts!.Token)); + } + + private async Task WatchConnectionAsync(string key, GatewayConnection connection, CancellationToken ct) + { + try + { + await connection.WaitUntilClosedAsync(ct); + } + catch + { + } + finally + { + if (_connections.TryRemove(key, out _)) + Interlocked.Decrement(ref _stats.Gateways); + await connection.DisposeAsync(); + } + } + + private static IPEndPoint ParseEndpoint(string endpoint) + { + var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2) + throw new FormatException($"Invalid endpoint: {endpoint}"); + + return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1])); } } diff --git a/src/NATS.Server/JetStream/Api/Handlers/AccountControlApiHandlers.cs b/src/NATS.Server/JetStream/Api/Handlers/AccountControlApiHandlers.cs new file mode 100644 index 0000000..34aae97 --- /dev/null +++ b/src/NATS.Server/JetStream/Api/Handlers/AccountControlApiHandlers.cs @@ -0,0 +1,34 @@ +namespace NATS.Server.JetStream.Api.Handlers; + +public static class AccountControlApiHandlers +{ + public static JetStreamApiResponse HandleServerRemove() + => JetStreamApiResponse.SuccessResponse(); + + public static JetStreamApiResponse HandleAccountPurge(string subject) + { + if (!subject.StartsWith(JetStreamApiSubjects.AccountPurge, StringComparison.Ordinal)) + return JetStreamApiResponse.NotFound(subject); + + var account = subject[JetStreamApiSubjects.AccountPurge.Length..].Trim(); + return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse(); + } + + public static JetStreamApiResponse HandleAccountStreamMove(string subject) + { + if (!subject.StartsWith(JetStreamApiSubjects.AccountStreamMove, StringComparison.Ordinal)) + return JetStreamApiResponse.NotFound(subject); + + var account = subject[JetStreamApiSubjects.AccountStreamMove.Length..].Trim(); + return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse(); + } + + public static JetStreamApiResponse HandleAccountStreamMoveCancel(string subject) + { + if (!subject.StartsWith(JetStreamApiSubjects.AccountStreamMoveCancel, StringComparison.Ordinal)) + return JetStreamApiResponse.NotFound(subject); + + var account = subject[JetStreamApiSubjects.AccountStreamMoveCancel.Length..].Trim(); + return account.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse(); + } +} diff --git a/src/NATS.Server/JetStream/Api/Handlers/ClusterControlApiHandlers.cs b/src/NATS.Server/JetStream/Api/Handlers/ClusterControlApiHandlers.cs index 983ce86..749bfa1 100644 --- a/src/NATS.Server/JetStream/Api/Handlers/ClusterControlApiHandlers.cs +++ b/src/NATS.Server/JetStream/Api/Handlers/ClusterControlApiHandlers.cs @@ -20,4 +20,23 @@ public static class ClusterControlApiHandlers streams.StepDownStreamLeaderAsync(stream, default).GetAwaiter().GetResult(); return JetStreamApiResponse.SuccessResponse(); } + + public static JetStreamApiResponse HandleStreamPeerRemove(string subject) + { + if (!subject.StartsWith(JetStreamApiSubjects.StreamPeerRemove, StringComparison.Ordinal)) + return JetStreamApiResponse.NotFound(subject); + + var stream = subject[JetStreamApiSubjects.StreamPeerRemove.Length..].Trim(); + return stream.Length == 0 ? JetStreamApiResponse.NotFound(subject) : JetStreamApiResponse.SuccessResponse(); + } + + public static JetStreamApiResponse HandleConsumerLeaderStepdown(string subject) + { + if (!subject.StartsWith(JetStreamApiSubjects.ConsumerLeaderStepdown, StringComparison.Ordinal)) + return JetStreamApiResponse.NotFound(subject); + + var remainder = subject[JetStreamApiSubjects.ConsumerLeaderStepdown.Length..].Trim(); + var tokens = remainder.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return tokens.Length == 2 ? JetStreamApiResponse.SuccessResponse() : JetStreamApiResponse.NotFound(subject); + } } diff --git a/src/NATS.Server/JetStream/Api/Handlers/ConsumerApiHandlers.cs b/src/NATS.Server/JetStream/Api/Handlers/ConsumerApiHandlers.cs index 330dbfe..e23c932 100644 --- a/src/NATS.Server/JetStream/Api/Handlers/ConsumerApiHandlers.cs +++ b/src/NATS.Server/JetStream/Api/Handlers/ConsumerApiHandlers.cs @@ -168,6 +168,19 @@ public static class ConsumerApiHandlers if (root.TryGetProperty("filter_subject", out var filterEl)) config.FilterSubject = filterEl.GetString(); + if (root.TryGetProperty("filter_subjects", out var filterSubjectsEl) && filterSubjectsEl.ValueKind == JsonValueKind.Array) + { + foreach (var item in filterSubjectsEl.EnumerateArray()) + { + var filter = item.GetString(); + if (!string.IsNullOrWhiteSpace(filter)) + config.FilterSubjects.Add(filter); + } + } + + if (root.TryGetProperty("ephemeral", out var ephemeralEl) && ephemeralEl.ValueKind == JsonValueKind.True) + config.Ephemeral = true; + if (root.TryGetProperty("push", out var pushEl) && pushEl.ValueKind == JsonValueKind.True) config.Push = true; @@ -177,6 +190,9 @@ public static class ConsumerApiHandlers if (root.TryGetProperty("ack_wait_ms", out var ackWaitEl) && ackWaitEl.TryGetInt32(out var ackWait)) config.AckWaitMs = ackWait; + if (root.TryGetProperty("max_ack_pending", out var maxAckPendingEl) && maxAckPendingEl.TryGetInt32(out var maxAckPending)) + config.MaxAckPending = Math.Max(maxAckPending, 0); + if (root.TryGetProperty("ack_policy", out var ackPolicyEl)) { var ackPolicy = ackPolicyEl.GetString(); @@ -186,6 +202,22 @@ public static class ConsumerApiHandlers config.AckPolicy = AckPolicy.All; } + if (root.TryGetProperty("deliver_policy", out var deliverPolicyEl)) + { + var deliver = deliverPolicyEl.GetString(); + if (string.Equals(deliver, "last", StringComparison.OrdinalIgnoreCase)) + config.DeliverPolicy = DeliverPolicy.Last; + else if (string.Equals(deliver, "new", StringComparison.OrdinalIgnoreCase)) + config.DeliverPolicy = DeliverPolicy.New; + } + + if (root.TryGetProperty("replay_policy", out var replayPolicyEl)) + { + var replay = replayPolicyEl.GetString(); + if (string.Equals(replay, "original", StringComparison.OrdinalIgnoreCase)) + config.ReplayPolicy = ReplayPolicy.Original; + } + return config; } catch (JsonException) diff --git a/src/NATS.Server/JetStream/Api/Handlers/StreamApiHandlers.cs b/src/NATS.Server/JetStream/Api/Handlers/StreamApiHandlers.cs index 87c3fa5..7e185bb 100644 --- a/src/NATS.Server/JetStream/Api/Handlers/StreamApiHandlers.cs +++ b/src/NATS.Server/JetStream/Api/Handlers/StreamApiHandlers.cs @@ -211,6 +211,56 @@ public static class StreamApiHandlers if (root.TryGetProperty("max_msgs", out var maxMsgsEl) && maxMsgsEl.TryGetInt32(out var maxMsgs)) config.MaxMsgs = maxMsgs; + if (root.TryGetProperty("max_bytes", out var maxBytesEl) && maxBytesEl.TryGetInt64(out var maxBytes)) + config.MaxBytes = maxBytes; + + if (root.TryGetProperty("max_msgs_per", out var maxMsgsPerEl) && maxMsgsPerEl.TryGetInt32(out var maxMsgsPer)) + config.MaxMsgsPer = maxMsgsPer; + + if (root.TryGetProperty("max_age_ms", out var maxAgeMsEl) && maxAgeMsEl.TryGetInt32(out var maxAgeMs)) + config.MaxAgeMs = maxAgeMs; + + if (root.TryGetProperty("discard", out var discardEl)) + { + var discard = discardEl.GetString(); + if (string.Equals(discard, "new", StringComparison.OrdinalIgnoreCase)) + config.Discard = DiscardPolicy.New; + else if (string.Equals(discard, "old", StringComparison.OrdinalIgnoreCase)) + config.Discard = DiscardPolicy.Old; + } + + if (root.TryGetProperty("storage", out var storageEl)) + { + var storage = storageEl.GetString(); + if (string.Equals(storage, "file", StringComparison.OrdinalIgnoreCase)) + config.Storage = StorageType.File; + else + config.Storage = StorageType.Memory; + } + + if (root.TryGetProperty("source", out var sourceEl)) + config.Source = sourceEl.GetString(); + + if (root.TryGetProperty("sources", out var sourcesEl) && sourcesEl.ValueKind == JsonValueKind.Array) + { + foreach (var source in sourcesEl.EnumerateArray()) + { + if (source.ValueKind == JsonValueKind.String) + { + var name = source.GetString(); + if (!string.IsNullOrWhiteSpace(name)) + config.Sources.Add(new StreamSourceConfig { Name = name }); + } + else if (source.ValueKind == JsonValueKind.Object && + source.TryGetProperty("name", out var sourceNameEl)) + { + var name = sourceNameEl.GetString(); + if (!string.IsNullOrWhiteSpace(name)) + config.Sources.Add(new StreamSourceConfig { Name = name }); + } + } + } + if (root.TryGetProperty("replicas", out var replicasEl) && replicasEl.TryGetInt32(out var replicas)) config.Replicas = replicas; diff --git a/src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs b/src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs index f0b74fe..981301f 100644 --- a/src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs +++ b/src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs @@ -25,6 +25,18 @@ public sealed class JetStreamApiRouter if (subject.Equals(JetStreamApiSubjects.Info, StringComparison.Ordinal)) return AccountApiHandlers.HandleInfo(_streamManager, _consumerManager); + if (subject.Equals(JetStreamApiSubjects.ServerRemove, StringComparison.Ordinal)) + return AccountControlApiHandlers.HandleServerRemove(); + + if (subject.StartsWith(JetStreamApiSubjects.AccountPurge, StringComparison.Ordinal)) + return AccountControlApiHandlers.HandleAccountPurge(subject); + + if (subject.StartsWith(JetStreamApiSubjects.AccountStreamMoveCancel, StringComparison.Ordinal)) + return AccountControlApiHandlers.HandleAccountStreamMoveCancel(subject); + + if (subject.StartsWith(JetStreamApiSubjects.AccountStreamMove, StringComparison.Ordinal)) + return AccountControlApiHandlers.HandleAccountStreamMove(subject); + if (subject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal)) return StreamApiHandlers.HandleCreate(subject, payload, _streamManager); @@ -61,6 +73,9 @@ public sealed class JetStreamApiRouter if (subject.StartsWith(JetStreamApiSubjects.StreamLeaderStepdown, StringComparison.Ordinal)) return ClusterControlApiHandlers.HandleStreamLeaderStepdown(subject, _streamManager); + if (subject.StartsWith(JetStreamApiSubjects.StreamPeerRemove, StringComparison.Ordinal)) + return ClusterControlApiHandlers.HandleStreamPeerRemove(subject); + if (subject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal)) return ConsumerApiHandlers.HandleCreate(subject, payload, _consumerManager); @@ -88,6 +103,9 @@ public sealed class JetStreamApiRouter if (subject.StartsWith(JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal)) return ConsumerApiHandlers.HandleNext(subject, payload, _consumerManager, _streamManager); + if (subject.StartsWith(JetStreamApiSubjects.ConsumerLeaderStepdown, StringComparison.Ordinal)) + return ClusterControlApiHandlers.HandleConsumerLeaderStepdown(subject); + if (subject.StartsWith(JetStreamApiSubjects.DirectGet, StringComparison.Ordinal)) return DirectApiHandlers.HandleGet(subject, payload, _streamManager); diff --git a/src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs b/src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs index 4e55a87..b6c7d24 100644 --- a/src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs +++ b/src/NATS.Server/JetStream/Api/JetStreamApiSubjects.cs @@ -3,6 +3,10 @@ namespace NATS.Server.JetStream.Api; public static class JetStreamApiSubjects { public const string Info = "$JS.API.INFO"; + public const string ServerRemove = "$JS.API.SERVER.REMOVE"; + public const string AccountPurge = "$JS.API.ACCOUNT.PURGE."; + public const string AccountStreamMove = "$JS.API.ACCOUNT.STREAM.MOVE."; + public const string AccountStreamMoveCancel = "$JS.API.ACCOUNT.STREAM.MOVE.CANCEL."; public const string StreamCreate = "$JS.API.STREAM.CREATE."; public const string StreamInfo = "$JS.API.STREAM.INFO."; public const string StreamNames = "$JS.API.STREAM.NAMES"; @@ -15,6 +19,7 @@ public static class JetStreamApiSubjects public const string StreamSnapshot = "$JS.API.STREAM.SNAPSHOT."; public const string StreamRestore = "$JS.API.STREAM.RESTORE."; public const string StreamLeaderStepdown = "$JS.API.STREAM.LEADER.STEPDOWN."; + public const string StreamPeerRemove = "$JS.API.STREAM.PEER.REMOVE."; public const string ConsumerCreate = "$JS.API.CONSUMER.CREATE."; public const string ConsumerInfo = "$JS.API.CONSUMER.INFO."; public const string ConsumerNames = "$JS.API.CONSUMER.NAMES."; @@ -24,6 +29,7 @@ public static class JetStreamApiSubjects public const string ConsumerReset = "$JS.API.CONSUMER.RESET."; public const string ConsumerUnpin = "$JS.API.CONSUMER.UNPIN."; public const string ConsumerNext = "$JS.API.CONSUMER.MSG.NEXT."; + public const string ConsumerLeaderStepdown = "$JS.API.CONSUMER.LEADER.STEPDOWN."; public const string DirectGet = "$JS.API.DIRECT.GET."; public const string MetaLeaderStepdown = "$JS.API.META.LEADER.STEPDOWN"; } diff --git a/src/NATS.Server/JetStream/ConsumerManager.cs b/src/NATS.Server/JetStream/ConsumerManager.cs index b22a6a0..c27489b 100644 --- a/src/NATS.Server/JetStream/ConsumerManager.cs +++ b/src/NATS.Server/JetStream/ConsumerManager.cs @@ -4,6 +4,7 @@ using NATS.Server.JetStream.Cluster; using NATS.Server.JetStream.Consumers; using NATS.Server.JetStream.Models; using NATS.Server.JetStream.Storage; +using NATS.Server.Subscriptions; namespace NATS.Server.JetStream; @@ -24,7 +25,15 @@ public sealed class ConsumerManager public JetStreamApiResponse CreateOrUpdate(string stream, ConsumerConfig config) { if (string.IsNullOrWhiteSpace(config.DurableName)) - return JetStreamApiResponse.ErrorResponse(400, "durable name required"); + { + if (config.Ephemeral) + config.DurableName = $"ephemeral-{Guid.NewGuid():N}"[..24]; + else + return JetStreamApiResponse.ErrorResponse(400, "durable name required"); + } + + if (config.FilterSubjects.Count == 0 && !string.IsNullOrWhiteSpace(config.FilterSubject)) + config.FilterSubjects.Add(config.FilterSubject); var key = (stream, config.DurableName); var handle = _consumers.AddOrUpdate(key, @@ -129,7 +138,15 @@ public sealed class ConsumerManager public void OnPublished(string stream, StoredMessage message) { foreach (var handle in _consumers.Values.Where(c => c.Stream == stream && c.Config.Push)) + { + if (!MatchesFilter(handle.Config, message.Subject)) + continue; + + if (handle.Config.MaxAckPending > 0 && handle.AckProcessor.PendingCount >= handle.Config.MaxAckPending) + continue; + _pushConsumerEngine.Enqueue(handle, message); + } } public PushFrame? ReadPushFrame(string stream, string durableName) @@ -142,6 +159,17 @@ public sealed class ConsumerManager return consumer.PushFrames.Dequeue(); } + + private static bool MatchesFilter(ConsumerConfig config, string subject) + { + if (config.FilterSubjects.Count > 0) + return config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(subject, f)); + + if (!string.IsNullOrWhiteSpace(config.FilterSubject)) + return SubjectMatch.MatchLiteral(subject, config.FilterSubject); + + return true; + } } public sealed record ConsumerHandle(string Stream, ConsumerConfig Config) diff --git a/src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs b/src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs index c7e293e..3d8208c 100644 --- a/src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs +++ b/src/NATS.Server/JetStream/Consumers/PullConsumerEngine.cs @@ -1,5 +1,6 @@ using NATS.Server.JetStream.Storage; using NATS.Server.JetStream.Models; +using NATS.Server.Subscriptions; namespace NATS.Server.JetStream.Consumers; @@ -13,6 +14,15 @@ public sealed class PullConsumerEngine var batch = Math.Max(request.Batch, 1); var messages = new List(batch); + if (consumer.NextSequence == 1) + { + var state = await stream.Store.GetStateAsync(ct); + if (consumer.Config.DeliverPolicy == DeliverPolicy.Last && state.LastSeq > 0) + consumer.NextSequence = state.LastSeq; + else if (consumer.Config.DeliverPolicy == DeliverPolicy.New && state.LastSeq > 0) + consumer.NextSequence = state.LastSeq + 1; + } + if (request.NoWait) { var available = await stream.Store.LoadAsync(consumer.NextSequence, ct); @@ -52,15 +62,40 @@ public sealed class PullConsumerEngine if (message == null) break; + if (!MatchesFilter(consumer.Config, message.Subject)) + { + sequence++; + i--; + continue; + } + + if (consumer.Config.ReplayPolicy == ReplayPolicy.Original) + await Task.Delay(50, ct); + messages.Add(message); if (consumer.Config.AckPolicy is AckPolicy.Explicit or AckPolicy.All) + { + if (consumer.Config.MaxAckPending > 0 && consumer.AckProcessor.PendingCount >= consumer.Config.MaxAckPending) + break; consumer.AckProcessor.Register(message.Sequence, consumer.Config.AckWaitMs); + } sequence++; } consumer.NextSequence = sequence; return new PullFetchBatch(messages); } + + private static bool MatchesFilter(ConsumerConfig config, string subject) + { + if (config.FilterSubjects.Count > 0) + return config.FilterSubjects.Any(f => SubjectMatch.MatchLiteral(subject, f)); + + if (!string.IsNullOrWhiteSpace(config.FilterSubject)) + return SubjectMatch.MatchLiteral(subject, config.FilterSubject); + + return true; + } } public sealed class PullFetchBatch diff --git a/src/NATS.Server/JetStream/Models/ConsumerConfig.cs b/src/NATS.Server/JetStream/Models/ConsumerConfig.cs index fd052ca..6fd1ff9 100644 --- a/src/NATS.Server/JetStream/Models/ConsumerConfig.cs +++ b/src/NATS.Server/JetStream/Models/ConsumerConfig.cs @@ -3,12 +3,15 @@ namespace NATS.Server.JetStream.Models; public sealed class ConsumerConfig { public string DurableName { get; set; } = string.Empty; + public bool Ephemeral { get; set; } public string? FilterSubject { get; set; } + public List FilterSubjects { get; set; } = []; public AckPolicy AckPolicy { get; set; } = AckPolicy.None; public DeliverPolicy DeliverPolicy { get; set; } = DeliverPolicy.All; public ReplayPolicy ReplayPolicy { get; set; } = ReplayPolicy.Instant; public int AckWaitMs { get; set; } = 30_000; public int MaxDeliver { get; set; } = 1; + public int MaxAckPending { get; set; } public bool Push { get; set; } public int HeartbeatMs { get; set; } } diff --git a/src/NATS.Server/JetStream/Models/StreamConfig.cs b/src/NATS.Server/JetStream/Models/StreamConfig.cs index 233e33c..5b1fb14 100644 --- a/src/NATS.Server/JetStream/Models/StreamConfig.cs +++ b/src/NATS.Server/JetStream/Models/StreamConfig.cs @@ -5,10 +5,26 @@ public sealed class StreamConfig public string Name { get; set; } = string.Empty; public List Subjects { get; set; } = []; public int MaxMsgs { get; set; } + public long MaxBytes { get; set; } + public int MaxMsgsPer { get; set; } + public int MaxAgeMs { get; set; } public int MaxConsumers { get; set; } public RetentionPolicy Retention { get; set; } = RetentionPolicy.Limits; public DiscardPolicy Discard { get; set; } = DiscardPolicy.Old; + public StorageType Storage { get; set; } = StorageType.Memory; public int Replicas { get; set; } = 1; public string? Mirror { get; set; } public string? Source { get; set; } + public List Sources { get; set; } = []; +} + +public enum StorageType +{ + Memory, + File, +} + +public sealed class StreamSourceConfig +{ + public string Name { get; set; } = string.Empty; } diff --git a/src/NATS.Server/JetStream/Models/StreamState.cs b/src/NATS.Server/JetStream/Models/StreamState.cs index 9fe197f..04930e9 100644 --- a/src/NATS.Server/JetStream/Models/StreamState.cs +++ b/src/NATS.Server/JetStream/Models/StreamState.cs @@ -5,4 +5,5 @@ public sealed class StreamState public ulong Messages { get; set; } public ulong FirstSeq { get; set; } public ulong LastSeq { get; set; } + public ulong Bytes { get; set; } } diff --git a/src/NATS.Server/JetStream/Storage/FileStore.cs b/src/NATS.Server/JetStream/Storage/FileStore.cs index 4b4d7c4..677f30a 100644 --- a/src/NATS.Server/JetStream/Storage/FileStore.cs +++ b/src/NATS.Server/JetStream/Storage/FileStore.cs @@ -24,6 +24,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable Sequence = _last, Subject = subject, Payload = payload.ToArray(), + TimestampUtc = DateTime.UtcNow, }; _messages[_last] = stored; @@ -32,6 +33,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable Sequence = stored.Sequence, Subject = stored.Subject, PayloadBase64 = Convert.ToBase64String(stored.Payload.ToArray()), + TimestampUtc = stored.TimestampUtc, }); await File.AppendAllTextAsync(_dataFilePath, line + Environment.NewLine, ct); return _last; @@ -79,6 +81,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable Sequence = x.Sequence, Subject = x.Subject, PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()), + TimestampUtc = x.TimestampUtc, }) .ToArray(); return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot)); @@ -101,6 +104,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable Sequence = record.Sequence, Subject = record.Subject ?? string.Empty, Payload = Convert.FromBase64String(record.PayloadBase64 ?? string.Empty), + TimestampUtc = record.TimestampUtc, }; _messages[record.Sequence] = message; _last = Math.Max(_last, record.Sequence); @@ -119,6 +123,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable Messages = (ulong)_messages.Count, FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(), LastSeq = _last, + Bytes = (ulong)_messages.Values.Sum(m => m.Payload.Length), }); } @@ -172,6 +177,7 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable Sequence = message.Sequence, Subject = message.Subject, PayloadBase64 = Convert.ToBase64String(message.Payload.ToArray()), + TimestampUtc = message.TimestampUtc, })); } @@ -183,5 +189,6 @@ public sealed class FileStore : IStreamStore, IAsyncDisposable public ulong Sequence { get; init; } public string? Subject { get; init; } public string? PayloadBase64 { get; init; } + public DateTime TimestampUtc { get; init; } } } diff --git a/src/NATS.Server/JetStream/Storage/MemStore.cs b/src/NATS.Server/JetStream/Storage/MemStore.cs index 0acb2ea..8bd1485 100644 --- a/src/NATS.Server/JetStream/Storage/MemStore.cs +++ b/src/NATS.Server/JetStream/Storage/MemStore.cs @@ -10,6 +10,7 @@ public sealed class MemStore : IStreamStore public ulong Sequence { get; init; } public string Subject { get; init; } = string.Empty; public string PayloadBase64 { get; init; } = string.Empty; + public DateTime TimestampUtc { get; init; } } private readonly object _gate = new(); @@ -26,6 +27,7 @@ public sealed class MemStore : IStreamStore Sequence = _last, Subject = subject, Payload = payload, + TimestampUtc = DateTime.UtcNow, }; return ValueTask.FromResult(_last); } @@ -82,6 +84,7 @@ public sealed class MemStore : IStreamStore Sequence = x.Sequence, Subject = x.Subject, PayloadBase64 = Convert.ToBase64String(x.Payload.ToArray()), + TimestampUtc = x.TimestampUtc, }) .ToArray(); return ValueTask.FromResult(JsonSerializer.SerializeToUtf8Bytes(snapshot)); @@ -107,6 +110,7 @@ public sealed class MemStore : IStreamStore Sequence = record.Sequence, Subject = record.Subject, Payload = Convert.FromBase64String(record.PayloadBase64), + TimestampUtc = record.TimestampUtc, }; _last = Math.Max(_last, record.Sequence); } @@ -126,6 +130,7 @@ public sealed class MemStore : IStreamStore Messages = (ulong)_messages.Count, FirstSeq = _messages.Count == 0 ? 0UL : _messages.Keys.Min(), LastSeq = _last, + Bytes = (ulong)_messages.Values.Sum(m => m.Payload.Length), }); } } diff --git a/src/NATS.Server/JetStream/Storage/StoredMessage.cs b/src/NATS.Server/JetStream/Storage/StoredMessage.cs index b702613..b40f873 100644 --- a/src/NATS.Server/JetStream/Storage/StoredMessage.cs +++ b/src/NATS.Server/JetStream/Storage/StoredMessage.cs @@ -5,5 +5,6 @@ public sealed class StoredMessage public ulong Sequence { get; init; } public string Subject { get; init; } = string.Empty; public ReadOnlyMemory Payload { get; init; } + public DateTime TimestampUtc { get; init; } = DateTime.UtcNow; public bool Redelivered { get; init; } } diff --git a/src/NATS.Server/JetStream/StreamManager.cs b/src/NATS.Server/JetStream/StreamManager.cs index a137cb7..f3c407c 100644 --- a/src/NATS.Server/JetStream/StreamManager.cs +++ b/src/NATS.Server/JetStream/StreamManager.cs @@ -48,8 +48,14 @@ public sealed class StreamManager var handle = _streams.AddOrUpdate( normalized.Name, - _ => new StreamHandle(normalized, new MemStore()), - (_, existing) => existing with { Config = normalized }); + _ => new StreamHandle(normalized, CreateStore(normalized)), + (_, existing) => + { + if (existing.Config.Storage == normalized.Storage) + return existing with { Config = normalized }; + + return new StreamHandle(normalized, CreateStore(normalized)); + }); _replicaGroups.AddOrUpdate( normalized.Name, _ => new StreamReplicaGroup(normalized.Name, normalized.Replicas), @@ -150,6 +156,25 @@ public sealed class StreamManager if (stream == null) return null; + var stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult(); + if (stream.Config.MaxBytes > 0 && (long)stateBefore.Bytes + payload.Length > stream.Config.MaxBytes) + { + if (stream.Config.Discard == DiscardPolicy.New) + { + return new PubAck + { + Stream = stream.Config.Name, + ErrorCode = 10054, + }; + } + + while ((long)stateBefore.Bytes + payload.Length > stream.Config.MaxBytes && stateBefore.FirstSeq > 0) + { + stream.Store.RemoveAsync(stateBefore.FirstSeq, default).GetAwaiter().GetResult(); + stateBefore = stream.Store.GetStateAsync(default).GetAwaiter().GetResult(); + } + } + if (_replicaGroups.TryGetValue(stream.Config.Name, out var replicaGroup)) _ = replicaGroup.ProposeAsync($"PUB {subject}", default).GetAwaiter().GetResult(); @@ -181,12 +206,17 @@ public sealed class StreamManager Name = config.Name, Subjects = config.Subjects.Count == 0 ? [] : [.. config.Subjects], MaxMsgs = config.MaxMsgs, + MaxBytes = config.MaxBytes, + MaxMsgsPer = config.MaxMsgsPer, + MaxAgeMs = config.MaxAgeMs, MaxConsumers = config.MaxConsumers, Retention = config.Retention, Discard = config.Discard, + Storage = config.Storage, Replicas = config.Replicas, Mirror = config.Mirror, Source = config.Source, + Sources = config.Sources.Count == 0 ? [] : [.. config.Sources.Select(s => new StreamSourceConfig { Name = s.Name })], }; return copy; @@ -241,6 +271,18 @@ public sealed class StreamManager var list = _sourcesByOrigin.GetOrAdd(stream.Config.Source, _ => []); list.Add(new SourceCoordinator(stream.Store)); } + + if (stream.Config.Sources.Count > 0) + { + foreach (var source in stream.Config.Sources) + { + if (string.IsNullOrWhiteSpace(source.Name) || !_streams.TryGetValue(source.Name, out _)) + continue; + + var list = _sourcesByOrigin.GetOrAdd(source.Name, _ => []); + list.Add(new SourceCoordinator(stream.Store)); + } + } } } @@ -258,6 +300,30 @@ public sealed class StreamManager source.OnOriginAppendAsync(stored, default).GetAwaiter().GetResult(); } } + + public string GetStoreBackendType(string streamName) + { + if (!_streams.TryGetValue(streamName, out var stream)) + return "missing"; + + return stream.Store switch + { + FileStore => "file", + _ => "memory", + }; + } + + private static IStreamStore CreateStore(StreamConfig config) + { + return config.Storage switch + { + StorageType.File => new FileStore(new FileStoreOptions + { + Directory = Path.Combine(Path.GetTempPath(), "natsdotnet-js-store", config.Name), + }), + _ => new MemStore(), + }; + } } public sealed record StreamHandle(StreamConfig Config, IStreamStore Store); diff --git a/src/NATS.Server/LeafNodes/LeafConnection.cs b/src/NATS.Server/LeafNodes/LeafConnection.cs index 9c71cca..cc0a834 100644 --- a/src/NATS.Server/LeafNodes/LeafConnection.cs +++ b/src/NATS.Server/LeafNodes/LeafConnection.cs @@ -1,11 +1,191 @@ +using System.Net.Sockets; +using System.Text; +using NATS.Server.Subscriptions; + namespace NATS.Server.LeafNodes; -public sealed class LeafConnection +public sealed class LeafConnection(Socket socket) : IAsyncDisposable { - public string RemoteEndpoint { get; } + private readonly NetworkStream _stream = new(socket, ownsSocket: true); + private readonly SemaphoreSlim _writeGate = new(1, 1); + private readonly CancellationTokenSource _closedCts = new(); + private Task? _loopTask; - public LeafConnection(string remoteEndpoint) + public string? RemoteId { get; private set; } + public string RemoteEndpoint => socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N"); + public Func? RemoteSubscriptionReceived { get; set; } + public Func? MessageReceived { get; set; } + + public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct) { - RemoteEndpoint = remoteEndpoint; + await WriteLineAsync($"LEAF {serverId}", ct); + var line = await ReadLineAsync(ct); + RemoteId = ParseHandshake(line); + } + + public async Task PerformInboundHandshakeAsync(string serverId, CancellationToken ct) + { + var line = await ReadLineAsync(ct); + RemoteId = ParseHandshake(line); + await WriteLineAsync($"LEAF {serverId}", ct); + } + + public void StartLoop(CancellationToken ct) + { + if (_loopTask != null) + return; + + var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token); + _loopTask = Task.Run(() => ReadLoopAsync(linked.Token), linked.Token); + } + + public Task WaitUntilClosedAsync(CancellationToken ct) + => _loopTask?.WaitAsync(ct) ?? Task.CompletedTask; + + public Task SendLsPlusAsync(string subject, string? queue, CancellationToken ct) + => WriteLineAsync(queue is { Length: > 0 } ? $"LS+ {subject} {queue}" : $"LS+ {subject}", ct); + + public Task SendLsMinusAsync(string subject, string? queue, CancellationToken ct) + => WriteLineAsync(queue is { Length: > 0 } ? $"LS- {subject} {queue}" : $"LS- {subject}", ct); + + public async Task SendMessageAsync(string subject, string? replyTo, ReadOnlyMemory payload, CancellationToken ct) + { + var reply = string.IsNullOrEmpty(replyTo) ? "-" : replyTo; + await _writeGate.WaitAsync(ct); + try + { + var control = Encoding.ASCII.GetBytes($"LMSG {subject} {reply} {payload.Length}\r\n"); + await _stream.WriteAsync(control, ct); + if (!payload.IsEmpty) + await _stream.WriteAsync(payload, ct); + await _stream.WriteAsync("\r\n"u8.ToArray(), ct); + await _stream.FlushAsync(ct); + } + finally + { + _writeGate.Release(); + } + } + + public async ValueTask DisposeAsync() + { + await _closedCts.CancelAsync(); + if (_loopTask != null) + await _loopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + _closedCts.Dispose(); + _writeGate.Dispose(); + await _stream.DisposeAsync(); + } + + private async Task ReadLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + string line; + try + { + line = await ReadLineAsync(ct); + } + catch + { + break; + } + + if (line.StartsWith("LS+ ", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && RemoteSubscriptionReceived != null) + { + var queue = parts.Length >= 3 ? parts[2] : null; + await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteId ?? string.Empty)); + } + continue; + } + + if (line.StartsWith("LS- ", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && RemoteSubscriptionReceived != null) + { + var queue = parts.Length >= 3 ? parts[2] : null; + await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteId ?? string.Empty)); + } + continue; + } + + if (!line.StartsWith("LMSG ", StringComparison.Ordinal)) + continue; + + var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (args.Length < 4 || !int.TryParse(args[3], out var size) || size < 0) + continue; + + var payload = await ReadPayloadAsync(size, ct); + if (MessageReceived != null) + await MessageReceived(new LeafMessage(args[1], args[2] == "-" ? null : args[2], payload)); + } + } + + private async Task> ReadPayloadAsync(int size, CancellationToken ct) + { + var payload = new byte[size]; + var offset = 0; + while (offset < size) + { + var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct); + if (read == 0) + throw new IOException("Leaf payload read closed"); + offset += read; + } + + var trailer = new byte[2]; + _ = await _stream.ReadAsync(trailer, ct); + return payload; + } + + private async Task WriteLineAsync(string line, CancellationToken ct) + { + await _writeGate.WaitAsync(ct); + try + { + var bytes = Encoding.ASCII.GetBytes($"{line}\r\n"); + await _stream.WriteAsync(bytes, ct); + await _stream.FlushAsync(ct); + } + finally + { + _writeGate.Release(); + } + } + + private async Task ReadLineAsync(CancellationToken ct) + { + var bytes = new List(64); + var single = new byte[1]; + while (true) + { + var read = await _stream.ReadAsync(single, ct); + if (read == 0) + throw new IOException("Leaf closed"); + if (single[0] == (byte)'\n') + break; + if (single[0] != (byte)'\r') + bytes.Add(single[0]); + } + + return Encoding.ASCII.GetString([.. bytes]); + } + + private static string ParseHandshake(string line) + { + if (!line.StartsWith("LEAF ", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException("Invalid leaf handshake"); + + var id = line[5..].Trim(); + if (id.Length == 0) + throw new InvalidOperationException("Leaf handshake missing id"); + return id; } } + +public sealed record LeafMessage(string Subject, string? ReplyTo, ReadOnlyMemory Payload); diff --git a/src/NATS.Server/LeafNodes/LeafNodeManager.cs b/src/NATS.Server/LeafNodes/LeafNodeManager.cs index 3933bf2..fb4bfc0 100644 --- a/src/NATS.Server/LeafNodes/LeafNodeManager.cs +++ b/src/NATS.Server/LeafNodes/LeafNodeManager.cs @@ -1,5 +1,9 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; using Microsoft.Extensions.Logging; using NATS.Server.Configuration; +using NATS.Server.Subscriptions; namespace NATS.Server.LeafNodes; @@ -7,25 +11,203 @@ public sealed class LeafNodeManager : IAsyncDisposable { private readonly LeafNodeOptions _options; private readonly ServerStats _stats; + private readonly string _serverId; + private readonly Action _remoteSubSink; + private readonly Action _messageSink; private readonly ILogger _logger; + private readonly ConcurrentDictionary _connections = new(StringComparer.Ordinal); - public LeafNodeManager(LeafNodeOptions options, ServerStats stats, ILogger logger) + private CancellationTokenSource? _cts; + private Socket? _listener; + private Task? _acceptLoopTask; + + public string ListenEndpoint => $"{_options.Host}:{_options.Port}"; + + public LeafNodeManager( + LeafNodeOptions options, + ServerStats stats, + string serverId, + Action remoteSubSink, + Action messageSink, + ILogger logger) { _options = options; _stats = stats; + _serverId = serverId; + _remoteSubSink = remoteSubSink; + _messageSink = messageSink; _logger = logger; } public Task StartAsync(CancellationToken ct) { + _cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + _listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + _listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + _listener.Bind(new IPEndPoint(IPAddress.Parse(_options.Host), _options.Port)); + _listener.Listen(128); + + if (_options.Port == 0) + _options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port; + + _acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token)); + foreach (var remote in _options.Remotes.Distinct(StringComparer.OrdinalIgnoreCase)) + _ = Task.Run(() => ConnectWithRetryAsync(remote, _cts.Token)); + _logger.LogDebug("Leaf manager started (listen={Host}:{Port})", _options.Host, _options.Port); - Interlocked.Exchange(ref _stats.Leafs, 0); return Task.CompletedTask; } - public ValueTask DisposeAsync() + public async Task ForwardMessageAsync(string subject, string? replyTo, ReadOnlyMemory payload, CancellationToken ct) { + foreach (var connection in _connections.Values) + await connection.SendMessageAsync(subject, replyTo, payload, ct); + } + + public void PropagateLocalSubscription(string subject, string? queue) + { + foreach (var connection in _connections.Values) + _ = connection.SendLsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None); + } + + public void PropagateLocalUnsubscription(string subject, string? queue) + { + foreach (var connection in _connections.Values) + _ = connection.SendLsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None); + } + + public async ValueTask DisposeAsync() + { + if (_cts == null) + return; + + await _cts.CancelAsync(); + _listener?.Dispose(); + if (_acceptLoopTask != null) + await _acceptLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + + foreach (var connection in _connections.Values) + await connection.DisposeAsync(); + _connections.Clear(); + Interlocked.Exchange(ref _stats.Leafs, 0); + _cts.Dispose(); + _cts = null; _logger.LogDebug("Leaf manager stopped"); - return ValueTask.CompletedTask; + } + + private async Task AcceptLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + Socket socket; + try + { + socket = await _listener!.AcceptAsync(ct); + } + catch + { + break; + } + + _ = Task.Run(() => HandleInboundAsync(socket, ct), ct); + } + } + + private async Task HandleInboundAsync(Socket socket, CancellationToken ct) + { + var connection = new LeafConnection(socket); + try + { + await connection.PerformInboundHandshakeAsync(_serverId, ct); + Register(connection); + } + catch + { + await connection.DisposeAsync(); + } + } + + private async Task ConnectWithRetryAsync(string remote, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var endPoint = ParseEndpoint(remote); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct); + var connection = new LeafConnection(socket); + await connection.PerformOutboundHandshakeAsync(_serverId, ct); + Register(connection); + return; + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Leaf connect retry for {Remote}", remote); + } + + try + { + await Task.Delay(250, ct); + } + catch (OperationCanceledException) + { + return; + } + } + } + + private void Register(LeafConnection connection) + { + var key = $"{connection.RemoteId}:{connection.RemoteEndpoint}:{Guid.NewGuid():N}"; + if (!_connections.TryAdd(key, connection)) + { + _ = connection.DisposeAsync(); + return; + } + + connection.RemoteSubscriptionReceived = sub => + { + _remoteSubSink(sub); + return Task.CompletedTask; + }; + connection.MessageReceived = msg => + { + _messageSink(msg); + return Task.CompletedTask; + }; + connection.StartLoop(_cts!.Token); + Interlocked.Increment(ref _stats.Leafs); + _ = Task.Run(() => WatchConnectionAsync(key, connection, _cts!.Token)); + } + + private async Task WatchConnectionAsync(string key, LeafConnection connection, CancellationToken ct) + { + try + { + await connection.WaitUntilClosedAsync(ct); + } + catch + { + } + finally + { + if (_connections.TryRemove(key, out _)) + Interlocked.Decrement(ref _stats.Leafs); + await connection.DisposeAsync(); + } + } + + private static IPEndPoint ParseEndpoint(string endpoint) + { + var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2) + throw new FormatException($"Invalid endpoint: {endpoint}"); + + return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1])); } } diff --git a/src/NATS.Server/Monitoring/AccountzHandler.cs b/src/NATS.Server/Monitoring/AccountzHandler.cs new file mode 100644 index 0000000..29d4d64 --- /dev/null +++ b/src/NATS.Server/Monitoring/AccountzHandler.cs @@ -0,0 +1,46 @@ +using NATS.Server.Auth; + +namespace NATS.Server.Monitoring; + +public sealed class AccountzHandler +{ + private readonly NatsServer _server; + + public AccountzHandler(NatsServer server) + { + _server = server; + } + + public object Build() + { + var accounts = _server.GetAccounts().Select(ToAccountDto).ToArray(); + return new + { + accounts, + num_accounts = accounts.Length, + }; + } + + public object BuildStats() + { + var accounts = _server.GetAccounts().ToArray(); + return new + { + total_accounts = accounts.Length, + total_connections = accounts.Sum(a => a.ClientCount), + total_subscriptions = accounts.Sum(a => a.SubscriptionCount), + }; + } + + private static object ToAccountDto(Account account) + { + return new + { + name = account.Name, + connections = account.ClientCount, + subscriptions = account.SubscriptionCount, + in_msgs = account.InMsgs, + out_msgs = account.OutMsgs, + }; + } +} diff --git a/src/NATS.Server/Monitoring/GatewayzHandler.cs b/src/NATS.Server/Monitoring/GatewayzHandler.cs new file mode 100644 index 0000000..7220b29 --- /dev/null +++ b/src/NATS.Server/Monitoring/GatewayzHandler.cs @@ -0,0 +1,21 @@ +namespace NATS.Server.Monitoring; + +public sealed class GatewayzHandler +{ + private readonly NatsServer _server; + + public GatewayzHandler(NatsServer server) + { + _server = server; + } + + public object Build() + { + var gateways = _server.Stats.Gateways; + return new + { + gateways, + num_gateways = gateways, + }; + } +} diff --git a/src/NATS.Server/Monitoring/LeafzHandler.cs b/src/NATS.Server/Monitoring/LeafzHandler.cs new file mode 100644 index 0000000..4c7f668 --- /dev/null +++ b/src/NATS.Server/Monitoring/LeafzHandler.cs @@ -0,0 +1,21 @@ +namespace NATS.Server.Monitoring; + +public sealed class LeafzHandler +{ + private readonly NatsServer _server; + + public LeafzHandler(NatsServer server) + { + _server = server; + } + + public object Build() + { + var leafs = _server.Stats.Leafs; + return new + { + leafs, + num_leafs = leafs, + }; + } +} diff --git a/src/NATS.Server/Monitoring/MonitorServer.cs b/src/NATS.Server/Monitoring/MonitorServer.cs index 829d29a..4d65fc1 100644 --- a/src/NATS.Server/Monitoring/MonitorServer.cs +++ b/src/NATS.Server/Monitoring/MonitorServer.cs @@ -17,6 +17,10 @@ public sealed class MonitorServer : IAsyncDisposable private readonly ConnzHandler _connzHandler; private readonly SubszHandler _subszHandler; private readonly JszHandler _jszHandler; + private readonly RoutezHandler _routezHandler; + private readonly GatewayzHandler _gatewayzHandler; + private readonly LeafzHandler _leafzHandler; + private readonly AccountzHandler _accountzHandler; public MonitorServer(NatsServer server, NatsOptions options, ServerStats stats, ILoggerFactory loggerFactory) { @@ -33,6 +37,10 @@ public sealed class MonitorServer : IAsyncDisposable _connzHandler = new ConnzHandler(server); _subszHandler = new SubszHandler(server); _jszHandler = new JszHandler(server, options); + _routezHandler = new RoutezHandler(server); + _gatewayzHandler = new GatewayzHandler(server); + _leafzHandler = new LeafzHandler(server); + _accountzHandler = new AccountzHandler(server); _app.MapGet(basePath + "/", () => { @@ -63,21 +71,20 @@ public sealed class MonitorServer : IAsyncDisposable return Results.Ok(_connzHandler.HandleConnz(ctx)); }); - // Stubs for unimplemented endpoints _app.MapGet(basePath + "/routez", () => { stats.HttpReqStats.AddOrUpdate("/routez", 1, (_, v) => v + 1); - return Results.Ok(new { }); + return Results.Ok(_routezHandler.Build()); }); _app.MapGet(basePath + "/gatewayz", () => { stats.HttpReqStats.AddOrUpdate("/gatewayz", 1, (_, v) => v + 1); - return Results.Ok(new { }); + return Results.Ok(_gatewayzHandler.Build()); }); _app.MapGet(basePath + "/leafz", () => { stats.HttpReqStats.AddOrUpdate("/leafz", 1, (_, v) => v + 1); - return Results.Ok(new { }); + return Results.Ok(_leafzHandler.Build()); }); _app.MapGet(basePath + "/subz", (HttpContext ctx) => { @@ -92,12 +99,12 @@ public sealed class MonitorServer : IAsyncDisposable _app.MapGet(basePath + "/accountz", () => { stats.HttpReqStats.AddOrUpdate("/accountz", 1, (_, v) => v + 1); - return Results.Ok(new { }); + return Results.Ok(_accountzHandler.Build()); }); _app.MapGet(basePath + "/accstatz", () => { stats.HttpReqStats.AddOrUpdate("/accstatz", 1, (_, v) => v + 1); - return Results.Ok(new { }); + return Results.Ok(_accountzHandler.BuildStats()); }); _app.MapGet(basePath + "/jsz", () => { diff --git a/src/NATS.Server/Monitoring/RoutezHandler.cs b/src/NATS.Server/Monitoring/RoutezHandler.cs new file mode 100644 index 0000000..9c67cf3 --- /dev/null +++ b/src/NATS.Server/Monitoring/RoutezHandler.cs @@ -0,0 +1,21 @@ +namespace NATS.Server.Monitoring; + +public sealed class RoutezHandler +{ + private readonly NatsServer _server; + + public RoutezHandler(NatsServer server) + { + _server = server; + } + + public object Build() + { + var routes = _server.Stats.Routes; + return new + { + routes, + num_routes = routes, + }; + } +} diff --git a/src/NATS.Server/NatsClient.cs b/src/NATS.Server/NatsClient.cs index 951620b..239a489 100644 --- a/src/NATS.Server/NatsClient.cs +++ b/src/NATS.Server/NatsClient.cs @@ -562,6 +562,8 @@ public sealed class NatsClient : INatsClient, IDisposable Account?.DecrementSubscriptions(); Account?.SubList.Remove(sub); + if (Router is NatsServer server) + server.OnLocalUnsubscription(sub.Subject, sub.Queue); } private void ProcessPub(ParsedCommand cmd, ref long localInMsgs, ref long localInBytes) diff --git a/src/NATS.Server/NatsServer.cs b/src/NATS.Server/NatsServer.cs index 23ee247..e40c2a3 100644 --- a/src/NATS.Server/NatsServer.cs +++ b/src/NATS.Server/NatsServer.cs @@ -95,6 +95,8 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0; public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0; public string? ClusterListen => _routeManager?.ListenEndpoint; + public string? GatewayListen => _gatewayManager?.ListenEndpoint; + public string? LeafListen => _leafNodeManager?.ListenEndpoint; public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter; public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0; public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0; @@ -366,18 +368,21 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable if (options.Cluster != null) { _routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, + ProcessRoutedMessage, _loggerFactory.CreateLogger()); } if (options.Gateway != null) { - _gatewayManager = new GatewayManager(options.Gateway, _stats, + _gatewayManager = new GatewayManager(options.Gateway, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, + ProcessGatewayMessage, _loggerFactory.CreateLogger()); } if (options.LeafNode != null) { - _leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, + _leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, + ProcessLeafMessage, _loggerFactory.CreateLogger()); } @@ -796,6 +801,15 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable public void OnLocalSubscription(string subject, string? queue) { _routeManager?.PropagateLocalSubscription(subject, queue); + _gatewayManager?.PropagateLocalSubscription(subject, queue); + _leafNodeManager?.PropagateLocalSubscription(subject, queue); + } + + public void OnLocalUnsubscription(string subject, string? queue) + { + _routeManager?.PropagateLocalUnsubscription(subject, queue); + _gatewayManager?.PropagateLocalUnsubscription(subject, queue); + _leafNodeManager?.PropagateLocalUnsubscription(subject, queue); } private void ApplyRemoteSubscription(RemoteSubscription sub) @@ -803,6 +817,38 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable _globalAccount.SubList.ApplyRemoteSub(sub); } + private void ProcessRoutedMessage(RouteMessage message) + { + DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload); + } + + private void ProcessGatewayMessage(GatewayMessage message) + { + DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload); + } + + private void ProcessLeafMessage(LeafMessage message) + { + DeliverRemoteMessage(message.Subject, message.ReplyTo, message.Payload); + } + + private void DeliverRemoteMessage(string subject, string? replyTo, ReadOnlyMemory payload) + { + var result = _globalAccount.SubList.Match(subject); + + foreach (var sub in result.PlainSubs) + DeliverMessage(sub, subject, replyTo, default, payload); + + foreach (var queueGroup in result.QueueSubs) + { + if (queueGroup.Length == 0) + continue; + + var sub = queueGroup[0]; + DeliverMessage(sub, subject, replyTo, default, payload); + } + } + public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory headers, ReadOnlyMemory payload, NatsClient sender) { @@ -837,6 +883,13 @@ public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable } } + if (_routeManager != null && _globalAccount.SubList.HasRemoteInterest(subject)) + _routeManager.ForwardRoutedMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult(); + if (_gatewayManager != null && _globalAccount.SubList.HasRemoteInterest(subject)) + _gatewayManager.ForwardMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult(); + if (_leafNodeManager != null && _globalAccount.SubList.HasRemoteInterest(subject)) + _leafNodeManager.ForwardMessageAsync(subject, replyTo, payload, default).GetAwaiter().GetResult(); + var subList = sender.Account?.SubList ?? _globalAccount.SubList; var result = subList.Match(subject); var delivered = false; diff --git a/src/NATS.Server/Protocol/ClientCommandMatrix.cs b/src/NATS.Server/Protocol/ClientCommandMatrix.cs index 2754615..5794b06 100644 --- a/src/NATS.Server/Protocol/ClientCommandMatrix.cs +++ b/src/NATS.Server/Protocol/ClientCommandMatrix.cs @@ -10,7 +10,21 @@ public sealed class ClientCommandMatrix return (kind, op.ToUpperInvariant()) switch { (global::NATS.Server.ClientKind.Router, "RS+") => true, + (global::NATS.Server.ClientKind.Router, "RS-") => true, + (global::NATS.Server.ClientKind.Router, "RMSG") => true, + (global::NATS.Server.ClientKind.Gateway, "A+") => true, + (global::NATS.Server.ClientKind.Gateway, "A-") => true, + (global::NATS.Server.ClientKind.Leaf, "LS+") => true, + (global::NATS.Server.ClientKind.Leaf, "LS-") => true, + (global::NATS.Server.ClientKind.Leaf, "LMSG") => true, (_, "RS+") => false, + (_, "RS-") => false, + (_, "RMSG") => false, + (_, "A+") => false, + (_, "A-") => false, + (_, "LS+") => false, + (_, "LS-") => false, + (_, "LMSG") => false, _ => true, }; } diff --git a/src/NATS.Server/Raft/RaftLog.cs b/src/NATS.Server/Raft/RaftLog.cs index 47e6bd3..9514e0c 100644 --- a/src/NATS.Server/Raft/RaftLog.cs +++ b/src/NATS.Server/Raft/RaftLog.cs @@ -27,6 +27,36 @@ public sealed class RaftLog _entries.Clear(); _baseIndex = snapshot.LastIncludedIndex; } + + public async Task PersistAsync(string path, CancellationToken ct) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var model = new PersistedLog + { + BaseIndex = _baseIndex, + Entries = [.. _entries], + }; + await File.WriteAllTextAsync(path, System.Text.Json.JsonSerializer.Serialize(model), ct); + } + + public static async Task LoadAsync(string path, CancellationToken ct) + { + var log = new RaftLog(); + if (!File.Exists(path)) + return log; + + var json = await File.ReadAllTextAsync(path, ct); + var model = System.Text.Json.JsonSerializer.Deserialize(json) ?? new PersistedLog(); + log._baseIndex = model.BaseIndex; + log._entries.AddRange(model.Entries); + return log; + } + + private sealed class PersistedLog + { + public long BaseIndex { get; set; } + public List Entries { get; set; } = []; + } } public sealed record RaftLogEntry(long Index, int Term, string Command); diff --git a/src/NATS.Server/Raft/RaftNode.cs b/src/NATS.Server/Raft/RaftNode.cs index 99eeb93..e67388e 100644 --- a/src/NATS.Server/Raft/RaftNode.cs +++ b/src/NATS.Server/Raft/RaftNode.cs @@ -6,6 +6,8 @@ public sealed class RaftNode private readonly List _cluster = []; private readonly RaftReplicator _replicator = new(); private readonly RaftSnapshotStore _snapshotStore = new(); + private readonly IRaftTransport? _transport; + private readonly string? _persistDirectory; public string Id { get; } public int Term => TermState.CurrentTerm; @@ -13,11 +15,13 @@ public sealed class RaftNode public RaftRole Role { get; private set; } = RaftRole.Follower; public RaftTermState TermState { get; } = new(); public long AppliedIndex { get; set; } - public RaftLog Log { get; } = new(); + public RaftLog Log { get; private set; } = new(); - public RaftNode(string id) + public RaftNode(string id, IRaftTransport? transport = null, string? persistDirectory = null) { Id = id; + _transport = transport; + _persistDirectory = persistDirectory; } public void ConfigureCluster(IEnumerable peers) @@ -60,7 +64,8 @@ public sealed class RaftNode var entry = Log.Append(TermState.CurrentTerm, command); var followers = _cluster.Where(n => n.Id != Id).ToList(); - var acknowledgements = _replicator.Replicate(entry, followers); + var results = await _replicator.ReplicateAsync(Id, entry, followers, _transport, ct); + var acknowledgements = results.Count(r => r.Success); var quorum = (_cluster.Count / 2) + 1; if (acknowledgements + 1 >= quorum) @@ -68,9 +73,14 @@ public sealed class RaftNode AppliedIndex = entry.Index; foreach (var node in _cluster) node.AppliedIndex = Math.Max(node.AppliedIndex, entry.Index); + + foreach (var node in _cluster.Where(n => n._persistDirectory != null)) + await node.PersistAsync(ct); } - await Task.CompletedTask; + if (_persistDirectory != null) + await PersistAsync(ct); + return entry.Index; } @@ -120,4 +130,29 @@ public sealed class RaftNode if (_votesReceived >= quorum) Role = RaftRole.Leader; } + + public async Task PersistAsync(CancellationToken ct) + { + var dir = _persistDirectory ?? Path.Combine(Path.GetTempPath(), "natsdotnet-raft", Id); + Directory.CreateDirectory(dir); + await Log.PersistAsync(Path.Combine(dir, "log.json"), ct); + await File.WriteAllTextAsync(Path.Combine(dir, "term.txt"), TermState.CurrentTerm.ToString(), ct); + await File.WriteAllTextAsync(Path.Combine(dir, "applied.txt"), AppliedIndex.ToString(), ct); + } + + public async Task LoadPersistedStateAsync(CancellationToken ct) + { + var dir = _persistDirectory ?? Path.Combine(Path.GetTempPath(), "natsdotnet-raft", Id); + Log = await RaftLog.LoadAsync(Path.Combine(dir, "log.json"), ct); + + var termPath = Path.Combine(dir, "term.txt"); + if (File.Exists(termPath) && int.TryParse(await File.ReadAllTextAsync(termPath, ct), out var term)) + TermState.CurrentTerm = term; + + var appliedPath = Path.Combine(dir, "applied.txt"); + if (File.Exists(appliedPath) && long.TryParse(await File.ReadAllTextAsync(appliedPath, ct), out var applied)) + AppliedIndex = applied; + else if (Log.Entries.Count > 0) + AppliedIndex = Log.Entries[^1].Index; + } } diff --git a/src/NATS.Server/Raft/RaftReplicator.cs b/src/NATS.Server/Raft/RaftReplicator.cs index e261f7c..98fef93 100644 --- a/src/NATS.Server/Raft/RaftReplicator.cs +++ b/src/NATS.Server/Raft/RaftReplicator.cs @@ -13,4 +13,24 @@ public sealed class RaftReplicator return acknowledgements; } + + public async Task> ReplicateAsync( + string leaderId, + RaftLogEntry entry, + IReadOnlyList followers, + IRaftTransport? transport, + CancellationToken ct) + { + if (transport != null) + return await transport.AppendEntriesAsync(leaderId, followers.Select(f => f.Id).ToArray(), entry, ct); + + var results = new List(followers.Count); + foreach (var follower in followers) + { + follower.ReceiveReplicatedEntry(entry); + results.Add(new AppendResult { FollowerId = follower.Id, Success = true }); + } + + return results; + } } diff --git a/src/NATS.Server/Raft/RaftRpcContracts.cs b/src/NATS.Server/Raft/RaftRpcContracts.cs index 8bad073..49ee72f 100644 --- a/src/NATS.Server/Raft/RaftRpcContracts.cs +++ b/src/NATS.Server/Raft/RaftRpcContracts.cs @@ -10,3 +10,9 @@ public sealed class VoteResponse { public bool Granted { get; init; } } + +public sealed class AppendResult +{ + public string FollowerId { get; init; } = string.Empty; + public bool Success { get; init; } +} diff --git a/src/NATS.Server/Raft/RaftTransport.cs b/src/NATS.Server/Raft/RaftTransport.cs new file mode 100644 index 0000000..6bf024f --- /dev/null +++ b/src/NATS.Server/Raft/RaftTransport.cs @@ -0,0 +1,44 @@ +namespace NATS.Server.Raft; + +public interface IRaftTransport +{ + Task> AppendEntriesAsync(string leaderId, IReadOnlyList followerIds, RaftLogEntry entry, CancellationToken ct); + Task RequestVoteAsync(string candidateId, string voterId, VoteRequest request, CancellationToken ct); +} + +public sealed class InMemoryRaftTransport : IRaftTransport +{ + private readonly Dictionary _nodes = new(StringComparer.Ordinal); + + public void Register(RaftNode node) + { + _nodes[node.Id] = node; + } + + public Task> AppendEntriesAsync(string leaderId, IReadOnlyList followerIds, RaftLogEntry entry, CancellationToken ct) + { + var results = new List(followerIds.Count); + foreach (var followerId in followerIds) + { + if (_nodes.TryGetValue(followerId, out var node)) + { + node.ReceiveReplicatedEntry(entry); + results.Add(new AppendResult { FollowerId = followerId, Success = true }); + } + else + { + results.Add(new AppendResult { FollowerId = followerId, Success = false }); + } + } + + return Task.FromResult>(results); + } + + public Task RequestVoteAsync(string candidateId, string voterId, VoteRequest request, CancellationToken ct) + { + if (_nodes.TryGetValue(voterId, out var node)) + return Task.FromResult(node.GrantVote(request.Term)); + + return Task.FromResult(new VoteResponse { Granted = false }); + } +} diff --git a/src/NATS.Server/Routes/RouteConnection.cs b/src/NATS.Server/Routes/RouteConnection.cs index 95595ae..c3aa18c 100644 --- a/src/NATS.Server/Routes/RouteConnection.cs +++ b/src/NATS.Server/Routes/RouteConnection.cs @@ -1,5 +1,6 @@ using System.Net.Sockets; using System.Text; +using NATS.Server.Subscriptions; namespace NATS.Server.Routes; @@ -7,9 +8,14 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable { private readonly Socket _socket = socket; private readonly NetworkStream _stream = new(socket, ownsSocket: true); + private readonly SemaphoreSlim _writeGate = new(1, 1); + private readonly CancellationTokenSource _closedCts = new(); + private Task? _frameLoopTask; public string? RemoteServerId { get; private set; } public string RemoteEndpoint => _socket.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString("N"); + public Func? RemoteSubscriptionReceived { get; set; } + public Func? RoutedMessageReceived { get; set; } public async Task PerformOutboundHandshakeAsync(string serverId, CancellationToken ct) { @@ -25,27 +31,168 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable await WriteLineAsync($"ROUTE {serverId}", ct); } + public void StartFrameLoop(CancellationToken ct) + { + if (_frameLoopTask != null) + return; + + var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token); + _frameLoopTask = Task.Run(() => ReadFramesAsync(linked.Token), linked.Token); + } + + public async Task SendRsPlusAsync(string subject, string? queue, CancellationToken ct) + { + var frame = queue is { Length: > 0 } + ? $"RS+ {subject} {queue}" + : $"RS+ {subject}"; + await WriteLineAsync(frame, ct); + } + + public async Task SendRsMinusAsync(string subject, string? queue, CancellationToken ct) + { + var frame = queue is { Length: > 0 } + ? $"RS- {subject} {queue}" + : $"RS- {subject}"; + await WriteLineAsync(frame, ct); + } + + public async Task SendRmsgAsync(string subject, string? replyTo, ReadOnlyMemory payload, CancellationToken ct) + { + var replyToken = string.IsNullOrEmpty(replyTo) ? "-" : replyTo; + await _writeGate.WaitAsync(ct); + try + { + var control = Encoding.ASCII.GetBytes($"RMSG {subject} {replyToken} {payload.Length}\r\n"); + await _stream.WriteAsync(control, ct); + if (!payload.IsEmpty) + await _stream.WriteAsync(payload, ct); + await _stream.WriteAsync("\r\n"u8.ToArray(), ct); + await _stream.FlushAsync(ct); + } + finally + { + _writeGate.Release(); + } + } + public async Task WaitUntilClosedAsync(CancellationToken ct) { - var buffer = new byte[1024]; - while (!ct.IsCancellationRequested) - { - var bytesRead = await _stream.ReadAsync(buffer, ct); - if (bytesRead == 0) - return; - } + if (_frameLoopTask == null) + return; + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _closedCts.Token); + await _frameLoopTask.WaitAsync(linked.Token); } public async ValueTask DisposeAsync() { + await _closedCts.CancelAsync(); + if (_frameLoopTask != null) + await _frameLoopTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + _closedCts.Dispose(); + _writeGate.Dispose(); await _stream.DisposeAsync(); } + private async Task ReadFramesAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + string line; + try + { + line = await ReadLineAsync(ct); + } + catch (OperationCanceledException) + { + break; + } + catch (IOException) + { + break; + } + + if (line.StartsWith("RS+ ", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && RemoteSubscriptionReceived != null) + { + var queue = parts.Length >= 3 ? parts[2] : null; + await RemoteSubscriptionReceived(new RemoteSubscription(parts[1], queue, RemoteServerId ?? string.Empty)); + } + continue; + } + + if (line.StartsWith("RS- ", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && RemoteSubscriptionReceived != null) + { + var queue = parts.Length >= 3 ? parts[2] : null; + await RemoteSubscriptionReceived(RemoteSubscription.Removal(parts[1], queue, RemoteServerId ?? string.Empty)); + } + continue; + } + + if (!line.StartsWith("RMSG ", StringComparison.Ordinal)) + continue; + + var args = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (args.Length < 4) + continue; + + var subject = args[1]; + var reply = args[2] == "-" ? null : args[2]; + if (!int.TryParse(args[3], out var size) || size < 0) + continue; + + var payload = await ReadPayloadAsync(size, ct); + if (RoutedMessageReceived != null) + await RoutedMessageReceived(new RouteMessage(subject, reply, payload)); + } + } + + private async Task> ReadPayloadAsync(int size, CancellationToken ct) + { + var payload = new byte[size]; + var offset = 0; + while (offset < size) + { + var read = await _stream.ReadAsync(payload.AsMemory(offset, size - offset), ct); + if (read == 0) + throw new IOException("Route connection closed during payload read"); + offset += read; + } + + var trailer = new byte[2]; + var trailerRead = 0; + while (trailerRead < 2) + { + var read = await _stream.ReadAsync(trailer.AsMemory(trailerRead, 2 - trailerRead), ct); + if (read == 0) + throw new IOException("Route connection closed during payload trailer read"); + trailerRead += read; + } + + if (trailer[0] != (byte)'\r' || trailer[1] != (byte)'\n') + throw new IOException("Invalid route payload trailer"); + + return payload; + } + private async Task WriteLineAsync(string line, CancellationToken ct) { - var bytes = Encoding.ASCII.GetBytes($"{line}\r\n"); - await _stream.WriteAsync(bytes, ct); - await _stream.FlushAsync(ct); + await _writeGate.WaitAsync(ct); + try + { + var bytes = Encoding.ASCII.GetBytes($"{line}\r\n"); + await _stream.WriteAsync(bytes, ct); + await _stream.FlushAsync(ct); + } + finally + { + _writeGate.Release(); + } } private async Task ReadLineAsync(CancellationToken ct) @@ -56,7 +203,7 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable { var read = await _stream.ReadAsync(single, ct); if (read == 0) - throw new IOException("Route connection closed during handshake"); + throw new IOException("Route connection closed"); if (single[0] == (byte)'\n') break; @@ -79,3 +226,5 @@ public sealed class RouteConnection(Socket socket) : IAsyncDisposable return id; } } + +public sealed record RouteMessage(string Subject, string? ReplyTo, ReadOnlyMemory Payload); diff --git a/src/NATS.Server/Routes/RouteManager.cs b/src/NATS.Server/Routes/RouteManager.cs index 4bf56df..35812a0 100644 --- a/src/NATS.Server/Routes/RouteManager.cs +++ b/src/NATS.Server/Routes/RouteManager.cs @@ -15,6 +15,7 @@ public sealed class RouteManager : IAsyncDisposable private readonly string _serverId; private readonly ILogger _logger; private readonly Action _remoteSubSink; + private readonly Action _routedMessageSink; private readonly ConcurrentDictionary _routes = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _connectedServerIds = new(StringComparer.Ordinal); @@ -29,12 +30,14 @@ public sealed class RouteManager : IAsyncDisposable ServerStats stats, string serverId, Action remoteSubSink, + Action routedMessageSink, ILogger logger) { _options = options; _stats = stats; _serverId = serverId; _remoteSubSink = remoteSubSink; + _routedMessageSink = routedMessageSink; _logger = logger; } @@ -51,8 +54,12 @@ public sealed class RouteManager : IAsyncDisposable _options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port; _acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token)); + var poolSize = Math.Max(_options.PoolSize, 1); foreach (var route in _options.Routes.Distinct(StringComparer.OrdinalIgnoreCase)) - _ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token)); + { + for (var i = 0; i < poolSize; i++) + _ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token)); + } return Task.CompletedTask; } @@ -81,17 +88,33 @@ public sealed class RouteManager : IAsyncDisposable public void PropagateLocalSubscription(string subject, string? queue) { - if (_connectedServerIds.IsEmpty) + if (_routes.IsEmpty) return; - var remoteSub = new RemoteSubscription(subject, queue, _serverId); - foreach (var peerId in _connectedServerIds.Keys) + foreach (var route in _routes.Values) { - if (Managers.TryGetValue(peerId, out var peer)) - peer.ReceiveRemoteSubscription(remoteSub); + _ = route.SendRsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None); } } + public void PropagateLocalUnsubscription(string subject, string? queue) + { + if (_routes.IsEmpty) + return; + + foreach (var route in _routes.Values) + _ = route.SendRsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None); + } + + public async Task ForwardRoutedMessageAsync(string subject, string? replyTo, ReadOnlyMemory payload, CancellationToken ct) + { + if (_routes.IsEmpty) + return; + + foreach (var route in _routes.Values) + await route.SendRmsgAsync(subject, replyTo, payload, ct); + } + private async Task AcceptLoopAsync(CancellationToken ct) { while (!ct.IsCancellationRequested) @@ -170,7 +193,7 @@ public sealed class RouteManager : IAsyncDisposable private void Register(RouteConnection route) { - var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}"; + var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}:{Guid.NewGuid():N}"; if (!_routes.TryAdd(key, route)) { _ = route.DisposeAsync(); @@ -180,6 +203,18 @@ public sealed class RouteManager : IAsyncDisposable if (route.RemoteServerId is { Length: > 0 } remoteServerId) _connectedServerIds[remoteServerId] = 0; + route.RemoteSubscriptionReceived = sub => + { + _remoteSubSink(sub); + return Task.CompletedTask; + }; + route.RoutedMessageReceived = msg => + { + _routedMessageSink(msg); + return Task.CompletedTask; + }; + route.StartFrameLoop(_cts!.Token); + Interlocked.Increment(ref _stats.Routes); _ = Task.Run(() => WatchRouteAsync(key, route, _cts!.Token)); } @@ -217,8 +252,5 @@ public sealed class RouteManager : IAsyncDisposable return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1])); } - private void ReceiveRemoteSubscription(RemoteSubscription sub) - { - _remoteSubSink(sub); - } + public int RouteCount => _routes.Count; } diff --git a/src/NATS.Server/Subscriptions/RemoteSubscription.cs b/src/NATS.Server/Subscriptions/RemoteSubscription.cs index b83bec5..75d4269 100644 --- a/src/NATS.Server/Subscriptions/RemoteSubscription.cs +++ b/src/NATS.Server/Subscriptions/RemoteSubscription.cs @@ -1,3 +1,7 @@ namespace NATS.Server.Subscriptions; -public sealed record RemoteSubscription(string Subject, string? Queue, string RouteId); +public sealed record RemoteSubscription(string Subject, string? Queue, string RouteId, bool IsRemoval = false) +{ + public static RemoteSubscription Removal(string subject, string? queue, string routeId) + => new(subject, queue, routeId, IsRemoval: true); +} diff --git a/src/NATS.Server/Subscriptions/SubList.cs b/src/NATS.Server/Subscriptions/SubList.cs index 0d5a216..96d51ae 100644 --- a/src/NATS.Server/Subscriptions/SubList.cs +++ b/src/NATS.Server/Subscriptions/SubList.cs @@ -103,7 +103,10 @@ public sealed class SubList : IDisposable try { var key = $"{sub.RouteId}|{sub.Subject}|{sub.Queue}"; - _remoteSubs[key] = sub; + if (sub.IsRemoval) + _remoteSubs.Remove(key); + else + _remoteSubs[key] = sub; Interlocked.Increment(ref _generation); } finally @@ -119,6 +122,9 @@ public sealed class SubList : IDisposable { foreach (var remoteSub in _remoteSubs.Values) { + if (remoteSub.IsRemoval) + continue; + if (SubjectMatch.MatchLiteral(subject, remoteSub.Subject)) return true; } diff --git a/tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs b/tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs new file mode 100644 index 0000000..c41420e --- /dev/null +++ b/tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs @@ -0,0 +1,29 @@ +using NATS.Server.Protocol; + +namespace NATS.Server.Tests; + +public class ClientKindProtocolRoutingTests +{ + [Theory] + [InlineData(ClientKind.Client, "RS+", false)] + [InlineData(ClientKind.Router, "RS+", true)] + [InlineData(ClientKind.Client, "RS-", false)] + [InlineData(ClientKind.Router, "RS-", true)] + [InlineData(ClientKind.Client, "RMSG", false)] + [InlineData(ClientKind.Router, "RMSG", true)] + [InlineData(ClientKind.Client, "A+", false)] + [InlineData(ClientKind.Gateway, "A+", true)] + [InlineData(ClientKind.Client, "A-", false)] + [InlineData(ClientKind.Gateway, "A-", true)] + [InlineData(ClientKind.Client, "LS+", false)] + [InlineData(ClientKind.Leaf, "LS+", true)] + [InlineData(ClientKind.Client, "LS-", false)] + [InlineData(ClientKind.Leaf, "LS-", true)] + [InlineData(ClientKind.Client, "LMSG", false)] + [InlineData(ClientKind.Leaf, "LMSG", true)] + public void Client_kind_protocol_matrix_enforces_inter_server_commands(ClientKind kind, string op, bool expected) + { + var matrix = new ClientCommandMatrix(); + matrix.IsAllowed(kind, op).ShouldBe(expected); + } +} diff --git a/tests/NATS.Server.Tests/GatewayProtocolTests.cs b/tests/NATS.Server.Tests/GatewayProtocolTests.cs new file mode 100644 index 0000000..c01efd7 --- /dev/null +++ b/tests/NATS.Server.Tests/GatewayProtocolTests.cs @@ -0,0 +1,153 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using NATS.Server.Configuration; + +namespace NATS.Server.Tests; + +public class GatewayProtocolTests +{ + [Fact] + public async Task Gateway_link_establishes_and_forwards_interested_message() + { + await using var fx = await GatewayFixture.StartTwoClustersAsync(); + await fx.SubscribeRemoteClusterAsync("g.>"); + await fx.PublishLocalClusterAsync("g.test", "hello"); + (await fx.ReadRemoteClusterMessageAsync()).ShouldContain("hello"); + } +} + +internal sealed class GatewayFixture : IAsyncDisposable +{ + private readonly NatsServer _local; + private readonly NatsServer _remote; + private readonly CancellationTokenSource _localCts; + private readonly CancellationTokenSource _remoteCts; + private Socket? _remoteSubscriber; + private Socket? _localPublisher; + + private GatewayFixture(NatsServer local, NatsServer remote, CancellationTokenSource localCts, CancellationTokenSource remoteCts) + { + _local = local; + _remote = remote; + _localCts = localCts; + _remoteCts = remoteCts; + } + + public static async Task StartTwoClustersAsync() + { + var localOptions = new NatsOptions + { + Host = "127.0.0.1", + Port = 0, + Gateway = new GatewayOptions + { + Name = "LOCAL", + Host = "127.0.0.1", + Port = 0, + }, + }; + + var local = new NatsServer(localOptions, NullLoggerFactory.Instance); + var localCts = new CancellationTokenSource(); + _ = local.StartAsync(localCts.Token); + await local.WaitForReadyAsync(); + + var remoteOptions = new NatsOptions + { + Host = "127.0.0.1", + Port = 0, + Gateway = new GatewayOptions + { + Name = "REMOTE", + Host = "127.0.0.1", + Port = 0, + Remotes = [local.GatewayListen!], + }, + }; + + var remote = new NatsServer(remoteOptions, NullLoggerFactory.Instance); + var remoteCts = new CancellationTokenSource(); + _ = remote.StartAsync(remoteCts.Token); + await remote.WaitForReadyAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!timeout.IsCancellationRequested && (local.Stats.Gateways == 0 || remote.Stats.Gateways == 0)) + await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default); + + return new GatewayFixture(local, remote, localCts, remoteCts); + } + + public async Task SubscribeRemoteClusterAsync(string subject) + { + var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(IPAddress.Loopback, _remote.Port); + _remoteSubscriber = sock; + + _ = await ReadLineAsync(sock); // INFO + await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + public async Task PublishLocalClusterAsync(string subject, string payload) + { + var sock = _localPublisher; + if (sock == null) + { + sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(IPAddress.Loopback, _local.Port); + _localPublisher = sock; + _ = await ReadLineAsync(sock); // INFO + await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + public Task ReadRemoteClusterMessageAsync() + { + if (_remoteSubscriber == null) + throw new InvalidOperationException("Remote subscriber was not initialized."); + + return ReadUntilAsync(_remoteSubscriber, "MSG "); + } + + public async ValueTask DisposeAsync() + { + _remoteSubscriber?.Dispose(); + _localPublisher?.Dispose(); + await _localCts.CancelAsync(); + await _remoteCts.CancelAsync(); + _local.Dispose(); + _remote.Dispose(); + _localCts.Dispose(); + _remoteCts.Dispose(); + } + + private static async Task ReadLineAsync(Socket sock) + { + var buf = new byte[4096]; + var n = await sock.ReceiveAsync(buf, SocketFlags.None); + return Encoding.ASCII.GetString(buf, 0, n); + } + + private static async Task ReadUntilAsync(Socket sock, string expected) + { + var sb = new StringBuilder(); + var buf = new byte[4096]; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + while (!sb.ToString().Contains(expected, StringComparison.Ordinal)) + { + var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token); + if (n == 0) + break; + sb.Append(Encoding.ASCII.GetString(buf, 0, n)); + } + + return sb.ToString(); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs b/tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs new file mode 100644 index 0000000..f4c461c --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs @@ -0,0 +1,17 @@ +using NATS.Server.JetStream; +using NATS.Server.JetStream.Api; + +namespace NATS.Server.Tests; + +public class JetStreamAccountControlApiTests +{ + [Fact] + public void Account_and_server_control_subjects_are_routable() + { + var router = new JetStreamApiRouter(new StreamManager(), new ConsumerManager()); + router.Route("$JS.API.SERVER.REMOVE", "{}"u8).Error.ShouldBeNull(); + router.Route("$JS.API.ACCOUNT.PURGE.ACC", "{}"u8).Error.ShouldBeNull(); + router.Route("$JS.API.ACCOUNT.STREAM.MOVE.ACC", "{}"u8).Error.ShouldBeNull(); + router.Route("$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.ACC", "{}"u8).Error.ShouldBeNull(); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamApiFixture.cs b/tests/NATS.Server.Tests/JetStreamApiFixture.cs index 8668c44..9607437 100644 --- a/tests/NATS.Server.Tests/JetStreamApiFixture.cs +++ b/tests/NATS.Server.Tests/JetStreamApiFixture.cs @@ -41,6 +41,20 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable return fixture; } + public static Task StartWithStreamConfigAsync(StreamConfig config) + { + var fixture = new JetStreamApiFixture(); + _ = fixture._streamManager.CreateOrUpdate(config); + return Task.FromResult(fixture); + } + + public static async Task StartWithStreamJsonAsync(string json) + { + var fixture = new JetStreamApiFixture(); + _ = await fixture.RequestLocalAsync("$JS.API.STREAM.CREATE.S", json); + return fixture; + } + public static async Task StartWithPullConsumerAsync() { var fixture = await StartWithStreamAsync("ORDERS", "orders.*"); @@ -82,6 +96,47 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable return fixture; } + public static async Task StartWithMultiFilterConsumerAsync() + { + var fixture = await StartWithStreamAsync("ORDERS", ">"); + _ = await fixture.CreateConsumerAsync("ORDERS", "CF", null, filterSubjects: ["orders.*"]); + return fixture; + } + + public static async Task StartWithReplayOriginalConsumerAsync() + { + var fixture = await StartWithStreamAsync("ORDERS", "orders.*"); + _ = await fixture.PublishAndGetAckAsync("orders.created", "1"); + _ = await fixture.CreateConsumerAsync("ORDERS", "RO", "orders.*", replayPolicy: ReplayPolicy.Original, ackPolicy: AckPolicy.Explicit); + return fixture; + } + + public static Task StartWithMultipleSourcesAsync() + { + var fixture = new JetStreamApiFixture(); + _ = fixture._streamManager.CreateOrUpdate(new StreamConfig + { + Name = "SRC1", + Subjects = ["a.>"], + }); + _ = fixture._streamManager.CreateOrUpdate(new StreamConfig + { + Name = "SRC2", + Subjects = ["b.>"], + }); + _ = fixture._streamManager.CreateOrUpdate(new StreamConfig + { + Name = "AGG", + Subjects = ["agg.>"], + Sources = + [ + new StreamSourceConfig { Name = "SRC1" }, + new StreamSourceConfig { Name = "SRC2" }, + ], + }); + return Task.FromResult(fixture); + } + public static Task StartJwtLimitedAccountAsync(int maxStreams) { var account = new Account("JWT-LIMITED") @@ -148,9 +203,45 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable return _streamManager.GetStateAsync(streamName, default).AsTask(); } - public Task CreateConsumerAsync(string stream, string durableName, string filterSubject, bool push = false, int heartbeatMs = 0, AckPolicy ackPolicy = AckPolicy.None, int ackWaitMs = 30_000) + public Task GetStreamBackendTypeAsync(string streamName) { - var payload = $@"{{""durable_name"":""{durableName}"",""filter_subject"":""{filterSubject}"",""push"":{push.ToString().ToLowerInvariant()},""heartbeat_ms"":{heartbeatMs},""ack_policy"":""{ackPolicy.ToString().ToLowerInvariant()}"",""ack_wait_ms"":{ackWaitMs}}}"; + return Task.FromResult(_streamManager.GetStoreBackendType(streamName)); + } + + public Task CreateConsumerAsync( + string stream, + string durableName, + string? filterSubject, + bool push = false, + int heartbeatMs = 0, + AckPolicy ackPolicy = AckPolicy.None, + int ackWaitMs = 30_000, + int maxAckPending = 0, + IReadOnlyList? filterSubjects = null, + ReplayPolicy replayPolicy = ReplayPolicy.Instant, + DeliverPolicy deliverPolicy = DeliverPolicy.All, + bool ephemeral = false) + { + var payloadObj = new + { + durable_name = durableName, + filter_subject = filterSubject, + filter_subjects = filterSubjects, + push, + heartbeat_ms = heartbeatMs, + ack_policy = ackPolicy.ToString().ToLowerInvariant(), + ack_wait_ms = ackWaitMs, + max_ack_pending = maxAckPending, + replay_policy = replayPolicy == ReplayPolicy.Original ? "original" : "instant", + deliver_policy = deliverPolicy switch + { + DeliverPolicy.Last => "last", + DeliverPolicy.New => "new", + _ => "all", + }, + ephemeral, + }; + var payload = JsonSerializer.Serialize(payloadObj); return RequestLocalAsync($"$JS.API.CONSUMER.CREATE.{stream}.{durableName}", payload); } @@ -206,6 +297,12 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable _ = await PublishAndGetAckAsync(subject, payload); } + public Task PublishToSourceAsync(string sourceStream, string subject, string payload) + { + _ = sourceStream; + return PublishAndGetAckAsync(subject, payload); + } + public Task AckAllAsync(string stream, string durableName, ulong sequence) { _consumerManager.AckAll(stream, durableName, sequence); diff --git a/tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs b/tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs new file mode 100644 index 0000000..f6dc43f --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs @@ -0,0 +1,88 @@ +using System.Diagnostics; +using System.Text.RegularExpressions; + +namespace NATS.Server.Tests; + +public class JetStreamApiGapInventoryTests +{ + [Fact] + public void Parity_map_has_no_unclassified_go_js_api_subjects() + { + var gap = JetStreamApiGapInventory.Load(); + gap.UnclassifiedSubjects.ShouldBeEmpty(); + } +} + +internal sealed class JetStreamApiGapInventory +{ + public IReadOnlyList UnclassifiedSubjects { get; } + + private JetStreamApiGapInventory(IReadOnlyList unclassifiedSubjects) + { + UnclassifiedSubjects = unclassifiedSubjects; + } + + public static JetStreamApiGapInventory Load() + { + var goSubjects = LoadGoSubjects(); + var mappedSubjects = LoadMappedSubjects(); + + var unclassified = goSubjects + .Where(s => !mappedSubjects.Contains(s)) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + + return new JetStreamApiGapInventory(unclassified); + } + + private static HashSet LoadGoSubjects() + { + var script = Path.Combine(AppContext.BaseDirectory, "../../../../../scripts/jetstream/extract-go-js-api.sh"); + script = Path.GetFullPath(script); + if (!File.Exists(script)) + throw new FileNotFoundException($"missing script: {script}"); + + var psi = new ProcessStartInfo + { + FileName = "bash", + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + psi.ArgumentList.Add(script); + + using var process = Process.Start(psi) ?? throw new InvalidOperationException("failed to start inventory script"); + var output = process.StandardOutput.ReadToEnd(); + var errors = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (process.ExitCode != 0) + throw new InvalidOperationException($"inventory script failed: {errors}"); + + return output + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(x => x.StartsWith("$JS.API.", StringComparison.Ordinal)) + .ToHashSet(StringComparer.Ordinal); + } + + private static HashSet LoadMappedSubjects() + { + var mapPath = Path.Combine(AppContext.BaseDirectory, "../../../../../docs/plans/2026-02-23-jetstream-remaining-parity-map.md"); + mapPath = Path.GetFullPath(mapPath); + if (!File.Exists(mapPath)) + throw new FileNotFoundException($"missing parity map: {mapPath}"); + + var subjectRegex = new Regex(@"^\|\s*(\$JS\.API[^\|]+?)\s*\|", RegexOptions.Compiled); + var subjects = new HashSet(StringComparer.Ordinal); + + foreach (var line in File.ReadLines(mapPath)) + { + var match = subjectRegex.Match(line); + if (!match.Success) + continue; + + subjects.Add(match.Groups[1].Value.Trim()); + } + + return subjects; + } +} diff --git a/tests/NATS.Server.Tests/JetStreamClusterControlExtendedApiTests.cs b/tests/NATS.Server.Tests/JetStreamClusterControlExtendedApiTests.cs new file mode 100644 index 0000000..e2e4694 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamClusterControlExtendedApiTests.cs @@ -0,0 +1,12 @@ +namespace NATS.Server.Tests; + +public class JetStreamClusterControlExtendedApiTests +{ + [Fact] + public async Task Peer_remove_and_consumer_stepdown_subjects_return_success_shape() + { + await using var fx = await JetStreamClusterFixture.StartAsync(nodes: 3); + (await fx.RequestAsync("$JS.API.STREAM.PEER.REMOVE.ORDERS", "{\"peer\":\"n2\"}")).Success.ShouldBeTrue(); + (await fx.RequestAsync("$JS.API.CONSUMER.LEADER.STEPDOWN.ORDERS.DUR", "{}")).Success.ShouldBeTrue(); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs b/tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs new file mode 100644 index 0000000..cf76efc --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs @@ -0,0 +1,16 @@ +namespace NATS.Server.Tests; + +public class JetStreamConsumerSemanticsTests +{ + [Fact] + public async Task Consumer_with_filter_subjects_only_receives_matching_messages() + { + await using var fx = await JetStreamApiFixture.StartWithMultiFilterConsumerAsync(); + await fx.PublishAndGetAckAsync("orders.created", "1"); + await fx.PublishAndGetAckAsync("payments.settled", "2"); + + var batch = await fx.FetchAsync("ORDERS", "CF", 10); + batch.Messages.ShouldNotBeEmpty(); + batch.Messages.All(m => m.Subject.StartsWith("orders.", StringComparison.Ordinal)).ShouldBeTrue(); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs b/tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs new file mode 100644 index 0000000..1bcc168 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs @@ -0,0 +1,15 @@ +using System.Diagnostics; + +namespace NATS.Server.Tests; + +public class JetStreamFlowReplayBackoffTests +{ + [Fact] + public async Task Replay_original_respects_message_timestamps_with_backoff_redelivery() + { + await using var fx = await JetStreamApiFixture.StartWithReplayOriginalConsumerAsync(); + var sw = Stopwatch.StartNew(); + _ = await fx.FetchAsync("ORDERS", "RO", 1); + sw.ElapsedMilliseconds.ShouldBeGreaterThanOrEqualTo(50); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamMirrorSourceAdvancedTests.cs b/tests/NATS.Server.Tests/JetStreamMirrorSourceAdvancedTests.cs new file mode 100644 index 0000000..d42f265 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamMirrorSourceAdvancedTests.cs @@ -0,0 +1,14 @@ +namespace NATS.Server.Tests; + +public class JetStreamMirrorSourceAdvancedTests +{ + [Fact] + public async Task Stream_with_multiple_sources_aggregates_messages_in_order() + { + await using var fx = await JetStreamApiFixture.StartWithMultipleSourcesAsync(); + await fx.PublishToSourceAsync("SRC1", "a.1", "1"); + await fx.PublishToSourceAsync("SRC2", "b.1", "2"); + + (await fx.GetStreamStateAsync("AGG")).Messages.ShouldBe((ulong)2); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs b/tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs new file mode 100644 index 0000000..290f215 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs @@ -0,0 +1,11 @@ +namespace NATS.Server.Tests; + +public class JetStreamStorageSelectionTests +{ + [Fact] + public async Task Stream_with_storage_file_uses_filestore_backend() + { + await using var fx = await JetStreamApiFixture.StartWithStreamJsonAsync("{\"name\":\"S\",\"subjects\":[\"s.*\"],\"storage\":\"file\"}"); + (await fx.GetStreamBackendTypeAsync("S")).ShouldBe("file"); + } +} diff --git a/tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs b/tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs new file mode 100644 index 0000000..e636503 --- /dev/null +++ b/tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs @@ -0,0 +1,21 @@ +using NATS.Server.JetStream.Models; + +namespace NATS.Server.Tests; + +public class JetStreamStreamPolicyRuntimeTests +{ + [Fact] + public async Task Discard_new_rejects_publish_when_max_bytes_exceeded() + { + await using var fx = await JetStreamApiFixture.StartWithStreamConfigAsync(new StreamConfig + { + Name = "S", + Subjects = ["s.*"], + MaxBytes = 2, + Discard = DiscardPolicy.New, + }); + + (await fx.PublishAndGetAckAsync("s.a", "12")).ErrorCode.ShouldBeNull(); + (await fx.PublishAndGetAckAsync("s.a", "34")).ErrorCode.ShouldNotBeNull(); + } +} diff --git a/tests/NATS.Server.Tests/LeafProtocolTests.cs b/tests/NATS.Server.Tests/LeafProtocolTests.cs new file mode 100644 index 0000000..25b6763 --- /dev/null +++ b/tests/NATS.Server.Tests/LeafProtocolTests.cs @@ -0,0 +1,151 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using NATS.Server.Configuration; + +namespace NATS.Server.Tests; + +public class LeafProtocolTests +{ + [Fact] + public async Task Leaf_link_propagates_subscription_and_message_flow() + { + await using var fx = await LeafFixture.StartHubSpokeAsync(); + await fx.SubscribeSpokeAsync("leaf.>"); + await fx.PublishHubAsync("leaf.msg", "x"); + (await fx.ReadSpokeMessageAsync()).ShouldContain("x"); + } +} + +internal sealed class LeafFixture : IAsyncDisposable +{ + private readonly NatsServer _hub; + private readonly NatsServer _spoke; + private readonly CancellationTokenSource _hubCts; + private readonly CancellationTokenSource _spokeCts; + private Socket? _spokeSubscriber; + private Socket? _hubPublisher; + + private LeafFixture(NatsServer hub, NatsServer spoke, CancellationTokenSource hubCts, CancellationTokenSource spokeCts) + { + _hub = hub; + _spoke = spoke; + _hubCts = hubCts; + _spokeCts = spokeCts; + } + + public static async Task StartHubSpokeAsync() + { + var hubOptions = new NatsOptions + { + Host = "127.0.0.1", + Port = 0, + LeafNode = new LeafNodeOptions + { + Host = "127.0.0.1", + Port = 0, + }, + }; + + var hub = new NatsServer(hubOptions, NullLoggerFactory.Instance); + var hubCts = new CancellationTokenSource(); + _ = hub.StartAsync(hubCts.Token); + await hub.WaitForReadyAsync(); + + var spokeOptions = new NatsOptions + { + Host = "127.0.0.1", + Port = 0, + LeafNode = new LeafNodeOptions + { + Host = "127.0.0.1", + Port = 0, + Remotes = [hub.LeafListen!], + }, + }; + + var spoke = new NatsServer(spokeOptions, NullLoggerFactory.Instance); + var spokeCts = new CancellationTokenSource(); + _ = spoke.StartAsync(spokeCts.Token); + await spoke.WaitForReadyAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!timeout.IsCancellationRequested && (hub.Stats.Leafs == 0 || spoke.Stats.Leafs == 0)) + await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default); + + return new LeafFixture(hub, spoke, hubCts, spokeCts); + } + + public async Task SubscribeSpokeAsync(string subject) + { + var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(IPAddress.Loopback, _spoke.Port); + _spokeSubscriber = sock; + + _ = await ReadLineAsync(sock); // INFO + await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + public async Task PublishHubAsync(string subject, string payload) + { + var sock = _hubPublisher; + if (sock == null) + { + sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(IPAddress.Loopback, _hub.Port); + _hubPublisher = sock; + _ = await ReadLineAsync(sock); // INFO + await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + public Task ReadSpokeMessageAsync() + { + if (_spokeSubscriber == null) + throw new InvalidOperationException("Spoke subscriber was not initialized."); + + return ReadUntilAsync(_spokeSubscriber, "MSG "); + } + + public async ValueTask DisposeAsync() + { + _spokeSubscriber?.Dispose(); + _hubPublisher?.Dispose(); + await _hubCts.CancelAsync(); + await _spokeCts.CancelAsync(); + _hub.Dispose(); + _spoke.Dispose(); + _hubCts.Dispose(); + _spokeCts.Dispose(); + } + + private static async Task ReadLineAsync(Socket sock) + { + var buf = new byte[4096]; + var n = await sock.ReceiveAsync(buf, SocketFlags.None); + return Encoding.ASCII.GetString(buf, 0, n); + } + + private static async Task ReadUntilAsync(Socket sock, string expected) + { + var sb = new StringBuilder(); + var buf = new byte[4096]; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + while (!sb.ToString().Contains(expected, StringComparison.Ordinal)) + { + var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token); + if (n == 0) + break; + sb.Append(Encoding.ASCII.GetString(buf, 0, n)); + } + + return sb.ToString(); + } +} diff --git a/tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs b/tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs new file mode 100644 index 0000000..f529f94 --- /dev/null +++ b/tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs @@ -0,0 +1,105 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Logging.Abstractions; +using NATS.Server.Configuration; + +namespace NATS.Server.Tests; + +public class MonitorClusterEndpointTests +{ + [Fact] + public async Task Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data() + { + await using var fx = await MonitorFixture.StartClusterEnabledAsync(); + (await fx.GetJsonAsync("/routez")).ShouldContain("routes"); + (await fx.GetJsonAsync("/gatewayz")).ShouldContain("gateways"); + (await fx.GetJsonAsync("/leafz")).ShouldContain("leafs"); + (await fx.GetJsonAsync("/accountz")).ShouldContain("accounts"); + } +} + +internal sealed class MonitorFixture : IAsyncDisposable +{ + private readonly NatsServer _server; + private readonly CancellationTokenSource _cts; + private readonly HttpClient _http; + private readonly int _monitorPort; + + private MonitorFixture(NatsServer server, CancellationTokenSource cts, HttpClient http, int monitorPort) + { + _server = server; + _cts = cts; + _http = http; + _monitorPort = monitorPort; + } + + public static async Task StartClusterEnabledAsync() + { + var monitorPort = GetFreePort(); + var options = new NatsOptions + { + Host = "127.0.0.1", + Port = 0, + MonitorPort = monitorPort, + Cluster = new ClusterOptions + { + Host = "127.0.0.1", + Port = 0, + }, + Gateway = new GatewayOptions + { + Host = "127.0.0.1", + Port = 0, + Name = "M", + }, + LeafNode = new LeafNodeOptions + { + Host = "127.0.0.1", + Port = 0, + }, + }; + + var server = new NatsServer(options, NullLoggerFactory.Instance); + var cts = new CancellationTokenSource(); + _ = server.StartAsync(cts.Token); + await server.WaitForReadyAsync(); + + var http = new HttpClient(); + for (var i = 0; i < 50; i++) + { + try + { + var response = await http.GetAsync($"http://127.0.0.1:{monitorPort}/healthz"); + if (response.IsSuccessStatusCode) + break; + } + catch + { + } + + await Task.Delay(50); + } + + return new MonitorFixture(server, cts, http, monitorPort); + } + + public Task GetJsonAsync(string path) + { + return _http.GetStringAsync($"http://127.0.0.1:{_monitorPort}{path}"); + } + + public async ValueTask DisposeAsync() + { + _http.Dispose(); + await _cts.CancelAsync(); + _server.Dispose(); + _cts.Dispose(); + } + + private static int GetFreePort() + { + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return ((IPEndPoint)socket.LocalEndPoint!).Port; + } +} diff --git a/tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs b/tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs new file mode 100644 index 0000000..f1b80d1 --- /dev/null +++ b/tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs @@ -0,0 +1,89 @@ +using NATS.Server.Raft; + +namespace NATS.Server.Tests; + +public class RaftTransportPersistenceTests +{ + [Fact] + public async Task Raft_node_recovers_log_and_term_after_restart() + { + await using var fx = await RaftFixture.StartPersistentClusterAsync(); + var idx = await fx.Leader.ProposeAsync("cmd", default); + await fx.RestartNodeAsync("n2"); + (await fx.ReadNodeAppliedIndexAsync("n2")).ShouldBeGreaterThanOrEqualTo(idx); + } +} + +internal sealed class RaftFixture : IAsyncDisposable +{ + private readonly string _root; + private readonly InMemoryRaftTransport _transport; + private readonly Dictionary _nodes; + + private RaftFixture(string root, InMemoryRaftTransport transport, Dictionary nodes) + { + _root = root; + _transport = transport; + _nodes = nodes; + } + + public RaftNode Leader => _nodes["n1"]; + + public static Task StartPersistentClusterAsync() + { + var root = Path.Combine(Path.GetTempPath(), $"nats-raft-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + + var transport = new InMemoryRaftTransport(); + var nodes = new Dictionary(StringComparer.Ordinal); + foreach (var id in new[] { "n1", "n2", "n3" }) + { + var node = new RaftNode(id, transport, Path.Combine(root, id)); + transport.Register(node); + nodes[id] = node; + } + + var all = nodes.Values.ToArray(); + foreach (var node in all) + node.ConfigureCluster(all); + + var leader = nodes["n1"]; + leader.StartElection(all.Length); + leader.ReceiveVote(nodes["n2"].GrantVote(leader.Term), all.Length); + leader.ReceiveVote(nodes["n3"].GrantVote(leader.Term), all.Length); + + return Task.FromResult(new RaftFixture(root, transport, nodes)); + } + + public async Task RestartNodeAsync(string id) + { + var nodePath = Path.Combine(_root, id); + var replacement = new RaftNode(id, _transport, nodePath); + await replacement.LoadPersistedStateAsync(default); + _transport.Register(replacement); + _nodes[id] = replacement; + + var all = _nodes.Values.ToArray(); + foreach (var node in all) + node.ConfigureCluster(all); + } + + public Task ReadNodeAppliedIndexAsync(string id) + { + return Task.FromResult(_nodes[id].AppliedIndex); + } + + public ValueTask DisposeAsync() + { + try + { + if (Directory.Exists(_root)) + Directory.Delete(_root, recursive: true); + } + catch + { + } + + return ValueTask.CompletedTask; + } +} diff --git a/tests/NATS.Server.Tests/RoutePoolTests.cs b/tests/NATS.Server.Tests/RoutePoolTests.cs new file mode 100644 index 0000000..d43dc7e --- /dev/null +++ b/tests/NATS.Server.Tests/RoutePoolTests.cs @@ -0,0 +1,11 @@ +namespace NATS.Server.Tests; + +public class RoutePoolTests +{ + [Fact] + public async Task Route_manager_establishes_default_pool_of_three_links_per_peer() + { + await using var fx = await RouteFixture.StartTwoNodeClusterAsync(); + (await fx.ServerARouteLinkCountToServerBAsync()).ShouldBe(3); + } +} diff --git a/tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs b/tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs new file mode 100644 index 0000000..ff91cf0 --- /dev/null +++ b/tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs @@ -0,0 +1,14 @@ +namespace NATS.Server.Tests; + +public class RouteRmsgForwardingTests +{ + [Fact] + public async Task Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG() + { + await using var fx = await RouteFixture.StartTwoNodeClusterAsync(); + await fx.SubscribeOnServerBAsync("foo.>"); + + await fx.PublishFromServerAAsync("foo.bar", "payload"); + (await fx.ReadServerBMessageAsync()).ShouldContain("payload"); + } +} diff --git a/tests/NATS.Server.Tests/RouteSubscriptionPropagationTests.cs b/tests/NATS.Server.Tests/RouteSubscriptionPropagationTests.cs index dbf9506..0f0173a 100644 --- a/tests/NATS.Server.Tests/RouteSubscriptionPropagationTests.cs +++ b/tests/NATS.Server.Tests/RouteSubscriptionPropagationTests.cs @@ -27,6 +27,8 @@ internal sealed class RouteFixture : IAsyncDisposable private readonly CancellationTokenSource _ctsA; private readonly CancellationTokenSource _ctsB; private Socket? _subscriberOnB; + private Socket? _publisherOnA; + private Socket? _manualRouteToA; private RouteFixture(NatsServer serverA, NatsServer serverB, CancellationTokenSource ctsA, CancellationTokenSource ctsB) { @@ -91,22 +93,82 @@ internal sealed class RouteFixture : IAsyncDisposable await ReadUntilAsync(sock, "PONG"); } - public async Task ServerAHasRemoteInterestAsync(string subject) + public async Task SendRouteSubFrameAsync(string subject) + { + var (host, port) = ParseHostPort(_serverA.ClusterListen!); + var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(IPAddress.Parse(host), port); + _manualRouteToA = sock; + + await sock.SendAsync(Encoding.ASCII.GetBytes("ROUTE test-remote\r\n")); + _ = await ReadLineAsync(sock); // ROUTE + await sock.SendAsync(Encoding.ASCII.GetBytes($"RS+ {subject}\r\n")); + } + + public async Task SendRouteUnsubFrameAsync(string subject) + { + if (_manualRouteToA == null) + throw new InvalidOperationException("Route frame socket not established."); + + await _manualRouteToA.SendAsync(Encoding.ASCII.GetBytes($"RS- {subject}\r\n")); + } + + public async Task PublishFromServerAAsync(string subject, string payload) + { + var sock = _publisherOnA; + if (sock == null) + { + sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(IPAddress.Loopback, _serverA.Port); + _publisherOnA = sock; + _ = await ReadLineAsync(sock); // INFO + await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n")); + await ReadUntilAsync(sock, "PONG"); + } + + public async Task ReadServerBMessageAsync() + { + if (_subscriberOnB == null) + throw new InvalidOperationException("No subscriber socket on server B."); + + return await ReadUntilAsync(_subscriberOnB, "MSG "); + } + + public async Task ServerAHasRemoteInterestAsync(string subject, bool expected = true) { using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); while (!timeout.IsCancellationRequested) { - if (_serverA.HasRemoteInterest(subject)) - return true; + if (_serverA.HasRemoteInterest(subject) == expected) + return expected; await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default); } - return false; + return !expected; + } + + public async Task ServerARouteLinkCountToServerBAsync() + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!timeout.IsCancellationRequested) + { + if (_serverA.Stats.Routes >= 3) + return (int)_serverA.Stats.Routes; + await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default); + } + + return (int)_serverA.Stats.Routes; } public async ValueTask DisposeAsync() { _subscriberOnB?.Dispose(); + _publisherOnA?.Dispose(); + _manualRouteToA?.Dispose(); await _ctsA.CancelAsync(); await _ctsB.CancelAsync(); _serverA.Dispose(); @@ -138,4 +200,10 @@ internal sealed class RouteFixture : IAsyncDisposable return sb.ToString(); } + + private static (string Host, int Port) ParseHostPort(string endpoint) + { + var parts = endpoint.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return (parts[0], int.Parse(parts[1])); + } } diff --git a/tests/NATS.Server.Tests/RouteWireSubscriptionProtocolTests.cs b/tests/NATS.Server.Tests/RouteWireSubscriptionProtocolTests.cs new file mode 100644 index 0000000..0002553 --- /dev/null +++ b/tests/NATS.Server.Tests/RouteWireSubscriptionProtocolTests.cs @@ -0,0 +1,16 @@ +namespace NATS.Server.Tests; + +public class RouteWireSubscriptionProtocolTests +{ + [Fact] + public async Task RSplus_RSminus_frames_propagate_remote_interest_over_socket() + { + await using var fx = await RouteFixture.StartTwoNodeClusterAsync(); + + await fx.SendRouteSubFrameAsync("foo.*"); + (await fx.ServerAHasRemoteInterestAsync("foo.bar")).ShouldBeTrue(); + + await fx.SendRouteUnsubFrameAsync("foo.*"); + (await fx.ServerAHasRemoteInterestAsync("foo.bar", expected: false)).ShouldBeFalse(); + } +}