feat: complete final jetstream parity transport and runtime baselines

src/NATS.Server/Routes/RouteManager.cs

@@ -15,6 +15,7 @@ public sealed class RouteManager : IAsyncDisposable
     private readonly string _serverId;
     private readonly ILogger<RouteManager> _logger;
     private readonly Action<RemoteSubscription> _remoteSubSink;
+    private readonly Action<RouteMessage> _routedMessageSink;
     private readonly ConcurrentDictionary<string, RouteConnection> _routes = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<string, byte> _connectedServerIds = new(StringComparer.Ordinal);
 
@@ -29,12 +30,14 @@ public sealed class RouteManager : IAsyncDisposable
         ServerStats stats,
         string serverId,
         Action<RemoteSubscription> remoteSubSink,
+        Action<RouteMessage> routedMessageSink,
         ILogger<RouteManager> logger)
     {
         _options = options;
         _stats = stats;
         _serverId = serverId;
         _remoteSubSink = remoteSubSink;
+        _routedMessageSink = routedMessageSink;
         _logger = logger;
     }
 
@@ -51,8 +54,12 @@ public sealed class RouteManager : IAsyncDisposable
             _options.Port = ((IPEndPoint)_listener.LocalEndPoint!).Port;
 
         _acceptLoopTask = Task.Run(() => AcceptLoopAsync(_cts.Token));
+        var poolSize = Math.Max(_options.PoolSize, 1);
         foreach (var route in _options.Routes.Distinct(StringComparer.OrdinalIgnoreCase))
-            _ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token));
+        {
+            for (var i = 0; i < poolSize; i++)
+                _ = Task.Run(() => ConnectToRouteWithRetryAsync(route, _cts.Token));
+        }
 
         return Task.CompletedTask;
     }
@@ -81,17 +88,33 @@ public sealed class RouteManager : IAsyncDisposable
 
     public void PropagateLocalSubscription(string subject, string? queue)
     {
-        if (_connectedServerIds.IsEmpty)
+        if (_routes.IsEmpty)
             return;
 
-        var remoteSub = new RemoteSubscription(subject, queue, _serverId);
-        foreach (var peerId in _connectedServerIds.Keys)
+        foreach (var route in _routes.Values)
         {
-            if (Managers.TryGetValue(peerId, out var peer))
-                peer.ReceiveRemoteSubscription(remoteSub);
+            _ = route.SendRsPlusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
         }
     }
 
+    public void PropagateLocalUnsubscription(string subject, string? queue)
+    {
+        if (_routes.IsEmpty)
+            return;
+
+        foreach (var route in _routes.Values)
+            _ = route.SendRsMinusAsync(subject, queue, _cts?.Token ?? CancellationToken.None);
+    }
+
+    public async Task ForwardRoutedMessageAsync(string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
+    {
+        if (_routes.IsEmpty)
+            return;
+
+        foreach (var route in _routes.Values)
+            await route.SendRmsgAsync(subject, replyTo, payload, ct);
+    }
+
     private async Task AcceptLoopAsync(CancellationToken ct)
     {
         while (!ct.IsCancellationRequested)
@@ -170,7 +193,7 @@ public sealed class RouteManager : IAsyncDisposable
 
     private void Register(RouteConnection route)
     {
-        var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}";
+        var key = $"{route.RemoteServerId}:{route.RemoteEndpoint}:{Guid.NewGuid():N}";
         if (!_routes.TryAdd(key, route))
         {
             _ = route.DisposeAsync();
@@ -180,6 +203,18 @@ public sealed class RouteManager : IAsyncDisposable
         if (route.RemoteServerId is { Length: > 0 } remoteServerId)
             _connectedServerIds[remoteServerId] = 0;
 
+        route.RemoteSubscriptionReceived = sub =>
+        {
+            _remoteSubSink(sub);
+            return Task.CompletedTask;
+        };
+        route.RoutedMessageReceived = msg =>
+        {
+            _routedMessageSink(msg);
+            return Task.CompletedTask;
+        };
+        route.StartFrameLoop(_cts!.Token);
+
         Interlocked.Increment(ref _stats.Routes);
         _ = Task.Run(() => WatchRouteAsync(key, route, _cts!.Token));
     }
@@ -217,8 +252,5 @@ public sealed class RouteManager : IAsyncDisposable
         return new IPEndPoint(IPAddress.Parse(parts[0]), int.Parse(parts[1]));
     }
 
-    private void ReceiveRemoteSubscription(RemoteSubscription sub)
-    {
-        _remoteSubSink(sub);
-    }
+    public int RouteCount => _routes.Count;
 }