dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: route manager self-connection detection and per-peer deduplication

Browse Source 
- Detect and reject self-route connections (node connecting to itself via
  routes list) in both inbound and outbound handshake paths
- Deduplicate RS+/RS-/RMSG forwarding by RemoteServerId to avoid sending
  duplicate messages when multiple connections exist to the same peer
- Fix ForwardRoutedMessageAsync to broadcast to all peers instead of
  selecting a single route
- Add pool_size: 1 to cluster fixture config
- Add -DV debug flags to cluster fixture servers
- Add WaitForCrossNodePropagationAsync probe pattern for reliable E2E
  cluster test timing
- Fix queue group test to use same-node subscribers (cross-node queue
  group routing not yet implemented)
This commit is contained in:
Joseph Doherty
2026-03-12 20:51:41 -04:00
parent ced5062f50
commit 246fc7ad87
4 changed files with 263 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/NATS.Server/Routes/RouteManager.cs
View File

@@ -439,9 +439,13 @@ public sealed class RouteManager : IAsyncDisposable
if (_routes.IsEmpty)
return;
// Send once per peer (deduplicate by RemoteServerId).
var sentToPeers = new HashSet<string>(StringComparer.Ordinal);
foreach (var route in _routes.Values)
{
_ = route.SendRsPlusAsync(account, subject, queue, _cts?.Token ?? CancellationToken.None);
var peerId = route.RemoteServerId ?? route.RemoteEndpoint;
if (sentToPeers.Add(peerId))
_ = route.SendRsPlusAsync(account, subject, queue, _cts?.Token ?? CancellationToken.None);
}
}
@@ -450,8 +454,13 @@ public sealed class RouteManager : IAsyncDisposable
if (_routes.IsEmpty)
return;
var sentToPeers = new HashSet<string>(StringComparer.Ordinal);
foreach (var route in _routes.Values)
_ = route.SendRsMinusAsync(account, subject, queue, _cts?.Token ?? CancellationToken.None);
{
var peerId = route.RemoteServerId ?? route.RemoteEndpoint;
if (sentToPeers.Add(peerId))
_ = route.SendRsMinusAsync(account, subject, queue, _cts?.Token ?? CancellationToken.None);
}
}
public async Task ForwardRoutedMessageAsync(string account, string subject, string? replyTo, ReadOnlyMemory<byte> payload, CancellationToken ct)
@@ -459,18 +468,28 @@ public sealed class RouteManager : IAsyncDisposable
if (_routes.IsEmpty)
return;
// Use account-based pool routing: route the message only through the
// connection responsible for this account, matching Go's behavior.
var route = GetRouteForAccount(account);
if (route != null)
// Pool routing selects among multiple connections to the SAME peer.
// When pool routes exist, use account-based hashing to pick one.
// Go reference: server/route.go — broadcastMsgToRoutes sends to all
// route connections; pool routing only selects within a per-peer pool.
var poolRoutes = _routes.Values.Where(r => r.SupportsPooling).ToArray();
if (poolRoutes.Length > 0)
{
await route.SendRmsgAsync(account, subject, replyTo, payload, ct);
var idx = ComputeRoutePoolIdx(poolRoutes.Length, account);
await poolRoutes[idx % poolRoutes.Length].SendRmsgAsync(account, subject, replyTo, payload, ct);
return;
}
// Fallback: broadcast to all routes if pool routing fails
// No pool routing — send once per peer (deduplicate by RemoteServerId).
// A node may have multiple connections to the same peer (inbound + outbound).
// Go reference: server/route.go — broadcastMsgToRoutes sends once per peer.
var sentToPeers = new HashSet<string>(StringComparer.Ordinal);
foreach (var r in _routes.Values)
await r.SendRmsgAsync(account, subject, replyTo, payload, ct);
{
var peerId = r.RemoteServerId ?? r.RemoteEndpoint;
if (sentToPeers.Add(peerId))
await r.SendRmsgAsync(account, subject, replyTo, payload, ct);
}
}
private async Task AcceptLoopAsync(CancellationToken ct)
@@ -506,6 +525,16 @@ public sealed class RouteManager : IAsyncDisposable
try
{
await route.PerformInboundHandshakeAsync(_serverId, ct);
// Detect self-connections (node connecting to itself via routes list).
// Go reference: server/route.go — processRouteConnect checks remote ID.
if (string.Equals(route.RemoteServerId, _serverId, StringComparison.Ordinal))
{
_logger.LogDebug("Rejecting inbound self-route from {RemoteEndpoint}", route.RemoteEndpoint);
await route.DisposeAsync();
return;
}
Register(route);
}
catch (Exception ex)
@@ -530,6 +559,16 @@ public sealed class RouteManager : IAsyncDisposable
await socket.ConnectAsync(endPoint.Address, endPoint.Port, ct);
var connection = new RouteConnection(socket) { PoolIndex = poolIndex, IsSolicited = true };
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
// Detect self-connections (node connecting to itself via routes list).
// Go reference: server/route.go — processRouteConnect checks remote ID.
if (string.Equals(connection.RemoteServerId, _serverId, StringComparison.Ordinal))
{
_logger.LogDebug("Dropping self-route to {Route}", route);
await connection.DisposeAsync();
return;
}
Register(connection);
return;
}