Fix E2E test gaps and add comprehensive E2E + parity test suites

- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status
This commit is contained in:
Joseph Doherty
2026-03-12 14:09:23 -04:00
parent 79c1ee8776
commit c30e67a69d
226 changed files with 17801 additions and 709 deletions
+82 -1
View File
@@ -3,6 +3,7 @@ using System.Net;
using System.Net.Sockets;
using Microsoft.Extensions.Logging;
using NATS.Server.Configuration;
using NATS.Server.Gateways;
using NATS.Server.Subscriptions;
namespace NATS.Server.LeafNodes;
@@ -16,6 +17,11 @@ namespace NATS.Server.LeafNodes;
/// </summary>
public sealed class LeafNodeManager : IAsyncDisposable
{
public static readonly TimeSpan LeafNodeReconnectDelayAfterLoopDetected = TimeSpan.FromSeconds(30);
public static readonly TimeSpan LeafNodeReconnectAfterPermViolation = TimeSpan.FromSeconds(30);
public static readonly TimeSpan LeafNodeReconnectDelayAfterClusterNameSame = TimeSpan.FromSeconds(30);
public static readonly TimeSpan LeafNodeWaitBeforeClose = TimeSpan.FromSeconds(5);
private readonly LeafNodeOptions _options;
private readonly ServerStats _stats;
private readonly string _serverId;
@@ -90,6 +96,27 @@ public sealed class LeafNodeManager : IAsyncDisposable
public bool IsLeafConnectDisabled(string remoteUrl)
=> IsGloballyDisabled || _disabledRemotes.ContainsKey(remoteUrl);
/// <summary>
/// Returns true when the remote URL is still configured and not disabled.
/// Go reference: leafnode.go remoteLeafNodeStillValid.
/// </summary>
internal bool RemoteLeafNodeStillValid(string remoteUrl)
{
if (IsLeafConnectDisabled(remoteUrl))
return false;
if (_options.Remotes.Any(r => string.Equals(r, remoteUrl, StringComparison.OrdinalIgnoreCase)))
return true;
foreach (var remote in _options.RemoteLeaves)
{
if (remote.Urls.Any(u => string.Equals(u, remoteUrl, StringComparison.OrdinalIgnoreCase)))
return true;
}
return false;
}
/// <summary>
/// Disables outbound leaf connections to the specified remote URL.
/// Has no effect if the remote is already disabled.
@@ -232,6 +259,8 @@ public sealed class LeafNodeManager : IAsyncDisposable
var connection = new LeafConnection(socket)
{
JetStreamDomain = _options.JetStreamDomain,
IsSolicited = true,
IsSpoke = true,
};
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
Register(connection);
@@ -263,6 +292,9 @@ public sealed class LeafNodeManager : IAsyncDisposable
}
public void PropagateLocalSubscription(string account, string subject, string? queue)
=> PropagateLocalSubscription(account, subject, queue, queueWeight: 0);
public void PropagateLocalSubscription(string account, string subject, string? queue, int queueWeight)
{
// Subscription propagation is also subject to export filtering:
// we don't propagate subscriptions for subjects that are denied.
@@ -273,7 +305,18 @@ public sealed class LeafNodeManager : IAsyncDisposable
}
foreach (var connection in _connections.Values)
_ = connection.SendLsPlusAsync(account, subject, queue, _cts?.Token ?? CancellationToken.None);
{
if (!CanSpokeSendSubscription(connection, subject))
{
_logger.LogDebug(
"Leaf subscription propagation denied for spoke connection {RemoteId} and subject {Subject} (subscribe permissions)",
connection.RemoteId ?? "<unknown>",
subject);
continue;
}
_ = connection.SendLsPlusAsync(account, subject, queue, queueWeight, _cts?.Token ?? CancellationToken.None);
}
}
public void PropagateLocalUnsubscription(string account, string subject, string? queue)
@@ -585,6 +628,9 @@ public sealed class LeafNodeManager : IAsyncDisposable
var attempt = 0;
while (!ct.IsCancellationRequested)
{
if (!RemoteLeafNodeStillValid(remote))
return;
try
{
var endPoint = ParseEndpoint(remote);
@@ -595,6 +641,8 @@ public sealed class LeafNodeManager : IAsyncDisposable
var connection = new LeafConnection(socket)
{
JetStreamDomain = jetStreamDomain,
IsSolicited = true,
IsSpoke = true,
};
await connection.PerformOutboundHandshakeAsync(_serverId, ct);
Register(connection);
@@ -736,6 +784,39 @@ public sealed class LeafNodeManager : IAsyncDisposable
return null;
}
private static bool CanSpokeSendSubscription(LeafConnection connection, string subject)
{
if (!connection.IsSpokeLeafNode())
return true;
if (ShouldBypassSpokeSubscribePermission(subject))
return true;
if (!connection.PermsSynced || connection.AllowedSubscribeSubjects.Count == 0)
return true;
for (var i = 0; i < connection.AllowedSubscribeSubjects.Count; i++)
{
if (SubjectMatch.MatchLiteral(subject, connection.AllowedSubscribeSubjects[i]))
return true;
}
return false;
}
private static bool ShouldBypassSpokeSubscribePermission(string subject)
{
if (string.IsNullOrEmpty(subject))
return false;
if (subject[0] != '$' && subject[0] != '_')
return false;
return subject.StartsWith("$LDS.", StringComparison.Ordinal)
|| subject.StartsWith(ReplyMapper.GatewayReplyPrefix, StringComparison.Ordinal)
|| subject.StartsWith(ReplyMapper.OldGatewayReplyPrefix, StringComparison.Ordinal);
}
private static IPEndPoint ParseEndpoint(string endpoint)
{
var parts = endpoint.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);