Fix E2E test gaps and add comprehensive E2E + parity test suites

- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status
This commit is contained in:
Joseph Doherty
2026-03-12 14:09:23 -04:00
parent 79c1ee8776
commit c30e67a69d
226 changed files with 17801 additions and 709 deletions

View File

@@ -8,6 +8,7 @@ public sealed class Account : IDisposable
{
public const string GlobalAccountName = "$G";
public const string SystemAccountName = "$SYS";
public const string ClientInfoHdr = "Nats-Request-Info";
public string Name { get; }
public SubList SubList { get; } = new();
@@ -789,6 +790,100 @@ public sealed class Account : IDisposable
return matchResult.PlainSubs.Length > 0 || matchResult.QueueSubs.Length > 0;
}
/// <summary>
/// Returns true if this account has at least one matching subscription for the given subject.
/// Go reference: accounts.go SubscriptionInterest.
/// </summary>
public bool SubscriptionInterest(string subject) => Interest(subject) > 0;
/// <summary>
/// Returns the total number of matching subscriptions (plain + queue) for the given subject.
/// Go reference: accounts.go Interest.
/// </summary>
public int Interest(string subject)
{
var (plainCount, queueCount) = SubList.NumInterest(subject);
return plainCount + queueCount;
}
/// <summary>
/// Returns the total number of outstanding response mappings for service exports.
/// Go reference: accounts.go NumPendingAllResponses.
/// </summary>
public int NumPendingAllResponses() => NumPendingResponses(string.Empty);
/// <summary>
/// Returns the number of outstanding response mappings for service exports.
/// When <paramref name="filter"/> is empty, counts all mappings.
/// Go reference: accounts.go NumPendingResponses.
/// </summary>
public int NumPendingResponses(string filter)
{
if (string.IsNullOrEmpty(filter))
return Exports.Responses.Count;
var se = GetServiceExportEntry(filter);
if (se == null)
return 0;
var count = 0;
foreach (var (_, si) in Exports.Responses)
{
if (ReferenceEquals(si.Export, se))
count++;
}
return count;
}
/// <summary>
/// Returns the number of configured service import subjects.
/// Go reference: accounts.go NumServiceImports.
/// </summary>
public int NumServiceImports() => Imports.Services.Count;
/// <summary>
/// Removes a response service import mapping.
/// Go reference: accounts.go removeRespServiceImport.
/// </summary>
public void RemoveRespServiceImport(ServiceImport? serviceImport, ResponseServiceImportRemovalReason reason = ResponseServiceImportRemovalReason.Ok)
{
if (serviceImport == null)
return;
string? replyPrefix = null;
foreach (var (prefix, si) in Exports.Responses)
{
if (ReferenceEquals(si, serviceImport))
{
replyPrefix = prefix;
break;
}
}
if (replyPrefix == null)
return;
// Current parity scope removes the response mapping. Reason-specific
// metrics/latency side effects are tracked separately.
ResponseRouter.CleanupResponse(this, replyPrefix, serviceImport);
_ = reason;
}
private ServiceExport? GetServiceExportEntry(string subject)
{
if (Exports.Services.TryGetValue(subject, out var exact))
return exact;
foreach (var (pattern, export) in Exports.Services)
{
if (SubjectMatch.MatchLiteral(subject, pattern))
return export;
}
return null;
}
/// <summary>
/// Returns all service import subjects registered on this account that are currently
/// shadowed by a local subscription in the SubList.
@@ -945,6 +1040,17 @@ public sealed record ActivationCheckResult(
DateTime? ExpiresAt,
TimeSpan? TimeToExpiry);
/// <summary>
/// Reason for removing a response service import.
/// Go reference: accounts.go rsiReason enum.
/// </summary>
public enum ResponseServiceImportRemovalReason
{
Ok = 0,
NoDelivery = 1,
Timeout = 2,
}
/// <summary>
/// Snapshot of account JWT claim fields used for hot-reload diff detection.
/// Go reference: server/accounts.go — AccountClaims / jwt.AccountClaims fields applied in updateAccountClaimsWithRefresh (~line 3374).

View File

@@ -1,4 +1,5 @@
using System.Security.Cryptography;
using NATS.Server.Protocol;
namespace NATS.Server.Auth;
@@ -33,11 +34,13 @@ public sealed class AuthService
var authRequired = false;
var nonceRequired = false;
Dictionary<string, User>? usersMap = null;
var users = NormalizeUsers(options.Users);
var nkeys = NormalizeNKeys(options.NKeys);
// TLS certificate mapping (highest priority when enabled)
if (options.TlsMap && options.TlsVerify && options.Users is { Count: > 0 })
if (options.TlsMap && options.TlsVerify && users is { Count: > 0 })
{
authenticators.Add(new TlsMapAuthenticator(options.Users));
authenticators.Add(new TlsMapAuthenticator(users));
authRequired = true;
}
@@ -63,19 +66,19 @@ public sealed class AuthService
// Priority order (matching Go): NKeys > Users > Token > SimpleUserPassword
if (options.NKeys is { Count: > 0 })
if (nkeys is { Count: > 0 })
{
authenticators.Add(new NKeyAuthenticator(options.NKeys));
authenticators.Add(new NKeyAuthenticator(nkeys));
authRequired = true;
nonceRequired = true;
}
if (options.Users is { Count: > 0 })
if (users is { Count: > 0 })
{
authenticators.Add(new UserPasswordAuthenticator(options.Users));
authenticators.Add(new UserPasswordAuthenticator(users));
authRequired = true;
usersMap = new Dictionary<string, User>(StringComparer.Ordinal);
foreach (var u in options.Users)
foreach (var u in users)
usersMap[u.Username] = u;
}
@@ -169,4 +172,77 @@ public sealed class AuthService
.Replace('+', '-')
.Replace('/', '_');
}
private static IReadOnlyList<User>? NormalizeUsers(IReadOnlyList<User>? users)
{
if (users is null)
return null;
var normalized = new List<User>(users.Count);
foreach (var user in users)
{
normalized.Add(new User
{
Username = user.Username,
Password = user.Password,
Account = string.IsNullOrWhiteSpace(user.Account) ? Account.GlobalAccountName : user.Account,
Permissions = NormalizePermissions(user.Permissions),
ConnectionDeadline = user.ConnectionDeadline,
AllowedConnectionTypes = user.AllowedConnectionTypes,
ProxyRequired = user.ProxyRequired,
});
}
return normalized;
}
private static IReadOnlyList<NKeyUser>? NormalizeNKeys(IReadOnlyList<NKeyUser>? nkeys)
{
if (nkeys is null)
return null;
var normalized = new List<NKeyUser>(nkeys.Count);
foreach (var nkey in nkeys)
{
normalized.Add(new NKeyUser
{
Nkey = nkey.Nkey,
Account = string.IsNullOrWhiteSpace(nkey.Account) ? Account.GlobalAccountName : nkey.Account,
Permissions = NormalizePermissions(nkey.Permissions),
SigningKey = nkey.SigningKey,
Issued = nkey.Issued,
AllowedConnectionTypes = nkey.AllowedConnectionTypes,
ProxyRequired = nkey.ProxyRequired,
});
}
return normalized;
}
private static Permissions? NormalizePermissions(Permissions? permissions)
{
if (permissions?.Response is null)
return permissions;
var publish = permissions.Publish;
if (publish?.Allow is null)
{
publish = new SubjectPermission
{
Allow = [],
Deny = publish?.Deny,
};
}
var response = permissions.Response;
var maxMsgs = response.MaxMsgs == 0 ? NatsProtocol.DefaultAllowResponseMaxMsgs : response.MaxMsgs;
var expires = response.Expires == TimeSpan.Zero ? NatsProtocol.DefaultAllowResponseExpiration : response.Expires;
return new Permissions
{
Publish = publish,
Subscribe = permissions.Subscribe,
Response = new ResponsePermission { MaxMsgs = maxMsgs, Expires = expires },
};
}
}

View File

@@ -2,6 +2,10 @@ namespace NATS.Server.Auth;
public sealed class ExternalAuthCalloutAuthenticator : IAuthenticator
{
public const string AuthCalloutSubject = "$SYS.REQ.USER.AUTH";
public const string AuthRequestSubject = "nats-authorization-request";
public const string AuthRequestXKeyHeader = "Nats-Server-Xkey";
private readonly IExternalAuthClient _client;
private readonly TimeSpan _timeout;

View File

@@ -6,4 +6,7 @@ public sealed class NKeyUser
public Permissions? Permissions { get; init; }
public string? Account { get; init; }
public string? SigningKey { get; init; }
public DateTimeOffset? Issued { get; init; }
public IReadOnlySet<string>? AllowedConnectionTypes { get; init; }
public bool ProxyRequired { get; init; }
}

View File

@@ -40,9 +40,88 @@ public sealed class TlsMapAuthenticator : IAuthenticator
if (cn != null && _usersByCn.TryGetValue(cn, out user))
return BuildResult(user);
// Try SAN-based values
var email = cert.GetNameInfo(X509NameType.EmailName, forIssuer: false);
if (!string.IsNullOrWhiteSpace(email) && _usersByCn.TryGetValue(email, out user))
return BuildResult(user);
var dns = cert.GetNameInfo(X509NameType.DnsName, forIssuer: false);
if (!string.IsNullOrWhiteSpace(dns) && _usersByCn.TryGetValue(dns, out user))
return BuildResult(user);
var uri = cert.GetNameInfo(X509NameType.UrlName, forIssuer: false);
if (!string.IsNullOrWhiteSpace(uri) && _usersByCn.TryGetValue(uri, out user))
return BuildResult(user);
// Match using full RDN + DC components if present.
var dcs = GetTlsAuthDcs(dn);
if (!string.IsNullOrEmpty(dcs))
{
var rdnWithDcs = string.IsNullOrEmpty(dnString) ? dcs : $"{dnString},{dcs}";
if (_usersByDn.TryGetValue(rdnWithDcs, out user))
return BuildResult(user);
}
return null;
}
internal static string GetTlsAuthDcs(X500DistinguishedName dn)
{
if (string.IsNullOrWhiteSpace(dn.Name))
return string.Empty;
var dcs = new List<string>();
foreach (var rdn in dn.Name.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
if (!rdn.StartsWith("DC=", StringComparison.OrdinalIgnoreCase))
continue;
dcs.Add("DC=" + rdn[3..].Trim());
}
return string.Join(",", dcs);
}
internal static string[] DnsAltNameLabels(string dnsAltName)
{
if (string.IsNullOrWhiteSpace(dnsAltName))
return [];
return dnsAltName.ToLowerInvariant().Split('.', StringSplitOptions.RemoveEmptyEntries);
}
internal static bool DnsAltNameMatches(string[] dnsAltNameLabels, IReadOnlyList<Uri?> urls)
{
foreach (var url in urls)
{
if (url == null)
continue;
var hostLabels = url.DnsSafeHost.ToLowerInvariant().Split('.', StringSplitOptions.RemoveEmptyEntries);
if (hostLabels.Length != dnsAltNameLabels.Length)
continue;
var i = 0;
if (dnsAltNameLabels.Length > 0 && dnsAltNameLabels[0] == "*")
i = 1;
var matched = true;
for (; i < dnsAltNameLabels.Length; i++)
{
if (!string.Equals(dnsAltNameLabels[i], hostLabels[i], StringComparison.Ordinal))
{
matched = false;
break;
}
}
if (matched)
return true;
}
return false;
}
private static string? ExtractCn(X500DistinguishedName dn)
{
var dnString = dn.Name;

View File

@@ -7,4 +7,6 @@ public sealed class User
public Permissions? Permissions { get; init; }
public string? Account { get; init; }
public DateTimeOffset? ConnectionDeadline { get; init; }
public IReadOnlySet<string>? AllowedConnectionTypes { get; init; }
public bool ProxyRequired { get; init; }
}