Fix E2E test gaps and add comprehensive E2E + parity test suites

- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status
This commit is contained in:
Joseph Doherty
2026-03-12 14:09:23 -04:00
parent 79c1ee8776
commit c30e67a69d
226 changed files with 17801 additions and 709 deletions

View File

@@ -0,0 +1,192 @@
using System.Security.Cryptography.X509Certificates;
using System.Text.Json;
using System.Text.Json.Serialization;
using NATS.Server.Configuration;
namespace NATS.Server.Tls;
[JsonConverter(typeof(StatusAssertionJsonConverter))]
public enum StatusAssertion
{
Good = 0,
Revoked = 1,
Unknown = 2,
}
public static class StatusAssertionMaps
{
public static readonly IReadOnlyDictionary<string, StatusAssertion> StatusAssertionStrToVal =
new Dictionary<string, StatusAssertion>(StringComparer.OrdinalIgnoreCase)
{
["good"] = StatusAssertion.Good,
["revoked"] = StatusAssertion.Revoked,
["unknown"] = StatusAssertion.Unknown,
};
public static readonly IReadOnlyDictionary<StatusAssertion, string> StatusAssertionValToStr =
new Dictionary<StatusAssertion, string>
{
[StatusAssertion.Good] = "good",
[StatusAssertion.Revoked] = "revoked",
[StatusAssertion.Unknown] = "unknown",
};
public static readonly IReadOnlyDictionary<int, StatusAssertion> StatusAssertionIntToVal =
new Dictionary<int, StatusAssertion>
{
[0] = StatusAssertion.Good,
[1] = StatusAssertion.Revoked,
[2] = StatusAssertion.Unknown,
};
public static string GetStatusAssertionStr(int sa)
{
var value = StatusAssertionIntToVal.TryGetValue(sa, out var mapped)
? mapped
: StatusAssertion.Unknown;
return StatusAssertionValToStr[value];
}
}
public sealed class StatusAssertionJsonConverter : JsonConverter<StatusAssertion>
{
public override StatusAssertion Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.String)
{
var str = reader.GetString();
if (str is not null && StatusAssertionMaps.StatusAssertionStrToVal.TryGetValue(str, out var mapped))
return mapped;
return StatusAssertion.Unknown;
}
if (reader.TokenType == JsonTokenType.Number && reader.TryGetInt32(out var v))
{
return StatusAssertionMaps.StatusAssertionIntToVal.TryGetValue(v, out var mapped)
? mapped
: StatusAssertion.Unknown;
}
return StatusAssertion.Unknown;
}
public override void Write(Utf8JsonWriter writer, StatusAssertion value, JsonSerializerOptions options)
{
if (!StatusAssertionMaps.StatusAssertionValToStr.TryGetValue(value, out var str))
str = StatusAssertionMaps.StatusAssertionValToStr[StatusAssertion.Unknown];
writer.WriteStringValue(str);
}
}
public sealed class ChainLink
{
public X509Certificate2? Leaf { get; set; }
public X509Certificate2? Issuer { get; set; }
public IReadOnlyList<Uri>? OCSPWebEndpoints { get; set; }
}
public sealed class OcspResponseInfo
{
public DateTime ThisUpdate { get; init; }
public DateTime? NextUpdate { get; init; }
}
public sealed class CertInfo
{
[JsonPropertyName("subject")]
public string Subject { get; init; } = string.Empty;
[JsonPropertyName("issuer")]
public string Issuer { get; init; } = string.Empty;
[JsonPropertyName("fingerprint")]
public string Fingerprint { get; init; } = string.Empty;
[JsonPropertyName("raw")]
public byte[] Raw { get; init; } = [];
}
public sealed class OCSPPeerConfig
{
public static readonly TimeSpan DefaultAllowedClockSkew = TimeSpan.FromSeconds(30);
public static readonly TimeSpan DefaultOCSPResponderTimeout = TimeSpan.FromSeconds(2);
public static readonly TimeSpan DefaultTTLUnsetNextUpdate = TimeSpan.FromHours(1);
public bool Verify { get; set; }
public double Timeout { get; set; } = DefaultOCSPResponderTimeout.TotalSeconds;
public double ClockSkew { get; set; } = DefaultAllowedClockSkew.TotalSeconds;
public bool WarnOnly { get; set; }
public bool UnknownIsGood { get; set; }
public bool AllowWhenCAUnreachable { get; set; }
public double TTLUnsetNextUpdate { get; set; } = DefaultTTLUnsetNextUpdate.TotalSeconds;
public static OCSPPeerConfig NewOCSPPeerConfig() => new();
public static OCSPPeerConfig Parse(IReadOnlyDictionary<string, object?> values)
{
var cfg = NewOCSPPeerConfig();
foreach (var (key, rawValue) in values)
{
switch (key.ToLowerInvariant())
{
case "verify":
cfg.Verify = ParseBool(rawValue, key);
break;
case "allowed_clockskew":
ApplyIfNonNegative(rawValue, key, v => cfg.ClockSkew = v);
break;
case "ca_timeout":
ApplyIfNonNegative(rawValue, key, v => cfg.Timeout = v);
break;
case "cache_ttl_when_next_update_unset":
ApplyIfNonNegative(rawValue, key, v => cfg.TTLUnsetNextUpdate = v);
break;
case "warn_only":
cfg.WarnOnly = ParseBool(rawValue, key);
break;
case "unknown_is_good":
cfg.UnknownIsGood = ParseBool(rawValue, key);
break;
case "allow_when_ca_unreachable":
cfg.AllowWhenCAUnreachable = ParseBool(rawValue, key);
break;
default:
throw new FormatException($"error parsing tls peer config, unknown field [{key}]");
}
}
return cfg;
}
private static bool ParseBool(object? rawValue, string key)
{
if (rawValue is bool b)
return b;
throw new FormatException($"error parsing tls peer config, unknown field [{key}]");
}
private static void ApplyIfNonNegative(object? rawValue, string key, Action<double> apply)
{
var parsed = ParseSeconds(rawValue, key);
if (parsed >= 0)
apply(parsed);
}
private static double ParseSeconds(object? rawValue, string key)
{
try
{
return rawValue switch
{
long l => l,
double d => d,
string s => ConfigProcessor.ParseDuration(s).TotalSeconds,
_ => throw new FormatException("unexpected type"),
};
}
catch (Exception ex)
{
throw new FormatException($"error parsing tls peer config, conversion error: {ex.Message}", ex);
}
}
}

View File

@@ -0,0 +1,85 @@
namespace NATS.Server.Tls;
public static class OcspPeerMessages
{
// Returned errors
public const string ErrIllegalPeerOptsConfig = "expected map to define OCSP peer options, got [%T]";
public const string ErrIllegalCacheOptsConfig = "expected map to define OCSP peer cache options, got [%T]";
public const string ErrParsingPeerOptFieldGeneric = "error parsing tls peer config, unknown field [%q]";
public const string ErrParsingPeerOptFieldTypeConversion = "error parsing tls peer config, conversion error: %s";
public const string ErrParsingCacheOptFieldTypeConversion = "error parsing OCSP peer cache config, conversion error: %s";
public const string ErrUnableToPlugTLSEmptyConfig = "unable to plug TLS verify connection, config is nil";
public const string ErrMTLSRequired = "OCSP peer verification for client connections requires TLS verify (mTLS) to be enabled";
public const string ErrUnableToPlugTLSClient = "unable to register client OCSP verification";
public const string ErrUnableToPlugTLSServer = "unable to register server OCSP verification";
public const string ErrCannotWriteCompressed = "error writing to compression writer: %w";
public const string ErrCannotReadCompressed = "error reading compression reader: %w";
public const string ErrTruncatedWrite = "short write on body (%d != %d)";
public const string ErrCannotCloseWriter = "error closing compression writer: %w";
public const string ErrParsingCacheOptFieldGeneric = "error parsing OCSP peer cache config, unknown field [%q]";
public const string ErrUnknownCacheType = "error parsing OCSP peer cache config, unknown type [%s]";
public const string ErrInvalidChainlink = "invalid chain link";
public const string ErrBadResponderHTTPStatus = "bad OCSP responder http status: [%d]";
public const string ErrNoAvailOCSPServers = "no available OCSP servers";
public const string ErrFailedWithAllRequests = "exhausted OCSP responders: %w";
// Direct logged errors
public const string ErrLoadCacheFail = "Unable to load OCSP peer cache: %s";
public const string ErrSaveCacheFail = "Unable to save OCSP peer cache: %s";
public const string ErrBadCacheTypeConfig = "Unimplemented OCSP peer cache type [%v]";
public const string ErrResponseCompressFail = "Unable to compress OCSP response for key [%s]: %s";
public const string ErrResponseDecompressFail = "Unable to decompress OCSP response for key [%s]: %s";
public const string ErrPeerEmptyNoEvent = "Peer certificate is nil, cannot send OCSP peer reject event";
public const string ErrPeerEmptyAutoReject = "Peer certificate is nil, rejecting OCSP peer";
// Debug information
public const string DbgPlugTLSForKind = "Plugging TLS OCSP peer for [%s]";
public const string DbgNumServerChains = "Peer OCSP enabled: %d TLS server chain(s) will be evaluated";
public const string DbgNumClientChains = "Peer OCSP enabled: %d TLS client chain(s) will be evaluated";
public const string DbgLinksInChain = "Chain [%d]: %d total link(s)";
public const string DbgSelfSignedValid = "Chain [%d] is self-signed, thus peer is valid";
public const string DbgValidNonOCSPChain = "Chain [%d] has no OCSP eligible links, thus peer is valid";
public const string DbgChainIsOCSPEligible = "Chain [%d] has %d OCSP eligible link(s)";
public const string DbgChainIsOCSPValid = "Chain [%d] is OCSP valid for all eligible links, thus peer is valid";
public const string DbgNoOCSPValidChains = "No OCSP valid chains, thus peer is invalid";
public const string DbgCheckingCacheForCert = "Checking OCSP peer cache for [%s], key [%s]";
public const string DbgCurrentResponseCached = "Cached OCSP response is current, status [%s]";
public const string DbgExpiredResponseCached = "Cached OCSP response is expired, status [%s]";
public const string DbgOCSPValidPeerLink = "OCSP verify pass for [%s]";
public const string DbgCachingResponse = "Caching OCSP response for [%s], key [%s]";
public const string DbgAchievedCompression = "OCSP response compression ratio: [%f]";
public const string DbgCacheHit = "OCSP peer cache hit for key [%s]";
public const string DbgCacheMiss = "OCSP peer cache miss for key [%s]";
public const string DbgPreservedRevocation = "Revoked OCSP response for key [%s] preserved by cache policy";
public const string DbgDeletingCacheResponse = "Deleting OCSP peer cached response for key [%s]";
public const string DbgStartingCache = "Starting OCSP peer cache";
public const string DbgStoppingCache = "Stopping OCSP peer cache";
public const string DbgLoadingCache = "Loading OCSP peer cache [%s]";
public const string DbgNoCacheFound = "No OCSP peer cache found, starting with empty cache";
public const string DbgSavingCache = "Saving OCSP peer cache [%s]";
public const string DbgCacheSaved = "Saved OCSP peer cache successfully (%d bytes)";
public const string DbgMakingCARequest = "Trying OCSP responder url [%s]";
public const string DbgResponseExpired = "OCSP response NextUpdate [%s] is before now [%s] with clockskew [%s]";
public const string DbgResponseTTLExpired = "OCSP response cache expiry [%s] is before now [%s] with clockskew [%s]";
public const string DbgResponseFutureDated = "OCSP response ThisUpdate [%s] is before now [%s] with clockskew [%s]";
public const string DbgCacheSaveTimerExpired = "OCSP peer cache save timer expired";
public const string DbgCacheDirtySave = "OCSP peer cache is dirty, saving";
public const string MsgTLSClientRejectConnection = "client not OCSP valid";
public const string MsgTLSServerRejectConnection = "server not OCSP valid";
public const string ErrCAResponderCalloutFail = "Attempt to obtain OCSP response from CA responder for [%s] failed: %s";
public const string ErrNewCAResponseNotCurrent = "New OCSP CA response obtained for [%s] but not current";
public const string ErrCAResponseParseFailed = "Could not parse OCSP CA response for [%s]: %s";
public const string ErrOCSPInvalidPeerLink = "OCSP verify fail for [%s] with CA status [%s]";
public const string MsgAllowWhenCAUnreachableOccurred = "Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; no cached revocation so allowing";
public const string MsgAllowWhenCAUnreachableOccurredCachedRevoke = "Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; cached revocation exists so rejecting";
public const string MsgAllowWarnOnlyOccurred = "OCSP verify fail for [%s] but WarnOnly is true so allowing";
public const string MsgCacheOnline = "OCSP peer cache online, type [%s]";
public const string MsgCacheOffline = "OCSP peer cache offline, type [%s]";
public const string MsgFailedOCSPResponseFetch = "Failed OCSP response fetch";
public const string MsgOCSPResponseNotEffective = "OCSP response not in effectivity window";
public const string MsgFailedOCSPResponseParse = "Failed OCSP response parse";
public const string MsgOCSPResponseInvalidStatus = "Invalid OCSP response status: %s";
public const string MsgOCSPResponseDelegationInvalid = "Invalid OCSP response delegation: %s";
public const string MsgCachedOCSPResponseInvalid = "Invalid cached OCSP response for [%s] with fingerprint [%s]";
}

View File

@@ -1,4 +1,6 @@
using System.Net.Security;
using System.Formats.Asn1;
using System.Text.RegularExpressions;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
@@ -7,6 +9,10 @@ namespace NATS.Server.Tls;
public static class TlsHelper
{
private const string AuthorityInfoAccessOid = "1.3.6.1.5.5.7.1.1";
private const string OcspAccessMethodOid = "1.3.6.1.5.5.7.48.1";
private const string OcspSigningEkuOid = "1.3.6.1.5.5.7.3.9";
public static X509Certificate2 LoadCertificate(string certPath, string? keyPath)
{
if (keyPath != null)
@@ -16,9 +22,48 @@ public static class TlsHelper
public static X509Certificate2Collection LoadCaCertificates(string caPath)
{
var collection = new X509Certificate2Collection();
collection.ImportFromPemFile(caPath);
return collection;
var pem = File.ReadAllText(caPath);
return ParseCertPem(pem);
}
/// <summary>
/// Parses one or more PEM blocks and requires all blocks to be CERTIFICATE.
/// Mirrors Go parseCertPEM behavior by rejecting unexpected block types.
/// </summary>
public static X509Certificate2Collection ParseCertPem(string pemData)
{
if (string.IsNullOrWhiteSpace(pemData))
throw new InvalidDataException("PEM data is empty.");
var beginMatches = Regex.Matches(pemData, "-----BEGIN ([^-]+)-----");
if (beginMatches.Count == 0)
throw new InvalidDataException("No PEM certificate block found.");
foreach (Match match in beginMatches)
{
var label = match.Groups[1].Value;
if (!string.Equals(label, "CERTIFICATE", StringComparison.Ordinal))
throw new InvalidDataException($"unexpected PEM certificate type: {label}");
}
var certs = new X509Certificate2Collection();
var certMatches = Regex.Matches(
pemData,
"-----BEGIN CERTIFICATE-----\\s*(?<body>[A-Za-z0-9+/=\\r\\n]+?)\\s*-----END CERTIFICATE-----",
RegexOptions.Singleline);
foreach (Match certMatch in certMatches)
{
var body = certMatch.Groups["body"].Value;
var normalized = Regex.Replace(body, "\\s+", "", RegexOptions.Singleline);
var der = Convert.FromBase64String(normalized);
certs.Add(X509CertificateLoader.LoadCertificate(der));
}
if (certs.Count == 0)
throw new InvalidDataException("No PEM certificate block found.");
return certs;
}
public static SslServerAuthenticationOptions BuildServerAuthOptions(NatsOptions opts)
@@ -92,9 +137,198 @@ public static class TlsHelper
return Convert.ToHexStringLower(hash);
}
public static string GenerateFingerprint(X509Certificate2 cert)
{
var hash = SHA256.HashData(cert.RawData);
return Convert.ToBase64String(hash);
}
public static IReadOnlyList<Uri> GetWebEndpoints(IEnumerable<string> uris)
{
var urls = new List<Uri>();
foreach (var uri in uris)
{
if (!Uri.TryCreate(uri, UriKind.Absolute, out var endpoint))
continue;
if (!string.Equals(endpoint.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) &&
!string.Equals(endpoint.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
continue;
urls.Add(endpoint);
}
return urls;
}
public static string GetSubjectDNForm(X509Certificate2? cert)
{
return cert?.SubjectName.Name ?? string.Empty;
}
public static string GetIssuerDNForm(X509Certificate2? cert)
{
return cert?.IssuerName.Name ?? string.Empty;
}
public static bool MatchesPinnedCert(X509Certificate2 cert, HashSet<string> pinned)
{
var hash = GetCertificateHash(cert);
return pinned.Contains(hash);
}
/// <summary>
/// Checks if a chain link is eligible for OCSP validation by ensuring the leaf
/// certificate includes at least one valid HTTP(S) OCSP AIA endpoint.
/// </summary>
public static bool CertOCSPEligible(ChainLink? link)
{
if (link?.Leaf is null)
return false;
if (link.Leaf.RawData is null || link.Leaf.RawData.Length == 0)
return false;
var aiaUris = GetOcspResponderUris(link.Leaf);
if (aiaUris.Count == 0)
return false;
var urls = GetWebEndpoints(aiaUris);
if (urls.Count == 0)
return false;
link.OCSPWebEndpoints = urls;
return true;
}
/// <summary>
/// Returns the positional issuer certificate for a leaf in a verified chain.
/// </summary>
public static X509Certificate2? GetLeafIssuerCert(IReadOnlyList<X509Certificate2>? chain, int leafPos)
{
if (chain is null || chain.Count == 0 || leafPos < 0 || leafPos >= chain.Count - 1)
return null;
return chain[leafPos + 1];
}
/// <summary>
/// Equivalent to Go certstore.GetLeafIssuer: verifies the leaf against the
/// supplied trust root and returns the first issuer in the verified chain.
/// </summary>
public static X509Certificate2? GetLeafIssuer(X509Certificate2 leaf, X509Certificate2 trustedRoot)
{
using var chain = new X509Chain();
chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
chain.ChainPolicy.CustomTrustStore.Add(trustedRoot);
chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
chain.ChainPolicy.VerificationFlags = X509VerificationFlags.NoFlag;
if (!chain.Build(leaf) || chain.ChainElements.Count < 2)
return null;
return X509CertificateLoader.LoadCertificate(chain.ChainElements[1].Certificate.RawData);
}
/// <summary>
/// Checks OCSP response currency semantics with clock skew and fallback TTL.
/// </summary>
public static bool OcspResponseCurrent(OcspResponseInfo response, OCSPPeerConfig opts)
{
var skew = TimeSpan.FromSeconds(opts.ClockSkew);
if (skew < TimeSpan.Zero)
skew = OCSPPeerConfig.DefaultAllowedClockSkew;
var now = DateTime.UtcNow;
if (response.NextUpdate.HasValue && response.NextUpdate.Value < now - skew)
return false;
if (!response.NextUpdate.HasValue)
{
var ttl = TimeSpan.FromSeconds(opts.TTLUnsetNextUpdate);
if (ttl < TimeSpan.Zero)
ttl = OCSPPeerConfig.DefaultTTLUnsetNextUpdate;
if (response.ThisUpdate + ttl < now - skew)
return false;
}
if (response.ThisUpdate > now + skew)
return false;
return true;
}
/// <summary>
/// Validates OCSP delegated signer semantics. Direct issuer signatures are valid;
/// delegated certificates must include id-kp-OCSPSigning EKU.
/// </summary>
public static bool ValidDelegationCheck(X509Certificate2? issuer, X509Certificate2? responderCertificate)
{
if (issuer is null)
return false;
if (responderCertificate is null)
return true;
if (responderCertificate.Thumbprint == issuer.Thumbprint)
return true;
foreach (var extension in responderCertificate.Extensions)
{
if (extension is not X509EnhancedKeyUsageExtension eku)
continue;
foreach (var oid in eku.EnhancedKeyUsages)
{
if (oid.Value == OcspSigningEkuOid)
return true;
}
}
return false;
}
[SlopwatchSuppress("SW003", "AsnContentException on a malformed AIA extension is intentionally swallowed; invalid extension shape means no usable OCSP URI")]
private static IReadOnlyList<string> GetOcspResponderUris(X509Certificate2 cert)
{
var uris = new List<string>();
foreach (var extension in cert.Extensions)
{
if (!string.Equals(extension.Oid?.Value, AuthorityInfoAccessOid, StringComparison.Ordinal))
continue;
try
{
var reader = new AsnReader(extension.RawData, AsnEncodingRules.DER);
var seq = reader.ReadSequence();
while (seq.HasData)
{
var accessDescription = seq.ReadSequence();
var accessMethod = accessDescription.ReadObjectIdentifier();
if (!string.Equals(accessMethod, OcspAccessMethodOid, StringComparison.Ordinal))
{
accessDescription.ThrowIfNotEmpty();
continue;
}
var uri = accessDescription.ReadCharacterString(
UniversalTagNumber.IA5String,
new Asn1Tag(TagClass.ContextSpecific, 6));
accessDescription.ThrowIfNotEmpty();
if (!string.IsNullOrWhiteSpace(uri))
uris.Add(uri);
}
seq.ThrowIfNotEmpty();
reader.ThrowIfNotEmpty();
}
catch (AsnContentException ex)
{
// Invalid AIA extension shape should behave as "no usable OCSP URI" — swallow is intentional.
_ = ex.Message;
}
}
return uris;
}
}