Fix E2E test gaps and add comprehensive E2E + parity test suites

- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status
This commit is contained in:
Joseph Doherty
2026-03-12 14:09:23 -04:00
parent 79c1ee8776
commit c30e67a69d
226 changed files with 17801 additions and 709 deletions

View File

@@ -1,22 +1,29 @@
using System.Security.Cryptography;
using System.Security.Authentication;
using System.Text;
using System.Text.Json;
using NATS.Server.Auth;
using NATS.Server.Configuration;
using NATS.Server.Protocol;
using NATS.Server.Tls;
namespace NATS.Server;
public sealed class NatsOptions
{
public string Host { get; set; } = "0.0.0.0";
public int Port { get; set; } = 4222;
private static bool _allowUnknownTopLevelFields;
private string _configDigest = string.Empty;
public string Host { get; set; } = NatsProtocol.DefaultHost;
public int Port { get; set; } = NatsProtocol.DefaultPort;
public string? ServerName { get; set; }
public int MaxPayload { get; set; } = 1024 * 1024;
public int MaxControlLine { get; set; } = 4096;
public int MaxConnections { get; set; } = 65536;
public int MaxConnections { get; set; } = NatsProtocol.DefaultMaxConnections;
public long MaxPending { get; set; } = 64 * 1024 * 1024; // 64MB, matching Go MAX_PENDING_SIZE
public TimeSpan WriteDeadline { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan PingInterval { get; set; } = TimeSpan.FromMinutes(2);
public int MaxPingsOut { get; set; } = 2;
public TimeSpan WriteDeadline { get; set; } = NatsProtocol.DefaultFlushDeadline;
public TimeSpan PingInterval { get; set; } = NatsProtocol.DefaultPingInterval;
public int MaxPingsOut { get; set; } = NatsProtocol.DefaultPingMaxOut;
// Subscription limits
public int MaxSubs { get; set; } // 0 = unlimited (per-connection)
@@ -45,7 +52,7 @@ public sealed class NatsOptions
public Auth.ProxyAuthOptions? ProxyAuth { get; set; }
// Auth timing
public TimeSpan AuthTimeout { get; set; } = TimeSpan.FromSeconds(2);
public TimeSpan AuthTimeout { get; set; } = NatsProtocol.AuthTimeout;
// Monitoring (0 = disabled; standard port is 8222)
public int MonitorPort { get; set; }
@@ -55,8 +62,8 @@ public sealed class NatsOptions
public int MonitorHttpsPort { get; set; }
// Lifecycle / lame-duck mode
public TimeSpan LameDuckDuration { get; set; } = TimeSpan.FromMinutes(2);
public TimeSpan LameDuckGracePeriod { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan LameDuckDuration { get; set; } = NatsProtocol.DefaultLameDuckDuration;
public TimeSpan LameDuckGracePeriod { get; set; } = NatsProtocol.DefaultLameDuckGracePeriod;
// File paths
public string? PidFile { get; set; }
@@ -82,10 +89,10 @@ public sealed class NatsOptions
public bool TraceVerbose { get; set; }
public int MaxTracedMsgLen { get; set; }
public bool DisableSublistCache { get; set; }
public int ConnectErrorReports { get; set; } = 3600;
public int ReconnectErrorReports { get; set; } = 1;
public int ConnectErrorReports { get; set; } = NatsProtocol.DefaultConnectErrorReports;
public int ReconnectErrorReports { get; set; } = NatsProtocol.DefaultReconnectErrorReports;
public bool NoHeaderSupport { get; set; }
public int MaxClosedClients { get; set; } = 10_000;
public int MaxClosedClients { get; set; } = NatsProtocol.DefaultMaxClosedClients;
public bool NoSystemAccount { get; set; }
public string? SystemAccount { get; set; }
@@ -98,9 +105,9 @@ public sealed class NatsOptions
public string? TlsCaCert { get; set; }
public bool TlsVerify { get; set; }
public bool TlsMap { get; set; }
public TimeSpan TlsTimeout { get; set; } = TimeSpan.FromSeconds(2);
public TimeSpan TlsTimeout { get; set; } = NatsProtocol.TlsTimeout;
public bool TlsHandshakeFirst { get; set; }
public TimeSpan TlsHandshakeFirstFallback { get; set; } = TimeSpan.FromMilliseconds(50);
public TimeSpan TlsHandshakeFirstFallback { get; set; } = NatsProtocol.DefaultTlsHandshakeFirstFallbackDelay;
public bool AllowNonTls { get; set; }
public long TlsRateLimit { get; set; }
public HashSet<string>? TlsPinnedCerts { get; set; }
@@ -133,6 +140,141 @@ public sealed class NatsOptions
// WebSocket
public WebSocketOptions WebSocket { get; set; } = new();
public static void NoErrOnUnknownFields(bool noError)
{
_allowUnknownTopLevelFields = noError;
}
internal static bool AllowUnknownTopLevelFields => _allowUnknownTopLevelFields;
public static List<Uri> RoutesFromStr(string routesStr)
{
if (string.IsNullOrWhiteSpace(routesStr))
return [];
var routes = new List<Uri>();
foreach (var route in routesStr.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
if (Uri.TryCreate(route, UriKind.Absolute, out var uri))
routes.Add(uri);
}
return routes;
}
public NatsOptions Clone()
{
try
{
var json = JsonSerializer.Serialize(this);
var clone = JsonSerializer.Deserialize<NatsOptions>(json) ?? new NatsOptions();
clone.InCmdLine.Clear();
foreach (var flag in InCmdLine)
clone.InCmdLine.Add(flag);
if (TlsPinnedCerts != null)
clone.TlsPinnedCerts = [.. TlsPinnedCerts];
return clone;
}
catch
{
var clone = new NatsOptions();
CopyFrom(clone, this);
clone.InCmdLine.Clear();
foreach (var flag in InCmdLine)
clone.InCmdLine.Add(flag);
if (Tags != null)
clone.Tags = new Dictionary<string, string>(Tags);
if (SubjectMappings != null)
clone.SubjectMappings = new Dictionary<string, string>(SubjectMappings);
if (TlsPinnedCerts != null)
clone.TlsPinnedCerts = [.. TlsPinnedCerts];
return clone;
}
}
public void ProcessConfigString(string data)
{
var parsed = ConfigProcessor.ProcessConfig(data);
CopyFrom(this, parsed);
_configDigest = ComputeDigest(data);
}
public string ConfigDigest() => _configDigest;
private static void CopyFrom(NatsOptions destination, NatsOptions source)
{
foreach (var prop in typeof(NatsOptions).GetProperties())
{
if (!prop.CanRead || !prop.CanWrite)
continue;
prop.SetValue(destination, prop.GetValue(source));
}
}
private static string ComputeDigest(string text)
{
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(text));
return Convert.ToHexString(hash).ToLowerInvariant();
}
}
public sealed class JSLimitOpts
{
public int MaxRequestBatch { get; set; }
public int MaxAckPending { get; set; }
public int MaxHAAssets { get; set; }
public TimeSpan Duplicates { get; set; }
public int MaxBatchInflightPerStream { get; set; }
public int MaxBatchInflightTotal { get; set; }
public int MaxBatchSize { get; set; }
public TimeSpan MaxBatchTimeout { get; set; }
}
public sealed class AuthCallout
{
public string Issuer { get; set; } = string.Empty;
public string Account { get; set; } = string.Empty;
public List<string> AuthUsers { get; set; } = [];
public string XKey { get; set; } = string.Empty;
public List<string> AllowedAccounts { get; set; } = [];
}
public sealed class ProxiesConfig
{
public List<ProxyConfig> Trusted { get; set; } = [];
}
public sealed class ProxyConfig
{
public string Key { get; set; } = string.Empty;
}
public sealed class Ports
{
public List<string> Nats { get; set; } = [];
public List<string> Monitoring { get; set; } = [];
public List<string> Cluster { get; set; } = [];
public List<string> Profile { get; set; } = [];
public List<string> WebSocket { get; set; } = [];
public List<string> LeafNodes { get; set; } = [];
}
public static class CompressionModes
{
public const string Off = "off";
public const string Accept = "accept";
public const string S2Fast = "s2_fast";
public const string S2Better = "s2_better";
public const string S2Best = "s2_best";
public const string S2Uncompressed = "s2_uncompressed";
public const string S2Auto = "s2_auto";
}
public sealed class CompressionOpts
{
public string Mode { get; set; } = CompressionModes.Off;
public List<int> RTTThresholds { get; set; } = [10, 50, 100, 250];
}
public sealed class WebSocketOptions
@@ -158,4 +300,8 @@ public sealed class WebSocketOptions
public TimeSpan HandshakeTimeout { get; set; } = TimeSpan.FromSeconds(2);
public TimeSpan? PingInterval { get; set; }
public Dictionary<string, string>? Headers { get; set; }
// Go websocket.go srvWebsocket.authOverride parity bit:
// true when websocket auth options override top-level auth config.
public bool AuthOverride { get; internal set; }
}