dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix E2E test gaps and add comprehensive E2E + parity test suites

Browse Source 
- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status
This commit is contained in:
Joseph Doherty
2026-03-12 14:09:23 -04:00
parent 79c1ee8776
commit c30e67a69d
226 changed files with 17801 additions and 709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
src/NATS.Server/Configuration/ConfigProcessor.cs
View File

@@ -4,6 +4,8 @@
using System.Globalization;
using System.Text.RegularExpressions;
using NATS.Server.Auth;
using NATS.Server.JetStream;
using NATS.Server.Tls;
namespace NATS.Server.Configuration;
@@ -43,12 +45,13 @@ public static class ConfigProcessor
public static void ApplyConfig(Dictionary<string, object?> config, NatsOptions opts)
{
var errors = new List<string>();
var warnings = new List<string>();
foreach (var (key, value) in config)
{
try
{
ProcessKey(key, value, opts, errors);
ProcessKey(key, value, opts, errors, warnings);
}
catch (Exception ex)
{
@@ -58,11 +61,16 @@ public static class ConfigProcessor
if (errors.Count > 0)
{
throw new ConfigProcessorException("Configuration errors", errors);
throw new ConfigProcessorException("Configuration errors", errors, warnings);
}
}
private static void ProcessKey(string key, object? value, NatsOptions opts, List<string> errors)
private static void ProcessKey(
string key,
object? value,
NatsOptions opts,
List<string> errors,
List<string> warnings)
{
// Keys are already case-insensitive from the parser (OrdinalIgnoreCase dictionaries),
// but we normalize here for the switch statement.
@@ -277,8 +285,15 @@ public static class ConfigProcessor
ParseWebSocket(wsDict, opts, errors);
break;
// Unknown keys silently ignored (accounts, resolver, operator, etc.)
// Accounts block — each key is an account name containing users/limits
case "accounts":
if (value is Dictionary<string, object?> accountsDict)
ParseAccounts(accountsDict, opts, errors);
break;
// Unknown keys silently ignored (resolver, operator, etc.)
default:
warnings.Add(new UnknownConfigFieldWarning(key).Message);
break;
}
}
@@ -745,6 +760,9 @@ public static class ConfigProcessor
case "store_dir":
options.StoreDir = ToString(value);
break;
case "domain":
options.Domain = ToString(value);
break;
case "max_mem_store":
try
{
@@ -766,6 +784,68 @@ public static class ConfigProcessor
errors.Add($"Invalid jetstream.max_file_store: {ex.Message}");
}
break;
case "sync_interval":
try
{
options.SyncInterval = ParseDuration(value);
}
catch (Exception ex)
{
errors.Add($"Invalid jetstream.sync_interval: {ex.Message}");
}
break;
case "sync_always":
options.SyncAlways = ToBool(value);
break;
case "compress_ok":
options.CompressOk = ToBool(value);
break;
case "unique_tag":
options.UniqueTag = ToString(value);
break;
case "strict":
options.Strict = ToBool(value);
break;
case "max_ack_pending":
options.MaxAckPending = ToInt(value);
break;
case "memory_max_stream_bytes":
try
{
options.MemoryMaxStreamBytes = ParseByteSize(value);
}
catch (Exception ex)
{
errors.Add($"Invalid jetstream.memory_max_stream_bytes: {ex.Message}");
}
break;
case "store_max_stream_bytes":
try
{
options.StoreMaxStreamBytes = ParseByteSize(value);
}
catch (Exception ex)
{
errors.Add($"Invalid jetstream.store_max_stream_bytes: {ex.Message}");
}
break;
case "max_bytes_required":
options.MaxBytesRequired = ToBool(value);
break;
case "tiers":
if (value is Dictionary<string, object?> tiers)
{
foreach (var (tierName, rawTier) in tiers)
{
if (rawTier is Dictionary<string, object?> tierDict)
options.Tiers[tierName] = ParseJetStreamTier(tierName, tierDict, errors);
}
}
break;
}
}
@@ -773,6 +853,47 @@ public static class ConfigProcessor
return options;
}
private static JetStreamTier ParseJetStreamTier(string tierName, Dictionary<string, object?> dict, List<string> errors)
{
var tier = new JetStreamTier { Name = tierName };
foreach (var (key, value) in dict)
{
switch (key.ToLowerInvariant())
{
case "memory" or "max_memory":
try
{
tier.Memory = ParseByteSize(value);
}
catch (Exception ex)
{
errors.Add($"Invalid jetstream.tiers.{tierName}.memory: {ex.Message}");
}
break;
case "store" or "max_store":
try
{
tier.Store = ParseByteSize(value);
}
catch (Exception ex)
{
errors.Add($"Invalid jetstream.tiers.{tierName}.store: {ex.Message}");
}
break;
case "streams" or "max_streams":
tier.Streams = ToInt(value);
break;
case "consumers" or "max_consumers":
tier.Consumers = ToInt(value);
break;
}
}
return tier;
}
// ─── Authorization parsing ─────────────────────────────────────
private static void ParseAuthorization(Dictionary<string, object?> dict, NatsOptions opts, List<string> errors)
@@ -831,12 +952,80 @@ public static class ConfigProcessor
}
}
// ─── Accounts parsing ──────────────────────────────────────────
/// <summary>
/// Parses the top-level "accounts" block. Each key is an account name, and each
/// value is a dictionary that may contain "users" (array) and account-level limits.
/// Users are stamped with the account name and appended to opts.Users / opts.NKeys.
/// Go reference: opts.go — configureAccounts / parseAccounts.
/// </summary>
private static void ParseAccounts(Dictionary<string, object?> accountsDict, NatsOptions opts, List<string> errors)
{
opts.Accounts ??= new Dictionary<string, AccountConfig>();
foreach (var (accountName, accountValue) in accountsDict)
{
if (accountValue is not Dictionary<string, object?> acctDict)
{
errors.Add($"Expected account '{accountName}' value to be a map");
continue;
}
int maxConnections = 0;
int maxSubscriptions = 0;
List<object?>? userList = null;
foreach (var (key, value) in acctDict)
{
switch (key.ToLowerInvariant())
{
case "users":
if (value is List<object?> ul)
userList = ul;
break;
case "max_connections" or "max_conns":
maxConnections = ToInt(value);
break;
case "max_subscriptions" or "max_subs":
maxSubscriptions = ToInt(value);
break;
}
}
opts.Accounts[accountName] = new AccountConfig
{
MaxConnections = maxConnections,
MaxSubscriptions = maxSubscriptions,
};
if (userList is not null)
{
var (plainUsers, nkeyUsers) = ParseUsersAndNkeys(userList, errors, defaultAccount: accountName);
if (plainUsers.Count > 0)
{
var existing = opts.Users?.ToList() ?? [];
existing.AddRange(plainUsers);
opts.Users = existing;
}
if (nkeyUsers.Count > 0)
{
var existing = opts.NKeys?.ToList() ?? [];
existing.AddRange(nkeyUsers);
opts.NKeys = existing;
}
}
}
}
/// <summary>
/// Splits a users array into plain users and NKey users.
/// An entry with an "nkey" field is an NKey user; entries with "user" are plain users.
/// Go reference: opts.go — parseUsers (lines ~2500-2700).
/// </summary>
private static (List<User> PlainUsers, List<Auth.NKeyUser> NkeyUsers) ParseUsersAndNkeys(List<object?> list, List<string> errors)
private static (List<User> PlainUsers, List<Auth.NKeyUser> NkeyUsers) ParseUsersAndNkeys(List<object?> list, List<string> errors, string? defaultAccount = null)
{
var plainUsers = new List<User>();
var nkeyUsers = new List<Auth.NKeyUser>();
@@ -888,7 +1077,7 @@ public static class ConfigProcessor
{
Nkey = nkey,
Permissions = permissions,
Account = account,
Account = account ?? defaultAccount,
});
continue;
}
@@ -903,7 +1092,7 @@ public static class ConfigProcessor
{
Username = username,
Password = password ?? string.Empty,
Account = account,
Account = account ?? defaultAccount,
Permissions = permissions,
});
}
@@ -1087,6 +1276,9 @@ public static class ConfigProcessor
case "handshake_first_fallback":
opts.TlsHandshakeFirstFallback = ParseDuration(value);
break;
case "ocsp_peer":
ParseOcspPeer(value, opts, errors);
break;
default:
// Unknown TLS keys silently ignored
break;
@@ -1094,6 +1286,31 @@ public static class ConfigProcessor
}
}
private static void ParseOcspPeer(object? value, NatsOptions opts, List<string> errors)
{
switch (value)
{
case bool verify:
opts.OcspPeerVerify = verify;
return;
case Dictionary<string, object?> dict:
try
{
var cfg = OCSPPeerConfig.Parse(dict);
opts.OcspPeerVerify = cfg.Verify;
}
catch (FormatException ex)
{
errors.Add(ex.Message);
}
return;
default:
errors.Add($"expected map to define OCSP peer options, got [{value?.GetType().Name ?? "null"}]");
return;
}
}
// ─── Tags parsing ──────────────────────────────────────────────
private static void ParseTags(Dictionary<string, object?> dict, NatsOptions opts)
@@ -1431,8 +1648,28 @@ public static class ConfigProcessor
/// Thrown when one or more configuration validation errors are detected.
/// All errors are collected rather than failing on the first one.
/// </summary>
public sealed class ConfigProcessorException(string message, List<string> errors)
public sealed class ConfigProcessorException(string message, List<string> errors, List<string>? warnings = null)
: Exception(message)
{
public IReadOnlyList<string> Errors => errors;
public IReadOnlyList<string> Warnings => warnings ?? [];
}
/// <summary>
/// Represents a non-fatal configuration warning.
/// Go reference: configWarningErr.
/// </summary>
public class ConfigWarningException(string message, string? source = null) : Exception(message)
{
public string? SourceLocation { get; } = source;
}
/// <summary>
/// Warning used when an unknown config field is encountered.
/// Go reference: unknownConfigFieldErr.
/// </summary>
public sealed class UnknownConfigFieldWarning(string field, string? source = null)
: ConfigWarningException($"unknown field {field}", source)
{
public string Field { get; } = field;
}