Files
natsdotnet/src/NATS.Server/NatsServer.cs
Joseph Doherty 4de691c9c5 perf: add FileStore buffered writes, O(1) state tracking, and eliminate redundant per-publish work
Implement Go-parity background flush loop (coalesce 16KB/8ms) in MsgBlock/FileStore,
replace O(n) GetStateAsync with incremental counters, skip PruneExpired/LoadAsync/
PrunePerSubject when not needed, and bypass RAFT for single-replica streams. Fix counter
tracking bugs in RemoveMsg/EraseMsg/TTL expiry and ObjectDisposedException races in
flush loop disposal. FileStore optimizations verified with 3112/3112 JetStream tests
passing; async publish benchmark remains at ~174 msg/s due to E2E protocol path bottleneck.
2026-03-13 03:11:11 -04:00

2547 lines
100 KiB
C#

using System.Collections.Concurrent;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Security;
using System.Net.Sockets;
using System.Runtime.InteropServices;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using NATS.NKeys;
using NATS.Server.Auth;
using NATS.Server.Auth.Jwt;
using NATS.Server.Configuration;
using NATS.Server.Events;
using NATS.Server.Gateways;
using NATS.Server.Imports;
using NATS.Server.JetStream;
using NATS.Server.JetStream.Api;
using NATS.Server.JetStream.Publish;
using NATS.Server.LeafNodes;
using NATS.Server.Monitoring;
using NATS.Server.Mqtt;
using NATS.Server.Protocol;
using NATS.Server.Routes;
using NATS.Server.Server;
using NATS.Server.Subscriptions;
using NATS.Server.Tls;
using NATS.Server.WebSocket;
namespace NATS.Server;
public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
{
private readonly NatsOptions _options;
private readonly ConcurrentDictionary<ulong, NatsClient> _clients = new();
private ClosedConnectionRingBuffer _closedClients;
private readonly ServerInfo _serverInfo;
private readonly ILogger<NatsServer> _logger;
private readonly ILoggerFactory _loggerFactory;
private readonly ServerStats _stats = new();
private readonly TaskCompletionSource _listeningStarted = new(TaskCreationOptions.RunContinuationsAsynchronously);
private AuthService _authService;
private readonly ConcurrentDictionary<string, Account> _accounts = new(StringComparer.Ordinal);
// Config reload state
private NatsOptions? _cliSnapshot;
private HashSet<string> _cliFlags = [];
private string? _configDigest;
private readonly SemaphoreSlim _reloadMu = new(1, 1);
private AcceptLoopErrorHandler? _acceptLoopErrorHandler;
private readonly Account _globalAccount;
private readonly Account _systemAccount;
private InternalEventSystem? _eventSystem;
private SslServerAuthenticationOptions? _sslOptions;
private readonly TlsRateLimiter? _tlsRateLimiter;
private readonly TlsCertificateProvider? _tlsCertProvider;
private readonly SubjectTransform[] _subjectTransforms;
private readonly RouteManager? _routeManager;
/// <summary>
/// Exposes the route manager for testing. Internal — visible to test project
/// via InternalsVisibleTo.
/// </summary>
internal RouteManager? RouteManager => _routeManager;
internal GatewayManager? GatewayManager => _gatewayManager;
private readonly GatewayManager? _gatewayManager;
private readonly LeafNodeManager? _leafNodeManager;
private readonly InternalClient? _jetStreamInternalClient;
private readonly JetStreamService? _jetStreamService;
private readonly JetStreamApiRouter? _jetStreamApiRouter;
private readonly StreamManager? _jetStreamStreamManager;
private readonly ConsumerManager? _jetStreamConsumerManager;
private readonly JetStreamPublisher? _jetStreamPublisher;
private MqttListener? _mqttListener;
private Socket? _listener;
private Socket? _wsListener;
private readonly TaskCompletionSource _wsAcceptLoopExited = new(TaskCreationOptions.RunContinuationsAsynchronously);
private MonitorServer? _monitorServer;
private ulong _nextClientId;
private long _startTimeTicks;
private readonly CancellationTokenSource _quitCts = new();
private readonly TaskCompletionSource _shutdownComplete = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _acceptLoopExited = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _shutdown;
private int _activeClientCount;
private int _lameDuck;
private byte[] _cachedInfoLine = [];
private readonly List<PosixSignalRegistration> _signalRegistrations = [];
private string? _portsFilePath;
private DateTime _configTime = DateTime.UtcNow;
private static readonly TimeSpan AcceptMinSleep = NatsProtocol.AcceptMinSleep;
private static readonly TimeSpan AcceptMaxSleep = NatsProtocol.AcceptMaxSleep;
private static readonly JsonSerializerOptions s_jetStreamJsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
};
public SubList SubList => _globalAccount.SubList;
public byte[] CachedInfoLine => _cachedInfoLine;
public ServerStats Stats => _stats;
public DateTime StartTime => new(Interlocked.Read(ref _startTimeTicks), DateTimeKind.Utc);
public string ServerId => _serverInfo.ServerId;
public string ServerName => _serverInfo.ServerName;
public int ClientCount => _clients.Count;
public int Port => _options.Port;
public Account SystemAccount => _systemAccount;
public string ServerNKey { get; }
public InternalEventSystem? EventSystem => _eventSystem;
public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0;
public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0;
public string? ClusterListen => _routeManager?.ListenEndpoint;
public string? GatewayListen => _gatewayManager?.ListenEndpoint;
public string? LeafListen => _leafNodeManager?.ListenEndpoint;
public bool IsProfilingEnabled => _options.ProfPort > 0;
public InternalClient? JetStreamInternalClient => _jetStreamInternalClient;
public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter;
public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0;
public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0;
public Action? ReOpenLogFile { get; set; }
public IEnumerable<NatsClient> GetClients() => _clients.Values;
public string? ClusterName() => _options.Cluster?.Name;
public IReadOnlyList<string> ActivePeers()
=> _routeManager?.BuildTopologySnapshot().ConnectedServerIds ?? [];
public bool StartProfiler()
{
if (_options.ProfPort <= 0)
return false;
_logger.LogWarning("Profiling endpoint not yet supported (port: {ProfPort})", _options.ProfPort);
return true;
}
public bool DisconnectClientByID(ulong clientId)
=> CloseClientById(clientId, minimalFlush: true);
public bool LDMClientByID(ulong clientId)
=> CloseClientById(clientId, minimalFlush: false);
public Ports PortsInfo()
{
var ports = new Ports();
AddEndpoint(ports.Nats, _options.Host, _options.Port);
AddEndpoint(ports.Monitoring, _options.MonitorHost, _options.MonitorPort);
if (_routeManager != null)
AddEndpoint(ports.Cluster, _routeManager.ListenEndpoint);
else if (_options.Cluster != null)
AddEndpoint(ports.Cluster, _options.Cluster.Host, _options.Cluster.Port);
AddEndpoint(ports.Profile, _options.Host, _options.ProfPort);
if (_options.WebSocket.Port >= 0)
AddEndpoint(ports.WebSocket, _options.WebSocket.Host, _options.WebSocket.Port);
if (_leafNodeManager != null)
AddEndpoint(ports.LeafNodes, _leafNodeManager.ListenEndpoint);
else if (_options.LeafNode != null)
AddEndpoint(ports.LeafNodes, _options.LeafNode.Host, _options.LeafNode.Port);
return ports;
}
public IReadOnlyList<string> GetConnectURLs()
{
if (!string.IsNullOrWhiteSpace(_options.ClientAdvertise))
return [NormalizeAdvertiseUrl(_options.ClientAdvertise!, "nats")];
var hosts = GetNonLocalIPsIfHostIsIPAny(_options.Host);
var result = new List<string>(hosts.Count);
foreach (var host in hosts)
result.Add($"nats://{host}:{_options.Port}");
return result;
}
public void UpdateServerINFOAndSendINFOToClients()
{
_serverInfo.ConnectUrls = [.. GetConnectURLs()];
BuildCachedInfo();
foreach (var client in _clients.Values)
{
if (client.ConnectReceived)
client.QueueOutbound(_cachedInfoLine);
}
}
public string ClientURL()
{
if (!string.IsNullOrWhiteSpace(_options.ClientAdvertise))
return NormalizeAdvertiseUrl(_options.ClientAdvertise!, "nats");
var host = IsWildcardHost(_options.Host) ? "127.0.0.1" : _options.Host;
return $"nats://{host}:{_options.Port}";
}
public string? WebsocketURL()
{
if (_options.WebSocket.Port < 0)
return null;
if (!string.IsNullOrWhiteSpace(_options.WebSocket.Advertise))
{
var scheme = _options.WebSocket.NoTls ? "ws" : "wss";
return NormalizeAdvertiseUrl(_options.WebSocket.Advertise!, scheme);
}
var wsHost = IsWildcardHost(_options.WebSocket.Host) ? "127.0.0.1" : _options.WebSocket.Host;
var wsScheme = _options.WebSocket.NoTls ? "ws" : "wss";
return $"{wsScheme}://{wsHost}:{_options.WebSocket.Port}";
}
public int NumRoutes() => (int)Interlocked.Read(ref _stats.Routes);
public int NumRemotes()
=> (int)(Interlocked.Read(ref _stats.Routes) + Interlocked.Read(ref _stats.Gateways) + Interlocked.Read(ref _stats.Leafs));
public int NumLeafNodes() => (int)Interlocked.Read(ref _stats.Leafs);
public int NumOutboundGateways() => _gatewayManager?.NumOutboundGateways() ?? 0;
public int NumInboundGateways() => _gatewayManager?.NumInboundGateways() ?? 0;
public int NumSubscriptions() => _accounts.Values.Sum(acc => acc.SubscriptionCount);
public bool JetStreamEnabled() => _jetStreamService?.IsRunning ?? false;
public JetStreamOptions? JetStreamConfig()
{
if (_options.JetStream is null)
return null;
return new JetStreamOptions
{
StoreDir = _options.JetStream.StoreDir,
MaxMemoryStore = _options.JetStream.MaxMemoryStore,
MaxFileStore = _options.JetStream.MaxFileStore,
MaxStreams = _options.JetStream.MaxStreams,
MaxConsumers = _options.JetStream.MaxConsumers,
Domain = _options.JetStream.Domain,
};
}
public string StoreDir() => _options.JetStream?.StoreDir ?? string.Empty;
public DateTime ConfigTime() => _configTime;
public string Addr() => $"{_options.Host}:{_options.Port}";
public string? MonitorAddr()
=> _options.MonitorPort > 0
? $"{_options.MonitorHost}:{_options.MonitorPort}"
: null;
public string? ClusterAddr() => _routeManager?.ListenEndpoint;
public string? GatewayAddr() => _gatewayManager?.ListenEndpoint;
public string? GetGatewayURL() => _gatewayManager?.ListenEndpoint;
public string? GetGatewayName() => _options.Gateway?.Name;
public string? ProfilerAddr()
=> _options.ProfPort > 0
? $"{_options.Host}:{_options.ProfPort}"
: null;
public int NumActiveAccounts() => _accounts.Values.Count(acc => acc.ClientCount > 0);
public int NumLoadedAccounts() => _accounts.Count;
public IReadOnlyList<ClosedClient> GetClosedClients() => _closedClients.GetAll();
public IEnumerable<Auth.Account> GetAccounts() => _accounts.Values;
public bool HasRemoteInterest(string subject) => _globalAccount.SubList.HasRemoteInterest(subject);
public bool HasRemoteInterest(string account, string subject)
=> GetOrCreateAccount(account).SubList.HasRemoteInterest(account, subject);
public bool TryCaptureJetStreamPublish(string subject, ReadOnlyMemory<byte> payload, out PubAck ack)
{
if (_jetStreamPublisher != null && _jetStreamPublisher.TryCapture(subject, payload, out ack))
{
// Only load the stored message for consumer notification when there are
// active consumers for this stream. Avoids unnecessary LoadAsync on the hot path.
if (ack.ErrorCode == null
&& _jetStreamConsumerManager != null
&& _jetStreamConsumerManager.HasConsumersForStream(ack.Stream)
&& _jetStreamStreamManager != null
&& _jetStreamStreamManager.TryGet(ack.Stream, out var streamHandle))
{
var stored = streamHandle.Store.LoadAsync(ack.Seq, default).GetAwaiter().GetResult();
if (stored != null)
_jetStreamConsumerManager.OnPublished(ack.Stream, stored);
}
return true;
}
ack = new PubAck();
return false;
}
/// <summary>
/// Replicates a successful JetStream mutating API call to cluster peers via internal subjects.
/// Maps API subjects to $JS.INTERNAL.* subjects and forwards to routes/leafnodes.
/// Go reference: jetstream_cluster.go — RAFT proposal broadcast.
/// </summary>
private void TryReplicateJetStreamMutation(string apiSubject, ReadOnlyMemory<byte> payload)
{
string? internalSubject = null;
if (apiSubject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal))
internalSubject = JetStreamApiSubjects.InternalStreamCreate + apiSubject[JetStreamApiSubjects.StreamCreate.Length..];
else if (apiSubject.StartsWith(JetStreamApiSubjects.StreamDelete, StringComparison.Ordinal))
internalSubject = JetStreamApiSubjects.InternalStreamDelete + apiSubject[JetStreamApiSubjects.StreamDelete.Length..];
else if (apiSubject.StartsWith(JetStreamApiSubjects.StreamPurge, StringComparison.Ordinal))
internalSubject = JetStreamApiSubjects.InternalStreamPurge + apiSubject[JetStreamApiSubjects.StreamPurge.Length..];
else if (apiSubject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal))
internalSubject = JetStreamApiSubjects.InternalConsumerCreate + apiSubject[JetStreamApiSubjects.ConsumerCreate.Length..];
else if (apiSubject.StartsWith(JetStreamApiSubjects.ConsumerDelete, StringComparison.Ordinal))
internalSubject = JetStreamApiSubjects.InternalConsumerDelete + apiSubject[JetStreamApiSubjects.ConsumerDelete.Length..];
if (internalSubject != null)
ReplicateJetStreamOperation("$G", internalSubject, null, payload);
}
/// <summary>
/// Forwards a JetStream replication message to all route and leaf node peers.
/// Bypasses interest checks since replication subjects have no client subscribers.
/// Go reference: jetstream_cluster.go — proposal broadcast to peers.
/// </summary>
private void ReplicateJetStreamOperation(string account, string subject, string? replyTo, ReadOnlyMemory<byte> payload)
{
_routeManager?.BroadcastRoutedMessageAsync(account, subject, replyTo, payload, default)
.GetAwaiter().GetResult();
if (_leafNodeManager != null)
{
var markedSubject = LeafLoopDetector.Mark(subject, ServerId);
_leafNodeManager.ForwardMessageAsync(account, markedSubject, replyTo, payload, default)
.GetAwaiter().GetResult();
}
}
/// <summary>
/// Handles incoming JetStream internal replication messages from cluster peers.
/// Dispatches to the appropriate handler based on the internal subject prefix.
/// Called from DeliverRemoteMessage for $JS.INTERNAL.* subjects.
/// Go reference: jetstream_cluster.go — RAFT proposal apply.
/// </summary>
private void HandleJetStreamReplication(string subject, ReadOnlyMemory<byte> payload)
{
if (_jetStreamStreamManager == null || _jetStreamConsumerManager == null)
return;
if (subject.StartsWith(JetStreamApiSubjects.InternalStreamCreate, StringComparison.Ordinal))
{
var apiSubject = JetStreamApiSubjects.StreamCreate + subject[JetStreamApiSubjects.InternalStreamCreate.Length..];
JetStream.Api.Handlers.StreamApiHandlers.HandleCreate(apiSubject, payload.Span, _jetStreamStreamManager);
}
else if (subject.StartsWith(JetStreamApiSubjects.InternalStreamDelete, StringComparison.Ordinal))
{
var streamName = subject[JetStreamApiSubjects.InternalStreamDelete.Length..];
_jetStreamStreamManager.Delete(streamName);
}
else if (subject.StartsWith(JetStreamApiSubjects.InternalStreamPurge, StringComparison.Ordinal))
{
var streamName = subject[JetStreamApiSubjects.InternalStreamPurge.Length..];
_jetStreamStreamManager.Purge(streamName);
}
else if (subject.StartsWith(JetStreamApiSubjects.InternalConsumerCreate, StringComparison.Ordinal))
{
var apiSubject = JetStreamApiSubjects.ConsumerCreate + subject[JetStreamApiSubjects.InternalConsumerCreate.Length..];
JetStream.Api.Handlers.ConsumerApiHandlers.HandleCreate(apiSubject, payload.Span, _jetStreamConsumerManager);
}
else if (subject.StartsWith(JetStreamApiSubjects.InternalConsumerDelete, StringComparison.Ordinal))
{
var parts = subject[JetStreamApiSubjects.InternalConsumerDelete.Length..].Split('.');
if (parts.Length >= 2)
_jetStreamConsumerManager.Delete(parts[0], parts[1]);
}
}
public Task WaitForReadyAsync() => _listeningStarted.Task;
public void WaitForShutdown() => _shutdownComplete.Task.GetAwaiter().GetResult();
internal TlsCertificateProvider? TlsCertProviderForTest => _tlsCertProvider;
internal Task AcquireReloadLockForTestAsync() => _reloadMu.WaitAsync();
internal void ReleaseReloadLockForTest() => _reloadMu.Release();
internal void SetAcceptLoopErrorHandlerForTest(AcceptLoopErrorHandler handler) => _acceptLoopErrorHandler = handler;
internal void NotifyAcceptErrorForTest(Exception ex, EndPoint? endpoint, TimeSpan delay) =>
_acceptLoopErrorHandler?.OnAcceptError(ex, endpoint, delay);
public async Task ShutdownAsync()
{
if (Interlocked.CompareExchange(ref _shutdown, 1, 0) != 0)
return; // Already shutting down
_logger.LogInformation("Initiating Shutdown...");
// Publish shutdown advisory before tearing down the event system
if (_eventSystem != null)
{
var shutdownSubject = string.Format(EventSubjects.ServerShutdown, _serverInfo.ServerId);
_eventSystem.Enqueue(new PublishMessage
{
Subject = shutdownSubject,
Body = new ShutdownEventMsg { Server = BuildEventServerInfo(), Reason = "Server Shutdown" },
IsLast = true,
});
// Give the send loop time to process the shutdown event
await Task.Delay(100);
await _eventSystem.DisposeAsync();
}
// Signal all internal loops to stop
await _quitCts.CancelAsync();
// Close listeners to stop accept loops
_listener?.Close();
_wsListener?.Close();
if (_routeManager != null)
await _routeManager.DisposeAsync();
if (_gatewayManager != null)
await _gatewayManager.DisposeAsync();
if (_leafNodeManager != null)
await _leafNodeManager.DisposeAsync();
if (_jetStreamService != null)
await _jetStreamService.DisposeAsync();
_stats.JetStreamEnabled = false;
// Wait for accept loops to exit
await _acceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
await _wsAcceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
// Close all client connections — flush first, then mark closed
var flushTasks = new List<Task>();
foreach (var client in _clients.Values)
{
client.MarkClosed(ClientClosedReason.ServerShutdown);
flushTasks.Add(client.FlushAndCloseAsync(minimalFlush: true));
}
await Task.WhenAll(flushTasks).WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
// Wait for active client tasks to drain (with timeout)
if (Volatile.Read(ref _activeClientCount) > 0)
{
using var drainCts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
try
{
while (Volatile.Read(ref _activeClientCount) > 0 && !drainCts.IsCancellationRequested)
await Task.Delay(50, drainCts.Token);
}
catch (OperationCanceledException) { }
}
// Stop monitor server
if (_monitorServer != null)
await _monitorServer.DisposeAsync();
if (_mqttListener != null)
await _mqttListener.DisposeAsync();
DeletePidFile();
DeletePortsFile();
_logger.LogInformation("Server Exiting..");
_shutdownComplete.TrySetResult();
}
public async Task LameDuckShutdownAsync()
{
if (IsShuttingDown || Interlocked.CompareExchange(ref _lameDuck, 1, 0) != 0)
return;
_logger.LogInformation("Entering lame duck mode, stop accepting new clients");
// Close listeners to stop accepting new connections
_listener?.Close();
_wsListener?.Close();
// Wait for accept loops to exit
await _acceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
await _wsAcceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
var gracePeriod = _options.LameDuckGracePeriod;
if (gracePeriod < TimeSpan.Zero) gracePeriod = -gracePeriod;
// If no clients, go straight to shutdown
if (_clients.IsEmpty)
{
await ShutdownAsync();
return;
}
// Wait grace period for clients to drain naturally
_logger.LogInformation("Waiting {GracePeriod}ms grace period", gracePeriod.TotalMilliseconds);
try
{
await Task.Delay(gracePeriod, _quitCts.Token);
}
catch (OperationCanceledException) { return; }
if (_clients.IsEmpty)
{
await ShutdownAsync();
return;
}
// Stagger-close remaining clients
var dur = _options.LameDuckDuration - gracePeriod;
if (dur <= TimeSpan.Zero) dur = TimeSpan.FromSeconds(1);
var clients = _clients.Values.ToList();
var numClients = clients.Count;
if (numClients > 0)
{
_logger.LogInformation("Closing {Count} existing clients over {Duration}ms",
numClients, dur.TotalMilliseconds);
var sleepInterval = dur.Ticks / numClients;
if (sleepInterval < TimeSpan.TicksPerMillisecond)
sleepInterval = TimeSpan.TicksPerMillisecond;
if (sleepInterval > TimeSpan.TicksPerSecond)
sleepInterval = TimeSpan.TicksPerSecond;
for (int i = 0; i < clients.Count; i++)
{
clients[i].MarkClosed(ClientClosedReason.ServerShutdown);
await clients[i].FlushAndCloseAsync(minimalFlush: true);
if (i < clients.Count - 1)
{
var jitter = Random.Shared.NextInt64(sleepInterval / 2, sleepInterval);
try
{
await Task.Delay(TimeSpan.FromTicks(jitter), _quitCts.Token);
}
catch (OperationCanceledException) { break; }
}
}
}
await ShutdownAsync();
}
/// <summary>
/// Registers Unix signal handlers.
/// SIGTERM → shutdown, SIGUSR2 → lame duck, SIGUSR1 → log reopen, SIGHUP → reload (stub).
/// </summary>
public void HandleSignals()
{
_signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGTERM, ctx =>
{
ctx.Cancel = true;
_logger.LogInformation("Trapped SIGTERM signal");
_ = Task.Run(async () => await ShutdownAsync());
}));
_signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGQUIT, ctx =>
{
ctx.Cancel = true;
_logger.LogInformation("Trapped SIGQUIT signal");
_ = Task.Run(async () => await ShutdownAsync());
}));
_signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGHUP, ctx =>
{
ctx.Cancel = true;
_logger.LogInformation("Trapped SIGHUP signal — reloading configuration");
_ = Task.Run(() => ReloadConfig());
}));
// SIGUSR1 and SIGUSR2 only on non-Windows
if (!OperatingSystem.IsWindows())
{
_signalRegistrations.Add(PosixSignalRegistration.Create((PosixSignal)10, ctx =>
{
ctx.Cancel = true;
_logger.LogInformation("Trapped SIGUSR1 signal — reopening log file");
ReOpenLogFile?.Invoke();
}));
_signalRegistrations.Add(PosixSignalRegistration.Create((PosixSignal)12, ctx =>
{
ctx.Cancel = true;
_logger.LogInformation("Trapped SIGUSR2 signal — entering lame duck mode");
_ = Task.Run(async () => await LameDuckShutdownAsync());
}));
}
}
public NatsServer(NatsOptions options, ILoggerFactory loggerFactory)
{
_options = options;
_loggerFactory = loggerFactory;
_logger = loggerFactory.CreateLogger<NatsServer>();
_closedClients = new ClosedConnectionRingBuffer(options.MaxClosedClients);
_authService = AuthService.Build(options);
_globalAccount = new Account(Account.GlobalAccountName);
_accounts[Account.GlobalAccountName] = _globalAccount;
// Create $SYS system account and mark it as the system account.
// Reference: Go server/server.go — initSystemAccount, accounts.go — isSystemAccount().
_systemAccount = new Account(Account.SystemAccountName) { IsSystemAccount = true };
_accounts[Account.SystemAccountName] = _systemAccount;
// If a user-defined system_account is configured, promote that account to be the
// system account. Events published to $SYS.* will be delivered to subscribers on
// this account. Go reference: server/server.go — configureAccounts / setSystemAccount.
if (!string.IsNullOrEmpty(options.SystemAccount) &&
!string.Equals(options.SystemAccount, Account.SystemAccountName, StringComparison.OrdinalIgnoreCase))
{
var userSysAccount = GetOrCreateAccount(options.SystemAccount);
userSysAccount.IsSystemAccount = true;
_systemAccount = userSysAccount;
}
// Create system internal client and event system
var sysClientId = Interlocked.Increment(ref _nextClientId);
var sysClient = new InternalClient(sysClientId, ClientKind.System, _systemAccount);
_eventSystem = new InternalEventSystem(
_systemAccount, sysClient,
options.ServerName ?? $"nats-dotnet-{Environment.MachineName}",
_loggerFactory.CreateLogger<InternalEventSystem>());
// Generate Ed25519 server NKey identity
using var serverKeyPair = KeyPair.CreatePair(PrefixByte.Server);
ServerNKey = serverKeyPair.GetPublicKey();
_serverInfo = new ServerInfo
{
ServerId = Guid.NewGuid().ToString("N")[..20].ToUpperInvariant(),
ServerName = options.ServerName ?? $"nats-dotnet-{Environment.MachineName}",
Version = NatsProtocol.Version,
Host = options.Host,
Port = options.Port,
MaxPayload = options.MaxPayload,
AuthRequired = _authService.IsAuthRequired,
};
if (options.Cluster != null)
{
_routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
ProcessRoutedMessage,
_loggerFactory.CreateLogger<RouteManager>());
_routeManager.OnRouteRemoved += RemoveRemoteSubscriptionsForRoute;
_routeManager.OnRouteAccountRemoved += RemoveRemoteSubscriptionsForRouteAccount;
}
if (options.Gateway != null)
{
_gatewayManager = new GatewayManager(options.Gateway, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
ProcessGatewayMessage,
_loggerFactory.CreateLogger<GatewayManager>());
}
if (options.LeafNode != null)
{
_leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
ProcessLeafMessage,
_loggerFactory.CreateLogger<LeafNodeManager>());
}
if (options.JetStream != null)
{
_jetStreamConsumerManager = new ConsumerManager();
_jetStreamStreamManager = new StreamManager(consumerManager: _jetStreamConsumerManager, storeDir: options.JetStream.StoreDir);
_jetStreamConsumerManager.StreamManager = _jetStreamStreamManager;
var jsClientId = Interlocked.Increment(ref _nextClientId);
_jetStreamInternalClient = new InternalClient(jsClientId, ClientKind.JetStream, _systemAccount);
_jetStreamService = new JetStreamService(options.JetStream, _jetStreamInternalClient);
_jetStreamApiRouter = new JetStreamApiRouter(_jetStreamStreamManager, _jetStreamConsumerManager);
_jetStreamPublisher = new JetStreamPublisher(_jetStreamStreamManager);
}
if (options.HasTls)
{
_tlsCertProvider = new TlsCertificateProvider(options.TlsCert!, options.TlsKey);
_sslOptions = TlsHelper.BuildServerAuthOptions(options);
_tlsCertProvider.SwapSslOptions(_sslOptions);
// OCSP stapling: build a certificate context so the runtime can
// fetch and cache a fresh OCSP response and staple it during the
// TLS handshake. offline:false tells the runtime to contact the
// OCSP responder; if the responder is unreachable we fall back to
// no stapling rather than refusing all connections.
var certContext = TlsHelper.BuildCertificateContext(options, offline: false);
if (certContext != null)
{
_sslOptions.ServerCertificateContext = certContext;
_logger.LogInformation("OCSP stapling enabled (mode: {OcspMode})", options.OcspConfig!.Mode);
}
_serverInfo.TlsRequired = !options.AllowNonTls;
_serverInfo.TlsAvailable = options.AllowNonTls;
_serverInfo.TlsVerify = options.TlsVerify;
if (options.TlsRateLimit > 0)
_tlsRateLimiter = new TlsRateLimiter(options.TlsRateLimit);
}
// Compile subject transforms
if (options.SubjectMappings is { Count: > 0 })
{
var transforms = new List<SubjectTransform>();
foreach (var (source, dest) in options.SubjectMappings)
{
var t = SubjectTransform.Create(source, dest);
if (t != null)
transforms.Add(t);
else
_logger.LogWarning("Invalid subject mapping: {Source} -> {Dest}", source, dest);
}
_subjectTransforms = transforms.ToArray();
if (_subjectTransforms.Length > 0)
_logger.LogInformation("Compiled {Count} subject transform(s)", _subjectTransforms.Length);
}
else
{
_subjectTransforms = [];
}
BuildCachedInfo();
// Store initial config digest for reload change detection
if (options.ConfigFile != null)
{
try
{
var (_, digest) = NatsConfParser.ParseFileWithDigest(options.ConfigFile);
_configDigest = digest;
_configTime = DateTime.UtcNow;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not compute initial config digest for {ConfigFile}", options.ConfigFile);
}
}
}
private void BuildCachedInfo()
{
var infoJson = System.Text.Json.JsonSerializer.Serialize(_serverInfo);
_cachedInfoLine = Encoding.ASCII.GetBytes($"INFO {infoJson}\r\n");
}
private static string NormalizeAdvertiseUrl(string advertise, string defaultScheme)
{
if (advertise.Contains("://", StringComparison.Ordinal))
return advertise;
return $"{defaultScheme}://{advertise}";
}
private static bool IsWildcardHost(string host)
=> host == "0.0.0.0" || host == "::";
internal static IReadOnlyList<string> GetNonLocalIPsIfHostIsIPAny(string host)
{
if (!IsWildcardHost(host))
return [host];
var addresses = new HashSet<string>(StringComparer.Ordinal);
foreach (var netIf in NetworkInterface.GetAllNetworkInterfaces())
{
if (netIf.OperationalStatus != OperationalStatus.Up)
continue;
IPInterfaceProperties? props;
try
{
props = netIf.GetIPProperties();
}
catch
{
continue;
}
foreach (var uni in props.UnicastAddresses)
{
var addr = uni.Address;
if (IPAddress.IsLoopback(addr) || addr.IsIPv6LinkLocal || addr.IsIPv6Multicast)
continue;
if (addr.AddressFamily is not (AddressFamily.InterNetwork or AddressFamily.InterNetworkV6))
continue;
addresses.Add(addr.ToString());
}
}
if (addresses.Count == 0)
addresses.Add("127.0.0.1");
return [.. addresses.OrderBy(static a => a, StringComparer.Ordinal)];
}
private bool CloseClientById(ulong clientId, bool minimalFlush)
{
if (!_clients.TryGetValue(clientId, out var client))
return false;
client.MarkClosed(ClientClosedReason.ServerShutdown);
_ = client.FlushAndCloseAsync(minimalFlush);
return true;
}
private static void AddEndpoint(List<string> targets, string? host, int port)
{
if (string.IsNullOrWhiteSpace(host) || port <= 0)
return;
targets.Add($"{host}:{port}");
}
private static void AddEndpoint(List<string> targets, string? endpoint)
{
if (!string.IsNullOrWhiteSpace(endpoint))
targets.Add(endpoint);
}
public async Task StartAsync(CancellationToken ct)
{
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _quitCts.Token);
_listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
_listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_listener.Bind(new IPEndPoint(
_options.Host == "0.0.0.0" ? IPAddress.Any : IPAddress.Parse(_options.Host),
_options.Port));
Interlocked.Exchange(ref _startTimeTicks, DateTime.UtcNow.Ticks);
_listener.Listen(128);
// Resolve ephemeral port if port=0
if (_options.Port == 0)
{
var actualPort = ((IPEndPoint)_listener.LocalEndPoint!).Port;
_options.Port = actualPort;
_serverInfo.Port = actualPort;
BuildCachedInfo();
}
_logger.LogInformation("Listening for client connections on {Host}:{Port}", _options.Host, _options.Port);
// Warn about stub features
StartProfiler();
if (_options.MonitorPort > 0)
{
_monitorServer = new MonitorServer(this, _options, _stats, _loggerFactory);
await _monitorServer.StartAsync(linked.Token);
}
WritePidFile();
WritePortsFile();
WsAuthConfig.Apply(_options.WebSocket);
var wsValidation = WebSocketOptionsValidator.Validate(_options);
if (!wsValidation.IsValid)
throw new InvalidOperationException($"Invalid websocket options: {string.Join("; ", wsValidation.Errors)}");
if (_options.WebSocket.Port >= 0)
{
_wsListener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
_wsListener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_wsListener.Bind(new IPEndPoint(
_options.WebSocket.Host == "0.0.0.0" ? IPAddress.Any : IPAddress.Parse(_options.WebSocket.Host),
_options.WebSocket.Port));
_wsListener.Listen(128);
if (_options.WebSocket.Port == 0)
{
_options.WebSocket.Port = ((IPEndPoint)_wsListener.LocalEndPoint!).Port;
}
_logger.LogInformation("Listening for WebSocket clients on {Host}:{Port}",
_options.WebSocket.Host, _options.WebSocket.Port);
if (_options.WebSocket.NoTls)
_logger.LogWarning("WebSocket not configured with TLS. DO NOT USE IN PRODUCTION!");
_ = RunWebSocketAcceptLoopAsync(linked.Token);
}
if (_routeManager != null)
await _routeManager.StartAsync(linked.Token);
if (_gatewayManager != null)
await _gatewayManager.StartAsync(linked.Token);
if (_leafNodeManager != null)
await _leafNodeManager.StartAsync(linked.Token);
if (_options.Mqtt is { Port: > 0 } mqttOptions)
{
var mqttHost = string.IsNullOrWhiteSpace(mqttOptions.Host) ? _options.Host : mqttOptions.Host;
_mqttListener = new MqttListener(
mqttHost,
mqttOptions.Port,
mqttOptions.Username,
mqttOptions.Password);
await _mqttListener.StartAsync(linked.Token);
}
if (_jetStreamService != null)
{
await _jetStreamService.StartAsync(linked.Token);
_stats.JetStreamEnabled = true;
}
_listeningStarted.TrySetResult();
_eventSystem?.Start(this);
_eventSystem?.InitEventTracking(this);
var tmpDelay = AcceptMinSleep;
try
{
while (!linked.Token.IsCancellationRequested)
{
Socket socket;
try
{
socket = await _listener!.AcceptAsync(linked.Token);
tmpDelay = AcceptMinSleep; // Reset on success
}
catch (OperationCanceledException)
{
break;
}
catch (ObjectDisposedException)
{
break;
}
catch (SocketException ex)
{
if (IsShuttingDown || IsLameDuckMode)
break;
_acceptLoopErrorHandler?.OnAcceptError(ex, _listener?.LocalEndPoint, tmpDelay);
_logger.LogError(ex, "Temporary accept error, sleeping {Delay}ms", tmpDelay.TotalMilliseconds);
try { await Task.Delay(tmpDelay, linked.Token); }
catch (OperationCanceledException) { break; }
tmpDelay = TimeSpan.FromTicks(Math.Min(tmpDelay.Ticks * 2, AcceptMaxSleep.Ticks));
continue;
}
// Check MaxConnections
if (_options.MaxConnections > 0 && _clients.Count >= _options.MaxConnections)
{
_logger.LogWarning("Client connection rejected: maximum connections ({MaxConnections}) exceeded",
_options.MaxConnections);
try
{
var stream = new NetworkStream(socket, ownsSocket: false);
var errBytes = Encoding.ASCII.GetBytes(
$"-ERR '{NatsProtocol.ErrMaxConnectionsExceeded}'\r\n");
await stream.WriteAsync(errBytes, linked.Token);
await stream.FlushAsync(linked.Token);
stream.Dispose();
}
catch (Exception ex2)
{
_logger.LogDebug(ex2, "Failed to send -ERR to rejected client");
}
finally
{
socket.Dispose();
}
continue;
}
var clientId = Interlocked.Increment(ref _nextClientId);
Interlocked.Increment(ref _stats.TotalConnections);
Interlocked.Increment(ref _activeClientCount);
_logger.LogDebug("Client {ClientId} connected from {RemoteEndpoint}", clientId, socket.RemoteEndPoint);
_ = AcceptClientAsync(socket, clientId, linked.Token);
}
}
catch (OperationCanceledException)
{
_logger.LogDebug("Accept loop cancelled, server shutting down");
}
finally
{
_acceptLoopExited.TrySetResult();
}
}
private async Task AcceptClientAsync(Socket socket, ulong clientId, CancellationToken ct)
{
var reloadLockHeld = false;
NatsClient? client = null;
try
{
await _reloadMu.WaitAsync(ct);
reloadLockHeld = true;
// Rate limit TLS handshakes
if (_tlsRateLimiter != null)
await _tlsRateLimiter.WaitAsync(ct);
var networkStream = new NetworkStream(socket, ownsSocket: false);
// TLS negotiation (no-op if not configured)
var (stream, infoAlreadySent) = await TlsConnectionWrapper.NegotiateAsync(
socket, networkStream, _options, _sslOptions, _serverInfo,
_loggerFactory.CreateLogger("NATS.Server.Tls"), ct);
// Extract TLS state
TlsConnectionState? tlsState = null;
if (stream is SslStream ssl)
{
tlsState = new TlsConnectionState(
ssl.SslProtocol.ToString(),
ssl.NegotiatedCipherSuite.ToString(),
ssl.RemoteCertificate as X509Certificate2);
}
// Build per-client ServerInfo with nonce if NKey auth is configured
byte[]? nonce = null;
var clientInfo = _serverInfo;
if (_authService.NonceRequired)
{
var rawNonce = _authService.GenerateNonce();
var nonceStr = _authService.EncodeNonce(rawNonce);
// The client signs the nonce string (ASCII), not the raw bytes
nonce = Encoding.ASCII.GetBytes(nonceStr);
clientInfo = new ServerInfo
{
ServerId = _serverInfo.ServerId,
ServerName = _serverInfo.ServerName,
Version = _serverInfo.Version,
Host = _serverInfo.Host,
Port = _serverInfo.Port,
MaxPayload = _serverInfo.MaxPayload,
AuthRequired = _serverInfo.AuthRequired,
TlsRequired = _serverInfo.TlsRequired,
TlsAvailable = _serverInfo.TlsAvailable,
TlsVerify = _serverInfo.TlsVerify,
Nonce = nonceStr,
};
}
var clientLogger = _loggerFactory.CreateLogger($"NATS.Server.NatsClient[{clientId}]");
client = new NatsClient(clientId, stream, socket, _options, clientInfo,
_authService, nonce, clientLogger, _stats);
client.Router = this;
client.TlsState = tlsState;
client.InfoAlreadySent = infoAlreadySent;
_clients[clientId] = client;
}
catch (Exception ex)
{
if (client is null)
{
var earlyReason = _options.HasTls
? ClientClosedReason.TlsHandshakeError
: ClientClosedReason.ReadError;
TrackEarlyClosedClient(socket, clientId, earlyReason);
}
_logger.LogDebug(ex, "Failed to accept client {ClientId}", clientId);
try { socket.Shutdown(SocketShutdown.Both); } catch { }
socket.Dispose();
return;
}
finally
{
if (reloadLockHeld)
_reloadMu.Release();
}
try
{
if (client != null)
await RunClientAsync(client, ct);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to accept client {ClientId}", clientId);
try { socket.Shutdown(SocketShutdown.Both); } catch { }
socket.Dispose();
}
}
private async Task RunWebSocketAcceptLoopAsync(CancellationToken ct)
{
var tmpDelay = AcceptMinSleep;
try
{
while (!ct.IsCancellationRequested)
{
Socket socket;
try
{
socket = await _wsListener!.AcceptAsync(ct);
tmpDelay = AcceptMinSleep;
}
catch (OperationCanceledException) { break; }
catch (ObjectDisposedException) { break; }
catch (SocketException ex)
{
if (IsShuttingDown || IsLameDuckMode) break;
_acceptLoopErrorHandler?.OnAcceptError(ex, _wsListener?.LocalEndPoint, tmpDelay);
_logger.LogError(ex, "Temporary WebSocket accept error, sleeping {Delay}ms", tmpDelay.TotalMilliseconds);
try { await Task.Delay(tmpDelay, ct); } catch (OperationCanceledException) { break; }
tmpDelay = TimeSpan.FromTicks(Math.Min(tmpDelay.Ticks * 2, AcceptMaxSleep.Ticks));
continue;
}
if (_options.MaxConnections > 0 && _clients.Count >= _options.MaxConnections)
{
socket.Dispose();
continue;
}
var clientId = Interlocked.Increment(ref _nextClientId);
Interlocked.Increment(ref _stats.TotalConnections);
Interlocked.Increment(ref _activeClientCount);
_ = AcceptWebSocketClientAsync(socket, clientId, ct);
}
}
finally
{
_wsAcceptLoopExited.TrySetResult();
}
}
private async Task AcceptWebSocketClientAsync(Socket socket, ulong clientId, CancellationToken ct)
{
try
{
var networkStream = new NetworkStream(socket, ownsSocket: false);
Stream stream = networkStream;
// TLS negotiation if configured
if (_sslOptions != null && !_options.WebSocket.NoTls)
{
var (tlsStream, _) = await TlsConnectionWrapper.NegotiateAsync(
socket, networkStream, _options, _sslOptions, _serverInfo,
_loggerFactory.CreateLogger("NATS.Server.Tls"), ct);
stream = tlsStream;
}
// HTTP upgrade handshake
var upgradeResult = await WsUpgrade.TryUpgradeAsync(stream, stream, _options.WebSocket, ct);
if (!upgradeResult.Success)
{
_logger.LogDebug("WebSocket upgrade failed for client {ClientId}", clientId);
socket.Dispose();
Interlocked.Decrement(ref _activeClientCount);
return;
}
// Create WsConnection wrapper
var wsConn = new WsConnection(stream,
compress: upgradeResult.Compress,
maskRead: upgradeResult.MaskRead,
maskWrite: upgradeResult.MaskWrite,
browser: upgradeResult.Browser,
noCompFrag: upgradeResult.NoCompFrag);
var clientLogger = _loggerFactory.CreateLogger($"NATS.Server.NatsClient[{clientId}]");
var client = new NatsClient(clientId, wsConn, socket, _options, _serverInfo,
_authService, null, clientLogger, _stats);
client.Router = this;
client.IsWebSocket = true;
client.WsInfo = upgradeResult;
_clients[clientId] = client;
await RunClientAsync(client, ct);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to accept WebSocket client {ClientId}", clientId);
try { socket.Shutdown(SocketShutdown.Both); } catch { }
socket.Dispose();
Interlocked.Decrement(ref _activeClientCount);
}
}
private async Task RunClientAsync(NatsClient client, CancellationToken ct)
{
try
{
await client.RunAsync(ct);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Client {ClientId} disconnected with error", client.Id);
}
finally
{
_logger.LogDebug("Client {ClientId} disconnected (reason: {CloseReason})", client.Id, client.CloseReason);
RemoveClient(client);
Interlocked.Decrement(ref _activeClientCount);
}
}
public void OnLocalSubscription(string account, string subject, string? queue)
{
_routeManager?.PropagateLocalSubscription(account, subject, queue);
_gatewayManager?.PropagateLocalSubscription(account, subject, queue);
_leafNodeManager?.PropagateLocalSubscription(account, subject, queue);
}
public void OnLocalUnsubscription(string account, string subject, string? queue)
{
_routeManager?.PropagateLocalUnsubscription(account, subject, queue);
_gatewayManager?.PropagateLocalUnsubscription(account, subject, queue);
_leafNodeManager?.PropagateLocalUnsubscription(account, subject, queue);
}
private void ApplyRemoteSubscription(RemoteSubscription sub)
{
var account = GetOrCreateAccount(sub.Account);
account.SubList.ApplyRemoteSub(sub);
}
private void RemoveRemoteSubscriptionsForRoute(string routeId)
{
foreach (var account in _accounts.Values)
account.SubList.RemoveRemoteSubs(routeId);
}
private void RemoveRemoteSubscriptionsForRouteAccount(string routeId, string accountName)
{
if (_accounts.TryGetValue(accountName, out var account))
account.SubList.RemoveRemoteSubsForAccount(routeId, accountName);
}
private void ProcessRoutedMessage(RouteMessage message)
{
DeliverRemoteMessage(message.Account, message.Subject, message.ReplyTo, message.Payload);
}
private void ProcessGatewayMessage(GatewayMessage message)
{
var replyTo = message.ReplyTo;
if (ReplyMapper.TryRestoreGatewayReply(replyTo, out var restoredReply))
replyTo = restoredReply;
else if (ReplyMapper.HasGatewayReplyPrefix(replyTo))
replyTo = null;
DeliverRemoteMessage(message.Account, message.Subject, replyTo, message.Payload);
}
private void ProcessLeafMessage(LeafMessage message)
{
if (LeafLoopDetector.IsLooped(message.Subject, ServerId))
return;
var subject = message.Subject;
if (LeafLoopDetector.TryUnmark(subject, out var unmarked))
subject = unmarked;
else if (LeafLoopDetector.HasLoopMarker(subject))
return;
DeliverRemoteMessage(message.Account, subject, message.ReplyTo, message.Payload);
}
private void DeliverRemoteMessage(string account, string subject, string? replyTo, ReadOnlyMemory<byte> payload)
{
// Handle internal JetStream cluster replication messages.
// Go reference: jetstream_cluster.go — RAFT proposal apply dispatches to handlers.
if (subject.StartsWith(JetStreamApiSubjects.InternalPrefix, StringComparison.Ordinal))
{
HandleJetStreamReplication(subject, payload);
return;
}
// Capture routed data messages into local JetStream streams.
// Skip $JS. subjects — those are API calls, not stream data.
if (!subject.StartsWith("$JS.", StringComparison.Ordinal))
TryCaptureJetStreamPublish(subject, payload, out _);
var targetAccount = GetOrCreateAccount(account);
var result = targetAccount.SubList.Match(subject);
foreach (var sub in result.PlainSubs)
DeliverMessage(sub, subject, replyTo, default, payload);
foreach (var queueGroup in result.QueueSubs)
{
if (queueGroup.Length == 0)
continue;
var sub = queueGroup[0];
DeliverMessage(sub, subject, replyTo, default, payload);
}
}
public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory<byte> headers,
ReadOnlyMemory<byte> payload, NatsClient sender)
{
if (replyTo != null
&& subject.StartsWith("$JS.API", StringComparison.Ordinal)
&& _jetStreamApiRouter != null)
{
// Pull consumer MSG.NEXT requires special handling: deliver individual
// HMSG messages to the client's reply inbox instead of a single JSON blob.
// Go reference: consumer.go:4276 processNextMsgRequest
if (subject.StartsWith(JetStream.Api.JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal)
&& _jetStreamConsumerManager != null
&& _jetStreamStreamManager != null)
{
Interlocked.Increment(ref _stats.JetStreamApiTotal);
DeliverPullFetchMessages(subject, replyTo, payload, sender);
return;
}
var response = _jetStreamApiRouter.Route(subject, payload.Span);
Interlocked.Increment(ref _stats.JetStreamApiTotal);
if (response.Error != null)
Interlocked.Increment(ref _stats.JetStreamApiErrors);
// Replicate successful mutating operations to cluster peers.
// Go reference: jetstream_cluster.go — RAFT proposal replication.
if (response.Error == null)
TryReplicateJetStreamMutation(subject, payload);
var data = JsonSerializer.SerializeToUtf8Bytes(response.ToWireFormat(), s_jetStreamJsonOptions);
ProcessMessage(replyTo, null, default, data, sender);
return;
}
if (TryCaptureJetStreamPublish(subject, payload, out var pubAck))
{
sender.RecordJetStreamPubAck(pubAck);
// Replicate data messages to cluster peers so their JetStream stores also capture them.
// Route forwarding below is gated on subscriber interest, which JetStream streams don't
// create, so we must explicitly push data to replicas.
// Go reference: jetstream_cluster.go — RAFT propose entry replication.
if (pubAck.ErrorCode == null)
ReplicateJetStreamOperation("$G", subject, null, payload);
// Send pub ack response to the reply subject (request-reply pattern).
// Go reference: server/jetstream.go — jsPubAckResponse sent to reply.
if (replyTo != null)
{
var ackData = JsonSerializer.SerializeToUtf8Bytes(pubAck, s_jetStreamJsonOptions);
ProcessMessage(replyTo, null, default, ackData, sender);
return;
}
}
// Apply subject transforms
if (_subjectTransforms.Length > 0)
{
foreach (var transform in _subjectTransforms)
{
var mapped = transform.Apply(subject);
if (mapped != null)
{
subject = mapped;
break; // First matching transform wins
}
}
}
var senderAccount = sender.Account ?? _globalAccount;
if (_routeManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject))
_routeManager.ForwardRoutedMessageAsync(senderAccount.Name, subject, replyTo, payload, default).GetAwaiter().GetResult();
if (_gatewayManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject))
{
var mappedReplyTo = ReplyMapper.ToGatewayReply(replyTo, ServerId);
_gatewayManager.ForwardMessageAsync(senderAccount.Name, subject, mappedReplyTo, payload, default).GetAwaiter().GetResult();
}
if (_leafNodeManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject))
{
var markedSubject = LeafLoopDetector.Mark(subject, ServerId);
_leafNodeManager.ForwardMessageAsync(senderAccount.Name, markedSubject, replyTo, payload, default).GetAwaiter().GetResult();
}
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
var result = subList.Match(subject);
var delivered = false;
// Deliver to plain subscribers
foreach (var sub in result.PlainSubs)
{
if (sub.Client == null || sub.Client == sender && !(sender.ClientOpts?.Echo ?? true))
continue;
DeliverMessage(sub, subject, replyTo, headers, payload);
delivered = true;
}
// Deliver to one member of each queue group (round-robin)
foreach (var queueGroup in result.QueueSubs)
{
if (queueGroup.Length == 0) continue;
// Simple round-robin -- pick based on total delivered across group
var idx = Math.Abs((int)Interlocked.Increment(ref sender.OutMsgs)) % queueGroup.Length;
// Undo the OutMsgs increment -- it will be incremented properly in SendMessage
Interlocked.Decrement(ref sender.OutMsgs);
for (int attempt = 0; attempt < queueGroup.Length; attempt++)
{
var sub = queueGroup[(idx + attempt) % queueGroup.Length];
if (sub.Client != null && (sub.Client != sender || (sender.ClientOpts?.Echo ?? true)))
{
DeliverMessage(sub, subject, replyTo, headers, payload);
delivered = true;
break;
}
}
}
// Check for service imports that match this subject.
// When a client in the importer account publishes to a subject
// that matches a service import "From" pattern, we forward the
// message to the destination (exporter) account's subscribers
// using the mapped "To" subject.
if (sender.Account != null)
{
foreach (var kvp in sender.Account.Imports.Services)
{
foreach (var si in kvp.Value)
{
if (si.Invalid) continue;
if (SubjectMatch.MatchLiteral(subject, si.From))
{
ProcessServiceImport(si, subject, replyTo, headers, payload, sender.Account);
delivered = true;
}
}
}
}
// No-responders: if nobody received the message and the publisher
// opted in, send back a 503 status HMSG on the reply subject.
if (!delivered && replyTo != null && sender.ClientOpts?.NoResponders == true)
{
SendNoResponders(sender, replyTo);
}
}
/// <summary>
/// Handles $JS.API.CONSUMER.MSG.NEXT by delivering individual HMSG messages
/// to the client's reply inbox. Go reference: consumer.go:4276 processNextMsgRequest.
/// </summary>
private void DeliverPullFetchMessages(string subject, string replyTo, ReadOnlyMemory<byte> payload, NatsClient sender)
{
var prefix = JetStream.Api.JetStreamApiSubjects.ConsumerNext;
var remainder = subject[prefix.Length..];
var split = remainder.Split('.', 2, StringSplitOptions.RemoveEmptyEntries);
if (split.Length != 2)
{
var notFoundHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n");
ProcessMessage(replyTo, null, (ReadOnlyMemory<byte>)notFoundHeader, default, sender);
return;
}
var (streamName, consumerName) = (split[0], split[1]);
// Parse batch request
int batch = 1;
int expiresMs = 0;
bool noWait = false;
if (payload.Length > 0)
{
try
{
using var doc = System.Text.Json.JsonDocument.Parse(payload);
if (doc.RootElement.TryGetProperty("batch", out var batchEl) && batchEl.TryGetInt32(out var b))
batch = Math.Max(b, 1);
if (doc.RootElement.TryGetProperty("no_wait", out var nwEl) && nwEl.ValueKind == System.Text.Json.JsonValueKind.True)
noWait = true;
if (doc.RootElement.TryGetProperty("expires", out var expEl) && expEl.TryGetInt64(out var expNs))
expiresMs = (int)(expNs / 1_000_000);
}
catch (System.Text.Json.JsonException ex)
{
_logger.LogDebug(ex, "Malformed JSON in pull request payload, using defaults");
}
}
var fetchResult = _jetStreamConsumerManager!.FetchAsync(
streamName, consumerName, new JetStream.Consumers.PullFetchRequest { Batch = batch, NoWait = noWait, ExpiresMs = expiresMs },
_jetStreamStreamManager!, default).GetAwaiter().GetResult();
// Find the sender's inbox subscription so we can deliver directly.
// Go reference: consumer.go deliverMsg — delivers directly to the client, bypassing pub/sub echo checks.
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
var matchResult = subList.Match(replyTo);
Subscription? inboxSub = null;
foreach (var sub in matchResult.PlainSubs)
{
if (sub.Client == sender)
{
inboxSub = sub;
break;
}
}
if (inboxSub == null)
return;
ReadOnlyMemory<byte> minHeaders = "NATS/1.0\r\n\r\n"u8.ToArray();
int deliverySeq = 0;
int numPending = fetchResult.Messages.Count;
foreach (var msg in fetchResult.Messages)
{
deliverySeq++;
numPending--;
var tsNanos = new DateTimeOffset(msg.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
var ackReply = $"$JS.ACK.{streamName}.{consumerName}.1.{msg.Sequence}.{deliverySeq}.{tsNanos}.{numPending}";
// Send with the ORIGINAL stream subject (not the inbox) so the NATS client
// can distinguish data messages from control/status messages.
// Go reference: consumer.go deliverMsg — uses original subject on wire, inbox SID.
DeliverMessage(inboxSub, msg.Subject, ackReply, minHeaders, msg.Payload);
}
// Send terminal status to end the fetch
ReadOnlyMemory<byte> statusHeader;
if (fetchResult.Messages.Count == 0 || noWait)
statusHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n");
else
statusHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 408 Request Timeout\r\n\r\n");
DeliverMessage(inboxSub, replyTo, null, statusHeader, default);
}
private void DeliverMessage(Subscription sub, string subject, string? replyTo,
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
{
var client = sub.Client;
if (client == null) return;
// Check auto-unsub
var count = Interlocked.Increment(ref sub.MessageCount);
if (sub.MaxMessages > 0 && count > sub.MaxMessages)
{
// Clean up exhausted subscription from trie and client tracking
var subList = client.Account?.SubList ?? _globalAccount.SubList;
subList.Remove(sub);
client.RemoveSubscription(sub.Sid);
return;
}
// Deny-list delivery filter
if (client.Permissions?.IsDeliveryAllowed(subject) == false)
return;
client.SendMessage(subject, sub.Sid, replyTo, headers, payload);
// Track reply subject for response permissions
if (replyTo != null && client.Permissions?.ResponseTracker != null)
{
if (client.Permissions.IsPublishAllowed(replyTo) == false)
client.Permissions.ResponseTracker.RegisterReply(replyTo);
}
}
/// <summary>
/// Processes a service import by transforming the subject from the importer's
/// subject space to the exporter's subject space, then delivering to matching
/// subscribers in the destination account.
/// Reference: Go server/accounts.go addServiceImport / processServiceImport.
/// </summary>
public void ProcessServiceImport(ServiceImport si, string subject, string? replyTo,
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload, Account? sourceAccount = null)
{
if (si.Invalid) return;
// Transform subject: map from importer subject space to exporter subject space
string targetSubject;
if (si.Transform != null)
{
var transformed = si.Transform.Apply(subject);
targetSubject = transformed ?? si.To;
}
else if (si.UsePub)
{
targetSubject = subject;
}
else
{
// Default: use the "To" subject from the import definition.
// For wildcard imports (e.g. "requests.>" -> "api.>"), we need
// to map the specific subject tokens from the source pattern to
// the destination pattern.
targetSubject = MapImportSubject(subject, si.From, si.To);
}
// Set up a temporary reverse service import so that responses from the
// destination (exporter) account can route back to the source (importer)
// account. This handles request-reply across account boundaries.
// Go reference: client.go setupResponseServiceImport
if (replyTo != null && sourceAccount != null && !si.IsResponse)
{
SetupResponseServiceImport(si.DestinationAccount, sourceAccount, replyTo, si.Export);
}
// Service latency tracking: when the response arrives back, compute elapsed
// time and publish a latency metric to the configured subject.
// Go reference: client.go processServiceImport — latency tracking path.
if (si.IsResponse && si.Tracking && si.TimestampTicks > 0)
{
var elapsed = TimeSpan.FromTicks(Environment.TickCount64 * TimeSpan.TicksPerMillisecond - si.TimestampTicks);
PublishServiceLatency(si, elapsed);
}
// Match against destination account's SubList
var destSubList = si.DestinationAccount.SubList;
var result = destSubList.Match(targetSubject);
// Deliver to plain subscribers in the destination account
foreach (var sub in result.PlainSubs)
{
if (sub.Client == null) continue;
DeliverMessage(sub, targetSubject, replyTo, headers, payload);
}
// Deliver to one member of each queue group
foreach (var queueGroup in result.QueueSubs)
{
if (queueGroup.Length == 0) continue;
var sub = queueGroup[0]; // Simple selection: first available
if (sub.Client != null)
DeliverMessage(sub, targetSubject, replyTo, headers, payload);
}
}
/// <summary>
/// Creates a temporary reverse service import in the exporter's account so that
/// when the exporter publishes a response to the reply subject, the message is
/// forwarded back to the importer's account where the reply subscription lives.
/// Go reference: client.go setupResponseServiceImport.
/// </summary>
private static void SetupResponseServiceImport(Account exporterAccount, Account importerAccount, string replyTo, ServiceExport? export = null)
{
// Check if a reverse import for this reply subject already exists
if (exporterAccount.Imports.Services.ContainsKey(replyTo))
return;
// Determine if we should track latency for this response
var shouldTrack = export?.Latency is { } latency && LatencyTracker.ShouldSample(latency);
var reverseImport = new ServiceImport
{
DestinationAccount = importerAccount,
From = replyTo,
To = replyTo,
IsResponse = true,
UsePub = true,
Export = export,
Tracking = shouldTrack,
// Store start time as TickCount64 (milliseconds) converted to ticks for elapsed computation
TimestampTicks = shouldTrack ? Environment.TickCount64 * TimeSpan.TicksPerMillisecond : 0,
};
exporterAccount.Imports.AddServiceImport(reverseImport);
}
/// <summary>
/// Maps a published subject from the import "From" pattern to the "To" pattern.
/// For example, if From="requests.>" and To="api.>" and subject="requests.test",
/// this returns "api.test".
/// </summary>
private static string MapImportSubject(string subject, string fromPattern, string toPattern)
{
// If "To" doesn't contain wildcards, use it directly
if (SubjectMatch.IsLiteral(toPattern))
return toPattern;
// For wildcard patterns, replace matching wildcard segments.
// Split into tokens and map from source to destination.
var subTokens = subject.Split('.');
var fromTokens = fromPattern.Split('.');
var toTokens = toPattern.Split('.');
var result = new string[toTokens.Length];
int subIdx = 0;
// Build a mapping: for each wildcard position in "from",
// capture the corresponding subject token(s)
var wildcardValues = new List<string>();
string? fwcValue = null;
for (int i = 0; i < fromTokens.Length && subIdx < subTokens.Length; i++)
{
if (fromTokens[i] == "*")
{
wildcardValues.Add(subTokens[subIdx]);
subIdx++;
}
else if (fromTokens[i] == ">")
{
// Capture all remaining tokens
fwcValue = string.Join(".", subTokens[subIdx..]);
subIdx = subTokens.Length;
}
else
{
subIdx++; // Skip literal match
}
}
// Now build the output using the "to" pattern
int wcIdx = 0;
var sb = new StringBuilder();
for (int i = 0; i < toTokens.Length; i++)
{
if (i > 0) sb.Append('.');
if (toTokens[i] == "*")
{
sb.Append(wcIdx < wildcardValues.Count ? wildcardValues[wcIdx] : "*");
wcIdx++;
}
else if (toTokens[i] == ">")
{
sb.Append(fwcValue ?? ">");
}
else
{
sb.Append(toTokens[i]);
}
}
return sb.ToString();
}
/// <summary>
/// Wires service import subscriptions for an account. Creates marker
/// subscriptions in the account's SubList so that the import paths
/// are tracked. The actual forwarding happens in ProcessMessage when
/// it checks the account's Imports.Services.
/// Reference: Go server/accounts.go addServiceImportSub.
/// </summary>
public void WireServiceImports(Account account)
{
foreach (var kvp in account.Imports.Services)
{
foreach (var si in kvp.Value)
{
if (si.Invalid) continue;
// Create a marker subscription in the importer account.
// This subscription doesn't directly deliver messages;
// the ProcessMessage method checks service imports after
// the regular SubList match.
_logger.LogDebug(
"Wired service import for account {Account}: {From} -> {To} (dest: {DestAccount})",
account.Name, si.From, si.To, si.DestinationAccount.Name);
}
}
}
private static void SendNoResponders(NatsClient sender, string replyTo)
{
// Find the sid for a subscription matching the reply subject
var sid = string.Empty;
foreach (var sub in sender.Subscriptions.Values)
{
if (SubjectMatch.MatchLiteral(replyTo, sub.Subject))
{
sid = sub.Sid;
break;
}
}
// Build: HMSG {replyTo} {sid} {hdrLen} {hdrLen}\r\n{headers}\r\n
var headerBlock = "NATS/1.0 503\r\n\r\n"u8;
var hdrLen = headerBlock.Length;
var controlLine = Encoding.ASCII.GetBytes($"HMSG {replyTo} {sid} {hdrLen} {hdrLen}\r\n");
var totalLen = controlLine.Length + hdrLen + NatsProtocol.CrLf.Length;
var msg = new byte[totalLen];
var offset = 0;
controlLine.CopyTo(msg.AsSpan(offset));
offset += controlLine.Length;
headerBlock.CopyTo(msg.AsSpan(offset));
offset += hdrLen;
NatsProtocol.CrLf.CopyTo(msg.AsSpan(offset));
sender.QueueOutbound(msg);
}
public Account GetOrCreateAccount(string name)
{
return _accounts.GetOrAdd(name, n =>
{
var acc = new Account(n);
if (_options.Accounts != null && _options.Accounts.TryGetValue(n, out var config))
{
acc.MaxConnections = config.MaxConnections;
acc.MaxSubscriptions = config.MaxSubscriptions;
acc.DefaultPermissions = config.DefaultPermissions;
// Wire exports from config
if (config.Exports != null)
{
foreach (var export in config.Exports)
{
if (export.Service is { Length: > 0 } svc)
{
ServiceLatency? latency = export.LatencySubject is { Length: > 0 }
? new ServiceLatency { Subject = export.LatencySubject, SamplingPercentage = export.LatencySampling }
: null;
acc.AddServiceExport(svc, Imports.ServiceResponseType.Singleton, approved: null, latency: latency);
}
else if (export.Stream is { Length: > 0 } strm)
{
acc.AddStreamExport(strm, approved: null);
}
}
}
// Wire imports from config (deferred — needs destination accounts resolved)
if (config.Imports != null)
WireAccountImports(acc, config.Imports);
}
return acc;
});
}
private void WireAccountImports(Account importer, List<Auth.ImportDefinition> imports)
{
foreach (var imp in imports)
{
if (imp.ServiceAccount is { Length: > 0 } svcAcct && imp.ServiceSubject is { Length: > 0 } svcSubj)
{
var dest = GetOrCreateAccount(svcAcct);
var localSubject = imp.To ?? svcSubj;
importer.AddServiceImport(dest, from: localSubject, to: svcSubj);
}
else if (imp.StreamAccount is { Length: > 0 } strmAcct && imp.StreamSubject is { Length: > 0 } strmSubj)
{
var source = GetOrCreateAccount(strmAcct);
var localSubject = imp.To ?? strmSubj;
importer.AddStreamImport(source, from: strmSubj, to: localSubject);
}
}
}
/// <summary>
/// Returns true if the subject belongs to the $SYS subject space.
/// Reference: Go server/server.go — isReservedSubject.
/// </summary>
public static bool IsSystemSubject(string subject)
=> subject.StartsWith("$SYS.", StringComparison.Ordinal) || subject == "$SYS";
/// <summary>
/// Checks whether the given account is allowed to subscribe to the specified subject.
/// Non-system accounts cannot subscribe to $SYS.> subjects.
/// Reference: Go server/accounts.go — isReservedForSys.
/// </summary>
public bool IsSubscriptionAllowed(Account? account, string subject)
{
if (!IsSystemSubject(subject))
return true;
// System account is always allowed
if (account != null && account.IsSystemAccount)
return true;
return false;
}
/// <summary>
/// Returns the SubList appropriate for a given subject: system account SubList
/// for $SYS.> subjects, or the provided account's SubList for everything else.
/// Reference: Go server/server.go — sublist routing for internal subjects.
/// </summary>
public SubList GetSubListForSubject(Account? account, string subject)
{
if (IsSystemSubject(subject))
return _systemAccount.SubList;
return account?.SubList ?? _globalAccount.SubList;
}
/// <summary>
/// Publishes a service latency metric message to the configured latency subject.
/// Go reference: client.go processServiceImport — trackLatency path.
/// </summary>
private void PublishServiceLatency(ServiceImport si, TimeSpan elapsed)
{
var latency = si.Export?.Latency;
if (latency == null || string.IsNullOrEmpty(latency.Subject))
return;
var msg = LatencyTracker.BuildLatencyMsg(
requestor: si.DestinationAccount.Name,
responder: si.Export?.Account?.Name ?? "unknown",
serviceLatency: elapsed,
totalLatency: elapsed);
SendInternalMsg(latency.Subject, reply: null, msg);
}
public void SendInternalMsg(string subject, string? reply, object? msg)
{
_eventSystem?.Enqueue(new PublishMessage { Subject = subject, Reply = reply, Body = msg });
}
public void SendInternalAccountMsg(Account account, string subject, object? msg)
{
_eventSystem?.Enqueue(new PublishMessage { Subject = subject, Body = msg });
}
/// <summary>
/// Handles $SYS.REQ.SERVER.{id}.VARZ requests.
/// Returns core server information including stats counters.
/// </summary>
public void HandleVarzRequest(string subject, string? reply)
{
if (reply == null) return;
var varz = new
{
server_id = _serverInfo.ServerId,
server_name = _serverInfo.ServerName,
version = NatsProtocol.Version,
host = _options.Host,
port = _options.Port,
max_payload = _options.MaxPayload,
connections = ClientCount,
total_connections = Interlocked.Read(ref _stats.TotalConnections),
in_msgs = Interlocked.Read(ref _stats.InMsgs),
out_msgs = Interlocked.Read(ref _stats.OutMsgs),
in_bytes = Interlocked.Read(ref _stats.InBytes),
out_bytes = Interlocked.Read(ref _stats.OutBytes),
};
SendInternalMsg(reply, null, varz);
}
/// <summary>
/// Handles $SYS.REQ.SERVER.{id}.HEALTHZ requests.
/// Returns a simple health status response.
/// </summary>
public void HandleHealthzRequest(string subject, string? reply)
{
if (reply == null) return;
SendInternalMsg(reply, null, new { status = "ok" });
}
/// <summary>
/// Handles $SYS.REQ.SERVER.{id}.SUBSZ requests.
/// Returns the current subscription count.
/// </summary>
public void HandleSubszRequest(string subject, string? reply)
{
if (reply == null) return;
SendInternalMsg(reply, null, new { num_subscriptions = SubList.Count });
}
/// <summary>
/// Handles $SYS.REQ.SERVER.{id}.STATSZ requests.
/// Publishes current server statistics through the event system.
/// </summary>
public void HandleStatszRequest(string subject, string? reply)
{
if (reply == null) return;
var process = System.Diagnostics.Process.GetCurrentProcess();
var statsMsg = new Events.ServerStatsMsg
{
Server = BuildEventServerInfo(),
Stats = new Events.ServerStatsData
{
Start = StartTime,
Mem = process.WorkingSet64,
Cores = Environment.ProcessorCount,
Connections = ClientCount,
TotalConnections = Interlocked.Read(ref _stats.TotalConnections),
Subscriptions = SubList.Count,
Sent = new Events.DataStats
{
Msgs = Interlocked.Read(ref _stats.OutMsgs),
Bytes = Interlocked.Read(ref _stats.OutBytes),
},
Received = new Events.DataStats
{
Msgs = Interlocked.Read(ref _stats.InMsgs),
Bytes = Interlocked.Read(ref _stats.InBytes),
},
InMsgs = Interlocked.Read(ref _stats.InMsgs),
OutMsgs = Interlocked.Read(ref _stats.OutMsgs),
InBytes = Interlocked.Read(ref _stats.InBytes),
OutBytes = Interlocked.Read(ref _stats.OutBytes),
SlowConsumers = Interlocked.Read(ref _stats.SlowConsumers),
},
};
SendInternalMsg(reply, null, statsMsg);
}
/// <summary>
/// Handles $SYS.REQ.SERVER.{id}.IDZ requests.
/// Returns basic server identity information.
/// </summary>
public void HandleIdzRequest(string subject, string? reply)
{
if (reply == null) return;
var idz = new
{
server_id = _serverInfo.ServerId,
server_name = _serverInfo.ServerName,
version = NatsProtocol.Version,
host = _options.Host,
port = _options.Port,
};
SendInternalMsg(reply, null, idz);
}
/// <summary>
/// Builds an EventServerInfo block for embedding in system event messages.
/// Maps to Go's serverInfo() helper used in events.go advisory publishing.
/// </summary>
public EventServerInfo BuildEventServerInfo()
{
var seq = _eventSystem?.NextSequence() ?? 0;
return new EventServerInfo
{
Name = _serverInfo.ServerName,
Host = _options.Host,
Id = _serverInfo.ServerId,
Version = NatsProtocol.Version,
Seq = seq,
};
}
private static EventClientInfo BuildEventClientInfo(NatsClient client)
{
return new EventClientInfo
{
Id = client.Id,
Host = client.RemoteIp,
Account = client.Account?.Name,
Name = client.ClientOpts?.Name,
Lang = client.ClientOpts?.Lang,
Version = client.ClientOpts?.Version,
Start = client.StartTime,
};
}
/// <summary>
/// Publishes a $SYS.ACCOUNT.{account}.CONNECT advisory when a client
/// completes authentication. Maps to Go's sendConnectEvent in events.go.
/// </summary>
public void PublishConnectEvent(NatsClient client)
{
if (_eventSystem == null) return;
var accountName = client.Account?.Name ?? Account.GlobalAccountName;
var subject = string.Format(EventSubjects.ConnectEvent, accountName);
var evt = new ConnectEventMsg
{
Id = Guid.NewGuid().ToString("N"),
Time = DateTime.UtcNow,
Server = BuildEventServerInfo(),
Client = BuildEventClientInfo(client),
};
SendInternalMsg(subject, null, evt);
}
/// <summary>
/// Publishes a $SYS.ACCOUNT.{account}.DISCONNECT advisory when a client
/// disconnects. Maps to Go's sendDisconnectEvent in events.go.
/// </summary>
public void PublishDisconnectEvent(NatsClient client)
{
if (_eventSystem == null) return;
var accountName = client.Account?.Name ?? Account.GlobalAccountName;
var subject = string.Format(EventSubjects.DisconnectEvent, accountName);
var evt = new DisconnectEventMsg
{
Id = Guid.NewGuid().ToString("N"),
Time = DateTime.UtcNow,
Server = BuildEventServerInfo(),
Client = BuildEventClientInfo(client),
Sent = new DataStats
{
Msgs = Interlocked.Read(ref client.OutMsgs),
Bytes = Interlocked.Read(ref client.OutBytes),
},
Received = new DataStats
{
Msgs = Interlocked.Read(ref client.InMsgs),
Bytes = Interlocked.Read(ref client.InBytes),
},
Reason = client.CloseReason.ToReasonString(),
};
SendInternalMsg(subject, null, evt);
}
public void RemoveClient(NatsClient client)
{
// Publish disconnect advisory before removing client state
if (client.ConnectReceived)
PublishDisconnectEvent(client);
_clients.TryRemove(client.Id, out _);
_logger.LogDebug("Removed client {ClientId}", client.Id);
var (tlsPeerCertSubject, tlsPeerCertSubjectPkSha256, tlsPeerCertSha256) =
TlsPeerCertMapper.ToClosedFields(client.TlsState?.PeerCert);
var (jwt, issuerKey, tags) = ExtractJwtMetadata(client.ClientOpts?.JWT);
var proxyKey = ExtractProxyKey(client.ClientOpts?.Username);
// Snapshot for closed-connections tracking (ring buffer auto-overwrites oldest when full)
_closedClients.Add(new ClosedClient
{
Cid = client.Id,
Ip = client.RemoteIp ?? "",
Port = client.RemotePort,
Start = client.StartTime,
Stop = DateTime.UtcNow,
Reason = client.CloseReason.ToReasonString(),
Name = client.ClientOpts?.Name ?? "",
Lang = client.ClientOpts?.Lang ?? "",
Version = client.ClientOpts?.Version ?? "",
AuthorizedUser = client.ClientOpts?.Username ?? "",
Account = client.Account?.Name ?? "",
InMsgs = Interlocked.Read(ref client.InMsgs),
OutMsgs = Interlocked.Read(ref client.OutMsgs),
InBytes = Interlocked.Read(ref client.InBytes),
OutBytes = Interlocked.Read(ref client.OutBytes),
NumSubs = (uint)client.Subscriptions.Count,
Rtt = client.Rtt,
TlsVersion = client.TlsState?.TlsVersion ?? "",
TlsCipherSuite = client.TlsState?.CipherSuite ?? "",
TlsPeerCertSubject = tlsPeerCertSubject,
TlsPeerCertSubjectPkSha256 = tlsPeerCertSubjectPkSha256,
TlsPeerCertSha256 = tlsPeerCertSha256,
MqttClient = "", // populated when MQTT transport is implemented
Stalls = 0,
Jwt = jwt,
IssuerKey = issuerKey,
NameTag = "",
Tags = tags,
ProxyKey = proxyKey,
});
var subList = client.Account?.SubList ?? _globalAccount.SubList;
client.RemoveAllSubscriptions(subList);
client.Account?.RemoveClient(client.Id);
}
private void TrackEarlyClosedClient(Socket socket, ulong clientId, ClientClosedReason reason)
{
string ip = "";
int port = 0;
if (socket.RemoteEndPoint is IPEndPoint endpoint)
{
ip = endpoint.Address.ToString();
port = endpoint.Port;
}
var now = DateTime.UtcNow;
_closedClients.Add(new ClosedClient
{
Cid = clientId,
Ip = ip,
Port = port,
Start = now,
Stop = now,
Reason = reason.ToReasonString(),
});
}
private static (string Jwt, string IssuerKey, string[] Tags) ExtractJwtMetadata(string? jwt)
{
if (string.IsNullOrWhiteSpace(jwt))
return ("", "", []);
var issuerKey = "";
var tags = Array.Empty<string>();
var claims = NatsJwt.DecodeUserClaims(jwt);
if (claims != null)
{
issuerKey = claims.Issuer ?? "";
tags = claims.Nats?.Tags ?? Array.Empty<string>();
}
return (jwt, issuerKey, tags);
}
private static string ExtractProxyKey(string? username)
{
if (string.IsNullOrWhiteSpace(username))
return "";
const string prefix = "proxy:";
return username.StartsWith(prefix, StringComparison.Ordinal)
? username[prefix.Length..]
: "";
}
private void WritePidFile()
{
if (string.IsNullOrEmpty(_options.PidFile)) return;
try
{
File.WriteAllText(_options.PidFile, Environment.ProcessId.ToString());
_logger.LogDebug("Wrote PID file {PidFile}", _options.PidFile);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error writing PID file {PidFile}", _options.PidFile);
}
}
private void DeletePidFile()
{
if (string.IsNullOrEmpty(_options.PidFile)) return;
try
{
if (File.Exists(_options.PidFile))
File.Delete(_options.PidFile);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting PID file {PidFile}", _options.PidFile);
}
}
private void WritePortsFile()
{
if (string.IsNullOrEmpty(_options.PortsFileDir)) return;
try
{
var exeName = Path.GetFileNameWithoutExtension(Environment.ProcessPath ?? "nats-server");
var fileName = $"{exeName}_{Environment.ProcessId}.ports";
_portsFilePath = Path.Combine(_options.PortsFileDir, fileName);
var ports = new { client = _options.Port, monitor = _options.MonitorPort > 0 ? _options.MonitorPort : (int?)null };
var json = System.Text.Json.JsonSerializer.Serialize(ports);
File.WriteAllText(_portsFilePath, json);
_logger.LogDebug("Wrote ports file {PortsFile}", _portsFilePath);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error writing ports file to {PortsFileDir}", _options.PortsFileDir);
}
}
private void DeletePortsFile()
{
if (_portsFilePath == null) return;
try
{
if (File.Exists(_portsFilePath))
File.Delete(_portsFilePath);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting ports file {PortsFile}", _portsFilePath);
}
}
/// <summary>
/// Stores the CLI snapshot and flags so that command-line overrides
/// always take precedence during config reload.
/// </summary>
public void SetCliSnapshot(NatsOptions cliSnapshot, HashSet<string> cliFlags)
{
_cliSnapshot = cliSnapshot;
_cliFlags = cliFlags;
}
/// <summary>
/// Reloads the configuration file, diffs against current options, validates
/// the changes, and applies reloadable settings. CLI overrides are preserved.
/// </summary>
public void ReloadConfig()
{
ReloadConfigCore(throwOnError: false);
}
public void ReloadConfigOrThrow()
{
ReloadConfigCore(throwOnError: true);
}
private void ReloadConfigCore(bool throwOnError)
{
if (_options.ConfigFile == null)
{
_logger.LogWarning("No config file specified, cannot reload");
if (throwOnError)
throw new InvalidOperationException("No config file specified.");
return;
}
try
{
var (newConfig, digest) = NatsConfParser.ParseFileWithDigest(_options.ConfigFile);
if (digest == _configDigest)
{
_logger.LogInformation("Config file unchanged, no reload needed");
return;
}
var newOpts = new NatsOptions { ConfigFile = _options.ConfigFile };
ConfigProcessor.ApplyConfig(newConfig, newOpts);
// CLI flags override config
if (_cliSnapshot != null)
ConfigReloader.MergeCliOverrides(newOpts, _cliSnapshot, _cliFlags);
var changes = ConfigReloader.Diff(_options, newOpts);
var errors = ConfigReloader.Validate(changes);
if (errors.Count > 0)
{
foreach (var err in errors)
_logger.LogError("Config reload error: {Error}", err);
if (throwOnError)
throw new InvalidOperationException(string.Join("; ", errors));
return;
}
// Apply changes to running options
ApplyConfigChanges(changes, newOpts);
_configDigest = digest;
_configTime = DateTime.UtcNow;
_logger.LogInformation("Config reloaded successfully ({Count} changes applied)", changes.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to reload config file: {ConfigFile}", _options.ConfigFile);
if (throwOnError)
throw;
}
}
private void ApplyConfigChanges(List<IConfigChange> changes, NatsOptions newOpts)
{
bool hasLoggingChanges = false;
bool hasAuthChanges = false;
bool hasTlsChanges = false;
foreach (var change in changes)
{
if (change.IsLoggingChange) hasLoggingChanges = true;
if (change.IsAuthChange) hasAuthChanges = true;
if (change.IsTlsChange) hasTlsChanges = true;
}
// Copy reloadable values from newOpts to _options
CopyReloadableOptions(newOpts);
// Trigger side effects
if (hasLoggingChanges)
{
ReOpenLogFile?.Invoke();
_logger.LogInformation("Logging configuration reloaded");
}
if (hasTlsChanges)
{
// Reload TLS certificates: new connections get the new cert,
// existing connections keep their original cert.
// Reference: golang/nats-server/server/reload.go — tlsOption.Apply.
if (ConfigReloader.ReloadTlsCertificate(_options, _tlsCertProvider))
{
_sslOptions = _tlsCertProvider!.GetCurrentSslOptions();
_logger.LogInformation("TLS configuration reloaded");
}
}
if (hasAuthChanges)
{
// Rebuild auth service with new options, then propagate changes to connected clients
var oldAuthService = _authService;
_authService = AuthService.Build(_options);
_logger.LogInformation("Authorization configuration reloaded");
// Re-evaluate connected clients against the new auth config.
// Clients that no longer pass authentication are disconnected with AUTH_EXPIRED.
// Reference: Go server/reload.go — applyOptions / reloadAuthorization.
PropagateAuthChanges();
}
}
/// <summary>
/// Re-evaluates all connected clients against the current auth configuration.
/// Clients whose credentials no longer pass authentication are disconnected
/// with an "Authorization Violation" error via SendErrAndCloseAsync, which
/// properly drains the outbound channel before closing the socket.
/// Reference: Go server/reload.go — reloadAuthorization, client.go — applyAccountLimits.
/// </summary>
internal void PropagateAuthChanges()
{
if (!_authService.IsAuthRequired)
{
// Auth was disabled — all existing clients are fine
return;
}
var clientsToDisconnect = new List<NatsClient>();
foreach (var client in _clients.Values)
{
if (client.ClientOpts == null)
continue; // Client hasn't sent CONNECT yet
var context = new ClientAuthContext
{
Opts = client.ClientOpts,
Nonce = [], // Nonce is only used at connect time; re-evaluation skips it
ClientCertificate = client.TlsState?.PeerCert,
};
var result = _authService.Authenticate(context);
if (result == null)
{
_logger.LogInformation(
"Client {ClientId} credentials no longer valid after auth reload, disconnecting",
client.Id);
clientsToDisconnect.Add(client);
}
}
// Disconnect clients that failed re-authentication.
// Use SendErrAndCloseAsync which queues the -ERR, completes the outbound channel,
// waits for the write loop to drain, then cancels the client.
var disconnectTasks = new List<Task>(clientsToDisconnect.Count);
foreach (var client in clientsToDisconnect)
{
disconnectTasks.Add(client.SendErrAndCloseAsync(
NatsProtocol.ErrAuthorizationViolation,
ClientClosedReason.AuthenticationExpired));
}
// Wait for all disconnects to complete (with timeout to avoid blocking reload)
if (disconnectTasks.Count > 0)
{
Task.WhenAll(disconnectTasks)
.WaitAsync(TimeSpan.FromSeconds(5))
.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing)
.GetAwaiter().GetResult();
_logger.LogInformation(
"Disconnected {Count} client(s) after auth configuration reload",
clientsToDisconnect.Count);
}
}
private void CopyReloadableOptions(NatsOptions newOpts)
{
// Logging
_options.Debug = newOpts.Debug;
_options.Trace = newOpts.Trace;
_options.TraceVerbose = newOpts.TraceVerbose;
_options.Logtime = newOpts.Logtime;
_options.LogtimeUTC = newOpts.LogtimeUTC;
_options.LogFile = newOpts.LogFile;
_options.LogSizeLimit = newOpts.LogSizeLimit;
_options.LogMaxFiles = newOpts.LogMaxFiles;
_options.Syslog = newOpts.Syslog;
_options.RemoteSyslog = newOpts.RemoteSyslog;
// Auth
_options.Username = newOpts.Username;
_options.Password = newOpts.Password;
_options.Authorization = newOpts.Authorization;
_options.Users = newOpts.Users;
_options.NKeys = newOpts.NKeys;
_options.NoAuthUser = newOpts.NoAuthUser;
_options.AuthTimeout = newOpts.AuthTimeout;
// Limits
_options.MaxConnections = newOpts.MaxConnections;
_options.MaxPayload = newOpts.MaxPayload;
_options.MaxPending = newOpts.MaxPending;
_options.WriteDeadline = newOpts.WriteDeadline;
_options.PingInterval = newOpts.PingInterval;
_options.MaxPingsOut = newOpts.MaxPingsOut;
_options.MaxControlLine = newOpts.MaxControlLine;
_options.MaxSubs = newOpts.MaxSubs;
_options.MaxSubTokens = newOpts.MaxSubTokens;
_options.MaxTracedMsgLen = newOpts.MaxTracedMsgLen;
if (newOpts.MaxClosedClients != _options.MaxClosedClients)
{
_options.MaxClosedClients = newOpts.MaxClosedClients;
_closedClients = new ClosedConnectionRingBuffer(newOpts.MaxClosedClients);
}
// TLS
_options.TlsCert = newOpts.TlsCert;
_options.TlsKey = newOpts.TlsKey;
_options.TlsCaCert = newOpts.TlsCaCert;
_options.TlsVerify = newOpts.TlsVerify;
_options.TlsMap = newOpts.TlsMap;
_options.TlsTimeout = newOpts.TlsTimeout;
_options.TlsHandshakeFirst = newOpts.TlsHandshakeFirst;
_options.TlsHandshakeFirstFallback = newOpts.TlsHandshakeFirstFallback;
_options.AllowNonTls = newOpts.AllowNonTls;
_options.TlsRateLimit = newOpts.TlsRateLimit;
_options.TlsPinnedCerts = newOpts.TlsPinnedCerts;
// Misc
_options.Tags = newOpts.Tags;
_options.LameDuckDuration = newOpts.LameDuckDuration;
_options.LameDuckGracePeriod = newOpts.LameDuckGracePeriod;
_options.ClientAdvertise = newOpts.ClientAdvertise;
_options.DisableSublistCache = newOpts.DisableSublistCache;
_options.ConnectErrorReports = newOpts.ConnectErrorReports;
_options.ReconnectErrorReports = newOpts.ReconnectErrorReports;
_options.NoHeaderSupport = newOpts.NoHeaderSupport;
_options.NoSystemAccount = newOpts.NoSystemAccount;
_options.SystemAccount = newOpts.SystemAccount;
}
public override string ToString()
=> $"NatsServer(ServerId={ServerId}, Name={ServerName}, Addr={Addr()}, Clients={ClientCount})";
public void Dispose()
{
if (!IsShuttingDown)
ShutdownAsync().GetAwaiter().GetResult();
foreach (var reg in _signalRegistrations)
reg.Dispose();
_quitCts.Dispose();
_tlsRateLimiter?.Dispose();
_tlsCertProvider?.Dispose();
_listener?.Dispose();
_wsListener?.Dispose();
_routeManager?.DisposeAsync().AsTask().GetAwaiter().GetResult();
_gatewayManager?.DisposeAsync().AsTask().GetAwaiter().GetResult();
_leafNodeManager?.DisposeAsync().AsTask().GetAwaiter().GetResult();
_mqttListener?.DisposeAsync().AsTask().GetAwaiter().GetResult();
_jetStreamService?.DisposeAsync().AsTask().GetAwaiter().GetResult();
_stats.JetStreamEnabled = false;
foreach (var client in _clients.Values)
client.Dispose();
foreach (var account in _accounts.Values)
account.Dispose();
}
}