using System.Collections.Concurrent; using System.Net; using System.Net.NetworkInformation; using System.Net.Security; using System.Net.Sockets; using System.Runtime.InteropServices; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.Json; using Microsoft.Extensions.Logging; using NATS.NKeys; using NATS.Server.Auth; using NATS.Server.Auth.Jwt; using NATS.Server.Configuration; using NATS.Server.Events; using NATS.Server.Gateways; using NATS.Server.Imports; using NATS.Server.JetStream; using NATS.Server.JetStream.Api; using NATS.Server.JetStream.Publish; using NATS.Server.LeafNodes; using NATS.Server.Monitoring; using NATS.Server.Mqtt; using NATS.Server.Protocol; using NATS.Server.Routes; using NATS.Server.Server; using NATS.Server.Subscriptions; using NATS.Server.Tls; using NATS.Server.WebSocket; namespace NATS.Server; public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable { private readonly NatsOptions _options; private readonly ConcurrentDictionary _clients = new(); private ClosedConnectionRingBuffer _closedClients; private readonly ServerInfo _serverInfo; private readonly ILogger _logger; private readonly ILoggerFactory _loggerFactory; private readonly ServerStats _stats = new(); // Per-client deferred flush set. Collects unique clients during fan-out delivery, // then flushes each once. Go reference: client.go addToPCD / flushClients. [ThreadStatic] private static HashSet? t_pcd; private readonly TaskCompletionSource _listeningStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); private AuthService _authService; private readonly ConcurrentDictionary _accounts = new(StringComparer.Ordinal); // Config reload state private NatsOptions? _cliSnapshot; private HashSet _cliFlags = []; private string? _configDigest; private readonly SemaphoreSlim _reloadMu = new(1, 1); private AcceptLoopErrorHandler? _acceptLoopErrorHandler; private readonly Account _globalAccount; private readonly Account _systemAccount; private InternalEventSystem? _eventSystem; private SslServerAuthenticationOptions? _sslOptions; private readonly TlsRateLimiter? _tlsRateLimiter; private readonly TlsCertificateProvider? _tlsCertProvider; private readonly SubjectTransform[] _subjectTransforms; private readonly RouteManager? _routeManager; /// /// Exposes the route manager for testing. Internal — visible to test project /// via InternalsVisibleTo. /// internal RouteManager? RouteManager => _routeManager; internal GatewayManager? GatewayManager => _gatewayManager; private readonly GatewayManager? _gatewayManager; private readonly LeafNodeManager? _leafNodeManager; private readonly InternalClient? _jetStreamInternalClient; private readonly JetStreamService? _jetStreamService; private readonly JetStreamApiRouter? _jetStreamApiRouter; private readonly StreamManager? _jetStreamStreamManager; private readonly ConsumerManager? _jetStreamConsumerManager; private readonly JetStreamPublisher? _jetStreamPublisher; private MqttListener? _mqttListener; private Socket? _listener; private Socket? _wsListener; private readonly TaskCompletionSource _wsAcceptLoopExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private MonitorServer? _monitorServer; private ulong _nextClientId; private long _startTimeTicks; private readonly CancellationTokenSource _quitCts = new(); private readonly TaskCompletionSource _shutdownComplete = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _acceptLoopExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _shutdown; private int _activeClientCount; private int _lameDuck; private byte[] _cachedInfoLine = []; private readonly List _signalRegistrations = []; private string? _portsFilePath; private DateTime _configTime = DateTime.UtcNow; private static readonly TimeSpan AcceptMinSleep = NatsProtocol.AcceptMinSleep; private static readonly TimeSpan AcceptMaxSleep = NatsProtocol.AcceptMaxSleep; private static readonly JsonSerializerOptions s_jetStreamJsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, }; public SubList SubList => _globalAccount.SubList; public byte[] CachedInfoLine => _cachedInfoLine; public ServerStats Stats => _stats; public DateTime StartTime => new(Interlocked.Read(ref _startTimeTicks), DateTimeKind.Utc); public string ServerId => _serverInfo.ServerId; public string ServerName => _serverInfo.ServerName; public int ClientCount => _clients.Count; public int Port => _options.Port; /// /// Returns the actual bound port of the MQTT listener, or null if MQTT is not enabled. /// Used by VarzHandler for monitoring. /// public int? MqttListenerPort => _mqttListener?.Port; /// /// Returns all active MQTT client adapters for monitoring (/connz). /// public IEnumerable GetMqttAdapters() => _mqttListener?.GetMqttAdapters() ?? []; public Account SystemAccount => _systemAccount; public string ServerNKey { get; } public InternalEventSystem? EventSystem => _eventSystem; public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0; public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0; public string? ClusterListen => _routeManager?.ListenEndpoint; public string? GatewayListen => _gatewayManager?.ListenEndpoint; public string? LeafListen => _leafNodeManager?.ListenEndpoint; public bool IsProfilingEnabled => _options.ProfPort > 0; public InternalClient? JetStreamInternalClient => _jetStreamInternalClient; public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter; public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0; public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0; public Action? ReOpenLogFile { get; set; } public IEnumerable GetClients() => _clients.Values; public string? ClusterName() => _options.Cluster?.Name; public IReadOnlyList ActivePeers() => _routeManager?.BuildTopologySnapshot().ConnectedServerIds ?? []; public bool StartProfiler() { if (_options.ProfPort <= 0) return false; _logger.LogWarning("Profiling endpoint not yet supported (port: {ProfPort})", _options.ProfPort); return true; } public bool DisconnectClientByID(ulong clientId) => CloseClientById(clientId, minimalFlush: true); public bool LDMClientByID(ulong clientId) => CloseClientById(clientId, minimalFlush: false); public Ports PortsInfo() { var ports = new Ports(); AddEndpoint(ports.Nats, _options.Host, _options.Port); AddEndpoint(ports.Monitoring, _options.MonitorHost, _options.MonitorPort); if (_routeManager != null) AddEndpoint(ports.Cluster, _routeManager.ListenEndpoint); else if (_options.Cluster != null) AddEndpoint(ports.Cluster, _options.Cluster.Host, _options.Cluster.Port); AddEndpoint(ports.Profile, _options.Host, _options.ProfPort); if (_options.WebSocket.Port >= 0) AddEndpoint(ports.WebSocket, _options.WebSocket.Host, _options.WebSocket.Port); if (_leafNodeManager != null) AddEndpoint(ports.LeafNodes, _leafNodeManager.ListenEndpoint); else if (_options.LeafNode != null) AddEndpoint(ports.LeafNodes, _options.LeafNode.Host, _options.LeafNode.Port); return ports; } public IReadOnlyList GetConnectURLs() { if (!string.IsNullOrWhiteSpace(_options.ClientAdvertise)) return [NormalizeAdvertiseUrl(_options.ClientAdvertise!, "nats")]; var hosts = GetNonLocalIPsIfHostIsIPAny(_options.Host); var result = new List(hosts.Count); foreach (var host in hosts) result.Add($"nats://{host}:{_options.Port}"); return result; } public void UpdateServerINFOAndSendINFOToClients() { _serverInfo.ConnectUrls = [.. GetConnectURLs()]; BuildCachedInfo(); foreach (var client in _clients.Values) { if (client.ConnectReceived) client.QueueOutbound(_cachedInfoLine); } } public string ClientURL() { if (!string.IsNullOrWhiteSpace(_options.ClientAdvertise)) return NormalizeAdvertiseUrl(_options.ClientAdvertise!, "nats"); var host = IsWildcardHost(_options.Host) ? "127.0.0.1" : _options.Host; return $"nats://{host}:{_options.Port}"; } public string? WebsocketURL() { if (_options.WebSocket.Port < 0) return null; if (!string.IsNullOrWhiteSpace(_options.WebSocket.Advertise)) { var scheme = _options.WebSocket.NoTls ? "ws" : "wss"; return NormalizeAdvertiseUrl(_options.WebSocket.Advertise!, scheme); } var wsHost = IsWildcardHost(_options.WebSocket.Host) ? "127.0.0.1" : _options.WebSocket.Host; var wsScheme = _options.WebSocket.NoTls ? "ws" : "wss"; return $"{wsScheme}://{wsHost}:{_options.WebSocket.Port}"; } public int NumRoutes() => (int)Interlocked.Read(ref _stats.Routes); public int NumRemotes() => (int)(Interlocked.Read(ref _stats.Routes) + Interlocked.Read(ref _stats.Gateways) + Interlocked.Read(ref _stats.Leafs)); public int NumLeafNodes() => (int)Interlocked.Read(ref _stats.Leafs); public int NumOutboundGateways() => _gatewayManager?.NumOutboundGateways() ?? 0; public int NumInboundGateways() => _gatewayManager?.NumInboundGateways() ?? 0; public int NumSubscriptions() => _accounts.Values.Sum(acc => acc.SubscriptionCount); public bool JetStreamEnabled() => _jetStreamService?.IsRunning ?? false; public JetStreamOptions? JetStreamConfig() { if (_options.JetStream is null) return null; return new JetStreamOptions { StoreDir = _options.JetStream.StoreDir, MaxMemoryStore = _options.JetStream.MaxMemoryStore, MaxFileStore = _options.JetStream.MaxFileStore, MaxStreams = _options.JetStream.MaxStreams, MaxConsumers = _options.JetStream.MaxConsumers, Domain = _options.JetStream.Domain, }; } public string StoreDir() => _options.JetStream?.StoreDir ?? string.Empty; public DateTime ConfigTime() => _configTime; public string Addr() => $"{_options.Host}:{_options.Port}"; public string? MonitorAddr() => _options.MonitorPort > 0 ? $"{_options.MonitorHost}:{_options.MonitorPort}" : null; public string? ClusterAddr() => _routeManager?.ListenEndpoint; public string? GatewayAddr() => _gatewayManager?.ListenEndpoint; public string? GetGatewayURL() => _gatewayManager?.ListenEndpoint; public string? GetGatewayName() => _options.Gateway?.Name; public string? ProfilerAddr() => _options.ProfPort > 0 ? $"{_options.Host}:{_options.ProfPort}" : null; public int NumActiveAccounts() => _accounts.Values.Count(acc => acc.ClientCount > 0); public int NumLoadedAccounts() => _accounts.Count; public IReadOnlyList GetClosedClients() => _closedClients.GetAll(); public IEnumerable GetAccounts() => _accounts.Values; public bool HasRemoteInterest(string subject) => _globalAccount.SubList.HasRemoteInterest(subject); public bool HasRemoteInterest(string account, string subject) => GetOrCreateAccount(account).SubList.HasRemoteInterest(account, subject); public bool TryCaptureJetStreamPublish(string subject, ReadOnlyMemory payload, out PubAck ack) { if (_jetStreamPublisher != null && _jetStreamPublisher.TryCapture(subject, payload, out ack)) { // Only load the stored message for consumer notification when there are // active consumers for this stream. Avoids unnecessary LoadAsync on the hot path. if (ack.ErrorCode == null && _jetStreamConsumerManager != null && _jetStreamConsumerManager.HasConsumersForStream(ack.Stream) && _jetStreamStreamManager != null && _jetStreamStreamManager.TryGet(ack.Stream, out var streamHandle)) { var stored = streamHandle.Store.LoadAsync(ack.Seq, default).GetAwaiter().GetResult(); if (stored != null) _jetStreamConsumerManager.OnPublished(ack.Stream, stored); } return true; } ack = new PubAck(); return false; } /// /// Replicates a successful JetStream mutating API call to cluster peers via internal subjects. /// Maps API subjects to $JS.INTERNAL.* subjects and forwards to routes/leafnodes. /// Go reference: jetstream_cluster.go — RAFT proposal broadcast. /// private void TryReplicateJetStreamMutation(string apiSubject, ReadOnlyMemory payload) { string? internalSubject = null; if (apiSubject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal)) internalSubject = JetStreamApiSubjects.InternalStreamCreate + apiSubject[JetStreamApiSubjects.StreamCreate.Length..]; else if (apiSubject.StartsWith(JetStreamApiSubjects.StreamDelete, StringComparison.Ordinal)) internalSubject = JetStreamApiSubjects.InternalStreamDelete + apiSubject[JetStreamApiSubjects.StreamDelete.Length..]; else if (apiSubject.StartsWith(JetStreamApiSubjects.StreamPurge, StringComparison.Ordinal)) internalSubject = JetStreamApiSubjects.InternalStreamPurge + apiSubject[JetStreamApiSubjects.StreamPurge.Length..]; else if (apiSubject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal)) internalSubject = JetStreamApiSubjects.InternalConsumerCreate + apiSubject[JetStreamApiSubjects.ConsumerCreate.Length..]; else if (apiSubject.StartsWith(JetStreamApiSubjects.ConsumerDelete, StringComparison.Ordinal)) internalSubject = JetStreamApiSubjects.InternalConsumerDelete + apiSubject[JetStreamApiSubjects.ConsumerDelete.Length..]; if (internalSubject != null) ReplicateJetStreamOperation("$G", internalSubject, null, payload); } /// /// Forwards a JetStream replication message to all route and leaf node peers. /// Bypasses interest checks since replication subjects have no client subscribers. /// Go reference: jetstream_cluster.go — proposal broadcast to peers. /// private void ReplicateJetStreamOperation(string account, string subject, string? replyTo, ReadOnlyMemory payload) { _routeManager?.BroadcastRoutedMessageAsync(account, subject, replyTo, payload, default) .GetAwaiter().GetResult(); if (_leafNodeManager != null) { var markedSubject = LeafLoopDetector.Mark(subject, ServerId); _leafNodeManager.ForwardMessageAsync(account, markedSubject, replyTo, payload, default) .GetAwaiter().GetResult(); } } /// /// Handles incoming JetStream internal replication messages from cluster peers. /// Dispatches to the appropriate handler based on the internal subject prefix. /// Called from DeliverRemoteMessage for $JS.INTERNAL.* subjects. /// Go reference: jetstream_cluster.go — RAFT proposal apply. /// private void HandleJetStreamReplication(string subject, ReadOnlyMemory payload) { if (_jetStreamStreamManager == null || _jetStreamConsumerManager == null) return; if (subject.StartsWith(JetStreamApiSubjects.InternalStreamCreate, StringComparison.Ordinal)) { var apiSubject = JetStreamApiSubjects.StreamCreate + subject[JetStreamApiSubjects.InternalStreamCreate.Length..]; JetStream.Api.Handlers.StreamApiHandlers.HandleCreate(apiSubject, payload.Span, _jetStreamStreamManager); } else if (subject.StartsWith(JetStreamApiSubjects.InternalStreamDelete, StringComparison.Ordinal)) { var streamName = subject[JetStreamApiSubjects.InternalStreamDelete.Length..]; _jetStreamStreamManager.Delete(streamName); } else if (subject.StartsWith(JetStreamApiSubjects.InternalStreamPurge, StringComparison.Ordinal)) { var streamName = subject[JetStreamApiSubjects.InternalStreamPurge.Length..]; _jetStreamStreamManager.Purge(streamName); } else if (subject.StartsWith(JetStreamApiSubjects.InternalConsumerCreate, StringComparison.Ordinal)) { var apiSubject = JetStreamApiSubjects.ConsumerCreate + subject[JetStreamApiSubjects.InternalConsumerCreate.Length..]; JetStream.Api.Handlers.ConsumerApiHandlers.HandleCreate(apiSubject, payload.Span, _jetStreamConsumerManager); } else if (subject.StartsWith(JetStreamApiSubjects.InternalConsumerDelete, StringComparison.Ordinal)) { var parts = subject[JetStreamApiSubjects.InternalConsumerDelete.Length..].Split('.'); if (parts.Length >= 2) _jetStreamConsumerManager.Delete(parts[0], parts[1]); } } public Task WaitForReadyAsync() => _listeningStarted.Task; public void WaitForShutdown() => _shutdownComplete.Task.GetAwaiter().GetResult(); internal TlsCertificateProvider? TlsCertProviderForTest => _tlsCertProvider; internal Task AcquireReloadLockForTestAsync() => _reloadMu.WaitAsync(); internal void ReleaseReloadLockForTest() => _reloadMu.Release(); internal void SetAcceptLoopErrorHandlerForTest(AcceptLoopErrorHandler handler) => _acceptLoopErrorHandler = handler; internal void NotifyAcceptErrorForTest(Exception ex, EndPoint? endpoint, TimeSpan delay) => _acceptLoopErrorHandler?.OnAcceptError(ex, endpoint, delay); public async Task ShutdownAsync() { if (Interlocked.CompareExchange(ref _shutdown, 1, 0) != 0) return; // Already shutting down _logger.LogInformation("Initiating Shutdown..."); // Publish shutdown advisory before tearing down the event system if (_eventSystem != null) { var shutdownSubject = string.Format(EventSubjects.ServerShutdown, _serverInfo.ServerId); _eventSystem.Enqueue(new PublishMessage { Subject = shutdownSubject, Body = new ShutdownEventMsg { Server = BuildEventServerInfo(), Reason = "Server Shutdown" }, IsLast = true, }); // Give the send loop time to process the shutdown event await Task.Delay(100); await _eventSystem.DisposeAsync(); } // Signal all internal loops to stop await _quitCts.CancelAsync(); // Close listeners to stop accept loops _listener?.Close(); _wsListener?.Close(); if (_routeManager != null) await _routeManager.DisposeAsync(); if (_gatewayManager != null) await _gatewayManager.DisposeAsync(); if (_leafNodeManager != null) await _leafNodeManager.DisposeAsync(); if (_jetStreamService != null) await _jetStreamService.DisposeAsync(); _stats.JetStreamEnabled = false; // If server was never started, accept loops never ran — signal immediately if (_listener == null) _acceptLoopExited.TrySetResult(); if (_wsListener == null) _wsAcceptLoopExited.TrySetResult(); // Wait for accept loops to exit (in parallel to avoid sequential 5s+5s waits) await Task.WhenAll( _acceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)), _wsAcceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5))).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); // Close all client connections — flush first, then mark closed var flushTasks = new List(); foreach (var client in _clients.Values) { client.MarkClosed(ClientClosedReason.ServerShutdown); flushTasks.Add(client.FlushAndCloseAsync(minimalFlush: true)); } await Task.WhenAll(flushTasks).WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); // Wait for active client tasks to drain (with timeout) if (Volatile.Read(ref _activeClientCount) > 0) { using var drainCts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); try { while (Volatile.Read(ref _activeClientCount) > 0 && !drainCts.IsCancellationRequested) await Task.Delay(50, drainCts.Token); } catch (OperationCanceledException) { } } // Stop monitor server if (_monitorServer != null) await _monitorServer.DisposeAsync(); if (_mqttListener != null) await _mqttListener.DisposeAsync(); DeletePidFile(); DeletePortsFile(); _logger.LogInformation("Server Exiting.."); _shutdownComplete.TrySetResult(); } public async Task LameDuckShutdownAsync() { if (IsShuttingDown || Interlocked.CompareExchange(ref _lameDuck, 1, 0) != 0) return; _logger.LogInformation("Entering lame duck mode, stop accepting new clients"); // Close listeners to stop accepting new connections _listener?.Close(); _wsListener?.Close(); // Wait for accept loops to exit (in parallel) await Task.WhenAll( _acceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)), _wsAcceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5))).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); var gracePeriod = _options.LameDuckGracePeriod; if (gracePeriod < TimeSpan.Zero) gracePeriod = -gracePeriod; // If no clients, go straight to shutdown if (_clients.IsEmpty) { await ShutdownAsync(); return; } // Wait grace period for clients to drain naturally _logger.LogInformation("Waiting {GracePeriod}ms grace period", gracePeriod.TotalMilliseconds); try { await Task.Delay(gracePeriod, _quitCts.Token); } catch (OperationCanceledException) { return; } if (_clients.IsEmpty) { await ShutdownAsync(); return; } // Stagger-close remaining clients var dur = _options.LameDuckDuration - gracePeriod; if (dur <= TimeSpan.Zero) dur = TimeSpan.FromSeconds(1); var clients = _clients.Values.ToList(); var numClients = clients.Count; if (numClients > 0) { _logger.LogInformation("Closing {Count} existing clients over {Duration}ms", numClients, dur.TotalMilliseconds); var sleepInterval = dur.Ticks / numClients; if (sleepInterval < TimeSpan.TicksPerMillisecond) sleepInterval = TimeSpan.TicksPerMillisecond; if (sleepInterval > TimeSpan.TicksPerSecond) sleepInterval = TimeSpan.TicksPerSecond; for (int i = 0; i < clients.Count; i++) { clients[i].MarkClosed(ClientClosedReason.ServerShutdown); await clients[i].FlushAndCloseAsync(minimalFlush: true); if (i < clients.Count - 1) { var jitter = Random.Shared.NextInt64(sleepInterval / 2, sleepInterval); try { await Task.Delay(TimeSpan.FromTicks(jitter), _quitCts.Token); } catch (OperationCanceledException) { break; } } } } await ShutdownAsync(); } /// /// Registers Unix signal handlers. /// SIGTERM → shutdown, SIGUSR2 → lame duck, SIGUSR1 → log reopen, SIGHUP → reload (stub). /// public void HandleSignals() { _signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGTERM, ctx => { ctx.Cancel = true; _logger.LogInformation("Trapped SIGTERM signal"); _ = Task.Run(async () => await ShutdownAsync()); })); _signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGQUIT, ctx => { ctx.Cancel = true; _logger.LogInformation("Trapped SIGQUIT signal"); _ = Task.Run(async () => await ShutdownAsync()); })); _signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGHUP, ctx => { ctx.Cancel = true; _logger.LogInformation("Trapped SIGHUP signal — reloading configuration"); _ = Task.Run(() => ReloadConfig()); })); // SIGUSR1 and SIGUSR2 only on non-Windows if (!OperatingSystem.IsWindows()) { _signalRegistrations.Add(PosixSignalRegistration.Create((PosixSignal)10, ctx => { ctx.Cancel = true; _logger.LogInformation("Trapped SIGUSR1 signal — reopening log file"); ReOpenLogFile?.Invoke(); })); _signalRegistrations.Add(PosixSignalRegistration.Create((PosixSignal)12, ctx => { ctx.Cancel = true; _logger.LogInformation("Trapped SIGUSR2 signal — entering lame duck mode"); _ = Task.Run(async () => await LameDuckShutdownAsync()); })); } } public NatsServer(NatsOptions options, ILoggerFactory loggerFactory) { _options = options; _loggerFactory = loggerFactory; _logger = loggerFactory.CreateLogger(); _closedClients = new ClosedConnectionRingBuffer(options.MaxClosedClients); _authService = AuthService.Build(options); _globalAccount = new Account(Account.GlobalAccountName); _accounts[Account.GlobalAccountName] = _globalAccount; // Create $SYS system account and mark it as the system account. // Reference: Go server/server.go — initSystemAccount, accounts.go — isSystemAccount(). _systemAccount = new Account(Account.SystemAccountName) { IsSystemAccount = true }; _accounts[Account.SystemAccountName] = _systemAccount; // If a user-defined system_account is configured, promote that account to be the // system account. Events published to $SYS.* will be delivered to subscribers on // this account. Go reference: server/server.go — configureAccounts / setSystemAccount. if (!string.IsNullOrEmpty(options.SystemAccount) && !string.Equals(options.SystemAccount, Account.SystemAccountName, StringComparison.OrdinalIgnoreCase)) { var userSysAccount = GetOrCreateAccount(options.SystemAccount); userSysAccount.IsSystemAccount = true; _systemAccount = userSysAccount; } // Create system internal client and event system var sysClientId = Interlocked.Increment(ref _nextClientId); var sysClient = new InternalClient(sysClientId, ClientKind.System, _systemAccount); _eventSystem = new InternalEventSystem( _systemAccount, sysClient, options.ServerName ?? $"nats-dotnet-{Environment.MachineName}", _loggerFactory.CreateLogger()); // Generate Ed25519 server NKey identity using var serverKeyPair = KeyPair.CreatePair(PrefixByte.Server); ServerNKey = serverKeyPair.GetPublicKey(); _serverInfo = new ServerInfo { ServerId = Guid.NewGuid().ToString("N")[..20].ToUpperInvariant(), ServerName = options.ServerName ?? $"nats-dotnet-{Environment.MachineName}", Version = NatsProtocol.Version, Host = options.Host, Port = options.Port, MaxPayload = options.MaxPayload, AuthRequired = _authService.IsAuthRequired, }; if (options.Cluster != null) { _routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, ProcessRoutedMessage, _loggerFactory.CreateLogger()); _routeManager.OnRouteRemoved += RemoveRemoteSubscriptionsForRoute; _routeManager.OnRouteAccountRemoved += RemoveRemoteSubscriptionsForRouteAccount; } if (options.Gateway != null) { _gatewayManager = new GatewayManager(options.Gateway, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, ProcessGatewayMessage, _loggerFactory.CreateLogger()); } if (options.LeafNode != null) { _leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _serverInfo.ServerId, ApplyRemoteSubscription, ProcessLeafMessage, _loggerFactory.CreateLogger()); } if (options.JetStream != null) { _jetStreamConsumerManager = new ConsumerManager(); _jetStreamStreamManager = new StreamManager(consumerManager: _jetStreamConsumerManager, storeDir: options.JetStream.StoreDir); _jetStreamConsumerManager.StreamManager = _jetStreamStreamManager; var jsClientId = Interlocked.Increment(ref _nextClientId); _jetStreamInternalClient = new InternalClient(jsClientId, ClientKind.JetStream, _systemAccount); _jetStreamService = new JetStreamService(options.JetStream, _jetStreamInternalClient); _jetStreamApiRouter = new JetStreamApiRouter(_jetStreamStreamManager, _jetStreamConsumerManager); _jetStreamPublisher = new JetStreamPublisher(_jetStreamStreamManager); } if (options.HasTls) { _tlsCertProvider = new TlsCertificateProvider(options.TlsCert!, options.TlsKey); _sslOptions = TlsHelper.BuildServerAuthOptions(options); _tlsCertProvider.SwapSslOptions(_sslOptions); // OCSP stapling: build a certificate context so the runtime can // fetch and cache a fresh OCSP response and staple it during the // TLS handshake. offline:false tells the runtime to contact the // OCSP responder; if the responder is unreachable we fall back to // no stapling rather than refusing all connections. var certContext = TlsHelper.BuildCertificateContext(options, offline: false); if (certContext != null) { _sslOptions.ServerCertificateContext = certContext; _logger.LogInformation("OCSP stapling enabled (mode: {OcspMode})", options.OcspConfig!.Mode); } _serverInfo.TlsRequired = !options.AllowNonTls; _serverInfo.TlsAvailable = options.AllowNonTls; _serverInfo.TlsVerify = options.TlsVerify; if (options.TlsRateLimit > 0) _tlsRateLimiter = new TlsRateLimiter(options.TlsRateLimit); } // Compile subject transforms if (options.SubjectMappings is { Count: > 0 }) { var transforms = new List(); foreach (var (source, dest) in options.SubjectMappings) { var t = SubjectTransform.Create(source, dest); if (t != null) transforms.Add(t); else _logger.LogWarning("Invalid subject mapping: {Source} -> {Dest}", source, dest); } _subjectTransforms = transforms.ToArray(); if (_subjectTransforms.Length > 0) _logger.LogInformation("Compiled {Count} subject transform(s)", _subjectTransforms.Length); } else { _subjectTransforms = []; } BuildCachedInfo(); // Store initial config digest for reload change detection if (options.ConfigFile != null) { try { var (_, digest) = NatsConfParser.ParseFileWithDigest(options.ConfigFile); _configDigest = digest; _configTime = DateTime.UtcNow; } catch (Exception ex) { _logger.LogWarning(ex, "Could not compute initial config digest for {ConfigFile}", options.ConfigFile); } } } private void BuildCachedInfo() { var infoJson = System.Text.Json.JsonSerializer.Serialize(_serverInfo); _cachedInfoLine = Encoding.ASCII.GetBytes($"INFO {infoJson}\r\n"); } private static string NormalizeAdvertiseUrl(string advertise, string defaultScheme) { if (advertise.Contains("://", StringComparison.Ordinal)) return advertise; return $"{defaultScheme}://{advertise}"; } private static bool IsWildcardHost(string host) => host == "0.0.0.0" || host == "::"; internal static IReadOnlyList GetNonLocalIPsIfHostIsIPAny(string host) { if (!IsWildcardHost(host)) return [host]; var addresses = new HashSet(StringComparer.Ordinal); foreach (var netIf in NetworkInterface.GetAllNetworkInterfaces()) { if (netIf.OperationalStatus != OperationalStatus.Up) continue; IPInterfaceProperties? props; try { props = netIf.GetIPProperties(); } catch { continue; } foreach (var uni in props.UnicastAddresses) { var addr = uni.Address; if (IPAddress.IsLoopback(addr) || addr.IsIPv6LinkLocal || addr.IsIPv6Multicast) continue; if (addr.AddressFamily is not (AddressFamily.InterNetwork or AddressFamily.InterNetworkV6)) continue; addresses.Add(addr.ToString()); } } if (addresses.Count == 0) addresses.Add("127.0.0.1"); return [.. addresses.OrderBy(static a => a, StringComparer.Ordinal)]; } private bool CloseClientById(ulong clientId, bool minimalFlush) { if (!_clients.TryGetValue(clientId, out var client)) return false; client.MarkClosed(ClientClosedReason.ServerShutdown); _ = client.FlushAndCloseAsync(minimalFlush); return true; } private static void AddEndpoint(List targets, string? host, int port) { if (string.IsNullOrWhiteSpace(host) || port <= 0) return; targets.Add($"{host}:{port}"); } private static void AddEndpoint(List targets, string? endpoint) { if (!string.IsNullOrWhiteSpace(endpoint)) targets.Add(endpoint); } public async Task StartAsync(CancellationToken ct) { using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _quitCts.Token); _listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); _listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); _listener.Bind(new IPEndPoint( _options.Host == "0.0.0.0" ? IPAddress.Any : IPAddress.Parse(_options.Host), _options.Port)); Interlocked.Exchange(ref _startTimeTicks, DateTime.UtcNow.Ticks); _listener.Listen(128); // Resolve ephemeral port if port=0 if (_options.Port == 0) { var actualPort = ((IPEndPoint)_listener.LocalEndPoint!).Port; _options.Port = actualPort; _serverInfo.Port = actualPort; BuildCachedInfo(); } _logger.LogInformation("Listening for client connections on {Host}:{Port}", _options.Host, _options.Port); // Warn about stub features StartProfiler(); if (_options.MonitorPort > 0) { _monitorServer = new MonitorServer(this, _options, _stats, _loggerFactory); await _monitorServer.StartAsync(linked.Token); } WritePidFile(); WritePortsFile(); WsAuthConfig.Apply(_options.WebSocket); var wsValidation = WebSocketOptionsValidator.Validate(_options); if (!wsValidation.IsValid) throw new InvalidOperationException($"Invalid websocket options: {string.Join("; ", wsValidation.Errors)}"); if (_options.WebSocket.Port >= 0) { _wsListener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); _wsListener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); _wsListener.Bind(new IPEndPoint( _options.WebSocket.Host == "0.0.0.0" ? IPAddress.Any : IPAddress.Parse(_options.WebSocket.Host), _options.WebSocket.Port)); _wsListener.Listen(128); if (_options.WebSocket.Port == 0) { _options.WebSocket.Port = ((IPEndPoint)_wsListener.LocalEndPoint!).Port; } _logger.LogInformation("Listening for WebSocket clients on {Host}:{Port}", _options.WebSocket.Host, _options.WebSocket.Port); if (_options.WebSocket.NoTls) _logger.LogWarning("WebSocket not configured with TLS. DO NOT USE IN PRODUCTION!"); _ = RunWebSocketAcceptLoopAsync(linked.Token); } else { // No WebSocket listener — signal immediately so shutdown doesn't wait _wsAcceptLoopExited.TrySetResult(); } if (_routeManager != null) await _routeManager.StartAsync(linked.Token); if (_gatewayManager != null) await _gatewayManager.StartAsync(linked.Token); if (_leafNodeManager != null) await _leafNodeManager.StartAsync(linked.Token); if (_options.Mqtt is { Port: > 0 } mqttOptions) { var mqttHost = string.IsNullOrWhiteSpace(mqttOptions.Host) ? _options.Host : mqttOptions.Host; // Create MQTT JetStream components if JetStream is enabled MqttStreamInitializer? mqttStreamInit = null; MqttConsumerManager? mqttConsumerMgr = null; if (_jetStreamStreamManager != null && _jetStreamConsumerManager != null) { mqttStreamInit = new Mqtt.MqttStreamInitializer(_jetStreamStreamManager); mqttConsumerMgr = new Mqtt.MqttConsumerManager(_jetStreamStreamManager, _jetStreamConsumerManager); } _mqttListener = new MqttListener( mqttHost, mqttOptions.Port, _authService, mqttOptions, mqttStreamInit, mqttConsumerMgr, router: this); _mqttListener.AllocateClientId = () => Interlocked.Increment(ref _nextClientId); _mqttListener.ResolveAccount = name => GetOrCreateAccount(name ?? Auth.Account.GlobalAccountName); await _mqttListener.StartAsync(linked.Token); } if (_jetStreamService != null) { await _jetStreamService.StartAsync(linked.Token); _stats.JetStreamEnabled = true; } _listeningStarted.TrySetResult(); _eventSystem?.Start(this); _eventSystem?.InitEventTracking(this); var tmpDelay = AcceptMinSleep; try { while (!linked.Token.IsCancellationRequested) { Socket socket; try { socket = await _listener!.AcceptAsync(linked.Token); tmpDelay = AcceptMinSleep; // Reset on success } catch (OperationCanceledException) { break; } catch (ObjectDisposedException) { break; } catch (SocketException ex) { if (IsShuttingDown || IsLameDuckMode) break; _acceptLoopErrorHandler?.OnAcceptError(ex, _listener?.LocalEndPoint, tmpDelay); _logger.LogError(ex, "Temporary accept error, sleeping {Delay}ms", tmpDelay.TotalMilliseconds); try { await Task.Delay(tmpDelay, linked.Token); } catch (OperationCanceledException) { break; } tmpDelay = TimeSpan.FromTicks(Math.Min(tmpDelay.Ticks * 2, AcceptMaxSleep.Ticks)); continue; } // Check MaxConnections if (_options.MaxConnections > 0 && _clients.Count >= _options.MaxConnections) { _logger.LogWarning("Client connection rejected: maximum connections ({MaxConnections}) exceeded", _options.MaxConnections); try { var stream = new NetworkStream(socket, ownsSocket: false); var errBytes = Encoding.ASCII.GetBytes( $"-ERR '{NatsProtocol.ErrMaxConnectionsExceeded}'\r\n"); await stream.WriteAsync(errBytes, linked.Token); await stream.FlushAsync(linked.Token); stream.Dispose(); } catch (Exception ex2) { _logger.LogDebug(ex2, "Failed to send -ERR to rejected client"); } finally { socket.Dispose(); } continue; } var clientId = Interlocked.Increment(ref _nextClientId); Interlocked.Increment(ref _stats.TotalConnections); Interlocked.Increment(ref _activeClientCount); _logger.LogDebug("Client {ClientId} connected from {RemoteEndpoint}", clientId, socket.RemoteEndPoint); _ = AcceptClientAsync(socket, clientId, linked.Token); } } catch (OperationCanceledException) { _logger.LogDebug("Accept loop cancelled, server shutting down"); } finally { _acceptLoopExited.TrySetResult(); } } private async Task AcceptClientAsync(Socket socket, ulong clientId, CancellationToken ct) { var reloadLockHeld = false; NatsClient? client = null; try { await _reloadMu.WaitAsync(ct); reloadLockHeld = true; // Rate limit TLS handshakes if (_tlsRateLimiter != null) await _tlsRateLimiter.WaitAsync(ct); var networkStream = new NetworkStream(socket, ownsSocket: false); // TLS negotiation (no-op if not configured) var (stream, infoAlreadySent) = await TlsConnectionWrapper.NegotiateAsync( socket, networkStream, _options, _sslOptions, _serverInfo, _loggerFactory.CreateLogger("NATS.Server.Tls"), ct); // Extract TLS state TlsConnectionState? tlsState = null; if (stream is SslStream ssl) { tlsState = new TlsConnectionState( ssl.SslProtocol.ToString(), ssl.NegotiatedCipherSuite.ToString(), ssl.RemoteCertificate as X509Certificate2); } // Build per-client ServerInfo with nonce if NKey auth is configured byte[]? nonce = null; var clientInfo = _serverInfo; if (_authService.NonceRequired) { var rawNonce = _authService.GenerateNonce(); var nonceStr = _authService.EncodeNonce(rawNonce); // The client signs the nonce string (ASCII), not the raw bytes nonce = Encoding.ASCII.GetBytes(nonceStr); clientInfo = new ServerInfo { ServerId = _serverInfo.ServerId, ServerName = _serverInfo.ServerName, Version = _serverInfo.Version, Host = _serverInfo.Host, Port = _serverInfo.Port, MaxPayload = _serverInfo.MaxPayload, AuthRequired = _serverInfo.AuthRequired, TlsRequired = _serverInfo.TlsRequired, TlsAvailable = _serverInfo.TlsAvailable, TlsVerify = _serverInfo.TlsVerify, Nonce = nonceStr, }; } var clientLogger = _loggerFactory.CreateLogger($"NATS.Server.NatsClient[{clientId}]"); client = new NatsClient(clientId, stream, socket, _options, clientInfo, _authService, nonce, clientLogger, _stats); client.Router = this; client.TlsState = tlsState; client.InfoAlreadySent = infoAlreadySent; _clients[clientId] = client; } catch (Exception ex) { if (client is null) { var earlyReason = _options.HasTls ? ClientClosedReason.TlsHandshakeError : ClientClosedReason.ReadError; TrackEarlyClosedClient(socket, clientId, earlyReason); } _logger.LogDebug(ex, "Failed to accept client {ClientId}", clientId); try { socket.Shutdown(SocketShutdown.Both); } catch { } socket.Dispose(); return; } finally { if (reloadLockHeld) _reloadMu.Release(); } try { if (client != null) await RunClientAsync(client, ct); } catch (Exception ex) { _logger.LogDebug(ex, "Failed to accept client {ClientId}", clientId); try { socket.Shutdown(SocketShutdown.Both); } catch { } socket.Dispose(); } } private async Task RunWebSocketAcceptLoopAsync(CancellationToken ct) { var tmpDelay = AcceptMinSleep; try { while (!ct.IsCancellationRequested) { Socket socket; try { socket = await _wsListener!.AcceptAsync(ct); tmpDelay = AcceptMinSleep; } catch (OperationCanceledException) { break; } catch (ObjectDisposedException) { break; } catch (SocketException ex) { if (IsShuttingDown || IsLameDuckMode) break; _acceptLoopErrorHandler?.OnAcceptError(ex, _wsListener?.LocalEndPoint, tmpDelay); _logger.LogError(ex, "Temporary WebSocket accept error, sleeping {Delay}ms", tmpDelay.TotalMilliseconds); try { await Task.Delay(tmpDelay, ct); } catch (OperationCanceledException) { break; } tmpDelay = TimeSpan.FromTicks(Math.Min(tmpDelay.Ticks * 2, AcceptMaxSleep.Ticks)); continue; } if (_options.MaxConnections > 0 && _clients.Count >= _options.MaxConnections) { socket.Dispose(); continue; } var clientId = Interlocked.Increment(ref _nextClientId); Interlocked.Increment(ref _stats.TotalConnections); Interlocked.Increment(ref _activeClientCount); _ = AcceptWebSocketClientAsync(socket, clientId, ct); } } finally { _wsAcceptLoopExited.TrySetResult(); } } private async Task AcceptWebSocketClientAsync(Socket socket, ulong clientId, CancellationToken ct) { try { var networkStream = new NetworkStream(socket, ownsSocket: false); Stream stream = networkStream; // TLS negotiation if configured if (_sslOptions != null && !_options.WebSocket.NoTls) { var (tlsStream, _) = await TlsConnectionWrapper.NegotiateAsync( socket, networkStream, _options, _sslOptions, _serverInfo, _loggerFactory.CreateLogger("NATS.Server.Tls"), ct); stream = tlsStream; } // HTTP upgrade handshake var upgradeResult = await WsUpgrade.TryUpgradeAsync(stream, stream, _options.WebSocket, ct); if (!upgradeResult.Success) { _logger.LogDebug("WebSocket upgrade failed for client {ClientId}", clientId); socket.Dispose(); Interlocked.Decrement(ref _activeClientCount); return; } // Create WsConnection wrapper var wsConn = new WsConnection(stream, compress: upgradeResult.Compress, maskRead: upgradeResult.MaskRead, maskWrite: upgradeResult.MaskWrite, browser: upgradeResult.Browser, noCompFrag: upgradeResult.NoCompFrag); var clientLogger = _loggerFactory.CreateLogger($"NATS.Server.NatsClient[{clientId}]"); var client = new NatsClient(clientId, wsConn, socket, _options, _serverInfo, _authService, null, clientLogger, _stats); client.Router = this; client.IsWebSocket = true; client.WsInfo = upgradeResult; _clients[clientId] = client; await RunClientAsync(client, ct); } catch (Exception ex) { _logger.LogDebug(ex, "Failed to accept WebSocket client {ClientId}", clientId); try { socket.Shutdown(SocketShutdown.Both); } catch { } socket.Dispose(); Interlocked.Decrement(ref _activeClientCount); } } private async Task RunClientAsync(NatsClient client, CancellationToken ct) { try { await client.RunAsync(ct); } catch (Exception ex) { _logger.LogDebug(ex, "Client {ClientId} disconnected with error", client.Id); } finally { _logger.LogDebug("Client {ClientId} disconnected (reason: {CloseReason})", client.Id, client.CloseReason); RemoveClient(client); Interlocked.Decrement(ref _activeClientCount); } } public void OnLocalSubscription(string account, string subject, string? queue) { _routeManager?.PropagateLocalSubscription(account, subject, queue); _gatewayManager?.PropagateLocalSubscription(account, subject, queue); _leafNodeManager?.PropagateLocalSubscription(account, subject, queue); } public void OnLocalUnsubscription(string account, string subject, string? queue) { _routeManager?.PropagateLocalUnsubscription(account, subject, queue); _gatewayManager?.PropagateLocalUnsubscription(account, subject, queue); _leafNodeManager?.PropagateLocalUnsubscription(account, subject, queue); } private void ApplyRemoteSubscription(RemoteSubscription sub) { var account = GetOrCreateAccount(sub.Account); account.SubList.ApplyRemoteSub(sub); } private void RemoveRemoteSubscriptionsForRoute(string routeId) { foreach (var account in _accounts.Values) account.SubList.RemoveRemoteSubs(routeId); } private void RemoveRemoteSubscriptionsForRouteAccount(string routeId, string accountName) { if (_accounts.TryGetValue(accountName, out var account)) account.SubList.RemoveRemoteSubsForAccount(routeId, accountName); } private void ProcessRoutedMessage(RouteMessage message) { DeliverRemoteMessage(message.Account, message.Subject, message.ReplyTo, message.Payload); } private void ProcessGatewayMessage(GatewayMessage message) { var replyTo = message.ReplyTo; if (ReplyMapper.TryRestoreGatewayReply(replyTo, out var restoredReply)) replyTo = restoredReply; else if (ReplyMapper.HasGatewayReplyPrefix(replyTo)) replyTo = null; DeliverRemoteMessage(message.Account, message.Subject, replyTo, message.Payload); } private void ProcessLeafMessage(LeafMessage message) { if (LeafLoopDetector.IsLooped(message.Subject, ServerId)) return; var subject = message.Subject; if (LeafLoopDetector.TryUnmark(subject, out var unmarked)) subject = unmarked; else if (LeafLoopDetector.HasLoopMarker(subject)) return; DeliverRemoteMessage(message.Account, subject, message.ReplyTo, message.Payload); } private void DeliverRemoteMessage(string account, string subject, string? replyTo, ReadOnlyMemory payload) { // Handle internal JetStream cluster replication messages. // Go reference: jetstream_cluster.go — RAFT proposal apply dispatches to handlers. if (subject.StartsWith(JetStreamApiSubjects.InternalPrefix, StringComparison.Ordinal)) { HandleJetStreamReplication(subject, payload); return; } // Capture routed data messages into local JetStream streams. // Skip $JS. subjects — those are API calls, not stream data. if (!subject.StartsWith("$JS.", StringComparison.Ordinal)) TryCaptureJetStreamPublish(subject, payload, out _); var targetAccount = GetOrCreateAccount(account); var result = targetAccount.SubList.Match(subject); foreach (var sub in result.PlainSubs) DeliverMessage(sub, subject, replyTo, default, payload); foreach (var queueGroup in result.QueueSubs) { if (queueGroup.Length == 0) continue; var sub = queueGroup[0]; DeliverMessage(sub, subject, replyTo, default, payload); } } public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory headers, ReadOnlyMemory payload, INatsClient sender) { // Cast to NatsClient for operations that require it (JetStream pub-ack, stats). // Non-NatsClient senders (e.g. MqttNatsClientAdapter) skip those code paths. var natsClient = sender as NatsClient; if (replyTo != null && subject.StartsWith("$JS.API", StringComparison.Ordinal) && _jetStreamApiRouter != null) { // Pull consumer MSG.NEXT requires special handling: deliver individual // HMSG messages to the client's reply inbox instead of a single JSON blob. // Go reference: consumer.go:4276 processNextMsgRequest if (subject.StartsWith(JetStream.Api.JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal) && _jetStreamConsumerManager != null && _jetStreamStreamManager != null && natsClient != null) { Interlocked.Increment(ref _stats.JetStreamApiTotal); DeliverPullFetchMessages(subject, replyTo, payload, natsClient); return; } var response = _jetStreamApiRouter.Route(subject, payload.Span); Interlocked.Increment(ref _stats.JetStreamApiTotal); if (response.Error != null) { Interlocked.Increment(ref _stats.JetStreamApiErrors); } // Replicate successful mutating operations to cluster peers. // Go reference: jetstream_cluster.go — RAFT proposal replication. if (response.Error == null) TryReplicateJetStreamMutation(subject, payload); var data = JsonSerializer.SerializeToUtf8Bytes(response.ToWireFormat(), s_jetStreamJsonOptions); ProcessMessage(replyTo, null, default, data, sender); return; } if (TryCaptureJetStreamPublish(subject, payload, out var pubAck)) { natsClient?.RecordJetStreamPubAck(pubAck); // Replicate data messages to cluster peers so their JetStream stores also capture them. // Route forwarding below is gated on subscriber interest, which JetStream streams don't // create, so we must explicitly push data to replicas. // Go reference: jetstream_cluster.go — RAFT propose entry replication. if (pubAck.ErrorCode == null) ReplicateJetStreamOperation("$G", subject, null, payload); // Send pub ack response to the reply subject (request-reply pattern). // Go reference: server/jetstream.go — jsPubAckResponse sent to reply. if (replyTo != null) { if (JetStream.Publish.JetStreamPubAckFormatter.IsSimpleSuccess(pubAck)) { // Fast path: hand-rolled UTF-8 formatter avoids JsonSerializer overhead. Span ackBuf = stackalloc byte[256]; var ackLen = JetStream.Publish.JetStreamPubAckFormatter.FormatSuccess(ackBuf, pubAck.Stream, pubAck.Seq); ProcessMessage(replyTo, null, default, ackBuf[..ackLen].ToArray(), sender); } else { var ackData = JsonSerializer.SerializeToUtf8Bytes(pubAck, s_jetStreamJsonOptions); ProcessMessage(replyTo, null, default, ackData, sender); } return; } } // Apply subject transforms if (_subjectTransforms.Length > 0) { foreach (var transform in _subjectTransforms) { var mapped = transform.Apply(subject); if (mapped != null) { subject = mapped; break; // First matching transform wins } } } var senderAccount = sender.Account ?? _globalAccount; if (_routeManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject)) _routeManager.ForwardRoutedMessageAsync(senderAccount.Name, subject, replyTo, payload, default).GetAwaiter().GetResult(); if (_gatewayManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject)) { var mappedReplyTo = ReplyMapper.ToGatewayReply(replyTo, ServerId); _gatewayManager.ForwardMessageAsync(senderAccount.Name, subject, mappedReplyTo, payload, default).GetAwaiter().GetResult(); } if (_leafNodeManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject)) { var markedSubject = LeafLoopDetector.Mark(subject, ServerId); _leafNodeManager.ForwardMessageAsync(senderAccount.Name, markedSubject, replyTo, payload, default).GetAwaiter().GetResult(); } var subList = sender.Account?.SubList ?? _globalAccount.SubList; var result = subList.Match(subject); var delivered = false; int deliveredCount = 0; // Pre-encode subject bytes once for all fan-out deliveries (one alloc per publish, not per delivery). var subjectBytes = Encoding.ASCII.GetBytes(subject); // Per-client deferred flush: collect unique clients during fan-out, signal each once. // Go reference: client.go:3905 addToPCD / client.go:1324 flushClients. var pcd = t_pcd ??= new HashSet(); pcd.Clear(); // Deliver to plain subscribers var messageSize = payload.Length + headers.Length; foreach (var sub in result.PlainSubs) { if (sub.Client == null || sub.Client == sender && !(sender.ClientOpts?.Echo ?? true)) continue; DeliverMessage(sub, subjectBytes, sub.SidBytes, subject, replyTo, headers, payload, pcd); delivered = true; deliveredCount++; } // Deliver to one member of each queue group (round-robin) foreach (var queueGroup in result.QueueSubs) { if (queueGroup.Length == 0) continue; // Simple round-robin -- pick based on total delivered across group if (natsClient != null) { var idx = Math.Abs((int)Interlocked.Increment(ref natsClient.OutMsgs)) % queueGroup.Length; // Undo the OutMsgs increment -- it will be incremented properly in SendMessageNoFlush Interlocked.Decrement(ref natsClient.OutMsgs); for (int attempt = 0; attempt < queueGroup.Length; attempt++) { var sub = queueGroup[(idx + attempt) % queueGroup.Length]; if (sub.Client != null && (sub.Client != sender || (sender.ClientOpts?.Echo ?? true))) { DeliverMessage(sub, subjectBytes, sub.SidBytes, subject, replyTo, headers, payload, pcd); delivered = true; deliveredCount++; break; } } } else { // Non-NatsClient sender: simple first-match foreach (var sub in queueGroup) { if (sub.Client != null && sub.Client != sender) { DeliverMessage(sub, subjectBytes, sub.SidBytes, subject, replyTo, headers, payload, pcd); delivered = true; deliveredCount++; break; } } } } // Batch server-wide stats once per publish (instead of per-delivery Interlocked ops). if (deliveredCount > 0) { Interlocked.Add(ref _stats.OutMsgs, (long)deliveredCount); Interlocked.Add(ref _stats.OutBytes, (long)messageSize * deliveredCount); } // Flush all unique clients once after fan-out. // Go reference: client.go:1324 flushClients — iterates pcd map, one signal per client. foreach (var client in pcd) client.SignalFlush(); pcd.Clear(); // Check for service imports that match this subject. // When a client in the importer account publishes to a subject // that matches a service import "From" pattern, we forward the // message to the destination (exporter) account's subscribers // using the mapped "To" subject. if (sender.Account != null) { foreach (var kvp in sender.Account.Imports.Services) { foreach (var si in kvp.Value) { if (si.Invalid) continue; if (SubjectMatch.MatchLiteral(subject, si.From)) { ProcessServiceImport(si, subject, replyTo, headers, payload, sender.Account); delivered = true; } } } } // No-responders: if nobody received the message and the publisher // opted in, send back a 503 status HMSG on the reply subject. if (!delivered && replyTo != null && sender.ClientOpts?.NoResponders == true && natsClient != null) { SendNoResponders(natsClient, replyTo); } } /// /// Handles $JS.API.CONSUMER.MSG.NEXT by delivering individual HMSG messages /// to the client's reply inbox. Go reference: consumer.go:4276 processNextMsgRequest. /// private void DeliverPullFetchMessages(string subject, string replyTo, ReadOnlyMemory payload, NatsClient sender) { var prefix = JetStream.Api.JetStreamApiSubjects.ConsumerNext; var remainder = subject[prefix.Length..]; var split = remainder.Split('.', 2, StringSplitOptions.RemoveEmptyEntries); if (split.Length != 2) { var notFoundHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n"); ProcessMessage(replyTo, null, (ReadOnlyMemory)notFoundHeader, default, sender); return; } var (streamName, consumerName) = (split[0], split[1]); // Parse batch request int batch = 1; int expiresMs = 0; bool noWait = false; int idleHeartbeatMs = 0; if (payload.Length > 0) { try { using var doc = System.Text.Json.JsonDocument.Parse(payload); if (doc.RootElement.TryGetProperty("batch", out var batchEl) && batchEl.TryGetInt32(out var b)) batch = Math.Max(b, 1); if (doc.RootElement.TryGetProperty("no_wait", out var nwEl) && nwEl.ValueKind == System.Text.Json.JsonValueKind.True) noWait = true; if (doc.RootElement.TryGetProperty("expires", out var expEl) && expEl.TryGetInt64(out var expNs)) expiresMs = (int)(expNs / 1_000_000); if (doc.RootElement.TryGetProperty("idle_heartbeat", out var hbEl) && hbEl.TryGetInt64(out var hbNs)) idleHeartbeatMs = (int)(hbNs / 1_000_000); } catch (System.Text.Json.JsonException ex) { _logger.LogDebug(ex, "Malformed JSON in pull request payload, using defaults"); } } // Find the sender's inbox subscription so we can deliver directly. // Go reference: consumer.go deliverMsg — delivers directly to the client, bypassing pub/sub echo checks. var subList = sender.Account?.SubList ?? _globalAccount.SubList; var matchResult = subList.Match(replyTo); Subscription? inboxSub = null; foreach (var sub in matchResult.PlainSubs) { if (sub.Client == sender) { inboxSub = sub; break; } } if (inboxSub == null) return; if (noWait || expiresMs <= 0) { // Synchronous path for no_wait (used by FetchAsync client path). // Fetch all immediately available messages and return. var fetchResult = _jetStreamConsumerManager!.FetchAsync( streamName, consumerName, new JetStream.Consumers.PullFetchRequest { Batch = batch, NoWait = true }, _jetStreamStreamManager!, default).GetAwaiter().GetResult(); DeliverFetchedMessages(inboxSub, streamName, consumerName, replyTo, fetchResult.Messages); // Send terminal status ReadOnlyMemory statusHeader = fetchResult.Messages.Count == 0 ? System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n") : System.Text.Encoding.UTF8.GetBytes("NATS/1.0 408 Request Timeout\r\n\r\n"); DeliverMessage(inboxSub, replyTo, null, statusHeader, default); } else { // Async path for ConsumeAsync: deliver messages incrementally without blocking // the client's read loop. Go reference: consumer.go processNextMsgRequest — // registers a waiting request and returns to the read loop; messages are delivered // asynchronously as they become available. var capturedSub = inboxSub; _ = Task.Run(() => DeliverPullFetchMessagesAsync( streamName, consumerName, batch, expiresMs, idleHeartbeatMs, replyTo, capturedSub, sender)); } } /// /// Background task that delivers pull fetch messages incrementally. /// Polls the stream store for messages and delivers each one as it becomes available. /// Sends idle heartbeats to keep the client connection alive. /// Go reference: consumer.go — waiting request fulfillment loop. /// private async Task DeliverPullFetchMessagesAsync( string streamName, string consumerName, int batch, int expiresMs, int idleHeartbeatMs, string replyTo, Subscription inboxSub, NatsClient sender) { using var expiresCts = new CancellationTokenSource(TimeSpan.FromMilliseconds(expiresMs)); var ct = expiresCts.Token; if (!_jetStreamConsumerManager!.TryGet(streamName, consumerName, out var consumer)) { DeliverMessage(inboxSub, replyTo, null, System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n"), default); return; } if (!_jetStreamStreamManager!.TryGet(streamName, out var streamHandle)) { DeliverMessage(inboxSub, replyTo, null, System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n"), default); return; } // Resolve initial sequence if needed if (consumer.NextSequence == 1) { var state = await streamHandle.Store.GetStateAsync(ct); consumer.NextSequence = consumer.Config.DeliverPolicy switch { JetStream.Models.DeliverPolicy.Last when state.LastSeq > 0 => state.LastSeq, JetStream.Models.DeliverPolicy.New when consumer.Config.OptStartSeq > 0 => consumer.Config.OptStartSeq, JetStream.Models.DeliverPolicy.New when state.LastSeq > 0 => state.LastSeq + 1, JetStream.Models.DeliverPolicy.ByStartSequence when consumer.Config.OptStartSeq > 0 => consumer.Config.OptStartSeq, _ => state.FirstSeq > 0 ? state.FirstSeq : 1, }; } // Use cached CompiledFilter from ConsumerHandle (avoids per-fetch allocation) var compiledFilter = consumer.CompiledFilter; var sequence = consumer.NextSequence; ReadOnlyMemory minHeaders = "NATS/1.0\r\n\r\n"u8.ToArray(); var ackPrefix = $"$JS.ACK.{streamName}.{consumerName}.1."; int deliverySeq = 0; int delivered = 0; var lastDeliveryTime = DateTime.UtcNow; var hbInterval = idleHeartbeatMs > 0 ? TimeSpan.FromMilliseconds(idleHeartbeatMs) : TimeSpan.FromSeconds(15); try { while (delivered < batch && !ct.IsCancellationRequested) { var message = await streamHandle.Store.LoadAsync(sequence, ct); if (message != null) { // Check filter if (!compiledFilter.Matches(message.Subject)) { sequence++; continue; } // Skip already-acked messages if (message.Sequence <= consumer.AckProcessor.AckFloor) { sequence++; continue; } deliverySeq++; delivered++; var tsNanos = new DateTimeOffset(message.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L; var numPending = batch - delivered; var ackReply = BuildAckReply(ackPrefix, message.Sequence, deliverySeq, tsNanos, numPending); // Bypass DeliverMessage — we already know the target client is sender. // Skip permission check and auto-unsub overhead for JS delivery inbox. // Go reference: consumer.go — batch delivery with deferred flush. sender.SendMessageNoFlush(message.Subject, inboxSub.Sid, ackReply, minHeaders, message.Payload); // Batch flush every 64 messages to amortize write-loop wakeup cost. if ((delivered & 63) == 0) sender.SignalFlush(); if (consumer.Config.AckPolicy is JetStream.Models.AckPolicy.Explicit or JetStream.Models.AckPolicy.All) { if (consumer.Config.MaxAckPending > 0 && consumer.AckProcessor.PendingCount >= consumer.Config.MaxAckPending) break; consumer.AckProcessor.Register(message.Sequence, consumer.Config.AckWaitMs); } sequence++; lastDeliveryTime = DateTime.UtcNow; } else { // Flush any buffered messages before blocking on the signal. // Without this, messages queued via SendMessageNoFlush would sit // in the buffer until the next batch boundary or loop exit. sender.SignalFlush(); // No message available — send idle heartbeat if needed if (DateTime.UtcNow - lastDeliveryTime >= hbInterval) { // Go reference: consumer.go sendIdleHeartbeat — status 100 with headers var hbHeader = System.Text.Encoding.UTF8.GetBytes( "NATS/1.0 100 Idle Heartbeat\r\nNats-Last-Consumer: " + consumerName + "\r\nNats-Last-Stream: " + streamName + "\r\n\r\n"); DeliverMessage(inboxSub, replyTo, null, hbHeader, default); lastDeliveryTime = DateTime.UtcNow; } // Wait for publisher to signal a new message, with a heartbeat-interval // timeout so the heartbeat check is re-evaluated periodically. // Go reference: consumer.go — channel signaling from publisher. try { await streamHandle.WaitForPublishAsync(ct).WaitAsync(hbInterval, ct).ConfigureAwait(false); } catch (TimeoutException) { // Heartbeat interval elapsed — loop back to re-check } } } } catch (OperationCanceledException) when (expiresCts.IsCancellationRequested) { // ExpiresMs timeout — expected } // Final flush for any remaining batched messages sender.SignalFlush(); consumer.NextSequence = sequence; // Send terminal status directly to sender (last message, includes flush) ReadOnlyMemory statusHeader = delivered == 0 ? System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n") : System.Text.Encoding.UTF8.GetBytes("NATS/1.0 408 Request Timeout\r\n\r\n"); sender.SendMessage(replyTo, inboxSub.Sid, null, statusHeader, default); } private void DeliverFetchedMessages(Subscription inboxSub, string streamName, string consumerName, string replyTo, IReadOnlyList messages) { ReadOnlyMemory minHeaders = "NATS/1.0\r\n\r\n"u8.ToArray(); int deliverySeq = 0; int numPending = messages.Count; // Pre-compute constant ack prefix to avoid per-message string interpolation. // Go reference: consumer.go — ack reply format is $JS.ACK...1.... var ackPrefix = $"$JS.ACK.{streamName}.{consumerName}.1."; // Use pcd pattern: all messages go to the same client, one flush after the loop. var pcd = t_pcd ??= new HashSet(); pcd.Clear(); foreach (var msg in messages) { deliverySeq++; numPending--; var tsNanos = new DateTimeOffset(msg.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L; var ackReply = BuildAckReply(ackPrefix, msg.Sequence, deliverySeq, tsNanos, numPending); DeliverMessage(inboxSub, msg.Subject, ackReply, minHeaders, msg.Payload, pcd); } // Flush once after all messages delivered foreach (var client in pcd) client.SignalFlush(); pcd.Clear(); } /// /// Fast-path overload using pre-encoded subject and SID bytes to avoid per-delivery encoding. /// Used by ProcessMessage fan-out loop. /// private void DeliverMessage(Subscription sub, ReadOnlySpan subjectBytes, ReadOnlySpan sidBytes, string subject, string? replyTo, ReadOnlyMemory headers, ReadOnlyMemory payload, HashSet? pcd = null) { var client = sub.Client; if (client == null) return; // Auto-unsub: only track when a limit is set (common case is MaxMessages == 0). if (sub.MaxMessages > 0) { var count = Interlocked.Increment(ref sub.MessageCount); if (count > sub.MaxMessages) { var subList = client.Account?.SubList ?? _globalAccount.SubList; subList.Remove(sub); client.RemoveSubscription(sub.Sid); return; } } if (client.Permissions?.IsDeliveryAllowed(subject) == false) return; if (pcd != null) { if (client is NatsClient nc) nc.SendMessageNoFlush(subjectBytes, sidBytes, replyTo, headers, payload); else client.SendMessageNoFlush(subject, sub.Sid, replyTo, headers, payload); pcd.Add(client); } else { client.SendMessage(subject, sub.Sid, replyTo, headers, payload); } if (replyTo != null && client.Permissions?.ResponseTracker != null) { if (client.Permissions.IsPublishAllowed(replyTo) == false) client.Permissions.ResponseTracker.RegisterReply(replyTo); } } private void DeliverMessage(Subscription sub, string subject, string? replyTo, ReadOnlyMemory headers, ReadOnlyMemory payload, HashSet? pcd = null) { var client = sub.Client; if (client == null) return; // Auto-unsub: only track when a limit is set (common case is MaxMessages == 0). if (sub.MaxMessages > 0) { var count = Interlocked.Increment(ref sub.MessageCount); if (count > sub.MaxMessages) { var subList = client.Account?.SubList ?? _globalAccount.SubList; subList.Remove(sub); client.RemoveSubscription(sub.Sid); return; } } if (client.Permissions?.IsDeliveryAllowed(subject) == false) return; if (pcd != null) { client.SendMessageNoFlush(subject, sub.Sid, replyTo, headers, payload); pcd.Add(client); } else { client.SendMessage(subject, sub.Sid, replyTo, headers, payload); } if (replyTo != null && client.Permissions?.ResponseTracker != null) { if (client.Permissions.IsPublishAllowed(replyTo) == false) client.Permissions.ResponseTracker.RegisterReply(replyTo); } } /// /// Builds an ack reply subject from pre-computed prefix and per-message values. /// Uses stack-based formatting to avoid string interpolation boxing/allocations. /// private static string BuildAckReply(string ackPrefix, ulong sequence, int deliverySeq, long tsNanos, int numPending) { // Max digits: ulong=20, int=11, long=20, int=11 + 3 dots = 65 chars max for suffix Span buf = stackalloc char[ackPrefix.Length + 65]; ackPrefix.AsSpan().CopyTo(buf); var pos = ackPrefix.Length; sequence.TryFormat(buf[pos..], out var w); pos += w; buf[pos++] = '.'; deliverySeq.TryFormat(buf[pos..], out w); pos += w; buf[pos++] = '.'; tsNanos.TryFormat(buf[pos..], out w); pos += w; buf[pos++] = '.'; numPending.TryFormat(buf[pos..], out w); pos += w; return new string(buf[..pos]); } /// /// Processes a service import by transforming the subject from the importer's /// subject space to the exporter's subject space, then delivering to matching /// subscribers in the destination account. /// Reference: Go server/accounts.go addServiceImport / processServiceImport. /// public void ProcessServiceImport(ServiceImport si, string subject, string? replyTo, ReadOnlyMemory headers, ReadOnlyMemory payload, Account? sourceAccount = null) { if (si.Invalid) return; // Transform subject: map from importer subject space to exporter subject space string targetSubject; if (si.Transform != null) { var transformed = si.Transform.Apply(subject); targetSubject = transformed ?? si.To; } else if (si.UsePub) { targetSubject = subject; } else { // Default: use the "To" subject from the import definition. // For wildcard imports (e.g. "requests.>" -> "api.>"), we need // to map the specific subject tokens from the source pattern to // the destination pattern. targetSubject = MapImportSubject(subject, si.From, si.To); } // Set up a temporary reverse service import so that responses from the // destination (exporter) account can route back to the source (importer) // account. This handles request-reply across account boundaries. // Go reference: client.go setupResponseServiceImport if (replyTo != null && sourceAccount != null && !si.IsResponse) { SetupResponseServiceImport(si.DestinationAccount, sourceAccount, replyTo, si.Export); } // Service latency tracking: when the response arrives back, compute elapsed // time and publish a latency metric to the configured subject. // Go reference: client.go processServiceImport — latency tracking path. if (si.IsResponse && si.Tracking && si.TimestampTicks > 0) { var elapsed = TimeSpan.FromTicks(Environment.TickCount64 * TimeSpan.TicksPerMillisecond - si.TimestampTicks); PublishServiceLatency(si, elapsed); } // Match against destination account's SubList var destSubList = si.DestinationAccount.SubList; var result = destSubList.Match(targetSubject); // Deliver to plain subscribers in the destination account foreach (var sub in result.PlainSubs) { if (sub.Client == null) continue; DeliverMessage(sub, targetSubject, replyTo, headers, payload); } // Deliver to one member of each queue group foreach (var queueGroup in result.QueueSubs) { if (queueGroup.Length == 0) continue; var sub = queueGroup[0]; // Simple selection: first available if (sub.Client != null) DeliverMessage(sub, targetSubject, replyTo, headers, payload); } } /// /// Creates a temporary reverse service import in the exporter's account so that /// when the exporter publishes a response to the reply subject, the message is /// forwarded back to the importer's account where the reply subscription lives. /// Go reference: client.go setupResponseServiceImport. /// private static void SetupResponseServiceImport(Account exporterAccount, Account importerAccount, string replyTo, ServiceExport? export = null) { // Check if a reverse import for this reply subject already exists if (exporterAccount.Imports.Services.ContainsKey(replyTo)) return; // Determine if we should track latency for this response var shouldTrack = export?.Latency is { } latency && LatencyTracker.ShouldSample(latency); var reverseImport = new ServiceImport { DestinationAccount = importerAccount, From = replyTo, To = replyTo, IsResponse = true, UsePub = true, Export = export, Tracking = shouldTrack, // Store start time as TickCount64 (milliseconds) converted to ticks for elapsed computation TimestampTicks = shouldTrack ? Environment.TickCount64 * TimeSpan.TicksPerMillisecond : 0, }; exporterAccount.Imports.AddServiceImport(reverseImport); } /// /// Maps a published subject from the import "From" pattern to the "To" pattern. /// For example, if From="requests.>" and To="api.>" and subject="requests.test", /// this returns "api.test". /// private static string MapImportSubject(string subject, string fromPattern, string toPattern) { // If "To" doesn't contain wildcards, use it directly if (SubjectMatch.IsLiteral(toPattern)) return toPattern; // For wildcard patterns, replace matching wildcard segments. // Split into tokens and map from source to destination. var subTokens = subject.Split('.'); var fromTokens = fromPattern.Split('.'); var toTokens = toPattern.Split('.'); var result = new string[toTokens.Length]; int subIdx = 0; // Build a mapping: for each wildcard position in "from", // capture the corresponding subject token(s) var wildcardValues = new List(); string? fwcValue = null; for (int i = 0; i < fromTokens.Length && subIdx < subTokens.Length; i++) { if (fromTokens[i] == "*") { wildcardValues.Add(subTokens[subIdx]); subIdx++; } else if (fromTokens[i] == ">") { // Capture all remaining tokens fwcValue = string.Join(".", subTokens[subIdx..]); subIdx = subTokens.Length; } else { subIdx++; // Skip literal match } } // Now build the output using the "to" pattern int wcIdx = 0; var sb = new StringBuilder(); for (int i = 0; i < toTokens.Length; i++) { if (i > 0) sb.Append('.'); if (toTokens[i] == "*") { sb.Append(wcIdx < wildcardValues.Count ? wildcardValues[wcIdx] : "*"); wcIdx++; } else if (toTokens[i] == ">") { sb.Append(fwcValue ?? ">"); } else { sb.Append(toTokens[i]); } } return sb.ToString(); } /// /// Wires service import subscriptions for an account. Creates marker /// subscriptions in the account's SubList so that the import paths /// are tracked. The actual forwarding happens in ProcessMessage when /// it checks the account's Imports.Services. /// Reference: Go server/accounts.go addServiceImportSub. /// public void WireServiceImports(Account account) { foreach (var kvp in account.Imports.Services) { foreach (var si in kvp.Value) { if (si.Invalid) continue; // Create a marker subscription in the importer account. // This subscription doesn't directly deliver messages; // the ProcessMessage method checks service imports after // the regular SubList match. _logger.LogDebug( "Wired service import for account {Account}: {From} -> {To} (dest: {DestAccount})", account.Name, si.From, si.To, si.DestinationAccount.Name); } } } private static void SendNoResponders(NatsClient sender, string replyTo) { // Find the sid for a subscription matching the reply subject var sid = string.Empty; foreach (var sub in sender.Subscriptions.Values) { if (SubjectMatch.MatchLiteral(replyTo, sub.Subject)) { sid = sub.Sid; break; } } // Build: HMSG {replyTo} {sid} {hdrLen} {hdrLen}\r\n{headers}\r\n var headerBlock = "NATS/1.0 503\r\n\r\n"u8; var hdrLen = headerBlock.Length; var controlLine = Encoding.ASCII.GetBytes($"HMSG {replyTo} {sid} {hdrLen} {hdrLen}\r\n"); var totalLen = controlLine.Length + hdrLen + NatsProtocol.CrLf.Length; var msg = new byte[totalLen]; var offset = 0; controlLine.CopyTo(msg.AsSpan(offset)); offset += controlLine.Length; headerBlock.CopyTo(msg.AsSpan(offset)); offset += hdrLen; NatsProtocol.CrLf.CopyTo(msg.AsSpan(offset)); sender.QueueOutbound(msg); } public Account GetOrCreateAccount(string name) { return _accounts.GetOrAdd(name, n => { var acc = new Account(n); if (_options.Accounts != null && _options.Accounts.TryGetValue(n, out var config)) { acc.MaxConnections = config.MaxConnections; acc.MaxSubscriptions = config.MaxSubscriptions; acc.DefaultPermissions = config.DefaultPermissions; // Wire exports from config if (config.Exports != null) { foreach (var export in config.Exports) { if (export.Service is { Length: > 0 } svc) { ServiceLatency? latency = export.LatencySubject is { Length: > 0 } ? new ServiceLatency { Subject = export.LatencySubject, SamplingPercentage = export.LatencySampling } : null; acc.AddServiceExport(svc, Imports.ServiceResponseType.Singleton, approved: null, latency: latency); } else if (export.Stream is { Length: > 0 } strm) { acc.AddStreamExport(strm, approved: null); } } } // Wire imports from config (deferred — needs destination accounts resolved) if (config.Imports != null) WireAccountImports(acc, config.Imports); } return acc; }); } private void WireAccountImports(Account importer, List imports) { foreach (var imp in imports) { if (imp.ServiceAccount is { Length: > 0 } svcAcct && imp.ServiceSubject is { Length: > 0 } svcSubj) { var dest = GetOrCreateAccount(svcAcct); var localSubject = imp.To ?? svcSubj; importer.AddServiceImport(dest, from: localSubject, to: svcSubj); } else if (imp.StreamAccount is { Length: > 0 } strmAcct && imp.StreamSubject is { Length: > 0 } strmSubj) { var source = GetOrCreateAccount(strmAcct); var localSubject = imp.To ?? strmSubj; importer.AddStreamImport(source, from: strmSubj, to: localSubject); } } } /// /// Returns true if the subject belongs to the $SYS subject space. /// Reference: Go server/server.go — isReservedSubject. /// public static bool IsSystemSubject(string subject) => subject.StartsWith("$SYS.", StringComparison.Ordinal) || subject == "$SYS"; /// /// Checks whether the given account is allowed to subscribe to the specified subject. /// Non-system accounts cannot subscribe to $SYS.> subjects. /// Reference: Go server/accounts.go — isReservedForSys. /// public bool IsSubscriptionAllowed(Account? account, string subject) { if (!IsSystemSubject(subject)) return true; // System account is always allowed if (account != null && account.IsSystemAccount) return true; return false; } /// /// Returns the SubList appropriate for a given subject: system account SubList /// for $SYS.> subjects, or the provided account's SubList for everything else. /// Reference: Go server/server.go — sublist routing for internal subjects. /// public SubList GetSubListForSubject(Account? account, string subject) { if (IsSystemSubject(subject)) return _systemAccount.SubList; return account?.SubList ?? _globalAccount.SubList; } /// /// Publishes a service latency metric message to the configured latency subject. /// Go reference: client.go processServiceImport — trackLatency path. /// private void PublishServiceLatency(ServiceImport si, TimeSpan elapsed) { var latency = si.Export?.Latency; if (latency == null || string.IsNullOrEmpty(latency.Subject)) return; var msg = LatencyTracker.BuildLatencyMsg( requestor: si.DestinationAccount.Name, responder: si.Export?.Account?.Name ?? "unknown", serviceLatency: elapsed, totalLatency: elapsed); SendInternalMsg(latency.Subject, reply: null, msg); } public void SendInternalMsg(string subject, string? reply, object? msg) { _eventSystem?.Enqueue(new PublishMessage { Subject = subject, Reply = reply, Body = msg }); } public void SendInternalAccountMsg(Account account, string subject, object? msg) { _eventSystem?.Enqueue(new PublishMessage { Subject = subject, Body = msg }); } /// /// Handles $SYS.REQ.SERVER.{id}.VARZ requests. /// Returns core server information including stats counters. /// public void HandleVarzRequest(string subject, string? reply) { if (reply == null) return; var varz = new { server_id = _serverInfo.ServerId, server_name = _serverInfo.ServerName, version = NatsProtocol.Version, host = _options.Host, port = _options.Port, max_payload = _options.MaxPayload, connections = ClientCount, total_connections = Interlocked.Read(ref _stats.TotalConnections), in_msgs = Interlocked.Read(ref _stats.InMsgs), out_msgs = Interlocked.Read(ref _stats.OutMsgs), in_bytes = Interlocked.Read(ref _stats.InBytes), out_bytes = Interlocked.Read(ref _stats.OutBytes), }; SendInternalMsg(reply, null, varz); } /// /// Handles $SYS.REQ.SERVER.{id}.HEALTHZ requests. /// Returns a simple health status response. /// public void HandleHealthzRequest(string subject, string? reply) { if (reply == null) return; SendInternalMsg(reply, null, new { status = "ok" }); } /// /// Handles $SYS.REQ.SERVER.{id}.SUBSZ requests. /// Returns the current subscription count. /// public void HandleSubszRequest(string subject, string? reply) { if (reply == null) return; SendInternalMsg(reply, null, new { num_subscriptions = SubList.Count }); } /// /// Handles $SYS.REQ.SERVER.{id}.STATSZ requests. /// Publishes current server statistics through the event system. /// public void HandleStatszRequest(string subject, string? reply) { if (reply == null) return; var process = System.Diagnostics.Process.GetCurrentProcess(); var statsMsg = new Events.ServerStatsMsg { Server = BuildEventServerInfo(), Stats = new Events.ServerStatsData { Start = StartTime, Mem = process.WorkingSet64, Cores = Environment.ProcessorCount, Connections = ClientCount, TotalConnections = Interlocked.Read(ref _stats.TotalConnections), Subscriptions = SubList.Count, Sent = new Events.DataStats { Msgs = Interlocked.Read(ref _stats.OutMsgs), Bytes = Interlocked.Read(ref _stats.OutBytes), }, Received = new Events.DataStats { Msgs = Interlocked.Read(ref _stats.InMsgs), Bytes = Interlocked.Read(ref _stats.InBytes), }, InMsgs = Interlocked.Read(ref _stats.InMsgs), OutMsgs = Interlocked.Read(ref _stats.OutMsgs), InBytes = Interlocked.Read(ref _stats.InBytes), OutBytes = Interlocked.Read(ref _stats.OutBytes), SlowConsumers = Interlocked.Read(ref _stats.SlowConsumers), }, }; SendInternalMsg(reply, null, statsMsg); } /// /// Handles $SYS.REQ.SERVER.{id}.IDZ requests. /// Returns basic server identity information. /// public void HandleIdzRequest(string subject, string? reply) { if (reply == null) return; var idz = new { server_id = _serverInfo.ServerId, server_name = _serverInfo.ServerName, version = NatsProtocol.Version, host = _options.Host, port = _options.Port, }; SendInternalMsg(reply, null, idz); } /// /// Builds an EventServerInfo block for embedding in system event messages. /// Maps to Go's serverInfo() helper used in events.go advisory publishing. /// public EventServerInfo BuildEventServerInfo() { var seq = _eventSystem?.NextSequence() ?? 0; return new EventServerInfo { Name = _serverInfo.ServerName, Host = _options.Host, Id = _serverInfo.ServerId, Version = NatsProtocol.Version, Seq = seq, }; } private static EventClientInfo BuildEventClientInfo(NatsClient client) { return new EventClientInfo { Id = client.Id, Host = client.RemoteIp, Account = client.Account?.Name, Name = client.ClientOpts?.Name, Lang = client.ClientOpts?.Lang, Version = client.ClientOpts?.Version, Start = client.StartTime, }; } /// /// Publishes a $SYS.ACCOUNT.{account}.CONNECT advisory when a client /// completes authentication. Maps to Go's sendConnectEvent in events.go. /// public void PublishConnectEvent(INatsClient client) { if (_eventSystem == null || client is not NatsClient natsClient) return; var accountName = client.Account?.Name ?? Account.GlobalAccountName; var subject = string.Format(EventSubjects.ConnectEvent, accountName); var evt = new ConnectEventMsg { Id = Guid.NewGuid().ToString("N"), Time = DateTime.UtcNow, Server = BuildEventServerInfo(), Client = BuildEventClientInfo(natsClient), }; SendInternalMsg(subject, null, evt); } /// /// Publishes a $SYS.ACCOUNT.{account}.DISCONNECT advisory when a client /// disconnects. Maps to Go's sendDisconnectEvent in events.go. /// public void PublishDisconnectEvent(INatsClient client) { if (_eventSystem == null || client is not NatsClient natsClient) return; var accountName = client.Account?.Name ?? Account.GlobalAccountName; var subject = string.Format(EventSubjects.DisconnectEvent, accountName); var evt = new DisconnectEventMsg { Id = Guid.NewGuid().ToString("N"), Time = DateTime.UtcNow, Server = BuildEventServerInfo(), Client = BuildEventClientInfo(natsClient), Sent = new DataStats { Msgs = Interlocked.Read(ref natsClient.OutMsgs), Bytes = Interlocked.Read(ref natsClient.OutBytes), }, Received = new DataStats { Msgs = Interlocked.Read(ref natsClient.InMsgs), Bytes = Interlocked.Read(ref natsClient.InBytes), }, Reason = natsClient.CloseReason.ToReasonString(), }; SendInternalMsg(subject, null, evt); } public void RemoveClient(INatsClient client) { if (client is not NatsClient natsClient) { // Non-NatsClient (e.g. MqttNatsClientAdapter) — basic cleanup _clients.TryRemove(client.Id, out _); var subList = client.Account?.SubList ?? _globalAccount.SubList; client.Account?.RemoveClient(client.Id); return; } // Publish disconnect advisory before removing client state if (natsClient.ConnectReceived) PublishDisconnectEvent(natsClient); _clients.TryRemove(natsClient.Id, out _); _logger.LogDebug("Removed client {ClientId}", natsClient.Id); var (tlsPeerCertSubject, tlsPeerCertSubjectPkSha256, tlsPeerCertSha256) = TlsPeerCertMapper.ToClosedFields(natsClient.TlsState?.PeerCert); var (jwt, issuerKey, tags) = ExtractJwtMetadata(natsClient.ClientOpts?.JWT); var proxyKey = ExtractProxyKey(natsClient.ClientOpts?.Username); // Snapshot for closed-connections tracking (ring buffer auto-overwrites oldest when full) _closedClients.Add(new ClosedClient { Cid = natsClient.Id, Ip = natsClient.RemoteIp ?? "", Port = natsClient.RemotePort, Start = natsClient.StartTime, Stop = DateTime.UtcNow, Reason = natsClient.CloseReason.ToReasonString(), Name = natsClient.ClientOpts?.Name ?? "", Lang = natsClient.ClientOpts?.Lang ?? "", Version = natsClient.ClientOpts?.Version ?? "", AuthorizedUser = natsClient.ClientOpts?.Username ?? "", Account = natsClient.Account?.Name ?? "", InMsgs = Interlocked.Read(ref natsClient.InMsgs), OutMsgs = Interlocked.Read(ref natsClient.OutMsgs), InBytes = Interlocked.Read(ref natsClient.InBytes), OutBytes = Interlocked.Read(ref natsClient.OutBytes), NumSubs = (uint)natsClient.Subscriptions.Count, Rtt = natsClient.Rtt, TlsVersion = natsClient.TlsState?.TlsVersion ?? "", TlsCipherSuite = natsClient.TlsState?.CipherSuite ?? "", TlsPeerCertSubject = tlsPeerCertSubject, TlsPeerCertSubjectPkSha256 = tlsPeerCertSubjectPkSha256, TlsPeerCertSha256 = tlsPeerCertSha256, MqttClient = natsClient.MqttClientId ?? "", Stalls = 0, Jwt = jwt, IssuerKey = issuerKey, NameTag = "", Tags = tags, ProxyKey = proxyKey, }); var ncSubList = natsClient.Account?.SubList ?? _globalAccount.SubList; natsClient.RemoveAllSubscriptions(ncSubList); natsClient.Account?.RemoveClient(natsClient.Id); } private void TrackEarlyClosedClient(Socket socket, ulong clientId, ClientClosedReason reason) { string ip = ""; int port = 0; if (socket.RemoteEndPoint is IPEndPoint endpoint) { ip = endpoint.Address.ToString(); port = endpoint.Port; } var now = DateTime.UtcNow; _closedClients.Add(new ClosedClient { Cid = clientId, Ip = ip, Port = port, Start = now, Stop = now, Reason = reason.ToReasonString(), }); } private static (string Jwt, string IssuerKey, string[] Tags) ExtractJwtMetadata(string? jwt) { if (string.IsNullOrWhiteSpace(jwt)) return ("", "", []); var issuerKey = ""; var tags = Array.Empty(); var claims = NatsJwt.DecodeUserClaims(jwt); if (claims != null) { issuerKey = claims.Issuer ?? ""; tags = claims.Nats?.Tags ?? Array.Empty(); } return (jwt, issuerKey, tags); } private static string ExtractProxyKey(string? username) { if (string.IsNullOrWhiteSpace(username)) return ""; const string prefix = "proxy:"; return username.StartsWith(prefix, StringComparison.Ordinal) ? username[prefix.Length..] : ""; } private void WritePidFile() { if (string.IsNullOrEmpty(_options.PidFile)) return; try { File.WriteAllText(_options.PidFile, Environment.ProcessId.ToString()); _logger.LogDebug("Wrote PID file {PidFile}", _options.PidFile); } catch (Exception ex) { _logger.LogError(ex, "Error writing PID file {PidFile}", _options.PidFile); } } private void DeletePidFile() { if (string.IsNullOrEmpty(_options.PidFile)) return; try { if (File.Exists(_options.PidFile)) File.Delete(_options.PidFile); } catch (Exception ex) { _logger.LogError(ex, "Error deleting PID file {PidFile}", _options.PidFile); } } private void WritePortsFile() { if (string.IsNullOrEmpty(_options.PortsFileDir)) return; try { var exeName = Path.GetFileNameWithoutExtension(Environment.ProcessPath ?? "nats-server"); var fileName = $"{exeName}_{Environment.ProcessId}.ports"; _portsFilePath = Path.Combine(_options.PortsFileDir, fileName); var ports = new { client = _options.Port, monitor = _options.MonitorPort > 0 ? _options.MonitorPort : (int?)null }; var json = System.Text.Json.JsonSerializer.Serialize(ports); File.WriteAllText(_portsFilePath, json); _logger.LogDebug("Wrote ports file {PortsFile}", _portsFilePath); } catch (Exception ex) { _logger.LogError(ex, "Error writing ports file to {PortsFileDir}", _options.PortsFileDir); } } private void DeletePortsFile() { if (_portsFilePath == null) return; try { if (File.Exists(_portsFilePath)) File.Delete(_portsFilePath); } catch (Exception ex) { _logger.LogError(ex, "Error deleting ports file {PortsFile}", _portsFilePath); } } /// /// Stores the CLI snapshot and flags so that command-line overrides /// always take precedence during config reload. /// public void SetCliSnapshot(NatsOptions cliSnapshot, HashSet cliFlags) { _cliSnapshot = cliSnapshot; _cliFlags = cliFlags; } /// /// Reloads the configuration file, diffs against current options, validates /// the changes, and applies reloadable settings. CLI overrides are preserved. /// public void ReloadConfig() { ReloadConfigCore(throwOnError: false); } public void ReloadConfigOrThrow() { ReloadConfigCore(throwOnError: true); } private void ReloadConfigCore(bool throwOnError) { if (_options.ConfigFile == null) { _logger.LogWarning("No config file specified, cannot reload"); if (throwOnError) throw new InvalidOperationException("No config file specified."); return; } try { var (newConfig, digest) = NatsConfParser.ParseFileWithDigest(_options.ConfigFile); if (digest == _configDigest) { _logger.LogInformation("Config file unchanged, no reload needed"); return; } var newOpts = new NatsOptions { ConfigFile = _options.ConfigFile }; ConfigProcessor.ApplyConfig(newConfig, newOpts); // CLI flags override config if (_cliSnapshot != null) ConfigReloader.MergeCliOverrides(newOpts, _cliSnapshot, _cliFlags); var changes = ConfigReloader.Diff(_options, newOpts); var errors = ConfigReloader.Validate(changes); if (errors.Count > 0) { foreach (var err in errors) _logger.LogError("Config reload error: {Error}", err); if (throwOnError) throw new InvalidOperationException(string.Join("; ", errors)); return; } // Apply changes to running options ApplyConfigChanges(changes, newOpts); _configDigest = digest; _configTime = DateTime.UtcNow; _logger.LogInformation("Config reloaded successfully ({Count} changes applied)", changes.Count); } catch (Exception ex) { _logger.LogError(ex, "Failed to reload config file: {ConfigFile}", _options.ConfigFile); if (throwOnError) throw; } } private void ApplyConfigChanges(List changes, NatsOptions newOpts) { bool hasLoggingChanges = false; bool hasAuthChanges = false; bool hasTlsChanges = false; foreach (var change in changes) { if (change.IsLoggingChange) hasLoggingChanges = true; if (change.IsAuthChange) hasAuthChanges = true; if (change.IsTlsChange) hasTlsChanges = true; } // Copy reloadable values from newOpts to _options CopyReloadableOptions(newOpts); // Trigger side effects if (hasLoggingChanges) { ReOpenLogFile?.Invoke(); _logger.LogInformation("Logging configuration reloaded"); } if (hasTlsChanges) { // Reload TLS certificates: new connections get the new cert, // existing connections keep their original cert. // Reference: golang/nats-server/server/reload.go — tlsOption.Apply. if (ConfigReloader.ReloadTlsCertificate(_options, _tlsCertProvider)) { _sslOptions = _tlsCertProvider!.GetCurrentSslOptions(); _logger.LogInformation("TLS configuration reloaded"); } } if (hasAuthChanges) { // Rebuild auth service with new options, then propagate changes to connected clients var oldAuthService = _authService; _authService = AuthService.Build(_options); _logger.LogInformation("Authorization configuration reloaded"); // Re-evaluate connected clients against the new auth config. // Clients that no longer pass authentication are disconnected with AUTH_EXPIRED. // Reference: Go server/reload.go — applyOptions / reloadAuthorization. PropagateAuthChanges(); } } /// /// Re-evaluates all connected clients against the current auth configuration. /// Clients whose credentials no longer pass authentication are disconnected /// with an "Authorization Violation" error via SendErrAndCloseAsync, which /// properly drains the outbound channel before closing the socket. /// Reference: Go server/reload.go — reloadAuthorization, client.go — applyAccountLimits. /// internal void PropagateAuthChanges() { if (!_authService.IsAuthRequired) { // Auth was disabled — all existing clients are fine return; } var clientsToDisconnect = new List(); foreach (var client in _clients.Values) { if (client.ClientOpts == null) continue; // Client hasn't sent CONNECT yet var context = new ClientAuthContext { Opts = client.ClientOpts, Nonce = [], // Nonce is only used at connect time; re-evaluation skips it ClientCertificate = client.TlsState?.PeerCert, }; var result = _authService.Authenticate(context); if (result == null) { _logger.LogInformation( "Client {ClientId} credentials no longer valid after auth reload, disconnecting", client.Id); clientsToDisconnect.Add(client); } } // Disconnect clients that failed re-authentication. // Use SendErrAndCloseAsync which queues the -ERR, completes the outbound channel, // waits for the write loop to drain, then cancels the client. var disconnectTasks = new List(clientsToDisconnect.Count); foreach (var client in clientsToDisconnect) { disconnectTasks.Add(client.SendErrAndCloseAsync( NatsProtocol.ErrAuthorizationViolation, ClientClosedReason.AuthenticationExpired)); } // Wait for all disconnects to complete (with timeout to avoid blocking reload) if (disconnectTasks.Count > 0) { Task.WhenAll(disconnectTasks) .WaitAsync(TimeSpan.FromSeconds(5)) .ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing) .GetAwaiter().GetResult(); _logger.LogInformation( "Disconnected {Count} client(s) after auth configuration reload", clientsToDisconnect.Count); } } private void CopyReloadableOptions(NatsOptions newOpts) { // Logging _options.Debug = newOpts.Debug; _options.Trace = newOpts.Trace; _options.TraceVerbose = newOpts.TraceVerbose; _options.Logtime = newOpts.Logtime; _options.LogtimeUTC = newOpts.LogtimeUTC; _options.LogFile = newOpts.LogFile; _options.LogSizeLimit = newOpts.LogSizeLimit; _options.LogMaxFiles = newOpts.LogMaxFiles; _options.Syslog = newOpts.Syslog; _options.RemoteSyslog = newOpts.RemoteSyslog; // Auth _options.Username = newOpts.Username; _options.Password = newOpts.Password; _options.Authorization = newOpts.Authorization; _options.Users = newOpts.Users; _options.NKeys = newOpts.NKeys; _options.NoAuthUser = newOpts.NoAuthUser; _options.AuthTimeout = newOpts.AuthTimeout; // Limits _options.MaxConnections = newOpts.MaxConnections; _options.MaxPayload = newOpts.MaxPayload; _options.MaxPending = newOpts.MaxPending; _options.WriteDeadline = newOpts.WriteDeadline; _options.PingInterval = newOpts.PingInterval; _options.MaxPingsOut = newOpts.MaxPingsOut; _options.MaxControlLine = newOpts.MaxControlLine; _options.MaxSubs = newOpts.MaxSubs; _options.MaxSubTokens = newOpts.MaxSubTokens; _options.MaxTracedMsgLen = newOpts.MaxTracedMsgLen; if (newOpts.MaxClosedClients != _options.MaxClosedClients) { _options.MaxClosedClients = newOpts.MaxClosedClients; _closedClients = new ClosedConnectionRingBuffer(newOpts.MaxClosedClients); } // TLS _options.TlsCert = newOpts.TlsCert; _options.TlsKey = newOpts.TlsKey; _options.TlsCaCert = newOpts.TlsCaCert; _options.TlsVerify = newOpts.TlsVerify; _options.TlsMap = newOpts.TlsMap; _options.TlsTimeout = newOpts.TlsTimeout; _options.TlsHandshakeFirst = newOpts.TlsHandshakeFirst; _options.TlsHandshakeFirstFallback = newOpts.TlsHandshakeFirstFallback; _options.AllowNonTls = newOpts.AllowNonTls; _options.TlsRateLimit = newOpts.TlsRateLimit; _options.TlsPinnedCerts = newOpts.TlsPinnedCerts; // Misc _options.Tags = newOpts.Tags; _options.LameDuckDuration = newOpts.LameDuckDuration; _options.LameDuckGracePeriod = newOpts.LameDuckGracePeriod; _options.ClientAdvertise = newOpts.ClientAdvertise; _options.DisableSublistCache = newOpts.DisableSublistCache; _options.ConnectErrorReports = newOpts.ConnectErrorReports; _options.ReconnectErrorReports = newOpts.ReconnectErrorReports; _options.NoHeaderSupport = newOpts.NoHeaderSupport; _options.NoSystemAccount = newOpts.NoSystemAccount; _options.SystemAccount = newOpts.SystemAccount; } public override string ToString() => $"NatsServer(ServerId={ServerId}, Name={ServerName}, Addr={Addr()}, Clients={ClientCount})"; public void Dispose() { if (!IsShuttingDown) ShutdownAsync().GetAwaiter().GetResult(); foreach (var reg in _signalRegistrations) reg.Dispose(); _quitCts.Dispose(); _tlsRateLimiter?.Dispose(); _tlsCertProvider?.Dispose(); _listener?.Dispose(); _wsListener?.Dispose(); _routeManager?.DisposeAsync().AsTask().GetAwaiter().GetResult(); _gatewayManager?.DisposeAsync().AsTask().GetAwaiter().GetResult(); _leafNodeManager?.DisposeAsync().AsTask().GetAwaiter().GetResult(); _mqttListener?.DisposeAsync().AsTask().GetAwaiter().GetResult(); _jetStreamService?.DisposeAsync().AsTask().GetAwaiter().GetResult(); _stats.JetStreamEnabled = false; foreach (var client in _clients.Values) client.Dispose(); foreach (var account in _accounts.Values) account.Dispose(); } }