Implement Go-parity background flush loop (coalesce 16KB/8ms) in MsgBlock/FileStore, replace O(n) GetStateAsync with incremental counters, skip PruneExpired/LoadAsync/ PrunePerSubject when not needed, and bypass RAFT for single-replica streams. Fix counter tracking bugs in RemoveMsg/EraseMsg/TTL expiry and ObjectDisposedException races in flush loop disposal. FileStore optimizations verified with 3112/3112 JetStream tests passing; async publish benchmark remains at ~174 msg/s due to E2E protocol path bottleneck.
2547 lines
100 KiB
C#
2547 lines
100 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Net;
|
|
using System.Net.NetworkInformation;
|
|
using System.Net.Security;
|
|
using System.Net.Sockets;
|
|
using System.Runtime.InteropServices;
|
|
using System.Security.Cryptography.X509Certificates;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.Logging;
|
|
using NATS.NKeys;
|
|
using NATS.Server.Auth;
|
|
using NATS.Server.Auth.Jwt;
|
|
using NATS.Server.Configuration;
|
|
using NATS.Server.Events;
|
|
using NATS.Server.Gateways;
|
|
using NATS.Server.Imports;
|
|
using NATS.Server.JetStream;
|
|
using NATS.Server.JetStream.Api;
|
|
using NATS.Server.JetStream.Publish;
|
|
using NATS.Server.LeafNodes;
|
|
using NATS.Server.Monitoring;
|
|
using NATS.Server.Mqtt;
|
|
using NATS.Server.Protocol;
|
|
using NATS.Server.Routes;
|
|
using NATS.Server.Server;
|
|
using NATS.Server.Subscriptions;
|
|
using NATS.Server.Tls;
|
|
using NATS.Server.WebSocket;
|
|
|
|
namespace NATS.Server;
|
|
|
|
public sealed class NatsServer : IMessageRouter, ISubListAccess, IDisposable
|
|
{
|
|
private readonly NatsOptions _options;
|
|
private readonly ConcurrentDictionary<ulong, NatsClient> _clients = new();
|
|
private ClosedConnectionRingBuffer _closedClients;
|
|
private readonly ServerInfo _serverInfo;
|
|
private readonly ILogger<NatsServer> _logger;
|
|
private readonly ILoggerFactory _loggerFactory;
|
|
private readonly ServerStats _stats = new();
|
|
private readonly TaskCompletionSource _listeningStarted = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private AuthService _authService;
|
|
private readonly ConcurrentDictionary<string, Account> _accounts = new(StringComparer.Ordinal);
|
|
|
|
// Config reload state
|
|
private NatsOptions? _cliSnapshot;
|
|
private HashSet<string> _cliFlags = [];
|
|
private string? _configDigest;
|
|
private readonly SemaphoreSlim _reloadMu = new(1, 1);
|
|
private AcceptLoopErrorHandler? _acceptLoopErrorHandler;
|
|
private readonly Account _globalAccount;
|
|
private readonly Account _systemAccount;
|
|
private InternalEventSystem? _eventSystem;
|
|
private SslServerAuthenticationOptions? _sslOptions;
|
|
private readonly TlsRateLimiter? _tlsRateLimiter;
|
|
private readonly TlsCertificateProvider? _tlsCertProvider;
|
|
private readonly SubjectTransform[] _subjectTransforms;
|
|
private readonly RouteManager? _routeManager;
|
|
|
|
/// <summary>
|
|
/// Exposes the route manager for testing. Internal — visible to test project
|
|
/// via InternalsVisibleTo.
|
|
/// </summary>
|
|
internal RouteManager? RouteManager => _routeManager;
|
|
internal GatewayManager? GatewayManager => _gatewayManager;
|
|
private readonly GatewayManager? _gatewayManager;
|
|
private readonly LeafNodeManager? _leafNodeManager;
|
|
private readonly InternalClient? _jetStreamInternalClient;
|
|
private readonly JetStreamService? _jetStreamService;
|
|
private readonly JetStreamApiRouter? _jetStreamApiRouter;
|
|
private readonly StreamManager? _jetStreamStreamManager;
|
|
private readonly ConsumerManager? _jetStreamConsumerManager;
|
|
private readonly JetStreamPublisher? _jetStreamPublisher;
|
|
private MqttListener? _mqttListener;
|
|
private Socket? _listener;
|
|
private Socket? _wsListener;
|
|
private readonly TaskCompletionSource _wsAcceptLoopExited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private MonitorServer? _monitorServer;
|
|
private ulong _nextClientId;
|
|
private long _startTimeTicks;
|
|
|
|
private readonly CancellationTokenSource _quitCts = new();
|
|
private readonly TaskCompletionSource _shutdownComplete = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private readonly TaskCompletionSource _acceptLoopExited = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private int _shutdown;
|
|
private int _activeClientCount;
|
|
|
|
private int _lameDuck;
|
|
|
|
private byte[] _cachedInfoLine = [];
|
|
|
|
private readonly List<PosixSignalRegistration> _signalRegistrations = [];
|
|
|
|
private string? _portsFilePath;
|
|
private DateTime _configTime = DateTime.UtcNow;
|
|
|
|
private static readonly TimeSpan AcceptMinSleep = NatsProtocol.AcceptMinSleep;
|
|
private static readonly TimeSpan AcceptMaxSleep = NatsProtocol.AcceptMaxSleep;
|
|
private static readonly JsonSerializerOptions s_jetStreamJsonOptions = new()
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
|
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
|
};
|
|
|
|
public SubList SubList => _globalAccount.SubList;
|
|
public byte[] CachedInfoLine => _cachedInfoLine;
|
|
public ServerStats Stats => _stats;
|
|
public DateTime StartTime => new(Interlocked.Read(ref _startTimeTicks), DateTimeKind.Utc);
|
|
public string ServerId => _serverInfo.ServerId;
|
|
public string ServerName => _serverInfo.ServerName;
|
|
public int ClientCount => _clients.Count;
|
|
public int Port => _options.Port;
|
|
public Account SystemAccount => _systemAccount;
|
|
public string ServerNKey { get; }
|
|
public InternalEventSystem? EventSystem => _eventSystem;
|
|
public bool IsShuttingDown => Volatile.Read(ref _shutdown) != 0;
|
|
public bool IsLameDuckMode => Volatile.Read(ref _lameDuck) != 0;
|
|
public string? ClusterListen => _routeManager?.ListenEndpoint;
|
|
public string? GatewayListen => _gatewayManager?.ListenEndpoint;
|
|
public string? LeafListen => _leafNodeManager?.ListenEndpoint;
|
|
public bool IsProfilingEnabled => _options.ProfPort > 0;
|
|
public InternalClient? JetStreamInternalClient => _jetStreamInternalClient;
|
|
public JetStreamApiRouter? JetStreamApiRouter => _jetStreamApiRouter;
|
|
public int JetStreamStreams => _jetStreamStreamManager?.StreamNames.Count ?? 0;
|
|
public int JetStreamConsumers => _jetStreamConsumerManager?.ConsumerCount ?? 0;
|
|
public Action? ReOpenLogFile { get; set; }
|
|
public IEnumerable<NatsClient> GetClients() => _clients.Values;
|
|
public string? ClusterName() => _options.Cluster?.Name;
|
|
public IReadOnlyList<string> ActivePeers()
|
|
=> _routeManager?.BuildTopologySnapshot().ConnectedServerIds ?? [];
|
|
|
|
public bool StartProfiler()
|
|
{
|
|
if (_options.ProfPort <= 0)
|
|
return false;
|
|
|
|
_logger.LogWarning("Profiling endpoint not yet supported (port: {ProfPort})", _options.ProfPort);
|
|
return true;
|
|
}
|
|
|
|
public bool DisconnectClientByID(ulong clientId)
|
|
=> CloseClientById(clientId, minimalFlush: true);
|
|
|
|
public bool LDMClientByID(ulong clientId)
|
|
=> CloseClientById(clientId, minimalFlush: false);
|
|
|
|
public Ports PortsInfo()
|
|
{
|
|
var ports = new Ports();
|
|
|
|
AddEndpoint(ports.Nats, _options.Host, _options.Port);
|
|
AddEndpoint(ports.Monitoring, _options.MonitorHost, _options.MonitorPort);
|
|
|
|
if (_routeManager != null)
|
|
AddEndpoint(ports.Cluster, _routeManager.ListenEndpoint);
|
|
else if (_options.Cluster != null)
|
|
AddEndpoint(ports.Cluster, _options.Cluster.Host, _options.Cluster.Port);
|
|
|
|
AddEndpoint(ports.Profile, _options.Host, _options.ProfPort);
|
|
|
|
if (_options.WebSocket.Port >= 0)
|
|
AddEndpoint(ports.WebSocket, _options.WebSocket.Host, _options.WebSocket.Port);
|
|
|
|
if (_leafNodeManager != null)
|
|
AddEndpoint(ports.LeafNodes, _leafNodeManager.ListenEndpoint);
|
|
else if (_options.LeafNode != null)
|
|
AddEndpoint(ports.LeafNodes, _options.LeafNode.Host, _options.LeafNode.Port);
|
|
|
|
return ports;
|
|
}
|
|
|
|
public IReadOnlyList<string> GetConnectURLs()
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(_options.ClientAdvertise))
|
|
return [NormalizeAdvertiseUrl(_options.ClientAdvertise!, "nats")];
|
|
|
|
var hosts = GetNonLocalIPsIfHostIsIPAny(_options.Host);
|
|
var result = new List<string>(hosts.Count);
|
|
foreach (var host in hosts)
|
|
result.Add($"nats://{host}:{_options.Port}");
|
|
|
|
return result;
|
|
}
|
|
|
|
public void UpdateServerINFOAndSendINFOToClients()
|
|
{
|
|
_serverInfo.ConnectUrls = [.. GetConnectURLs()];
|
|
BuildCachedInfo();
|
|
|
|
foreach (var client in _clients.Values)
|
|
{
|
|
if (client.ConnectReceived)
|
|
client.QueueOutbound(_cachedInfoLine);
|
|
}
|
|
}
|
|
|
|
public string ClientURL()
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(_options.ClientAdvertise))
|
|
return NormalizeAdvertiseUrl(_options.ClientAdvertise!, "nats");
|
|
|
|
var host = IsWildcardHost(_options.Host) ? "127.0.0.1" : _options.Host;
|
|
return $"nats://{host}:{_options.Port}";
|
|
}
|
|
|
|
public string? WebsocketURL()
|
|
{
|
|
if (_options.WebSocket.Port < 0)
|
|
return null;
|
|
|
|
if (!string.IsNullOrWhiteSpace(_options.WebSocket.Advertise))
|
|
{
|
|
var scheme = _options.WebSocket.NoTls ? "ws" : "wss";
|
|
return NormalizeAdvertiseUrl(_options.WebSocket.Advertise!, scheme);
|
|
}
|
|
|
|
var wsHost = IsWildcardHost(_options.WebSocket.Host) ? "127.0.0.1" : _options.WebSocket.Host;
|
|
var wsScheme = _options.WebSocket.NoTls ? "ws" : "wss";
|
|
return $"{wsScheme}://{wsHost}:{_options.WebSocket.Port}";
|
|
}
|
|
|
|
public int NumRoutes() => (int)Interlocked.Read(ref _stats.Routes);
|
|
|
|
public int NumRemotes()
|
|
=> (int)(Interlocked.Read(ref _stats.Routes) + Interlocked.Read(ref _stats.Gateways) + Interlocked.Read(ref _stats.Leafs));
|
|
|
|
public int NumLeafNodes() => (int)Interlocked.Read(ref _stats.Leafs);
|
|
public int NumOutboundGateways() => _gatewayManager?.NumOutboundGateways() ?? 0;
|
|
public int NumInboundGateways() => _gatewayManager?.NumInboundGateways() ?? 0;
|
|
|
|
public int NumSubscriptions() => _accounts.Values.Sum(acc => acc.SubscriptionCount);
|
|
public bool JetStreamEnabled() => _jetStreamService?.IsRunning ?? false;
|
|
|
|
public JetStreamOptions? JetStreamConfig()
|
|
{
|
|
if (_options.JetStream is null)
|
|
return null;
|
|
|
|
return new JetStreamOptions
|
|
{
|
|
StoreDir = _options.JetStream.StoreDir,
|
|
MaxMemoryStore = _options.JetStream.MaxMemoryStore,
|
|
MaxFileStore = _options.JetStream.MaxFileStore,
|
|
MaxStreams = _options.JetStream.MaxStreams,
|
|
MaxConsumers = _options.JetStream.MaxConsumers,
|
|
Domain = _options.JetStream.Domain,
|
|
};
|
|
}
|
|
|
|
public string StoreDir() => _options.JetStream?.StoreDir ?? string.Empty;
|
|
|
|
public DateTime ConfigTime() => _configTime;
|
|
|
|
public string Addr() => $"{_options.Host}:{_options.Port}";
|
|
|
|
public string? MonitorAddr()
|
|
=> _options.MonitorPort > 0
|
|
? $"{_options.MonitorHost}:{_options.MonitorPort}"
|
|
: null;
|
|
|
|
public string? ClusterAddr() => _routeManager?.ListenEndpoint;
|
|
public string? GatewayAddr() => _gatewayManager?.ListenEndpoint;
|
|
public string? GetGatewayURL() => _gatewayManager?.ListenEndpoint;
|
|
public string? GetGatewayName() => _options.Gateway?.Name;
|
|
|
|
public string? ProfilerAddr()
|
|
=> _options.ProfPort > 0
|
|
? $"{_options.Host}:{_options.ProfPort}"
|
|
: null;
|
|
|
|
public int NumActiveAccounts() => _accounts.Values.Count(acc => acc.ClientCount > 0);
|
|
|
|
public int NumLoadedAccounts() => _accounts.Count;
|
|
|
|
public IReadOnlyList<ClosedClient> GetClosedClients() => _closedClients.GetAll();
|
|
|
|
public IEnumerable<Auth.Account> GetAccounts() => _accounts.Values;
|
|
public bool HasRemoteInterest(string subject) => _globalAccount.SubList.HasRemoteInterest(subject);
|
|
public bool HasRemoteInterest(string account, string subject)
|
|
=> GetOrCreateAccount(account).SubList.HasRemoteInterest(account, subject);
|
|
public bool TryCaptureJetStreamPublish(string subject, ReadOnlyMemory<byte> payload, out PubAck ack)
|
|
{
|
|
if (_jetStreamPublisher != null && _jetStreamPublisher.TryCapture(subject, payload, out ack))
|
|
{
|
|
// Only load the stored message for consumer notification when there are
|
|
// active consumers for this stream. Avoids unnecessary LoadAsync on the hot path.
|
|
if (ack.ErrorCode == null
|
|
&& _jetStreamConsumerManager != null
|
|
&& _jetStreamConsumerManager.HasConsumersForStream(ack.Stream)
|
|
&& _jetStreamStreamManager != null
|
|
&& _jetStreamStreamManager.TryGet(ack.Stream, out var streamHandle))
|
|
{
|
|
var stored = streamHandle.Store.LoadAsync(ack.Seq, default).GetAwaiter().GetResult();
|
|
if (stored != null)
|
|
_jetStreamConsumerManager.OnPublished(ack.Stream, stored);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
ack = new PubAck();
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Replicates a successful JetStream mutating API call to cluster peers via internal subjects.
|
|
/// Maps API subjects to $JS.INTERNAL.* subjects and forwards to routes/leafnodes.
|
|
/// Go reference: jetstream_cluster.go — RAFT proposal broadcast.
|
|
/// </summary>
|
|
private void TryReplicateJetStreamMutation(string apiSubject, ReadOnlyMemory<byte> payload)
|
|
{
|
|
string? internalSubject = null;
|
|
|
|
if (apiSubject.StartsWith(JetStreamApiSubjects.StreamCreate, StringComparison.Ordinal))
|
|
internalSubject = JetStreamApiSubjects.InternalStreamCreate + apiSubject[JetStreamApiSubjects.StreamCreate.Length..];
|
|
else if (apiSubject.StartsWith(JetStreamApiSubjects.StreamDelete, StringComparison.Ordinal))
|
|
internalSubject = JetStreamApiSubjects.InternalStreamDelete + apiSubject[JetStreamApiSubjects.StreamDelete.Length..];
|
|
else if (apiSubject.StartsWith(JetStreamApiSubjects.StreamPurge, StringComparison.Ordinal))
|
|
internalSubject = JetStreamApiSubjects.InternalStreamPurge + apiSubject[JetStreamApiSubjects.StreamPurge.Length..];
|
|
else if (apiSubject.StartsWith(JetStreamApiSubjects.ConsumerCreate, StringComparison.Ordinal))
|
|
internalSubject = JetStreamApiSubjects.InternalConsumerCreate + apiSubject[JetStreamApiSubjects.ConsumerCreate.Length..];
|
|
else if (apiSubject.StartsWith(JetStreamApiSubjects.ConsumerDelete, StringComparison.Ordinal))
|
|
internalSubject = JetStreamApiSubjects.InternalConsumerDelete + apiSubject[JetStreamApiSubjects.ConsumerDelete.Length..];
|
|
|
|
if (internalSubject != null)
|
|
ReplicateJetStreamOperation("$G", internalSubject, null, payload);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Forwards a JetStream replication message to all route and leaf node peers.
|
|
/// Bypasses interest checks since replication subjects have no client subscribers.
|
|
/// Go reference: jetstream_cluster.go — proposal broadcast to peers.
|
|
/// </summary>
|
|
private void ReplicateJetStreamOperation(string account, string subject, string? replyTo, ReadOnlyMemory<byte> payload)
|
|
{
|
|
_routeManager?.BroadcastRoutedMessageAsync(account, subject, replyTo, payload, default)
|
|
.GetAwaiter().GetResult();
|
|
if (_leafNodeManager != null)
|
|
{
|
|
var markedSubject = LeafLoopDetector.Mark(subject, ServerId);
|
|
_leafNodeManager.ForwardMessageAsync(account, markedSubject, replyTo, payload, default)
|
|
.GetAwaiter().GetResult();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles incoming JetStream internal replication messages from cluster peers.
|
|
/// Dispatches to the appropriate handler based on the internal subject prefix.
|
|
/// Called from DeliverRemoteMessage for $JS.INTERNAL.* subjects.
|
|
/// Go reference: jetstream_cluster.go — RAFT proposal apply.
|
|
/// </summary>
|
|
private void HandleJetStreamReplication(string subject, ReadOnlyMemory<byte> payload)
|
|
{
|
|
if (_jetStreamStreamManager == null || _jetStreamConsumerManager == null)
|
|
return;
|
|
|
|
if (subject.StartsWith(JetStreamApiSubjects.InternalStreamCreate, StringComparison.Ordinal))
|
|
{
|
|
var apiSubject = JetStreamApiSubjects.StreamCreate + subject[JetStreamApiSubjects.InternalStreamCreate.Length..];
|
|
JetStream.Api.Handlers.StreamApiHandlers.HandleCreate(apiSubject, payload.Span, _jetStreamStreamManager);
|
|
}
|
|
else if (subject.StartsWith(JetStreamApiSubjects.InternalStreamDelete, StringComparison.Ordinal))
|
|
{
|
|
var streamName = subject[JetStreamApiSubjects.InternalStreamDelete.Length..];
|
|
_jetStreamStreamManager.Delete(streamName);
|
|
}
|
|
else if (subject.StartsWith(JetStreamApiSubjects.InternalStreamPurge, StringComparison.Ordinal))
|
|
{
|
|
var streamName = subject[JetStreamApiSubjects.InternalStreamPurge.Length..];
|
|
_jetStreamStreamManager.Purge(streamName);
|
|
}
|
|
else if (subject.StartsWith(JetStreamApiSubjects.InternalConsumerCreate, StringComparison.Ordinal))
|
|
{
|
|
var apiSubject = JetStreamApiSubjects.ConsumerCreate + subject[JetStreamApiSubjects.InternalConsumerCreate.Length..];
|
|
JetStream.Api.Handlers.ConsumerApiHandlers.HandleCreate(apiSubject, payload.Span, _jetStreamConsumerManager);
|
|
}
|
|
else if (subject.StartsWith(JetStreamApiSubjects.InternalConsumerDelete, StringComparison.Ordinal))
|
|
{
|
|
var parts = subject[JetStreamApiSubjects.InternalConsumerDelete.Length..].Split('.');
|
|
if (parts.Length >= 2)
|
|
_jetStreamConsumerManager.Delete(parts[0], parts[1]);
|
|
}
|
|
}
|
|
|
|
public Task WaitForReadyAsync() => _listeningStarted.Task;
|
|
|
|
public void WaitForShutdown() => _shutdownComplete.Task.GetAwaiter().GetResult();
|
|
|
|
internal TlsCertificateProvider? TlsCertProviderForTest => _tlsCertProvider;
|
|
|
|
internal Task AcquireReloadLockForTestAsync() => _reloadMu.WaitAsync();
|
|
|
|
internal void ReleaseReloadLockForTest() => _reloadMu.Release();
|
|
|
|
internal void SetAcceptLoopErrorHandlerForTest(AcceptLoopErrorHandler handler) => _acceptLoopErrorHandler = handler;
|
|
|
|
internal void NotifyAcceptErrorForTest(Exception ex, EndPoint? endpoint, TimeSpan delay) =>
|
|
_acceptLoopErrorHandler?.OnAcceptError(ex, endpoint, delay);
|
|
|
|
public async Task ShutdownAsync()
|
|
{
|
|
if (Interlocked.CompareExchange(ref _shutdown, 1, 0) != 0)
|
|
return; // Already shutting down
|
|
|
|
_logger.LogInformation("Initiating Shutdown...");
|
|
|
|
// Publish shutdown advisory before tearing down the event system
|
|
if (_eventSystem != null)
|
|
{
|
|
var shutdownSubject = string.Format(EventSubjects.ServerShutdown, _serverInfo.ServerId);
|
|
_eventSystem.Enqueue(new PublishMessage
|
|
{
|
|
Subject = shutdownSubject,
|
|
Body = new ShutdownEventMsg { Server = BuildEventServerInfo(), Reason = "Server Shutdown" },
|
|
IsLast = true,
|
|
});
|
|
// Give the send loop time to process the shutdown event
|
|
await Task.Delay(100);
|
|
await _eventSystem.DisposeAsync();
|
|
}
|
|
|
|
// Signal all internal loops to stop
|
|
await _quitCts.CancelAsync();
|
|
|
|
// Close listeners to stop accept loops
|
|
_listener?.Close();
|
|
_wsListener?.Close();
|
|
if (_routeManager != null)
|
|
await _routeManager.DisposeAsync();
|
|
if (_gatewayManager != null)
|
|
await _gatewayManager.DisposeAsync();
|
|
if (_leafNodeManager != null)
|
|
await _leafNodeManager.DisposeAsync();
|
|
if (_jetStreamService != null)
|
|
await _jetStreamService.DisposeAsync();
|
|
_stats.JetStreamEnabled = false;
|
|
|
|
// Wait for accept loops to exit
|
|
await _acceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
|
await _wsAcceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
|
|
|
// Close all client connections — flush first, then mark closed
|
|
var flushTasks = new List<Task>();
|
|
foreach (var client in _clients.Values)
|
|
{
|
|
client.MarkClosed(ClientClosedReason.ServerShutdown);
|
|
flushTasks.Add(client.FlushAndCloseAsync(minimalFlush: true));
|
|
}
|
|
await Task.WhenAll(flushTasks).WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
|
|
|
// Wait for active client tasks to drain (with timeout)
|
|
if (Volatile.Read(ref _activeClientCount) > 0)
|
|
{
|
|
using var drainCts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
|
try
|
|
{
|
|
while (Volatile.Read(ref _activeClientCount) > 0 && !drainCts.IsCancellationRequested)
|
|
await Task.Delay(50, drainCts.Token);
|
|
}
|
|
catch (OperationCanceledException) { }
|
|
}
|
|
|
|
// Stop monitor server
|
|
if (_monitorServer != null)
|
|
await _monitorServer.DisposeAsync();
|
|
if (_mqttListener != null)
|
|
await _mqttListener.DisposeAsync();
|
|
|
|
DeletePidFile();
|
|
DeletePortsFile();
|
|
|
|
_logger.LogInformation("Server Exiting..");
|
|
_shutdownComplete.TrySetResult();
|
|
}
|
|
|
|
public async Task LameDuckShutdownAsync()
|
|
{
|
|
if (IsShuttingDown || Interlocked.CompareExchange(ref _lameDuck, 1, 0) != 0)
|
|
return;
|
|
|
|
_logger.LogInformation("Entering lame duck mode, stop accepting new clients");
|
|
|
|
// Close listeners to stop accepting new connections
|
|
_listener?.Close();
|
|
_wsListener?.Close();
|
|
|
|
// Wait for accept loops to exit
|
|
await _acceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
|
await _wsAcceptLoopExited.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
|
|
|
var gracePeriod = _options.LameDuckGracePeriod;
|
|
if (gracePeriod < TimeSpan.Zero) gracePeriod = -gracePeriod;
|
|
|
|
// If no clients, go straight to shutdown
|
|
if (_clients.IsEmpty)
|
|
{
|
|
await ShutdownAsync();
|
|
return;
|
|
}
|
|
|
|
// Wait grace period for clients to drain naturally
|
|
_logger.LogInformation("Waiting {GracePeriod}ms grace period", gracePeriod.TotalMilliseconds);
|
|
try
|
|
{
|
|
await Task.Delay(gracePeriod, _quitCts.Token);
|
|
}
|
|
catch (OperationCanceledException) { return; }
|
|
|
|
if (_clients.IsEmpty)
|
|
{
|
|
await ShutdownAsync();
|
|
return;
|
|
}
|
|
|
|
// Stagger-close remaining clients
|
|
var dur = _options.LameDuckDuration - gracePeriod;
|
|
if (dur <= TimeSpan.Zero) dur = TimeSpan.FromSeconds(1);
|
|
|
|
var clients = _clients.Values.ToList();
|
|
var numClients = clients.Count;
|
|
|
|
if (numClients > 0)
|
|
{
|
|
_logger.LogInformation("Closing {Count} existing clients over {Duration}ms",
|
|
numClients, dur.TotalMilliseconds);
|
|
|
|
var sleepInterval = dur.Ticks / numClients;
|
|
if (sleepInterval < TimeSpan.TicksPerMillisecond)
|
|
sleepInterval = TimeSpan.TicksPerMillisecond;
|
|
if (sleepInterval > TimeSpan.TicksPerSecond)
|
|
sleepInterval = TimeSpan.TicksPerSecond;
|
|
|
|
for (int i = 0; i < clients.Count; i++)
|
|
{
|
|
clients[i].MarkClosed(ClientClosedReason.ServerShutdown);
|
|
await clients[i].FlushAndCloseAsync(minimalFlush: true);
|
|
|
|
if (i < clients.Count - 1)
|
|
{
|
|
var jitter = Random.Shared.NextInt64(sleepInterval / 2, sleepInterval);
|
|
try
|
|
{
|
|
await Task.Delay(TimeSpan.FromTicks(jitter), _quitCts.Token);
|
|
}
|
|
catch (OperationCanceledException) { break; }
|
|
}
|
|
}
|
|
}
|
|
|
|
await ShutdownAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Registers Unix signal handlers.
|
|
/// SIGTERM → shutdown, SIGUSR2 → lame duck, SIGUSR1 → log reopen, SIGHUP → reload (stub).
|
|
/// </summary>
|
|
public void HandleSignals()
|
|
{
|
|
_signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGTERM, ctx =>
|
|
{
|
|
ctx.Cancel = true;
|
|
_logger.LogInformation("Trapped SIGTERM signal");
|
|
_ = Task.Run(async () => await ShutdownAsync());
|
|
}));
|
|
|
|
_signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGQUIT, ctx =>
|
|
{
|
|
ctx.Cancel = true;
|
|
_logger.LogInformation("Trapped SIGQUIT signal");
|
|
_ = Task.Run(async () => await ShutdownAsync());
|
|
}));
|
|
|
|
_signalRegistrations.Add(PosixSignalRegistration.Create(PosixSignal.SIGHUP, ctx =>
|
|
{
|
|
ctx.Cancel = true;
|
|
_logger.LogInformation("Trapped SIGHUP signal — reloading configuration");
|
|
_ = Task.Run(() => ReloadConfig());
|
|
}));
|
|
|
|
// SIGUSR1 and SIGUSR2 only on non-Windows
|
|
if (!OperatingSystem.IsWindows())
|
|
{
|
|
_signalRegistrations.Add(PosixSignalRegistration.Create((PosixSignal)10, ctx =>
|
|
{
|
|
ctx.Cancel = true;
|
|
_logger.LogInformation("Trapped SIGUSR1 signal — reopening log file");
|
|
ReOpenLogFile?.Invoke();
|
|
}));
|
|
|
|
_signalRegistrations.Add(PosixSignalRegistration.Create((PosixSignal)12, ctx =>
|
|
{
|
|
ctx.Cancel = true;
|
|
_logger.LogInformation("Trapped SIGUSR2 signal — entering lame duck mode");
|
|
_ = Task.Run(async () => await LameDuckShutdownAsync());
|
|
}));
|
|
}
|
|
}
|
|
|
|
public NatsServer(NatsOptions options, ILoggerFactory loggerFactory)
|
|
{
|
|
_options = options;
|
|
_loggerFactory = loggerFactory;
|
|
_logger = loggerFactory.CreateLogger<NatsServer>();
|
|
_closedClients = new ClosedConnectionRingBuffer(options.MaxClosedClients);
|
|
_authService = AuthService.Build(options);
|
|
_globalAccount = new Account(Account.GlobalAccountName);
|
|
_accounts[Account.GlobalAccountName] = _globalAccount;
|
|
|
|
// Create $SYS system account and mark it as the system account.
|
|
// Reference: Go server/server.go — initSystemAccount, accounts.go — isSystemAccount().
|
|
_systemAccount = new Account(Account.SystemAccountName) { IsSystemAccount = true };
|
|
_accounts[Account.SystemAccountName] = _systemAccount;
|
|
|
|
// If a user-defined system_account is configured, promote that account to be the
|
|
// system account. Events published to $SYS.* will be delivered to subscribers on
|
|
// this account. Go reference: server/server.go — configureAccounts / setSystemAccount.
|
|
if (!string.IsNullOrEmpty(options.SystemAccount) &&
|
|
!string.Equals(options.SystemAccount, Account.SystemAccountName, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
var userSysAccount = GetOrCreateAccount(options.SystemAccount);
|
|
userSysAccount.IsSystemAccount = true;
|
|
_systemAccount = userSysAccount;
|
|
}
|
|
|
|
// Create system internal client and event system
|
|
var sysClientId = Interlocked.Increment(ref _nextClientId);
|
|
var sysClient = new InternalClient(sysClientId, ClientKind.System, _systemAccount);
|
|
_eventSystem = new InternalEventSystem(
|
|
_systemAccount, sysClient,
|
|
options.ServerName ?? $"nats-dotnet-{Environment.MachineName}",
|
|
_loggerFactory.CreateLogger<InternalEventSystem>());
|
|
|
|
// Generate Ed25519 server NKey identity
|
|
using var serverKeyPair = KeyPair.CreatePair(PrefixByte.Server);
|
|
ServerNKey = serverKeyPair.GetPublicKey();
|
|
|
|
_serverInfo = new ServerInfo
|
|
{
|
|
ServerId = Guid.NewGuid().ToString("N")[..20].ToUpperInvariant(),
|
|
ServerName = options.ServerName ?? $"nats-dotnet-{Environment.MachineName}",
|
|
Version = NatsProtocol.Version,
|
|
Host = options.Host,
|
|
Port = options.Port,
|
|
MaxPayload = options.MaxPayload,
|
|
AuthRequired = _authService.IsAuthRequired,
|
|
};
|
|
|
|
if (options.Cluster != null)
|
|
{
|
|
_routeManager = new RouteManager(options.Cluster, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
|
|
ProcessRoutedMessage,
|
|
_loggerFactory.CreateLogger<RouteManager>());
|
|
_routeManager.OnRouteRemoved += RemoveRemoteSubscriptionsForRoute;
|
|
_routeManager.OnRouteAccountRemoved += RemoveRemoteSubscriptionsForRouteAccount;
|
|
}
|
|
|
|
if (options.Gateway != null)
|
|
{
|
|
_gatewayManager = new GatewayManager(options.Gateway, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
|
|
ProcessGatewayMessage,
|
|
_loggerFactory.CreateLogger<GatewayManager>());
|
|
}
|
|
|
|
if (options.LeafNode != null)
|
|
{
|
|
_leafNodeManager = new LeafNodeManager(options.LeafNode, _stats, _serverInfo.ServerId, ApplyRemoteSubscription,
|
|
ProcessLeafMessage,
|
|
_loggerFactory.CreateLogger<LeafNodeManager>());
|
|
}
|
|
|
|
if (options.JetStream != null)
|
|
{
|
|
_jetStreamConsumerManager = new ConsumerManager();
|
|
_jetStreamStreamManager = new StreamManager(consumerManager: _jetStreamConsumerManager, storeDir: options.JetStream.StoreDir);
|
|
_jetStreamConsumerManager.StreamManager = _jetStreamStreamManager;
|
|
var jsClientId = Interlocked.Increment(ref _nextClientId);
|
|
_jetStreamInternalClient = new InternalClient(jsClientId, ClientKind.JetStream, _systemAccount);
|
|
_jetStreamService = new JetStreamService(options.JetStream, _jetStreamInternalClient);
|
|
_jetStreamApiRouter = new JetStreamApiRouter(_jetStreamStreamManager, _jetStreamConsumerManager);
|
|
_jetStreamPublisher = new JetStreamPublisher(_jetStreamStreamManager);
|
|
}
|
|
|
|
if (options.HasTls)
|
|
{
|
|
_tlsCertProvider = new TlsCertificateProvider(options.TlsCert!, options.TlsKey);
|
|
_sslOptions = TlsHelper.BuildServerAuthOptions(options);
|
|
_tlsCertProvider.SwapSslOptions(_sslOptions);
|
|
|
|
// OCSP stapling: build a certificate context so the runtime can
|
|
// fetch and cache a fresh OCSP response and staple it during the
|
|
// TLS handshake. offline:false tells the runtime to contact the
|
|
// OCSP responder; if the responder is unreachable we fall back to
|
|
// no stapling rather than refusing all connections.
|
|
var certContext = TlsHelper.BuildCertificateContext(options, offline: false);
|
|
if (certContext != null)
|
|
{
|
|
_sslOptions.ServerCertificateContext = certContext;
|
|
_logger.LogInformation("OCSP stapling enabled (mode: {OcspMode})", options.OcspConfig!.Mode);
|
|
}
|
|
|
|
_serverInfo.TlsRequired = !options.AllowNonTls;
|
|
_serverInfo.TlsAvailable = options.AllowNonTls;
|
|
_serverInfo.TlsVerify = options.TlsVerify;
|
|
|
|
if (options.TlsRateLimit > 0)
|
|
_tlsRateLimiter = new TlsRateLimiter(options.TlsRateLimit);
|
|
}
|
|
|
|
// Compile subject transforms
|
|
if (options.SubjectMappings is { Count: > 0 })
|
|
{
|
|
var transforms = new List<SubjectTransform>();
|
|
foreach (var (source, dest) in options.SubjectMappings)
|
|
{
|
|
var t = SubjectTransform.Create(source, dest);
|
|
if (t != null)
|
|
transforms.Add(t);
|
|
else
|
|
_logger.LogWarning("Invalid subject mapping: {Source} -> {Dest}", source, dest);
|
|
}
|
|
_subjectTransforms = transforms.ToArray();
|
|
if (_subjectTransforms.Length > 0)
|
|
_logger.LogInformation("Compiled {Count} subject transform(s)", _subjectTransforms.Length);
|
|
}
|
|
else
|
|
{
|
|
_subjectTransforms = [];
|
|
}
|
|
|
|
BuildCachedInfo();
|
|
|
|
// Store initial config digest for reload change detection
|
|
if (options.ConfigFile != null)
|
|
{
|
|
try
|
|
{
|
|
var (_, digest) = NatsConfParser.ParseFileWithDigest(options.ConfigFile);
|
|
_configDigest = digest;
|
|
_configTime = DateTime.UtcNow;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Could not compute initial config digest for {ConfigFile}", options.ConfigFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void BuildCachedInfo()
|
|
{
|
|
var infoJson = System.Text.Json.JsonSerializer.Serialize(_serverInfo);
|
|
_cachedInfoLine = Encoding.ASCII.GetBytes($"INFO {infoJson}\r\n");
|
|
}
|
|
|
|
private static string NormalizeAdvertiseUrl(string advertise, string defaultScheme)
|
|
{
|
|
if (advertise.Contains("://", StringComparison.Ordinal))
|
|
return advertise;
|
|
|
|
return $"{defaultScheme}://{advertise}";
|
|
}
|
|
|
|
private static bool IsWildcardHost(string host)
|
|
=> host == "0.0.0.0" || host == "::";
|
|
|
|
internal static IReadOnlyList<string> GetNonLocalIPsIfHostIsIPAny(string host)
|
|
{
|
|
if (!IsWildcardHost(host))
|
|
return [host];
|
|
|
|
var addresses = new HashSet<string>(StringComparer.Ordinal);
|
|
foreach (var netIf in NetworkInterface.GetAllNetworkInterfaces())
|
|
{
|
|
if (netIf.OperationalStatus != OperationalStatus.Up)
|
|
continue;
|
|
|
|
IPInterfaceProperties? props;
|
|
try
|
|
{
|
|
props = netIf.GetIPProperties();
|
|
}
|
|
catch
|
|
{
|
|
continue;
|
|
}
|
|
|
|
foreach (var uni in props.UnicastAddresses)
|
|
{
|
|
var addr = uni.Address;
|
|
if (IPAddress.IsLoopback(addr) || addr.IsIPv6LinkLocal || addr.IsIPv6Multicast)
|
|
continue;
|
|
if (addr.AddressFamily is not (AddressFamily.InterNetwork or AddressFamily.InterNetworkV6))
|
|
continue;
|
|
addresses.Add(addr.ToString());
|
|
}
|
|
}
|
|
|
|
if (addresses.Count == 0)
|
|
addresses.Add("127.0.0.1");
|
|
|
|
return [.. addresses.OrderBy(static a => a, StringComparer.Ordinal)];
|
|
}
|
|
|
|
private bool CloseClientById(ulong clientId, bool minimalFlush)
|
|
{
|
|
if (!_clients.TryGetValue(clientId, out var client))
|
|
return false;
|
|
|
|
client.MarkClosed(ClientClosedReason.ServerShutdown);
|
|
_ = client.FlushAndCloseAsync(minimalFlush);
|
|
return true;
|
|
}
|
|
|
|
private static void AddEndpoint(List<string> targets, string? host, int port)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(host) || port <= 0)
|
|
return;
|
|
|
|
targets.Add($"{host}:{port}");
|
|
}
|
|
|
|
private static void AddEndpoint(List<string> targets, string? endpoint)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(endpoint))
|
|
targets.Add(endpoint);
|
|
}
|
|
|
|
public async Task StartAsync(CancellationToken ct)
|
|
{
|
|
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _quitCts.Token);
|
|
|
|
_listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
|
_listener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
|
_listener.Bind(new IPEndPoint(
|
|
_options.Host == "0.0.0.0" ? IPAddress.Any : IPAddress.Parse(_options.Host),
|
|
_options.Port));
|
|
Interlocked.Exchange(ref _startTimeTicks, DateTime.UtcNow.Ticks);
|
|
_listener.Listen(128);
|
|
|
|
// Resolve ephemeral port if port=0
|
|
if (_options.Port == 0)
|
|
{
|
|
var actualPort = ((IPEndPoint)_listener.LocalEndPoint!).Port;
|
|
_options.Port = actualPort;
|
|
_serverInfo.Port = actualPort;
|
|
BuildCachedInfo();
|
|
}
|
|
|
|
_logger.LogInformation("Listening for client connections on {Host}:{Port}", _options.Host, _options.Port);
|
|
|
|
// Warn about stub features
|
|
StartProfiler();
|
|
|
|
if (_options.MonitorPort > 0)
|
|
{
|
|
_monitorServer = new MonitorServer(this, _options, _stats, _loggerFactory);
|
|
await _monitorServer.StartAsync(linked.Token);
|
|
}
|
|
|
|
WritePidFile();
|
|
WritePortsFile();
|
|
|
|
WsAuthConfig.Apply(_options.WebSocket);
|
|
var wsValidation = WebSocketOptionsValidator.Validate(_options);
|
|
if (!wsValidation.IsValid)
|
|
throw new InvalidOperationException($"Invalid websocket options: {string.Join("; ", wsValidation.Errors)}");
|
|
|
|
if (_options.WebSocket.Port >= 0)
|
|
{
|
|
_wsListener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
|
_wsListener.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
|
_wsListener.Bind(new IPEndPoint(
|
|
_options.WebSocket.Host == "0.0.0.0" ? IPAddress.Any : IPAddress.Parse(_options.WebSocket.Host),
|
|
_options.WebSocket.Port));
|
|
_wsListener.Listen(128);
|
|
|
|
if (_options.WebSocket.Port == 0)
|
|
{
|
|
_options.WebSocket.Port = ((IPEndPoint)_wsListener.LocalEndPoint!).Port;
|
|
}
|
|
|
|
_logger.LogInformation("Listening for WebSocket clients on {Host}:{Port}",
|
|
_options.WebSocket.Host, _options.WebSocket.Port);
|
|
|
|
if (_options.WebSocket.NoTls)
|
|
_logger.LogWarning("WebSocket not configured with TLS. DO NOT USE IN PRODUCTION!");
|
|
|
|
_ = RunWebSocketAcceptLoopAsync(linked.Token);
|
|
}
|
|
|
|
if (_routeManager != null)
|
|
await _routeManager.StartAsync(linked.Token);
|
|
if (_gatewayManager != null)
|
|
await _gatewayManager.StartAsync(linked.Token);
|
|
if (_leafNodeManager != null)
|
|
await _leafNodeManager.StartAsync(linked.Token);
|
|
if (_options.Mqtt is { Port: > 0 } mqttOptions)
|
|
{
|
|
var mqttHost = string.IsNullOrWhiteSpace(mqttOptions.Host) ? _options.Host : mqttOptions.Host;
|
|
_mqttListener = new MqttListener(
|
|
mqttHost,
|
|
mqttOptions.Port,
|
|
mqttOptions.Username,
|
|
mqttOptions.Password);
|
|
await _mqttListener.StartAsync(linked.Token);
|
|
}
|
|
if (_jetStreamService != null)
|
|
{
|
|
await _jetStreamService.StartAsync(linked.Token);
|
|
_stats.JetStreamEnabled = true;
|
|
}
|
|
|
|
_listeningStarted.TrySetResult();
|
|
|
|
_eventSystem?.Start(this);
|
|
_eventSystem?.InitEventTracking(this);
|
|
|
|
var tmpDelay = AcceptMinSleep;
|
|
|
|
try
|
|
{
|
|
while (!linked.Token.IsCancellationRequested)
|
|
{
|
|
Socket socket;
|
|
try
|
|
{
|
|
socket = await _listener!.AcceptAsync(linked.Token);
|
|
tmpDelay = AcceptMinSleep; // Reset on success
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
break;
|
|
}
|
|
catch (ObjectDisposedException)
|
|
{
|
|
break;
|
|
}
|
|
catch (SocketException ex)
|
|
{
|
|
if (IsShuttingDown || IsLameDuckMode)
|
|
break;
|
|
|
|
_acceptLoopErrorHandler?.OnAcceptError(ex, _listener?.LocalEndPoint, tmpDelay);
|
|
_logger.LogError(ex, "Temporary accept error, sleeping {Delay}ms", tmpDelay.TotalMilliseconds);
|
|
try { await Task.Delay(tmpDelay, linked.Token); }
|
|
catch (OperationCanceledException) { break; }
|
|
|
|
tmpDelay = TimeSpan.FromTicks(Math.Min(tmpDelay.Ticks * 2, AcceptMaxSleep.Ticks));
|
|
continue;
|
|
}
|
|
|
|
// Check MaxConnections
|
|
if (_options.MaxConnections > 0 && _clients.Count >= _options.MaxConnections)
|
|
{
|
|
_logger.LogWarning("Client connection rejected: maximum connections ({MaxConnections}) exceeded",
|
|
_options.MaxConnections);
|
|
try
|
|
{
|
|
var stream = new NetworkStream(socket, ownsSocket: false);
|
|
var errBytes = Encoding.ASCII.GetBytes(
|
|
$"-ERR '{NatsProtocol.ErrMaxConnectionsExceeded}'\r\n");
|
|
await stream.WriteAsync(errBytes, linked.Token);
|
|
await stream.FlushAsync(linked.Token);
|
|
stream.Dispose();
|
|
}
|
|
catch (Exception ex2)
|
|
{
|
|
_logger.LogDebug(ex2, "Failed to send -ERR to rejected client");
|
|
}
|
|
finally
|
|
{
|
|
socket.Dispose();
|
|
}
|
|
continue;
|
|
}
|
|
|
|
var clientId = Interlocked.Increment(ref _nextClientId);
|
|
Interlocked.Increment(ref _stats.TotalConnections);
|
|
Interlocked.Increment(ref _activeClientCount);
|
|
|
|
_logger.LogDebug("Client {ClientId} connected from {RemoteEndpoint}", clientId, socket.RemoteEndPoint);
|
|
|
|
_ = AcceptClientAsync(socket, clientId, linked.Token);
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
_logger.LogDebug("Accept loop cancelled, server shutting down");
|
|
}
|
|
finally
|
|
{
|
|
_acceptLoopExited.TrySetResult();
|
|
}
|
|
}
|
|
|
|
private async Task AcceptClientAsync(Socket socket, ulong clientId, CancellationToken ct)
|
|
{
|
|
var reloadLockHeld = false;
|
|
NatsClient? client = null;
|
|
try
|
|
{
|
|
await _reloadMu.WaitAsync(ct);
|
|
reloadLockHeld = true;
|
|
|
|
// Rate limit TLS handshakes
|
|
if (_tlsRateLimiter != null)
|
|
await _tlsRateLimiter.WaitAsync(ct);
|
|
|
|
var networkStream = new NetworkStream(socket, ownsSocket: false);
|
|
|
|
// TLS negotiation (no-op if not configured)
|
|
var (stream, infoAlreadySent) = await TlsConnectionWrapper.NegotiateAsync(
|
|
socket, networkStream, _options, _sslOptions, _serverInfo,
|
|
_loggerFactory.CreateLogger("NATS.Server.Tls"), ct);
|
|
|
|
// Extract TLS state
|
|
TlsConnectionState? tlsState = null;
|
|
if (stream is SslStream ssl)
|
|
{
|
|
tlsState = new TlsConnectionState(
|
|
ssl.SslProtocol.ToString(),
|
|
ssl.NegotiatedCipherSuite.ToString(),
|
|
ssl.RemoteCertificate as X509Certificate2);
|
|
}
|
|
|
|
// Build per-client ServerInfo with nonce if NKey auth is configured
|
|
byte[]? nonce = null;
|
|
var clientInfo = _serverInfo;
|
|
if (_authService.NonceRequired)
|
|
{
|
|
var rawNonce = _authService.GenerateNonce();
|
|
var nonceStr = _authService.EncodeNonce(rawNonce);
|
|
// The client signs the nonce string (ASCII), not the raw bytes
|
|
nonce = Encoding.ASCII.GetBytes(nonceStr);
|
|
clientInfo = new ServerInfo
|
|
{
|
|
ServerId = _serverInfo.ServerId,
|
|
ServerName = _serverInfo.ServerName,
|
|
Version = _serverInfo.Version,
|
|
Host = _serverInfo.Host,
|
|
Port = _serverInfo.Port,
|
|
MaxPayload = _serverInfo.MaxPayload,
|
|
AuthRequired = _serverInfo.AuthRequired,
|
|
TlsRequired = _serverInfo.TlsRequired,
|
|
TlsAvailable = _serverInfo.TlsAvailable,
|
|
TlsVerify = _serverInfo.TlsVerify,
|
|
Nonce = nonceStr,
|
|
};
|
|
}
|
|
|
|
var clientLogger = _loggerFactory.CreateLogger($"NATS.Server.NatsClient[{clientId}]");
|
|
client = new NatsClient(clientId, stream, socket, _options, clientInfo,
|
|
_authService, nonce, clientLogger, _stats);
|
|
client.Router = this;
|
|
client.TlsState = tlsState;
|
|
client.InfoAlreadySent = infoAlreadySent;
|
|
_clients[clientId] = client;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (client is null)
|
|
{
|
|
var earlyReason = _options.HasTls
|
|
? ClientClosedReason.TlsHandshakeError
|
|
: ClientClosedReason.ReadError;
|
|
TrackEarlyClosedClient(socket, clientId, earlyReason);
|
|
}
|
|
|
|
_logger.LogDebug(ex, "Failed to accept client {ClientId}", clientId);
|
|
try { socket.Shutdown(SocketShutdown.Both); } catch { }
|
|
socket.Dispose();
|
|
return;
|
|
}
|
|
finally
|
|
{
|
|
if (reloadLockHeld)
|
|
_reloadMu.Release();
|
|
}
|
|
|
|
try
|
|
{
|
|
if (client != null)
|
|
await RunClientAsync(client, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Failed to accept client {ClientId}", clientId);
|
|
try { socket.Shutdown(SocketShutdown.Both); } catch { }
|
|
socket.Dispose();
|
|
}
|
|
}
|
|
|
|
private async Task RunWebSocketAcceptLoopAsync(CancellationToken ct)
|
|
{
|
|
var tmpDelay = AcceptMinSleep;
|
|
try
|
|
{
|
|
while (!ct.IsCancellationRequested)
|
|
{
|
|
Socket socket;
|
|
try
|
|
{
|
|
socket = await _wsListener!.AcceptAsync(ct);
|
|
tmpDelay = AcceptMinSleep;
|
|
}
|
|
catch (OperationCanceledException) { break; }
|
|
catch (ObjectDisposedException) { break; }
|
|
catch (SocketException ex)
|
|
{
|
|
if (IsShuttingDown || IsLameDuckMode) break;
|
|
_acceptLoopErrorHandler?.OnAcceptError(ex, _wsListener?.LocalEndPoint, tmpDelay);
|
|
_logger.LogError(ex, "Temporary WebSocket accept error, sleeping {Delay}ms", tmpDelay.TotalMilliseconds);
|
|
try { await Task.Delay(tmpDelay, ct); } catch (OperationCanceledException) { break; }
|
|
tmpDelay = TimeSpan.FromTicks(Math.Min(tmpDelay.Ticks * 2, AcceptMaxSleep.Ticks));
|
|
continue;
|
|
}
|
|
|
|
if (_options.MaxConnections > 0 && _clients.Count >= _options.MaxConnections)
|
|
{
|
|
socket.Dispose();
|
|
continue;
|
|
}
|
|
|
|
var clientId = Interlocked.Increment(ref _nextClientId);
|
|
Interlocked.Increment(ref _stats.TotalConnections);
|
|
Interlocked.Increment(ref _activeClientCount);
|
|
|
|
_ = AcceptWebSocketClientAsync(socket, clientId, ct);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
_wsAcceptLoopExited.TrySetResult();
|
|
}
|
|
}
|
|
|
|
private async Task AcceptWebSocketClientAsync(Socket socket, ulong clientId, CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
var networkStream = new NetworkStream(socket, ownsSocket: false);
|
|
Stream stream = networkStream;
|
|
|
|
// TLS negotiation if configured
|
|
if (_sslOptions != null && !_options.WebSocket.NoTls)
|
|
{
|
|
var (tlsStream, _) = await TlsConnectionWrapper.NegotiateAsync(
|
|
socket, networkStream, _options, _sslOptions, _serverInfo,
|
|
_loggerFactory.CreateLogger("NATS.Server.Tls"), ct);
|
|
stream = tlsStream;
|
|
}
|
|
|
|
// HTTP upgrade handshake
|
|
var upgradeResult = await WsUpgrade.TryUpgradeAsync(stream, stream, _options.WebSocket, ct);
|
|
if (!upgradeResult.Success)
|
|
{
|
|
_logger.LogDebug("WebSocket upgrade failed for client {ClientId}", clientId);
|
|
socket.Dispose();
|
|
Interlocked.Decrement(ref _activeClientCount);
|
|
return;
|
|
}
|
|
|
|
// Create WsConnection wrapper
|
|
var wsConn = new WsConnection(stream,
|
|
compress: upgradeResult.Compress,
|
|
maskRead: upgradeResult.MaskRead,
|
|
maskWrite: upgradeResult.MaskWrite,
|
|
browser: upgradeResult.Browser,
|
|
noCompFrag: upgradeResult.NoCompFrag);
|
|
|
|
var clientLogger = _loggerFactory.CreateLogger($"NATS.Server.NatsClient[{clientId}]");
|
|
var client = new NatsClient(clientId, wsConn, socket, _options, _serverInfo,
|
|
_authService, null, clientLogger, _stats);
|
|
client.Router = this;
|
|
client.IsWebSocket = true;
|
|
client.WsInfo = upgradeResult;
|
|
_clients[clientId] = client;
|
|
|
|
await RunClientAsync(client, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Failed to accept WebSocket client {ClientId}", clientId);
|
|
try { socket.Shutdown(SocketShutdown.Both); } catch { }
|
|
socket.Dispose();
|
|
Interlocked.Decrement(ref _activeClientCount);
|
|
}
|
|
}
|
|
|
|
private async Task RunClientAsync(NatsClient client, CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
await client.RunAsync(ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Client {ClientId} disconnected with error", client.Id);
|
|
}
|
|
finally
|
|
{
|
|
_logger.LogDebug("Client {ClientId} disconnected (reason: {CloseReason})", client.Id, client.CloseReason);
|
|
RemoveClient(client);
|
|
Interlocked.Decrement(ref _activeClientCount);
|
|
}
|
|
}
|
|
|
|
public void OnLocalSubscription(string account, string subject, string? queue)
|
|
{
|
|
_routeManager?.PropagateLocalSubscription(account, subject, queue);
|
|
_gatewayManager?.PropagateLocalSubscription(account, subject, queue);
|
|
_leafNodeManager?.PropagateLocalSubscription(account, subject, queue);
|
|
}
|
|
|
|
public void OnLocalUnsubscription(string account, string subject, string? queue)
|
|
{
|
|
_routeManager?.PropagateLocalUnsubscription(account, subject, queue);
|
|
_gatewayManager?.PropagateLocalUnsubscription(account, subject, queue);
|
|
_leafNodeManager?.PropagateLocalUnsubscription(account, subject, queue);
|
|
}
|
|
|
|
private void ApplyRemoteSubscription(RemoteSubscription sub)
|
|
{
|
|
var account = GetOrCreateAccount(sub.Account);
|
|
account.SubList.ApplyRemoteSub(sub);
|
|
}
|
|
|
|
private void RemoveRemoteSubscriptionsForRoute(string routeId)
|
|
{
|
|
foreach (var account in _accounts.Values)
|
|
account.SubList.RemoveRemoteSubs(routeId);
|
|
}
|
|
|
|
private void RemoveRemoteSubscriptionsForRouteAccount(string routeId, string accountName)
|
|
{
|
|
if (_accounts.TryGetValue(accountName, out var account))
|
|
account.SubList.RemoveRemoteSubsForAccount(routeId, accountName);
|
|
}
|
|
|
|
private void ProcessRoutedMessage(RouteMessage message)
|
|
{
|
|
DeliverRemoteMessage(message.Account, message.Subject, message.ReplyTo, message.Payload);
|
|
}
|
|
|
|
private void ProcessGatewayMessage(GatewayMessage message)
|
|
{
|
|
var replyTo = message.ReplyTo;
|
|
if (ReplyMapper.TryRestoreGatewayReply(replyTo, out var restoredReply))
|
|
replyTo = restoredReply;
|
|
else if (ReplyMapper.HasGatewayReplyPrefix(replyTo))
|
|
replyTo = null;
|
|
|
|
DeliverRemoteMessage(message.Account, message.Subject, replyTo, message.Payload);
|
|
}
|
|
|
|
private void ProcessLeafMessage(LeafMessage message)
|
|
{
|
|
if (LeafLoopDetector.IsLooped(message.Subject, ServerId))
|
|
return;
|
|
|
|
var subject = message.Subject;
|
|
if (LeafLoopDetector.TryUnmark(subject, out var unmarked))
|
|
subject = unmarked;
|
|
else if (LeafLoopDetector.HasLoopMarker(subject))
|
|
return;
|
|
|
|
DeliverRemoteMessage(message.Account, subject, message.ReplyTo, message.Payload);
|
|
}
|
|
|
|
private void DeliverRemoteMessage(string account, string subject, string? replyTo, ReadOnlyMemory<byte> payload)
|
|
{
|
|
// Handle internal JetStream cluster replication messages.
|
|
// Go reference: jetstream_cluster.go — RAFT proposal apply dispatches to handlers.
|
|
if (subject.StartsWith(JetStreamApiSubjects.InternalPrefix, StringComparison.Ordinal))
|
|
{
|
|
HandleJetStreamReplication(subject, payload);
|
|
return;
|
|
}
|
|
|
|
// Capture routed data messages into local JetStream streams.
|
|
// Skip $JS. subjects — those are API calls, not stream data.
|
|
if (!subject.StartsWith("$JS.", StringComparison.Ordinal))
|
|
TryCaptureJetStreamPublish(subject, payload, out _);
|
|
|
|
var targetAccount = GetOrCreateAccount(account);
|
|
var result = targetAccount.SubList.Match(subject);
|
|
|
|
foreach (var sub in result.PlainSubs)
|
|
DeliverMessage(sub, subject, replyTo, default, payload);
|
|
|
|
foreach (var queueGroup in result.QueueSubs)
|
|
{
|
|
if (queueGroup.Length == 0)
|
|
continue;
|
|
|
|
var sub = queueGroup[0];
|
|
DeliverMessage(sub, subject, replyTo, default, payload);
|
|
}
|
|
}
|
|
|
|
public void ProcessMessage(string subject, string? replyTo, ReadOnlyMemory<byte> headers,
|
|
ReadOnlyMemory<byte> payload, NatsClient sender)
|
|
{
|
|
if (replyTo != null
|
|
&& subject.StartsWith("$JS.API", StringComparison.Ordinal)
|
|
&& _jetStreamApiRouter != null)
|
|
{
|
|
// Pull consumer MSG.NEXT requires special handling: deliver individual
|
|
// HMSG messages to the client's reply inbox instead of a single JSON blob.
|
|
// Go reference: consumer.go:4276 processNextMsgRequest
|
|
if (subject.StartsWith(JetStream.Api.JetStreamApiSubjects.ConsumerNext, StringComparison.Ordinal)
|
|
&& _jetStreamConsumerManager != null
|
|
&& _jetStreamStreamManager != null)
|
|
{
|
|
Interlocked.Increment(ref _stats.JetStreamApiTotal);
|
|
DeliverPullFetchMessages(subject, replyTo, payload, sender);
|
|
return;
|
|
}
|
|
|
|
var response = _jetStreamApiRouter.Route(subject, payload.Span);
|
|
Interlocked.Increment(ref _stats.JetStreamApiTotal);
|
|
if (response.Error != null)
|
|
Interlocked.Increment(ref _stats.JetStreamApiErrors);
|
|
|
|
// Replicate successful mutating operations to cluster peers.
|
|
// Go reference: jetstream_cluster.go — RAFT proposal replication.
|
|
if (response.Error == null)
|
|
TryReplicateJetStreamMutation(subject, payload);
|
|
|
|
var data = JsonSerializer.SerializeToUtf8Bytes(response.ToWireFormat(), s_jetStreamJsonOptions);
|
|
ProcessMessage(replyTo, null, default, data, sender);
|
|
return;
|
|
}
|
|
|
|
if (TryCaptureJetStreamPublish(subject, payload, out var pubAck))
|
|
{
|
|
sender.RecordJetStreamPubAck(pubAck);
|
|
|
|
// Replicate data messages to cluster peers so their JetStream stores also capture them.
|
|
// Route forwarding below is gated on subscriber interest, which JetStream streams don't
|
|
// create, so we must explicitly push data to replicas.
|
|
// Go reference: jetstream_cluster.go — RAFT propose entry replication.
|
|
if (pubAck.ErrorCode == null)
|
|
ReplicateJetStreamOperation("$G", subject, null, payload);
|
|
|
|
// Send pub ack response to the reply subject (request-reply pattern).
|
|
// Go reference: server/jetstream.go — jsPubAckResponse sent to reply.
|
|
if (replyTo != null)
|
|
{
|
|
var ackData = JsonSerializer.SerializeToUtf8Bytes(pubAck, s_jetStreamJsonOptions);
|
|
ProcessMessage(replyTo, null, default, ackData, sender);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Apply subject transforms
|
|
if (_subjectTransforms.Length > 0)
|
|
{
|
|
foreach (var transform in _subjectTransforms)
|
|
{
|
|
var mapped = transform.Apply(subject);
|
|
if (mapped != null)
|
|
{
|
|
subject = mapped;
|
|
break; // First matching transform wins
|
|
}
|
|
}
|
|
}
|
|
|
|
var senderAccount = sender.Account ?? _globalAccount;
|
|
if (_routeManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject))
|
|
_routeManager.ForwardRoutedMessageAsync(senderAccount.Name, subject, replyTo, payload, default).GetAwaiter().GetResult();
|
|
if (_gatewayManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject))
|
|
{
|
|
var mappedReplyTo = ReplyMapper.ToGatewayReply(replyTo, ServerId);
|
|
_gatewayManager.ForwardMessageAsync(senderAccount.Name, subject, mappedReplyTo, payload, default).GetAwaiter().GetResult();
|
|
}
|
|
|
|
if (_leafNodeManager != null && senderAccount.SubList.HasRemoteInterest(senderAccount.Name, subject))
|
|
{
|
|
var markedSubject = LeafLoopDetector.Mark(subject, ServerId);
|
|
_leafNodeManager.ForwardMessageAsync(senderAccount.Name, markedSubject, replyTo, payload, default).GetAwaiter().GetResult();
|
|
}
|
|
|
|
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
|
|
var result = subList.Match(subject);
|
|
var delivered = false;
|
|
|
|
// Deliver to plain subscribers
|
|
foreach (var sub in result.PlainSubs)
|
|
{
|
|
if (sub.Client == null || sub.Client == sender && !(sender.ClientOpts?.Echo ?? true))
|
|
continue;
|
|
|
|
DeliverMessage(sub, subject, replyTo, headers, payload);
|
|
delivered = true;
|
|
}
|
|
|
|
// Deliver to one member of each queue group (round-robin)
|
|
foreach (var queueGroup in result.QueueSubs)
|
|
{
|
|
if (queueGroup.Length == 0) continue;
|
|
|
|
// Simple round-robin -- pick based on total delivered across group
|
|
var idx = Math.Abs((int)Interlocked.Increment(ref sender.OutMsgs)) % queueGroup.Length;
|
|
// Undo the OutMsgs increment -- it will be incremented properly in SendMessage
|
|
Interlocked.Decrement(ref sender.OutMsgs);
|
|
|
|
for (int attempt = 0; attempt < queueGroup.Length; attempt++)
|
|
{
|
|
var sub = queueGroup[(idx + attempt) % queueGroup.Length];
|
|
if (sub.Client != null && (sub.Client != sender || (sender.ClientOpts?.Echo ?? true)))
|
|
{
|
|
DeliverMessage(sub, subject, replyTo, headers, payload);
|
|
delivered = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check for service imports that match this subject.
|
|
// When a client in the importer account publishes to a subject
|
|
// that matches a service import "From" pattern, we forward the
|
|
// message to the destination (exporter) account's subscribers
|
|
// using the mapped "To" subject.
|
|
if (sender.Account != null)
|
|
{
|
|
foreach (var kvp in sender.Account.Imports.Services)
|
|
{
|
|
foreach (var si in kvp.Value)
|
|
{
|
|
if (si.Invalid) continue;
|
|
if (SubjectMatch.MatchLiteral(subject, si.From))
|
|
{
|
|
ProcessServiceImport(si, subject, replyTo, headers, payload, sender.Account);
|
|
delivered = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// No-responders: if nobody received the message and the publisher
|
|
// opted in, send back a 503 status HMSG on the reply subject.
|
|
if (!delivered && replyTo != null && sender.ClientOpts?.NoResponders == true)
|
|
{
|
|
SendNoResponders(sender, replyTo);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles $JS.API.CONSUMER.MSG.NEXT by delivering individual HMSG messages
|
|
/// to the client's reply inbox. Go reference: consumer.go:4276 processNextMsgRequest.
|
|
/// </summary>
|
|
private void DeliverPullFetchMessages(string subject, string replyTo, ReadOnlyMemory<byte> payload, NatsClient sender)
|
|
{
|
|
var prefix = JetStream.Api.JetStreamApiSubjects.ConsumerNext;
|
|
var remainder = subject[prefix.Length..];
|
|
var split = remainder.Split('.', 2, StringSplitOptions.RemoveEmptyEntries);
|
|
if (split.Length != 2)
|
|
{
|
|
var notFoundHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n");
|
|
ProcessMessage(replyTo, null, (ReadOnlyMemory<byte>)notFoundHeader, default, sender);
|
|
return;
|
|
}
|
|
|
|
var (streamName, consumerName) = (split[0], split[1]);
|
|
|
|
// Parse batch request
|
|
int batch = 1;
|
|
int expiresMs = 0;
|
|
bool noWait = false;
|
|
if (payload.Length > 0)
|
|
{
|
|
try
|
|
{
|
|
using var doc = System.Text.Json.JsonDocument.Parse(payload);
|
|
if (doc.RootElement.TryGetProperty("batch", out var batchEl) && batchEl.TryGetInt32(out var b))
|
|
batch = Math.Max(b, 1);
|
|
if (doc.RootElement.TryGetProperty("no_wait", out var nwEl) && nwEl.ValueKind == System.Text.Json.JsonValueKind.True)
|
|
noWait = true;
|
|
if (doc.RootElement.TryGetProperty("expires", out var expEl) && expEl.TryGetInt64(out var expNs))
|
|
expiresMs = (int)(expNs / 1_000_000);
|
|
}
|
|
catch (System.Text.Json.JsonException ex)
|
|
{
|
|
_logger.LogDebug(ex, "Malformed JSON in pull request payload, using defaults");
|
|
}
|
|
}
|
|
|
|
var fetchResult = _jetStreamConsumerManager!.FetchAsync(
|
|
streamName, consumerName, new JetStream.Consumers.PullFetchRequest { Batch = batch, NoWait = noWait, ExpiresMs = expiresMs },
|
|
_jetStreamStreamManager!, default).GetAwaiter().GetResult();
|
|
|
|
// Find the sender's inbox subscription so we can deliver directly.
|
|
// Go reference: consumer.go deliverMsg — delivers directly to the client, bypassing pub/sub echo checks.
|
|
var subList = sender.Account?.SubList ?? _globalAccount.SubList;
|
|
var matchResult = subList.Match(replyTo);
|
|
Subscription? inboxSub = null;
|
|
foreach (var sub in matchResult.PlainSubs)
|
|
{
|
|
if (sub.Client == sender)
|
|
{
|
|
inboxSub = sub;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (inboxSub == null)
|
|
return;
|
|
|
|
ReadOnlyMemory<byte> minHeaders = "NATS/1.0\r\n\r\n"u8.ToArray();
|
|
int deliverySeq = 0;
|
|
int numPending = fetchResult.Messages.Count;
|
|
|
|
foreach (var msg in fetchResult.Messages)
|
|
{
|
|
deliverySeq++;
|
|
numPending--;
|
|
|
|
var tsNanos = new DateTimeOffset(msg.TimestampUtc).ToUnixTimeMilliseconds() * 1_000_000L;
|
|
var ackReply = $"$JS.ACK.{streamName}.{consumerName}.1.{msg.Sequence}.{deliverySeq}.{tsNanos}.{numPending}";
|
|
|
|
// Send with the ORIGINAL stream subject (not the inbox) so the NATS client
|
|
// can distinguish data messages from control/status messages.
|
|
// Go reference: consumer.go deliverMsg — uses original subject on wire, inbox SID.
|
|
DeliverMessage(inboxSub, msg.Subject, ackReply, minHeaders, msg.Payload);
|
|
}
|
|
|
|
// Send terminal status to end the fetch
|
|
ReadOnlyMemory<byte> statusHeader;
|
|
if (fetchResult.Messages.Count == 0 || noWait)
|
|
statusHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 404 No Messages\r\n\r\n");
|
|
else
|
|
statusHeader = System.Text.Encoding.UTF8.GetBytes("NATS/1.0 408 Request Timeout\r\n\r\n");
|
|
DeliverMessage(inboxSub, replyTo, null, statusHeader, default);
|
|
}
|
|
|
|
private void DeliverMessage(Subscription sub, string subject, string? replyTo,
|
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload)
|
|
{
|
|
var client = sub.Client;
|
|
if (client == null) return;
|
|
|
|
// Check auto-unsub
|
|
var count = Interlocked.Increment(ref sub.MessageCount);
|
|
if (sub.MaxMessages > 0 && count > sub.MaxMessages)
|
|
{
|
|
// Clean up exhausted subscription from trie and client tracking
|
|
var subList = client.Account?.SubList ?? _globalAccount.SubList;
|
|
subList.Remove(sub);
|
|
client.RemoveSubscription(sub.Sid);
|
|
return;
|
|
}
|
|
|
|
// Deny-list delivery filter
|
|
if (client.Permissions?.IsDeliveryAllowed(subject) == false)
|
|
return;
|
|
|
|
client.SendMessage(subject, sub.Sid, replyTo, headers, payload);
|
|
|
|
// Track reply subject for response permissions
|
|
if (replyTo != null && client.Permissions?.ResponseTracker != null)
|
|
{
|
|
if (client.Permissions.IsPublishAllowed(replyTo) == false)
|
|
client.Permissions.ResponseTracker.RegisterReply(replyTo);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Processes a service import by transforming the subject from the importer's
|
|
/// subject space to the exporter's subject space, then delivering to matching
|
|
/// subscribers in the destination account.
|
|
/// Reference: Go server/accounts.go addServiceImport / processServiceImport.
|
|
/// </summary>
|
|
public void ProcessServiceImport(ServiceImport si, string subject, string? replyTo,
|
|
ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload, Account? sourceAccount = null)
|
|
{
|
|
if (si.Invalid) return;
|
|
|
|
// Transform subject: map from importer subject space to exporter subject space
|
|
string targetSubject;
|
|
if (si.Transform != null)
|
|
{
|
|
var transformed = si.Transform.Apply(subject);
|
|
targetSubject = transformed ?? si.To;
|
|
}
|
|
else if (si.UsePub)
|
|
{
|
|
targetSubject = subject;
|
|
}
|
|
else
|
|
{
|
|
// Default: use the "To" subject from the import definition.
|
|
// For wildcard imports (e.g. "requests.>" -> "api.>"), we need
|
|
// to map the specific subject tokens from the source pattern to
|
|
// the destination pattern.
|
|
targetSubject = MapImportSubject(subject, si.From, si.To);
|
|
}
|
|
|
|
// Set up a temporary reverse service import so that responses from the
|
|
// destination (exporter) account can route back to the source (importer)
|
|
// account. This handles request-reply across account boundaries.
|
|
// Go reference: client.go setupResponseServiceImport
|
|
if (replyTo != null && sourceAccount != null && !si.IsResponse)
|
|
{
|
|
SetupResponseServiceImport(si.DestinationAccount, sourceAccount, replyTo, si.Export);
|
|
}
|
|
|
|
// Service latency tracking: when the response arrives back, compute elapsed
|
|
// time and publish a latency metric to the configured subject.
|
|
// Go reference: client.go processServiceImport — latency tracking path.
|
|
if (si.IsResponse && si.Tracking && si.TimestampTicks > 0)
|
|
{
|
|
var elapsed = TimeSpan.FromTicks(Environment.TickCount64 * TimeSpan.TicksPerMillisecond - si.TimestampTicks);
|
|
PublishServiceLatency(si, elapsed);
|
|
}
|
|
|
|
// Match against destination account's SubList
|
|
var destSubList = si.DestinationAccount.SubList;
|
|
var result = destSubList.Match(targetSubject);
|
|
|
|
// Deliver to plain subscribers in the destination account
|
|
foreach (var sub in result.PlainSubs)
|
|
{
|
|
if (sub.Client == null) continue;
|
|
DeliverMessage(sub, targetSubject, replyTo, headers, payload);
|
|
}
|
|
|
|
// Deliver to one member of each queue group
|
|
foreach (var queueGroup in result.QueueSubs)
|
|
{
|
|
if (queueGroup.Length == 0) continue;
|
|
var sub = queueGroup[0]; // Simple selection: first available
|
|
if (sub.Client != null)
|
|
DeliverMessage(sub, targetSubject, replyTo, headers, payload);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a temporary reverse service import in the exporter's account so that
|
|
/// when the exporter publishes a response to the reply subject, the message is
|
|
/// forwarded back to the importer's account where the reply subscription lives.
|
|
/// Go reference: client.go setupResponseServiceImport.
|
|
/// </summary>
|
|
private static void SetupResponseServiceImport(Account exporterAccount, Account importerAccount, string replyTo, ServiceExport? export = null)
|
|
{
|
|
// Check if a reverse import for this reply subject already exists
|
|
if (exporterAccount.Imports.Services.ContainsKey(replyTo))
|
|
return;
|
|
|
|
// Determine if we should track latency for this response
|
|
var shouldTrack = export?.Latency is { } latency && LatencyTracker.ShouldSample(latency);
|
|
|
|
var reverseImport = new ServiceImport
|
|
{
|
|
DestinationAccount = importerAccount,
|
|
From = replyTo,
|
|
To = replyTo,
|
|
IsResponse = true,
|
|
UsePub = true,
|
|
Export = export,
|
|
Tracking = shouldTrack,
|
|
// Store start time as TickCount64 (milliseconds) converted to ticks for elapsed computation
|
|
TimestampTicks = shouldTrack ? Environment.TickCount64 * TimeSpan.TicksPerMillisecond : 0,
|
|
};
|
|
exporterAccount.Imports.AddServiceImport(reverseImport);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Maps a published subject from the import "From" pattern to the "To" pattern.
|
|
/// For example, if From="requests.>" and To="api.>" and subject="requests.test",
|
|
/// this returns "api.test".
|
|
/// </summary>
|
|
private static string MapImportSubject(string subject, string fromPattern, string toPattern)
|
|
{
|
|
// If "To" doesn't contain wildcards, use it directly
|
|
if (SubjectMatch.IsLiteral(toPattern))
|
|
return toPattern;
|
|
|
|
// For wildcard patterns, replace matching wildcard segments.
|
|
// Split into tokens and map from source to destination.
|
|
var subTokens = subject.Split('.');
|
|
var fromTokens = fromPattern.Split('.');
|
|
var toTokens = toPattern.Split('.');
|
|
|
|
var result = new string[toTokens.Length];
|
|
int subIdx = 0;
|
|
|
|
// Build a mapping: for each wildcard position in "from",
|
|
// capture the corresponding subject token(s)
|
|
var wildcardValues = new List<string>();
|
|
string? fwcValue = null;
|
|
|
|
for (int i = 0; i < fromTokens.Length && subIdx < subTokens.Length; i++)
|
|
{
|
|
if (fromTokens[i] == "*")
|
|
{
|
|
wildcardValues.Add(subTokens[subIdx]);
|
|
subIdx++;
|
|
}
|
|
else if (fromTokens[i] == ">")
|
|
{
|
|
// Capture all remaining tokens
|
|
fwcValue = string.Join(".", subTokens[subIdx..]);
|
|
subIdx = subTokens.Length;
|
|
}
|
|
else
|
|
{
|
|
subIdx++; // Skip literal match
|
|
}
|
|
}
|
|
|
|
// Now build the output using the "to" pattern
|
|
int wcIdx = 0;
|
|
var sb = new StringBuilder();
|
|
for (int i = 0; i < toTokens.Length; i++)
|
|
{
|
|
if (i > 0) sb.Append('.');
|
|
|
|
if (toTokens[i] == "*")
|
|
{
|
|
sb.Append(wcIdx < wildcardValues.Count ? wildcardValues[wcIdx] : "*");
|
|
wcIdx++;
|
|
}
|
|
else if (toTokens[i] == ">")
|
|
{
|
|
sb.Append(fwcValue ?? ">");
|
|
}
|
|
else
|
|
{
|
|
sb.Append(toTokens[i]);
|
|
}
|
|
}
|
|
|
|
return sb.ToString();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Wires service import subscriptions for an account. Creates marker
|
|
/// subscriptions in the account's SubList so that the import paths
|
|
/// are tracked. The actual forwarding happens in ProcessMessage when
|
|
/// it checks the account's Imports.Services.
|
|
/// Reference: Go server/accounts.go addServiceImportSub.
|
|
/// </summary>
|
|
public void WireServiceImports(Account account)
|
|
{
|
|
foreach (var kvp in account.Imports.Services)
|
|
{
|
|
foreach (var si in kvp.Value)
|
|
{
|
|
if (si.Invalid) continue;
|
|
|
|
// Create a marker subscription in the importer account.
|
|
// This subscription doesn't directly deliver messages;
|
|
// the ProcessMessage method checks service imports after
|
|
// the regular SubList match.
|
|
_logger.LogDebug(
|
|
"Wired service import for account {Account}: {From} -> {To} (dest: {DestAccount})",
|
|
account.Name, si.From, si.To, si.DestinationAccount.Name);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void SendNoResponders(NatsClient sender, string replyTo)
|
|
{
|
|
// Find the sid for a subscription matching the reply subject
|
|
var sid = string.Empty;
|
|
foreach (var sub in sender.Subscriptions.Values)
|
|
{
|
|
if (SubjectMatch.MatchLiteral(replyTo, sub.Subject))
|
|
{
|
|
sid = sub.Sid;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Build: HMSG {replyTo} {sid} {hdrLen} {hdrLen}\r\n{headers}\r\n
|
|
var headerBlock = "NATS/1.0 503\r\n\r\n"u8;
|
|
var hdrLen = headerBlock.Length;
|
|
var controlLine = Encoding.ASCII.GetBytes($"HMSG {replyTo} {sid} {hdrLen} {hdrLen}\r\n");
|
|
|
|
var totalLen = controlLine.Length + hdrLen + NatsProtocol.CrLf.Length;
|
|
var msg = new byte[totalLen];
|
|
var offset = 0;
|
|
controlLine.CopyTo(msg.AsSpan(offset));
|
|
offset += controlLine.Length;
|
|
headerBlock.CopyTo(msg.AsSpan(offset));
|
|
offset += hdrLen;
|
|
NatsProtocol.CrLf.CopyTo(msg.AsSpan(offset));
|
|
|
|
sender.QueueOutbound(msg);
|
|
}
|
|
|
|
public Account GetOrCreateAccount(string name)
|
|
{
|
|
return _accounts.GetOrAdd(name, n =>
|
|
{
|
|
var acc = new Account(n);
|
|
if (_options.Accounts != null && _options.Accounts.TryGetValue(n, out var config))
|
|
{
|
|
acc.MaxConnections = config.MaxConnections;
|
|
acc.MaxSubscriptions = config.MaxSubscriptions;
|
|
acc.DefaultPermissions = config.DefaultPermissions;
|
|
|
|
// Wire exports from config
|
|
if (config.Exports != null)
|
|
{
|
|
foreach (var export in config.Exports)
|
|
{
|
|
if (export.Service is { Length: > 0 } svc)
|
|
{
|
|
ServiceLatency? latency = export.LatencySubject is { Length: > 0 }
|
|
? new ServiceLatency { Subject = export.LatencySubject, SamplingPercentage = export.LatencySampling }
|
|
: null;
|
|
acc.AddServiceExport(svc, Imports.ServiceResponseType.Singleton, approved: null, latency: latency);
|
|
}
|
|
else if (export.Stream is { Length: > 0 } strm)
|
|
{
|
|
acc.AddStreamExport(strm, approved: null);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Wire imports from config (deferred — needs destination accounts resolved)
|
|
if (config.Imports != null)
|
|
WireAccountImports(acc, config.Imports);
|
|
}
|
|
|
|
return acc;
|
|
});
|
|
}
|
|
|
|
private void WireAccountImports(Account importer, List<Auth.ImportDefinition> imports)
|
|
{
|
|
foreach (var imp in imports)
|
|
{
|
|
if (imp.ServiceAccount is { Length: > 0 } svcAcct && imp.ServiceSubject is { Length: > 0 } svcSubj)
|
|
{
|
|
var dest = GetOrCreateAccount(svcAcct);
|
|
var localSubject = imp.To ?? svcSubj;
|
|
importer.AddServiceImport(dest, from: localSubject, to: svcSubj);
|
|
}
|
|
else if (imp.StreamAccount is { Length: > 0 } strmAcct && imp.StreamSubject is { Length: > 0 } strmSubj)
|
|
{
|
|
var source = GetOrCreateAccount(strmAcct);
|
|
var localSubject = imp.To ?? strmSubj;
|
|
importer.AddStreamImport(source, from: strmSubj, to: localSubject);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if the subject belongs to the $SYS subject space.
|
|
/// Reference: Go server/server.go — isReservedSubject.
|
|
/// </summary>
|
|
public static bool IsSystemSubject(string subject)
|
|
=> subject.StartsWith("$SYS.", StringComparison.Ordinal) || subject == "$SYS";
|
|
|
|
/// <summary>
|
|
/// Checks whether the given account is allowed to subscribe to the specified subject.
|
|
/// Non-system accounts cannot subscribe to $SYS.> subjects.
|
|
/// Reference: Go server/accounts.go — isReservedForSys.
|
|
/// </summary>
|
|
public bool IsSubscriptionAllowed(Account? account, string subject)
|
|
{
|
|
if (!IsSystemSubject(subject))
|
|
return true;
|
|
|
|
// System account is always allowed
|
|
if (account != null && account.IsSystemAccount)
|
|
return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the SubList appropriate for a given subject: system account SubList
|
|
/// for $SYS.> subjects, or the provided account's SubList for everything else.
|
|
/// Reference: Go server/server.go — sublist routing for internal subjects.
|
|
/// </summary>
|
|
public SubList GetSubListForSubject(Account? account, string subject)
|
|
{
|
|
if (IsSystemSubject(subject))
|
|
return _systemAccount.SubList;
|
|
|
|
return account?.SubList ?? _globalAccount.SubList;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Publishes a service latency metric message to the configured latency subject.
|
|
/// Go reference: client.go processServiceImport — trackLatency path.
|
|
/// </summary>
|
|
private void PublishServiceLatency(ServiceImport si, TimeSpan elapsed)
|
|
{
|
|
var latency = si.Export?.Latency;
|
|
if (latency == null || string.IsNullOrEmpty(latency.Subject))
|
|
return;
|
|
|
|
var msg = LatencyTracker.BuildLatencyMsg(
|
|
requestor: si.DestinationAccount.Name,
|
|
responder: si.Export?.Account?.Name ?? "unknown",
|
|
serviceLatency: elapsed,
|
|
totalLatency: elapsed);
|
|
|
|
SendInternalMsg(latency.Subject, reply: null, msg);
|
|
}
|
|
|
|
public void SendInternalMsg(string subject, string? reply, object? msg)
|
|
{
|
|
_eventSystem?.Enqueue(new PublishMessage { Subject = subject, Reply = reply, Body = msg });
|
|
}
|
|
|
|
public void SendInternalAccountMsg(Account account, string subject, object? msg)
|
|
{
|
|
_eventSystem?.Enqueue(new PublishMessage { Subject = subject, Body = msg });
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles $SYS.REQ.SERVER.{id}.VARZ requests.
|
|
/// Returns core server information including stats counters.
|
|
/// </summary>
|
|
public void HandleVarzRequest(string subject, string? reply)
|
|
{
|
|
if (reply == null) return;
|
|
var varz = new
|
|
{
|
|
server_id = _serverInfo.ServerId,
|
|
server_name = _serverInfo.ServerName,
|
|
version = NatsProtocol.Version,
|
|
host = _options.Host,
|
|
port = _options.Port,
|
|
max_payload = _options.MaxPayload,
|
|
connections = ClientCount,
|
|
total_connections = Interlocked.Read(ref _stats.TotalConnections),
|
|
in_msgs = Interlocked.Read(ref _stats.InMsgs),
|
|
out_msgs = Interlocked.Read(ref _stats.OutMsgs),
|
|
in_bytes = Interlocked.Read(ref _stats.InBytes),
|
|
out_bytes = Interlocked.Read(ref _stats.OutBytes),
|
|
};
|
|
SendInternalMsg(reply, null, varz);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles $SYS.REQ.SERVER.{id}.HEALTHZ requests.
|
|
/// Returns a simple health status response.
|
|
/// </summary>
|
|
public void HandleHealthzRequest(string subject, string? reply)
|
|
{
|
|
if (reply == null) return;
|
|
SendInternalMsg(reply, null, new { status = "ok" });
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles $SYS.REQ.SERVER.{id}.SUBSZ requests.
|
|
/// Returns the current subscription count.
|
|
/// </summary>
|
|
public void HandleSubszRequest(string subject, string? reply)
|
|
{
|
|
if (reply == null) return;
|
|
SendInternalMsg(reply, null, new { num_subscriptions = SubList.Count });
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles $SYS.REQ.SERVER.{id}.STATSZ requests.
|
|
/// Publishes current server statistics through the event system.
|
|
/// </summary>
|
|
public void HandleStatszRequest(string subject, string? reply)
|
|
{
|
|
if (reply == null) return;
|
|
var process = System.Diagnostics.Process.GetCurrentProcess();
|
|
var statsMsg = new Events.ServerStatsMsg
|
|
{
|
|
Server = BuildEventServerInfo(),
|
|
Stats = new Events.ServerStatsData
|
|
{
|
|
Start = StartTime,
|
|
Mem = process.WorkingSet64,
|
|
Cores = Environment.ProcessorCount,
|
|
Connections = ClientCount,
|
|
TotalConnections = Interlocked.Read(ref _stats.TotalConnections),
|
|
Subscriptions = SubList.Count,
|
|
Sent = new Events.DataStats
|
|
{
|
|
Msgs = Interlocked.Read(ref _stats.OutMsgs),
|
|
Bytes = Interlocked.Read(ref _stats.OutBytes),
|
|
},
|
|
Received = new Events.DataStats
|
|
{
|
|
Msgs = Interlocked.Read(ref _stats.InMsgs),
|
|
Bytes = Interlocked.Read(ref _stats.InBytes),
|
|
},
|
|
InMsgs = Interlocked.Read(ref _stats.InMsgs),
|
|
OutMsgs = Interlocked.Read(ref _stats.OutMsgs),
|
|
InBytes = Interlocked.Read(ref _stats.InBytes),
|
|
OutBytes = Interlocked.Read(ref _stats.OutBytes),
|
|
SlowConsumers = Interlocked.Read(ref _stats.SlowConsumers),
|
|
},
|
|
};
|
|
SendInternalMsg(reply, null, statsMsg);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Handles $SYS.REQ.SERVER.{id}.IDZ requests.
|
|
/// Returns basic server identity information.
|
|
/// </summary>
|
|
public void HandleIdzRequest(string subject, string? reply)
|
|
{
|
|
if (reply == null) return;
|
|
var idz = new
|
|
{
|
|
server_id = _serverInfo.ServerId,
|
|
server_name = _serverInfo.ServerName,
|
|
version = NatsProtocol.Version,
|
|
host = _options.Host,
|
|
port = _options.Port,
|
|
};
|
|
SendInternalMsg(reply, null, idz);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Builds an EventServerInfo block for embedding in system event messages.
|
|
/// Maps to Go's serverInfo() helper used in events.go advisory publishing.
|
|
/// </summary>
|
|
public EventServerInfo BuildEventServerInfo()
|
|
{
|
|
var seq = _eventSystem?.NextSequence() ?? 0;
|
|
return new EventServerInfo
|
|
{
|
|
Name = _serverInfo.ServerName,
|
|
Host = _options.Host,
|
|
Id = _serverInfo.ServerId,
|
|
Version = NatsProtocol.Version,
|
|
Seq = seq,
|
|
};
|
|
}
|
|
|
|
private static EventClientInfo BuildEventClientInfo(NatsClient client)
|
|
{
|
|
return new EventClientInfo
|
|
{
|
|
Id = client.Id,
|
|
Host = client.RemoteIp,
|
|
Account = client.Account?.Name,
|
|
Name = client.ClientOpts?.Name,
|
|
Lang = client.ClientOpts?.Lang,
|
|
Version = client.ClientOpts?.Version,
|
|
Start = client.StartTime,
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Publishes a $SYS.ACCOUNT.{account}.CONNECT advisory when a client
|
|
/// completes authentication. Maps to Go's sendConnectEvent in events.go.
|
|
/// </summary>
|
|
public void PublishConnectEvent(NatsClient client)
|
|
{
|
|
if (_eventSystem == null) return;
|
|
var accountName = client.Account?.Name ?? Account.GlobalAccountName;
|
|
var subject = string.Format(EventSubjects.ConnectEvent, accountName);
|
|
var evt = new ConnectEventMsg
|
|
{
|
|
Id = Guid.NewGuid().ToString("N"),
|
|
Time = DateTime.UtcNow,
|
|
Server = BuildEventServerInfo(),
|
|
Client = BuildEventClientInfo(client),
|
|
};
|
|
SendInternalMsg(subject, null, evt);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Publishes a $SYS.ACCOUNT.{account}.DISCONNECT advisory when a client
|
|
/// disconnects. Maps to Go's sendDisconnectEvent in events.go.
|
|
/// </summary>
|
|
public void PublishDisconnectEvent(NatsClient client)
|
|
{
|
|
if (_eventSystem == null) return;
|
|
var accountName = client.Account?.Name ?? Account.GlobalAccountName;
|
|
var subject = string.Format(EventSubjects.DisconnectEvent, accountName);
|
|
var evt = new DisconnectEventMsg
|
|
{
|
|
Id = Guid.NewGuid().ToString("N"),
|
|
Time = DateTime.UtcNow,
|
|
Server = BuildEventServerInfo(),
|
|
Client = BuildEventClientInfo(client),
|
|
Sent = new DataStats
|
|
{
|
|
Msgs = Interlocked.Read(ref client.OutMsgs),
|
|
Bytes = Interlocked.Read(ref client.OutBytes),
|
|
},
|
|
Received = new DataStats
|
|
{
|
|
Msgs = Interlocked.Read(ref client.InMsgs),
|
|
Bytes = Interlocked.Read(ref client.InBytes),
|
|
},
|
|
Reason = client.CloseReason.ToReasonString(),
|
|
};
|
|
SendInternalMsg(subject, null, evt);
|
|
}
|
|
|
|
public void RemoveClient(NatsClient client)
|
|
{
|
|
// Publish disconnect advisory before removing client state
|
|
if (client.ConnectReceived)
|
|
PublishDisconnectEvent(client);
|
|
|
|
_clients.TryRemove(client.Id, out _);
|
|
_logger.LogDebug("Removed client {ClientId}", client.Id);
|
|
|
|
var (tlsPeerCertSubject, tlsPeerCertSubjectPkSha256, tlsPeerCertSha256) =
|
|
TlsPeerCertMapper.ToClosedFields(client.TlsState?.PeerCert);
|
|
var (jwt, issuerKey, tags) = ExtractJwtMetadata(client.ClientOpts?.JWT);
|
|
var proxyKey = ExtractProxyKey(client.ClientOpts?.Username);
|
|
|
|
// Snapshot for closed-connections tracking (ring buffer auto-overwrites oldest when full)
|
|
_closedClients.Add(new ClosedClient
|
|
{
|
|
Cid = client.Id,
|
|
Ip = client.RemoteIp ?? "",
|
|
Port = client.RemotePort,
|
|
Start = client.StartTime,
|
|
Stop = DateTime.UtcNow,
|
|
Reason = client.CloseReason.ToReasonString(),
|
|
Name = client.ClientOpts?.Name ?? "",
|
|
Lang = client.ClientOpts?.Lang ?? "",
|
|
Version = client.ClientOpts?.Version ?? "",
|
|
AuthorizedUser = client.ClientOpts?.Username ?? "",
|
|
Account = client.Account?.Name ?? "",
|
|
InMsgs = Interlocked.Read(ref client.InMsgs),
|
|
OutMsgs = Interlocked.Read(ref client.OutMsgs),
|
|
InBytes = Interlocked.Read(ref client.InBytes),
|
|
OutBytes = Interlocked.Read(ref client.OutBytes),
|
|
NumSubs = (uint)client.Subscriptions.Count,
|
|
Rtt = client.Rtt,
|
|
TlsVersion = client.TlsState?.TlsVersion ?? "",
|
|
TlsCipherSuite = client.TlsState?.CipherSuite ?? "",
|
|
TlsPeerCertSubject = tlsPeerCertSubject,
|
|
TlsPeerCertSubjectPkSha256 = tlsPeerCertSubjectPkSha256,
|
|
TlsPeerCertSha256 = tlsPeerCertSha256,
|
|
MqttClient = "", // populated when MQTT transport is implemented
|
|
Stalls = 0,
|
|
Jwt = jwt,
|
|
IssuerKey = issuerKey,
|
|
NameTag = "",
|
|
Tags = tags,
|
|
ProxyKey = proxyKey,
|
|
});
|
|
|
|
var subList = client.Account?.SubList ?? _globalAccount.SubList;
|
|
client.RemoveAllSubscriptions(subList);
|
|
client.Account?.RemoveClient(client.Id);
|
|
}
|
|
|
|
private void TrackEarlyClosedClient(Socket socket, ulong clientId, ClientClosedReason reason)
|
|
{
|
|
string ip = "";
|
|
int port = 0;
|
|
|
|
if (socket.RemoteEndPoint is IPEndPoint endpoint)
|
|
{
|
|
ip = endpoint.Address.ToString();
|
|
port = endpoint.Port;
|
|
}
|
|
|
|
var now = DateTime.UtcNow;
|
|
_closedClients.Add(new ClosedClient
|
|
{
|
|
Cid = clientId,
|
|
Ip = ip,
|
|
Port = port,
|
|
Start = now,
|
|
Stop = now,
|
|
Reason = reason.ToReasonString(),
|
|
});
|
|
}
|
|
|
|
private static (string Jwt, string IssuerKey, string[] Tags) ExtractJwtMetadata(string? jwt)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(jwt))
|
|
return ("", "", []);
|
|
|
|
var issuerKey = "";
|
|
var tags = Array.Empty<string>();
|
|
|
|
var claims = NatsJwt.DecodeUserClaims(jwt);
|
|
if (claims != null)
|
|
{
|
|
issuerKey = claims.Issuer ?? "";
|
|
tags = claims.Nats?.Tags ?? Array.Empty<string>();
|
|
}
|
|
|
|
return (jwt, issuerKey, tags);
|
|
}
|
|
|
|
private static string ExtractProxyKey(string? username)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(username))
|
|
return "";
|
|
|
|
const string prefix = "proxy:";
|
|
return username.StartsWith(prefix, StringComparison.Ordinal)
|
|
? username[prefix.Length..]
|
|
: "";
|
|
}
|
|
|
|
private void WritePidFile()
|
|
{
|
|
if (string.IsNullOrEmpty(_options.PidFile)) return;
|
|
try
|
|
{
|
|
File.WriteAllText(_options.PidFile, Environment.ProcessId.ToString());
|
|
_logger.LogDebug("Wrote PID file {PidFile}", _options.PidFile);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Error writing PID file {PidFile}", _options.PidFile);
|
|
}
|
|
}
|
|
|
|
private void DeletePidFile()
|
|
{
|
|
if (string.IsNullOrEmpty(_options.PidFile)) return;
|
|
try
|
|
{
|
|
if (File.Exists(_options.PidFile))
|
|
File.Delete(_options.PidFile);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Error deleting PID file {PidFile}", _options.PidFile);
|
|
}
|
|
}
|
|
|
|
private void WritePortsFile()
|
|
{
|
|
if (string.IsNullOrEmpty(_options.PortsFileDir)) return;
|
|
try
|
|
{
|
|
var exeName = Path.GetFileNameWithoutExtension(Environment.ProcessPath ?? "nats-server");
|
|
var fileName = $"{exeName}_{Environment.ProcessId}.ports";
|
|
_portsFilePath = Path.Combine(_options.PortsFileDir, fileName);
|
|
|
|
var ports = new { client = _options.Port, monitor = _options.MonitorPort > 0 ? _options.MonitorPort : (int?)null };
|
|
var json = System.Text.Json.JsonSerializer.Serialize(ports);
|
|
File.WriteAllText(_portsFilePath, json);
|
|
_logger.LogDebug("Wrote ports file {PortsFile}", _portsFilePath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Error writing ports file to {PortsFileDir}", _options.PortsFileDir);
|
|
}
|
|
}
|
|
|
|
private void DeletePortsFile()
|
|
{
|
|
if (_portsFilePath == null) return;
|
|
try
|
|
{
|
|
if (File.Exists(_portsFilePath))
|
|
File.Delete(_portsFilePath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Error deleting ports file {PortsFile}", _portsFilePath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stores the CLI snapshot and flags so that command-line overrides
|
|
/// always take precedence during config reload.
|
|
/// </summary>
|
|
public void SetCliSnapshot(NatsOptions cliSnapshot, HashSet<string> cliFlags)
|
|
{
|
|
_cliSnapshot = cliSnapshot;
|
|
_cliFlags = cliFlags;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reloads the configuration file, diffs against current options, validates
|
|
/// the changes, and applies reloadable settings. CLI overrides are preserved.
|
|
/// </summary>
|
|
public void ReloadConfig()
|
|
{
|
|
ReloadConfigCore(throwOnError: false);
|
|
}
|
|
|
|
public void ReloadConfigOrThrow()
|
|
{
|
|
ReloadConfigCore(throwOnError: true);
|
|
}
|
|
|
|
private void ReloadConfigCore(bool throwOnError)
|
|
{
|
|
if (_options.ConfigFile == null)
|
|
{
|
|
_logger.LogWarning("No config file specified, cannot reload");
|
|
if (throwOnError)
|
|
throw new InvalidOperationException("No config file specified.");
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var (newConfig, digest) = NatsConfParser.ParseFileWithDigest(_options.ConfigFile);
|
|
if (digest == _configDigest)
|
|
{
|
|
_logger.LogInformation("Config file unchanged, no reload needed");
|
|
return;
|
|
}
|
|
|
|
var newOpts = new NatsOptions { ConfigFile = _options.ConfigFile };
|
|
ConfigProcessor.ApplyConfig(newConfig, newOpts);
|
|
|
|
// CLI flags override config
|
|
if (_cliSnapshot != null)
|
|
ConfigReloader.MergeCliOverrides(newOpts, _cliSnapshot, _cliFlags);
|
|
|
|
var changes = ConfigReloader.Diff(_options, newOpts);
|
|
var errors = ConfigReloader.Validate(changes);
|
|
if (errors.Count > 0)
|
|
{
|
|
foreach (var err in errors)
|
|
_logger.LogError("Config reload error: {Error}", err);
|
|
if (throwOnError)
|
|
throw new InvalidOperationException(string.Join("; ", errors));
|
|
return;
|
|
}
|
|
|
|
// Apply changes to running options
|
|
ApplyConfigChanges(changes, newOpts);
|
|
_configDigest = digest;
|
|
_configTime = DateTime.UtcNow;
|
|
_logger.LogInformation("Config reloaded successfully ({Count} changes applied)", changes.Count);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to reload config file: {ConfigFile}", _options.ConfigFile);
|
|
if (throwOnError)
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private void ApplyConfigChanges(List<IConfigChange> changes, NatsOptions newOpts)
|
|
{
|
|
bool hasLoggingChanges = false;
|
|
bool hasAuthChanges = false;
|
|
bool hasTlsChanges = false;
|
|
|
|
foreach (var change in changes)
|
|
{
|
|
if (change.IsLoggingChange) hasLoggingChanges = true;
|
|
if (change.IsAuthChange) hasAuthChanges = true;
|
|
if (change.IsTlsChange) hasTlsChanges = true;
|
|
}
|
|
|
|
// Copy reloadable values from newOpts to _options
|
|
CopyReloadableOptions(newOpts);
|
|
|
|
// Trigger side effects
|
|
if (hasLoggingChanges)
|
|
{
|
|
ReOpenLogFile?.Invoke();
|
|
_logger.LogInformation("Logging configuration reloaded");
|
|
}
|
|
|
|
if (hasTlsChanges)
|
|
{
|
|
// Reload TLS certificates: new connections get the new cert,
|
|
// existing connections keep their original cert.
|
|
// Reference: golang/nats-server/server/reload.go — tlsOption.Apply.
|
|
if (ConfigReloader.ReloadTlsCertificate(_options, _tlsCertProvider))
|
|
{
|
|
_sslOptions = _tlsCertProvider!.GetCurrentSslOptions();
|
|
_logger.LogInformation("TLS configuration reloaded");
|
|
}
|
|
}
|
|
|
|
if (hasAuthChanges)
|
|
{
|
|
// Rebuild auth service with new options, then propagate changes to connected clients
|
|
var oldAuthService = _authService;
|
|
_authService = AuthService.Build(_options);
|
|
_logger.LogInformation("Authorization configuration reloaded");
|
|
|
|
// Re-evaluate connected clients against the new auth config.
|
|
// Clients that no longer pass authentication are disconnected with AUTH_EXPIRED.
|
|
// Reference: Go server/reload.go — applyOptions / reloadAuthorization.
|
|
PropagateAuthChanges();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Re-evaluates all connected clients against the current auth configuration.
|
|
/// Clients whose credentials no longer pass authentication are disconnected
|
|
/// with an "Authorization Violation" error via SendErrAndCloseAsync, which
|
|
/// properly drains the outbound channel before closing the socket.
|
|
/// Reference: Go server/reload.go — reloadAuthorization, client.go — applyAccountLimits.
|
|
/// </summary>
|
|
internal void PropagateAuthChanges()
|
|
{
|
|
if (!_authService.IsAuthRequired)
|
|
{
|
|
// Auth was disabled — all existing clients are fine
|
|
return;
|
|
}
|
|
|
|
var clientsToDisconnect = new List<NatsClient>();
|
|
|
|
foreach (var client in _clients.Values)
|
|
{
|
|
if (client.ClientOpts == null)
|
|
continue; // Client hasn't sent CONNECT yet
|
|
|
|
var context = new ClientAuthContext
|
|
{
|
|
Opts = client.ClientOpts,
|
|
Nonce = [], // Nonce is only used at connect time; re-evaluation skips it
|
|
ClientCertificate = client.TlsState?.PeerCert,
|
|
};
|
|
|
|
var result = _authService.Authenticate(context);
|
|
if (result == null)
|
|
{
|
|
_logger.LogInformation(
|
|
"Client {ClientId} credentials no longer valid after auth reload, disconnecting",
|
|
client.Id);
|
|
clientsToDisconnect.Add(client);
|
|
}
|
|
}
|
|
|
|
// Disconnect clients that failed re-authentication.
|
|
// Use SendErrAndCloseAsync which queues the -ERR, completes the outbound channel,
|
|
// waits for the write loop to drain, then cancels the client.
|
|
var disconnectTasks = new List<Task>(clientsToDisconnect.Count);
|
|
foreach (var client in clientsToDisconnect)
|
|
{
|
|
disconnectTasks.Add(client.SendErrAndCloseAsync(
|
|
NatsProtocol.ErrAuthorizationViolation,
|
|
ClientClosedReason.AuthenticationExpired));
|
|
}
|
|
|
|
// Wait for all disconnects to complete (with timeout to avoid blocking reload)
|
|
if (disconnectTasks.Count > 0)
|
|
{
|
|
Task.WhenAll(disconnectTasks)
|
|
.WaitAsync(TimeSpan.FromSeconds(5))
|
|
.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing)
|
|
.GetAwaiter().GetResult();
|
|
|
|
_logger.LogInformation(
|
|
"Disconnected {Count} client(s) after auth configuration reload",
|
|
clientsToDisconnect.Count);
|
|
}
|
|
}
|
|
|
|
private void CopyReloadableOptions(NatsOptions newOpts)
|
|
{
|
|
// Logging
|
|
_options.Debug = newOpts.Debug;
|
|
_options.Trace = newOpts.Trace;
|
|
_options.TraceVerbose = newOpts.TraceVerbose;
|
|
_options.Logtime = newOpts.Logtime;
|
|
_options.LogtimeUTC = newOpts.LogtimeUTC;
|
|
_options.LogFile = newOpts.LogFile;
|
|
_options.LogSizeLimit = newOpts.LogSizeLimit;
|
|
_options.LogMaxFiles = newOpts.LogMaxFiles;
|
|
_options.Syslog = newOpts.Syslog;
|
|
_options.RemoteSyslog = newOpts.RemoteSyslog;
|
|
|
|
// Auth
|
|
_options.Username = newOpts.Username;
|
|
_options.Password = newOpts.Password;
|
|
_options.Authorization = newOpts.Authorization;
|
|
_options.Users = newOpts.Users;
|
|
_options.NKeys = newOpts.NKeys;
|
|
_options.NoAuthUser = newOpts.NoAuthUser;
|
|
_options.AuthTimeout = newOpts.AuthTimeout;
|
|
|
|
// Limits
|
|
_options.MaxConnections = newOpts.MaxConnections;
|
|
_options.MaxPayload = newOpts.MaxPayload;
|
|
_options.MaxPending = newOpts.MaxPending;
|
|
_options.WriteDeadline = newOpts.WriteDeadline;
|
|
_options.PingInterval = newOpts.PingInterval;
|
|
_options.MaxPingsOut = newOpts.MaxPingsOut;
|
|
_options.MaxControlLine = newOpts.MaxControlLine;
|
|
_options.MaxSubs = newOpts.MaxSubs;
|
|
_options.MaxSubTokens = newOpts.MaxSubTokens;
|
|
_options.MaxTracedMsgLen = newOpts.MaxTracedMsgLen;
|
|
if (newOpts.MaxClosedClients != _options.MaxClosedClients)
|
|
{
|
|
_options.MaxClosedClients = newOpts.MaxClosedClients;
|
|
_closedClients = new ClosedConnectionRingBuffer(newOpts.MaxClosedClients);
|
|
}
|
|
|
|
// TLS
|
|
_options.TlsCert = newOpts.TlsCert;
|
|
_options.TlsKey = newOpts.TlsKey;
|
|
_options.TlsCaCert = newOpts.TlsCaCert;
|
|
_options.TlsVerify = newOpts.TlsVerify;
|
|
_options.TlsMap = newOpts.TlsMap;
|
|
_options.TlsTimeout = newOpts.TlsTimeout;
|
|
_options.TlsHandshakeFirst = newOpts.TlsHandshakeFirst;
|
|
_options.TlsHandshakeFirstFallback = newOpts.TlsHandshakeFirstFallback;
|
|
_options.AllowNonTls = newOpts.AllowNonTls;
|
|
_options.TlsRateLimit = newOpts.TlsRateLimit;
|
|
_options.TlsPinnedCerts = newOpts.TlsPinnedCerts;
|
|
|
|
// Misc
|
|
_options.Tags = newOpts.Tags;
|
|
_options.LameDuckDuration = newOpts.LameDuckDuration;
|
|
_options.LameDuckGracePeriod = newOpts.LameDuckGracePeriod;
|
|
_options.ClientAdvertise = newOpts.ClientAdvertise;
|
|
_options.DisableSublistCache = newOpts.DisableSublistCache;
|
|
_options.ConnectErrorReports = newOpts.ConnectErrorReports;
|
|
_options.ReconnectErrorReports = newOpts.ReconnectErrorReports;
|
|
_options.NoHeaderSupport = newOpts.NoHeaderSupport;
|
|
_options.NoSystemAccount = newOpts.NoSystemAccount;
|
|
_options.SystemAccount = newOpts.SystemAccount;
|
|
}
|
|
|
|
public override string ToString()
|
|
=> $"NatsServer(ServerId={ServerId}, Name={ServerName}, Addr={Addr()}, Clients={ClientCount})";
|
|
|
|
public void Dispose()
|
|
{
|
|
if (!IsShuttingDown)
|
|
ShutdownAsync().GetAwaiter().GetResult();
|
|
foreach (var reg in _signalRegistrations)
|
|
reg.Dispose();
|
|
_quitCts.Dispose();
|
|
_tlsRateLimiter?.Dispose();
|
|
_tlsCertProvider?.Dispose();
|
|
_listener?.Dispose();
|
|
_wsListener?.Dispose();
|
|
_routeManager?.DisposeAsync().AsTask().GetAwaiter().GetResult();
|
|
_gatewayManager?.DisposeAsync().AsTask().GetAwaiter().GetResult();
|
|
_leafNodeManager?.DisposeAsync().AsTask().GetAwaiter().GetResult();
|
|
_mqttListener?.DisposeAsync().AsTask().GetAwaiter().GetResult();
|
|
_jetStreamService?.DisposeAsync().AsTask().GetAwaiter().GetResult();
|
|
_stats.JetStreamEnabled = false;
|
|
foreach (var client in _clients.Values)
|
|
client.Dispose();
|
|
foreach (var account in _accounts.Values)
|
|
account.Dispose();
|
|
}
|
|
}
|