Files
natsnet/dotnet/src/ZB.MOM.NatsNet.Server/NatsServer.Reload.cs
2026-02-28 12:12:50 -05:00

1346 lines
53 KiB
C#

// Copyright 2012-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Adapted from server/reload.go in the NATS server Go source.
using System.Reflection;
using System.Text.Json;
using ZB.MOM.NatsNet.Server.Auth;
using ZB.MOM.NatsNet.Server.Internal;
namespace ZB.MOM.NatsNet.Server;
public sealed partial class NatsServer
{
/// <summary>
/// Applies supported changes from <paramref name="newOptions"/> to the running server.
/// Mirrors Go <c>Server.ReloadOptions()</c>.
/// </summary>
public Exception? ReloadOptions(ServerOptions newOptions)
{
ArgumentNullException.ThrowIfNull(newOptions);
_reloadMu.EnterWriteLock();
try
{
var currentOptions = GetOpts().Clone();
// Match Go behavior: when operators are configured, trusted keys are cleared.
if (currentOptions.TrustedOperators.Count > 0 && currentOptions.TrustedKeys.Count > 0)
currentOptions.TrustedKeys = [];
var clientOriginalPort = currentOptions.Port;
var clusterOriginalPort = currentOptions.Cluster.Port;
var gatewayOriginalPort = currentOptions.Gateway.Port;
var leafOriginalPort = currentOptions.LeafNode.Port;
var websocketOriginalPort = currentOptions.Websocket.Port;
var mqttOriginalPort = currentOptions.Mqtt.Port;
if (!string.IsNullOrEmpty(currentOptions.Cluster.ListenStr))
{
newOptions.Cluster.ListenStr = currentOptions.Cluster.ListenStr;
var overrideError = newOptions.OverrideCluster();
if (overrideError != null)
return overrideError;
}
newOptions = ServerOptions.MergeOptions(newOptions, ServerOptions.FlagSnapshot);
if (ServerOptions.FlagSnapshot != null)
ConfigReloader.ApplyBoolFlags(newOptions, ServerOptions.FlagSnapshot);
newOptions.SetBaselineOptions();
// Preserve already-bound listener ports on reload when RANDOM was used.
if (newOptions.Port == 0)
newOptions.Port = clientOriginalPort;
if (newOptions.Cluster.Port == ServerConstants.RandomPort)
newOptions.Cluster.Port = clusterOriginalPort;
if (newOptions.Gateway.Port == ServerConstants.RandomPort)
newOptions.Gateway.Port = gatewayOriginalPort;
if (newOptions.LeafNode.Port == ServerConstants.RandomPort)
newOptions.LeafNode.Port = leafOriginalPort;
if (newOptions.Websocket.Port == ServerConstants.RandomPort)
newOptions.Websocket.Port = websocketOriginalPort;
if (newOptions.Mqtt.Port == ServerConstants.RandomPort)
newOptions.Mqtt.Port = mqttOriginalPort;
var reloadError = ReloadOptionsInternal(currentOptions, newOptions);
if (reloadError != null)
return reloadError;
RecheckPinnedCerts(currentOptions, newOptions);
_mu.EnterWriteLock();
try
{
_configTime = DateTime.UtcNow;
}
finally
{
_mu.ExitWriteLock();
}
return null;
}
finally
{
_reloadMu.ExitWriteLock();
}
}
/// <summary>
/// Applies immutable/runtime-only fields and executes diff + apply.
/// Mirrors Go <c>Server.reloadOptions()</c>.
/// </summary>
internal Exception? ReloadOptionsInternal(ServerOptions currentOptions, ServerOptions newOptions)
{
ArgumentNullException.ThrowIfNull(currentOptions);
ArgumentNullException.ThrowIfNull(newOptions);
newOptions.CustomClientAuthentication = currentOptions.CustomClientAuthentication;
newOptions.CustomRouterAuthentication = currentOptions.CustomRouterAuthentication;
var (changed, diffError) = DiffOptions(newOptions);
if (diffError != null)
return diffError;
if (changed.Count > 0)
{
var validateError = ValidateOptions(newOptions);
if (validateError != null)
return validateError;
}
SetOpts(newOptions);
ApplyOptions(new ReloadContext(currentOptions.Cluster.Permissions), changed);
return null;
}
/// <summary>
/// Diffs current and new options and returns ordered reload actions.
/// Mirrors Go <c>Server.diffOptions()</c>.
/// </summary>
internal (List<IReloadOption> Changed, Exception? Error) DiffOptions(ServerOptions newOptions)
{
ArgumentNullException.ThrowIfNull(newOptions);
var oldOptions = GetOpts().Clone();
var diffOptions = new List<IReloadOption>();
var jsEnabled = oldOptions.JetStream;
var disableJetStream = false;
var jsMemLimitsChanged = false;
var jsStoreLimitsChanged = false;
var jsStoreDirChanged = false;
foreach (var property in typeof(ServerOptions).GetProperties(BindingFlags.Instance | BindingFlags.Public))
{
if (!property.CanRead)
continue;
var oldValue = property.GetValue(oldOptions);
var newValue = property.GetValue(newOptions);
var oldImposeError = ConfigReloader.ImposeOrder(oldValue);
if (oldImposeError != null)
return ([], oldImposeError);
var newImposeError = ConfigReloader.ImposeOrder(newValue);
if (newImposeError != null)
return ([], newImposeError);
var optionName = property.Name;
var changed = optionName is not nameof(ServerOptions.Accounts) and not nameof(ServerOptions.Users)
? !OptionValueEquals(optionName, oldValue, newValue)
: true;
if (!changed)
{
if (optionName == nameof(ServerOptions.JetStream) && newValue is bool jsWanted && jsWanted && !jsEnabled)
diffOptions.Add(new JetStreamReloadOption(true));
continue;
}
switch (optionName)
{
case nameof(ServerOptions.TraceVerbose):
diffOptions.Add(new TraceVerboseReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.TraceHeaders):
diffOptions.Add(new TraceHeadersReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.Trace):
diffOptions.Add(new TraceReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.Debug):
diffOptions.Add(new DebugReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.Logtime):
diffOptions.Add(new LogtimeReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.LogtimeUtc):
diffOptions.Add(new LogtimeUTCOption((bool)newValue!));
break;
case nameof(ServerOptions.LogFile):
diffOptions.Add(new LogfileOption((string)newValue!));
break;
case nameof(ServerOptions.Syslog):
diffOptions.Add(new SyslogReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.RemoteSyslog):
diffOptions.Add(new RemoteSyslogReloadOption((string)newValue!));
break;
case nameof(ServerOptions.TlsConfig):
diffOptions.Add(new TlsReloadOption(newValue));
break;
case nameof(ServerOptions.TlsTimeout):
diffOptions.Add(new TlsTimeoutReloadOption((double)newValue!));
break;
case nameof(ServerOptions.TlsPinnedCerts):
diffOptions.Add(new TlsPinnedCertReloadOption(newValue));
break;
case nameof(ServerOptions.TlsHandshakeFirst):
diffOptions.Add(new TlsHandshakeFirstReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.TlsHandshakeFirstFallback):
diffOptions.Add(new TlsHandshakeFirstFallbackReloadOption((TimeSpan)newValue!));
break;
case nameof(ServerOptions.Username):
diffOptions.Add(new UsernameReloadOption());
break;
case nameof(ServerOptions.Password):
diffOptions.Add(new PasswordReloadOption());
break;
case nameof(ServerOptions.Tags):
diffOptions.Add(new TagsReloadOption());
break;
case nameof(ServerOptions.Metadata):
diffOptions.Add(new MetadataReloadOption());
break;
case nameof(ServerOptions.Authorization):
diffOptions.Add(new AuthorizationReloadOption());
break;
case nameof(ServerOptions.AuthTimeout):
diffOptions.Add(new AuthTimeoutReloadOption((double)newValue!));
break;
case nameof(ServerOptions.Users):
diffOptions.Add(new UsersReloadOption());
break;
case nameof(ServerOptions.Nkeys):
diffOptions.Add(new NkeysReloadOption());
break;
case nameof(ServerOptions.Cluster):
{
var oldCluster = (ClusterOpts)oldValue!;
var newCluster = (ClusterOpts)newValue!;
var clusterError = ConfigReloader.ValidateClusterOpts(oldCluster, newCluster);
if (clusterError != null)
return ([], clusterError);
var clusterOption = new ClusterReloadOption(
newCluster,
permsChanged: !OptionValueEquals(nameof(ClusterOpts.Permissions), oldCluster.Permissions, newCluster.Permissions),
poolSizeChanged: false,
compressChanged: !CompressOptsEqual(oldCluster.Compression, newCluster.Compression),
accsAdded: [],
accsRemoved: []);
clusterOption.DiffPoolAndAccounts(oldCluster);
foreach (var accountName in clusterOption.AccountsAdded)
{
var (_, lookupError) = LookupAccount(accountName);
if (lookupError != null)
{
return ([], new InvalidOperationException(
$"unable to add account '{accountName}' to dedicated routes: {lookupError.Message}",
lookupError));
}
}
diffOptions.Add(clusterOption);
break;
}
case nameof(ServerOptions.Routes):
{
var oldRoutes = oldValue as IReadOnlyList<Uri> ?? [];
var newRoutes = newValue as IReadOnlyList<Uri> ?? [];
var (add, remove) = ConfigReloader.DiffRoutes(oldRoutes, newRoutes);
diffOptions.Add(new RoutesReloadOption([.. add.Cast<object>()], [.. remove.Cast<object>()]));
break;
}
case nameof(ServerOptions.MaxConn):
diffOptions.Add(new MaxConnReloadOption((int)newValue!));
break;
case nameof(ServerOptions.PidFile):
diffOptions.Add(new PidFileReloadOption((string)newValue!));
break;
case nameof(ServerOptions.PortsFileDir):
diffOptions.Add(new PortsFileDirReloadOption((string)oldValue!, (string)newValue!));
break;
case nameof(ServerOptions.MaxControlLine):
diffOptions.Add(new MaxControlLineReloadOption((int)newValue!));
break;
case nameof(ServerOptions.MaxPayload):
diffOptions.Add(new MaxPayloadReloadOption((int)newValue!));
break;
case nameof(ServerOptions.PingInterval):
diffOptions.Add(new PingIntervalReloadOption((TimeSpan)newValue!));
break;
case nameof(ServerOptions.MaxPingsOut):
diffOptions.Add(new MaxPingsOutReloadOption((int)newValue!));
break;
case nameof(ServerOptions.WriteDeadline):
diffOptions.Add(new WriteDeadlineReloadOption((TimeSpan)newValue!));
break;
case nameof(ServerOptions.ClientAdvertise):
{
var clientAdvertise = (string)newValue!;
if (!string.IsNullOrEmpty(clientAdvertise))
{
var (_, _, parseError) = ServerUtilities.ParseHostPort(clientAdvertise, 0);
if (parseError != null)
{
return ([], new InvalidOperationException(
$"invalid ClientAdvertise value '{clientAdvertise}': {parseError.Message}",
parseError));
}
}
diffOptions.Add(new ClientAdvertiseReloadOption(clientAdvertise));
break;
}
case nameof(ServerOptions.Accounts):
diffOptions.Add(new AccountsReloadOption());
break;
case nameof(ServerOptions.AccountResolver):
if ((oldValue is null) != (newValue is null))
return ([], new InvalidOperationException("config reload does not support moving to or from an account resolver"));
diffOptions.Add(new AccountsReloadOption());
break;
case nameof(ServerOptions.AccountResolverTlsConfig):
diffOptions.Add(new AccountsReloadOption());
break;
case nameof(ServerOptions.Gateway):
{
var oldGateway = (GatewayOpts)oldValue!;
var newGateway = (GatewayOpts)newValue!;
if (!GatewayEquivalentForReload(oldGateway, newGateway))
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldGateway}, new={newGateway}"));
}
break;
}
case nameof(ServerOptions.LeafNode):
{
var oldLeaf = (LeafNodeOpts)oldValue!;
var newLeaf = (LeafNodeOpts)newValue!;
var (equivalent, tlsFirstChanged, compressionChanged, disabledChanged) =
LeafNodeReloadCompare(oldLeaf, newLeaf);
if (!equivalent)
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldLeaf}, new={newLeaf}"));
}
diffOptions.Add(new LeafNodeReloadOption(tlsFirstChanged, compressionChanged, disabledChanged));
break;
}
case nameof(ServerOptions.JetStream):
{
var newJsValue = (bool)newValue!;
var oldJsValue = (bool)oldValue!;
if (newJsValue != oldJsValue)
diffOptions.Add(new JetStreamReloadOption(newJsValue));
disableJetStream = !newJsValue;
break;
}
case nameof(ServerOptions.StoreDir):
{
var oldStoreDir = (string)oldValue!;
var newStoreDir = (string)newValue!;
if (jsEnabled && !string.Equals(oldStoreDir, newStoreDir, StringComparison.Ordinal))
{
if (string.IsNullOrEmpty(newStoreDir))
jsStoreDirChanged = true;
else
return ([], new InvalidOperationException("config reload not supported for jetstream storage directory"));
}
break;
}
case nameof(ServerOptions.JetStreamMaxMemory):
case nameof(ServerOptions.JetStreamMaxStore):
{
var oldLimit = (long)oldValue!;
var newLimit = (long)newValue!;
if (jsEnabled && oldLimit != newLimit)
{
var fromUnset = oldLimit == -1;
var toUnset = newLimit == -1;
if (!fromUnset && toUnset)
{
if (optionName == nameof(ServerOptions.JetStreamMaxMemory))
jsMemLimitsChanged = true;
else
jsStoreLimitsChanged = true;
}
else if (fromUnset && !toUnset)
{
return ([], new InvalidOperationException(
"config reload not supported for jetstream dynamic max memory and store"));
}
else
{
return ([], new InvalidOperationException(
"config reload not supported for jetstream max memory and store"));
}
}
break;
}
case nameof(ServerOptions.Websocket):
{
var oldWebsocket = CloneByJson((WebsocketOpts)oldValue!);
var newWebsocket = CloneByJson((WebsocketOpts)newValue!);
oldWebsocket.TlsConfig = null;
oldWebsocket.TlsConfigOpts = null;
newWebsocket.TlsConfig = null;
newWebsocket.TlsConfigOpts = null;
if (!JsonEquivalent(oldWebsocket, newWebsocket))
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
}
break;
}
case nameof(ServerOptions.Mqtt):
{
var oldMqtt = CloneByJson((MqttOpts)oldValue!);
var newMqtt = CloneByJson((MqttOpts)newValue!);
diffOptions.Add(new MqttAckWaitReloadOption(newMqtt.AckWait));
diffOptions.Add(new MqttMaxAckPendingReloadOption(newMqtt.MaxAckPending));
diffOptions.Add(new MqttStreamReplicasReloadOption(newMqtt.StreamReplicas));
diffOptions.Add(new MqttConsumerReplicasReloadOption(newMqtt.ConsumerReplicas));
diffOptions.Add(new MqttConsumerMemoryStorageReloadOption(newMqtt.ConsumerMemoryStorage));
diffOptions.Add(new MqttInactiveThresholdReloadOption(newMqtt.ConsumerInactiveThreshold));
oldMqtt.TlsConfig = null;
oldMqtt.TlsConfigOpts = null;
oldMqtt.AckWait = default;
oldMqtt.MaxAckPending = default;
oldMqtt.StreamReplicas = default;
oldMqtt.ConsumerReplicas = default;
oldMqtt.ConsumerMemoryStorage = default;
oldMqtt.ConsumerInactiveThreshold = default;
newMqtt.TlsConfig = null;
newMqtt.TlsConfigOpts = null;
newMqtt.AckWait = default;
newMqtt.MaxAckPending = default;
newMqtt.StreamReplicas = default;
newMqtt.ConsumerReplicas = default;
newMqtt.ConsumerMemoryStorage = default;
newMqtt.ConsumerInactiveThreshold = default;
if (!JsonEquivalent(oldMqtt, newMqtt))
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
}
break;
}
case nameof(ServerOptions.ConnectErrorReports):
diffOptions.Add(new ConnectErrorReportsReloadOption((int)newValue!));
break;
case nameof(ServerOptions.ReconnectErrorReports):
diffOptions.Add(new ReconnectErrorReportsReloadOption((int)newValue!));
break;
case nameof(ServerOptions.NoLog):
case nameof(ServerOptions.NoSigs):
break;
case nameof(ServerOptions.DisableShortFirstPing):
newOptions.DisableShortFirstPing = (bool)oldValue!;
break;
case nameof(ServerOptions.MaxTracedMsgLen):
diffOptions.Add(new MaxTracedMsgLenReloadOption((int)newValue!));
break;
case nameof(ServerOptions.Port):
if ((int)newValue! == 0)
break;
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
case nameof(ServerOptions.NoAuthUser):
{
var oldNoAuth = (string)oldValue!;
var newNoAuth = (string)newValue!;
if (!string.IsNullOrEmpty(oldNoAuth) && string.IsNullOrEmpty(newNoAuth))
{
var matchesUser = newOptions.Users?.Any(u =>
string.Equals(u.Username, oldNoAuth, StringComparison.Ordinal)) == true;
if (matchesUser)
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
}
}
else
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
}
break;
}
case nameof(ServerOptions.DefaultSentinel):
diffOptions.Add(new DefaultSentinelReloadOption((string)newValue!));
break;
case nameof(ServerOptions.SystemAccount):
{
var oldSystem = (string)oldValue!;
var newSystem = (string)newValue!;
if (!string.Equals(oldSystem, ServerConstants.DefaultSystemAccount, StringComparison.Ordinal) ||
!string.IsNullOrEmpty(newSystem))
{
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
}
break;
}
case nameof(ServerOptions.OcspConfig):
diffOptions.Add(new OcspReloadOption(newValue));
break;
case nameof(ServerOptions.OcspCacheConfig):
diffOptions.Add(new OcspResponseCacheReloadOption(newValue));
break;
case nameof(ServerOptions.ProfBlockRate):
diffOptions.Add(new ProfBlockRateReloadOption((int)newValue!));
break;
case nameof(ServerOptions.ConfigDigestValue):
break;
case nameof(ServerOptions.NoFastProducerStall):
diffOptions.Add(new NoFastProducerStallReloadOption((bool)newValue!));
break;
case nameof(ServerOptions.Proxies):
{
var oldProxies = oldValue as ProxiesConfig;
var newProxies = newValue as ProxiesConfig;
var (add, del) = ConfigReloader.DiffProxiesTrustedKeys(oldProxies?.Trusted, newProxies?.Trusted);
if (add.Count > 0 || del.Count > 0)
diffOptions.Add(new ProxiesReloadOption([.. add], [.. del]));
break;
}
default:
return ([], new InvalidOperationException(
$"config reload not supported for {optionName}: old={oldValue}, new={newValue}"));
}
}
if (!disableJetStream)
{
if (jsMemLimitsChanged || jsStoreLimitsChanged)
return ([], new InvalidOperationException(
"config reload not supported for jetstream max memory and max store"));
if (jsStoreDirChanged)
return ([], new InvalidOperationException(
"config reload not supported for jetstream storage dir"));
}
return (diffOptions, null);
}
/// <summary>
/// Applies reload options and executes post-processing hooks.
/// Mirrors Go <c>Server.applyOptions()</c>.
/// </summary>
internal void ApplyOptions(ReloadContext context, IReadOnlyList<IReloadOption> options)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(options);
var reloadLogging = false;
var reloadAuthorization = false;
var reloadClusterPermissions = false;
var reloadClientTraceLevel = false;
var statsChanged = false;
var jetStreamChanged = false;
var jetStreamEnabled = false;
ClusterReloadOption? clusterOption = null;
foreach (var option in options)
{
option.Apply(this);
if (option.IsLoggingChange())
reloadLogging = true;
if (option.IsTraceLevelChange())
reloadClientTraceLevel = true;
if (option.IsAuthChange())
reloadAuthorization = true;
if (option.IsClusterPermsChange())
reloadClusterPermissions = true;
if (option.IsClusterPoolSizeOrAccountsChange() && option is ClusterReloadOption co)
clusterOption = co;
if (option.IsStatszChange())
statsChanged = true;
if (option.IsJetStreamChange() && option is JetStreamReloadOption js)
{
jetStreamChanged = true;
jetStreamEnabled = js.Value;
}
}
if (reloadLogging)
ConfigureLogger();
if (reloadClientTraceLevel)
ReloadClientTraceLevel();
if (reloadAuthorization)
ReloadAuthorization();
if (reloadClusterPermissions)
ReloadClusterPermissions(context.OldClusterPermissions);
var updatedOptions = GetOpts();
if (clusterOption != null)
ReloadClusterPoolAndAccounts(clusterOption, updatedOptions);
if (jetStreamChanged && !jetStreamEnabled)
Noticef("Reloaded: JetStream disabled");
if (statsChanged)
Noticef("Reloaded: stats-related settings updated");
var digest = updatedOptions.ConfigDigest();
if (string.IsNullOrEmpty(digest))
Noticef("Reloaded server configuration");
else
Noticef("Reloaded server configuration ({0})", digest);
}
/// <summary>
/// Disconnects clients that no longer satisfy pinned-certificate policy.
/// Mirrors Go <c>Server.recheckPinnedCerts()</c>.
/// </summary>
internal void RecheckPinnedCerts(ServerOptions currentOptions, ServerOptions newOptions)
{
ArgumentNullException.ThrowIfNull(currentOptions);
ArgumentNullException.ThrowIfNull(newOptions);
var disconnectClients = new List<ClientConnection>();
var protocolPinnedSets = new Dictionary<ClientConnectionType, PinnedCertSet?>();
if (!PinnedCertSetEquals(newOptions.TlsPinnedCerts, currentOptions.TlsPinnedCerts))
protocolPinnedSets[ClientConnectionType.Nats] = newOptions.TlsPinnedCerts;
if (!PinnedCertSetEquals(newOptions.Mqtt.TlsPinnedCerts, currentOptions.Mqtt.TlsPinnedCerts))
protocolPinnedSets[ClientConnectionType.Mqtt] = newOptions.Mqtt.TlsPinnedCerts;
if (!PinnedCertSetEquals(newOptions.Websocket.TlsPinnedCerts, currentOptions.Websocket.TlsPinnedCerts))
protocolPinnedSets[ClientConnectionType.WebSocket] = newOptions.Websocket.TlsPinnedCerts;
_mu.EnterReadLock();
try
{
foreach (var client in _clients.Values)
{
if (client.Kind != ClientKind.Client)
continue;
if (protocolPinnedSets.TryGetValue(client.ClientType(), out var pinned) &&
!MatchesPinnedCert(client, pinned))
{
disconnectClients.Add(client);
}
}
if (!PinnedCertSetEquals(newOptions.LeafNode.TlsPinnedCerts, currentOptions.LeafNode.TlsPinnedCerts))
{
foreach (var leaf in _leafs.Values)
{
if (leaf.Kind == ClientKind.Leaf && !MatchesPinnedCert(leaf, newOptions.LeafNode.TlsPinnedCerts))
disconnectClients.Add(leaf);
}
}
if (!PinnedCertSetEquals(newOptions.Cluster.TlsPinnedCerts, currentOptions.Cluster.TlsPinnedCerts))
{
ForEachRoute(route =>
{
if (!MatchesPinnedCert(route, newOptions.Cluster.TlsPinnedCerts))
disconnectClients.Add(route);
});
}
}
finally
{
_mu.ExitReadLock();
}
if (_gateway.Enabled &&
!PinnedCertSetEquals(newOptions.Gateway.TlsPinnedCerts, currentOptions.Gateway.TlsPinnedCerts))
{
_gateway.AcquireReadLock();
try
{
foreach (var outbound in _gateway.Out.Values)
{
if (!MatchesPinnedCert(outbound, newOptions.Gateway.TlsPinnedCerts))
disconnectClients.Add(outbound);
}
foreach (var inbound in _gateway.In.Values)
{
if (inbound.Kind == ClientKind.Gateway &&
!MatchesPinnedCert(inbound, newOptions.Gateway.TlsPinnedCerts))
{
disconnectClients.Add(inbound);
}
}
}
finally
{
_gateway.ReleaseReadLock();
}
}
var unique = disconnectClients
.Where(static c => c != null)
.DistinctBy(static c => c.Cid)
.ToList();
if (unique.Count == 0)
return;
Noticef("Disconnect {0} clients due to pinned certs reload", unique.Count);
foreach (var client in unique)
client.CloseConnection(ClosedState.TlsHandshakeError);
}
/// <summary>
/// Signals the internal event-send loop to refresh state.
/// Mirrors Go <c>Server.resetInternalLoopInfo()</c>.
/// </summary>
internal void ResetInternalLoopInfo()
{
System.Threading.Channels.Channel<bool>? resetChannel;
_mu.EnterReadLock();
try
{
resetChannel = _sys?.ResetChannel;
}
finally
{
_mu.ExitReadLock();
}
if (resetChannel != null)
resetChannel.Writer.TryWrite(true);
}
/// <summary>
/// Recomputes per-client trace flags after logging-level changes.
/// Mirrors Go <c>Server.reloadClientTraceLevel()</c>.
/// </summary>
internal void ReloadClientTraceLevel()
{
var opts = GetOpts();
if (opts.NoLog)
return;
var clients = new List<ClientConnection>();
_mu.EnterReadLock();
try
{
clients.AddRange(_clients.Values);
clients.AddRange(_leafs.Values);
ForEachRoute(route => clients.Add(route));
}
finally
{
_mu.ExitReadLock();
}
lock (_grMu)
{
clients.AddRange(_grTmpClients.Values);
}
_gateway.AcquireReadLock();
try
{
clients.AddRange(_gateway.In.Values);
clients.AddRange(_gateway.Outo);
}
finally
{
_gateway.ReleaseReadLock();
}
foreach (var client in clients.DistinctBy(static c => c.Cid))
client.SetTraceLevel();
}
/// <summary>
/// Reloads authorization state and disconnects no-longer-authorized clients.
/// Mirrors Go <c>Server.reloadAuthorization()</c>.
/// </summary>
internal void ReloadAuthorization()
{
var opts = GetOpts();
var deletedAccounts = new List<Account>();
var movedClients = new List<ClientConnection>();
var clientsToCheck = new List<ClientConnection>();
var routes = new List<ClientConnection>();
System.Threading.Channels.Channel<bool>? resetChannel = null;
_mu.EnterWriteLock();
try
{
if (_trustedKeys == null)
{
var configuredAccounts = new HashSet<string>(
opts.Accounts.Select(static a => a.GetName()),
StringComparer.Ordinal);
foreach (var kvp in _accounts.ToArray())
{
var accountName = kvp.Key;
var account = kvp.Value;
if (accountName == ServerConstants.DefaultGlobalAccount ||
accountName == ServerConstants.DefaultSystemAccount)
{
continue;
}
if (!configuredAccounts.Contains(accountName))
{
deletedAccounts.Add(account);
_accounts.TryRemove(accountName, out _);
}
}
var (_, configureError) = ConfigureAccounts(true);
if (configureError != null)
Errorf("reloadAuthorization: configureAccounts failed: {0}", configureError.Message);
ConfigureAuthorization();
}
else if (opts.AccountResolver != null)
{
var resolverError = ConfigureResolver();
if (resolverError != null)
Errorf("reloadAuthorization: configureResolver failed: {0}", resolverError.Message);
if (_accResolver is MemoryAccountResolver)
CheckResolvePreloads();
}
foreach (var client in _clients.Values)
{
if (ClientHasMovedToDifferentAccount(client))
movedClients.Add(client);
else
clientsToCheck.Add(client);
}
ForEachRoute(route => routes.Add(route));
if (_sys?.Account != null && !opts.NoSystemAccount)
_accounts[_sys.Account.Name] = _sys.Account;
foreach (var account in _accounts.Values)
{
foreach (var client in account.GetClients())
{
if (client.Kind != ClientKind.Client && client.Kind != ClientKind.Leaf)
clientsToCheck.Add(client);
}
}
resetChannel = _sys?.ResetChannel;
}
finally
{
_mu.ExitWriteLock();
}
foreach (var account in deletedAccounts)
account.ClearEventing();
if (resetChannel != null)
resetChannel.Writer.TryWrite(true);
foreach (var client in movedClients.DistinctBy(static c => c.Cid))
client.CloseConnection(ClosedState.ClientClosed);
foreach (var client in clientsToCheck.DistinctBy(static c => c.Cid))
{
if ((client.Kind == ClientKind.Client || client.Kind == ClientKind.Leaf) && !IsClientAuthorized(client))
{
client.AuthViolation();
continue;
}
if (client.Kind == ClientKind.Router && !IsRouterAuthorized(client))
{
client.SetNoReconnect();
client.AuthViolation();
}
}
foreach (var route in routes.DistinctBy(static c => c.Cid))
{
if (!IsRouterAuthorized(route))
{
route.SetNoReconnect();
route.AuthViolation();
}
}
AccountResolver()?.Reload();
}
/// <summary>
/// Returns true if a client's configured user/nkey now maps to a different account.
/// Mirrors Go <c>Server.clientHasMovedToDifferentAccount()</c>.
/// </summary>
internal bool ClientHasMovedToDifferentAccount(ClientConnection connection)
{
ArgumentNullException.ThrowIfNull(connection);
NkeyUser? newNkeyUser = null;
User? newUser = null;
var nkey = connection.GetNkey();
if (!string.IsNullOrEmpty(nkey))
{
if (_nkeys != null)
_nkeys.TryGetValue(nkey, out newNkeyUser);
}
else
{
var username = connection.GetUsername();
if (!string.IsNullOrEmpty(username) && _users != null)
_users.TryGetValue(username, out newUser);
else if (string.IsNullOrEmpty(username))
return false;
}
var currentAccountName = connection.GetAccount()?.Name ?? string.Empty;
if (newNkeyUser?.Account != null)
return !string.Equals(currentAccountName, newNkeyUser.Account.Name, StringComparison.Ordinal);
if (newUser?.Account != null)
return !string.Equals(currentAccountName, newUser.Account.Name, StringComparison.Ordinal);
// User or nkey no longer exists in current config.
return true;
}
/// <summary>
/// Reloads cluster route permissions and propagates updated INFO to routes.
/// Mirrors Go <c>Server.reloadClusterPermissions()</c>.
/// </summary>
internal void ReloadClusterPermissions(RoutePermissions? oldPermissions)
{
_ = oldPermissions;
RoutePermissions? newPermissions;
List<ClientConnection> routes = [];
byte[] infoJson;
_mu.EnterWriteLock();
try
{
newPermissions = GetOpts().Cluster.Permissions;
_routeInfo.Import = newPermissions?.Import;
_routeInfo.Export = newPermissions?.Export;
infoJson = GenerateInfoJson(_routeInfo);
ForEachRoute(route => routes.Add(route));
}
finally
{
_mu.ExitWriteLock();
}
foreach (var route in routes.DistinctBy(static c => c.Cid))
{
var protocol = route.GetOpts().Protocol;
if (protocol < ServerProtocol.RouteProtoInfo)
{
route.CloseConnection(ClosedState.RouteRemoved);
continue;
}
route.EnqueueProto(infoJson);
}
}
/// <summary>
/// Reloads route-pool size and pinned-account route assignments.
/// Mirrors Go <c>Server.reloadClusterPoolAndAccounts()</c>.
/// </summary>
internal void ReloadClusterPoolAndAccounts(ClusterReloadOption clusterOption, ServerOptions options)
{
ArgumentNullException.ThrowIfNull(clusterOption);
ArgumentNullException.ThrowIfNull(options);
var routesToClose = new HashSet<ClientConnection>();
_mu.EnterWriteLock();
try
{
_routesReject = true;
if (clusterOption.AccountsAdded.Count > 0)
{
_accRoutes ??= new Dictionary<string, Dictionary<string, ClientConnection>>(StringComparer.Ordinal);
foreach (var accountName in clusterOption.AccountsAdded)
{
if (!_accRoutes.ContainsKey(accountName))
_accRoutes[accountName] = new Dictionary<string, ClientConnection>(StringComparer.Ordinal);
}
}
if (clusterOption.AccountsRemoved.Count > 0 && _accRoutes != null)
{
foreach (var accountName in clusterOption.AccountsRemoved)
{
if (_accRoutes.TryGetValue(accountName, out var remoteRoutes))
{
foreach (var route in remoteRoutes.Values)
{
route.SetNoReconnect();
routesToClose.Add(route);
}
}
}
}
if (clusterOption.PoolSizeChanged)
{
ForEachRoute(route =>
{
route.SetNoReconnect();
routesToClose.Add(route);
});
}
}
finally
{
_mu.ExitWriteLock();
}
foreach (var route in routesToClose)
route.CloseConnection(ClosedState.RouteRemoved);
_mu.EnterWriteLock();
try
{
_accAddedReqId = string.Empty;
_accAddedCh = null;
if (_accRoutes != null)
{
foreach (var accountName in clusterOption.AccountsRemoved)
_accRoutes.Remove(accountName);
}
if (options.Cluster.PinnedAccounts.Count == 0)
_accRoutes = null;
if (options.Cluster.PoolSize > 0)
{
_routesPoolSize = options.Cluster.PoolSize;
_routeInfo.RoutePoolSize = options.Cluster.PoolSize;
}
else
{
_routesPoolSize = 1;
_routeInfo.RoutePoolSize = 0;
}
if (clusterOption.PoolSizeChanged || clusterOption.AccountsRemoved.Count > 0)
{
foreach (var account in _accounts.Values)
SetRouteInfo(account);
}
_routesReject = false;
}
finally
{
_mu.ExitWriteLock();
}
}
private static bool OptionValueEquals(string optionName, object? oldValue, object? newValue)
{
if (ReferenceEquals(oldValue, newValue))
return true;
if (oldValue is null || newValue is null)
return false;
switch (oldValue)
{
case string oldString when newValue is string newString:
return string.Equals(oldString, newString, StringComparison.Ordinal);
case bool oldBool when newValue is bool newBool:
return oldBool == newBool;
case int oldInt when newValue is int newInt:
return oldInt == newInt;
case long oldLong when newValue is long newLong:
return oldLong == newLong;
case ulong oldUlong when newValue is ulong newUlong:
return oldUlong == newUlong;
case double oldDouble when newValue is double newDouble:
return oldDouble.Equals(newDouble);
case TimeSpan oldTimeSpan when newValue is TimeSpan newTimeSpan:
return oldTimeSpan == newTimeSpan;
case List<string> oldStrings when newValue is List<string> newStrings:
return oldStrings.SequenceEqual(newStrings, StringComparer.Ordinal);
case Dictionary<string, string> oldMap when newValue is Dictionary<string, string> newMap:
return oldMap.Count == newMap.Count &&
oldMap.All(entry =>
newMap.TryGetValue(entry.Key, out var value) &&
string.Equals(value, entry.Value, StringComparison.Ordinal));
case List<Uri> oldUris when newValue is List<Uri> newUris:
return oldUris.Count == newUris.Count &&
oldUris.Zip(newUris, static (oldUri, newUri) => ServerUtilities.UrlsAreEqual(oldUri, newUri))
.All(static same => same);
case PinnedCertSet oldPinned when newValue is PinnedCertSet newPinned:
return PinnedCertSetEquals(oldPinned, newPinned);
case CompressionOpts oldCompression when newValue is CompressionOpts newCompression:
return CompressOptsEqual(oldCompression, newCompression);
case GatewayOpts oldGateway when newValue is GatewayOpts newGateway:
return GatewayEquivalentForReload(oldGateway, newGateway);
case LeafNodeOpts oldLeaf when newValue is LeafNodeOpts newLeaf:
return LeafNodeReloadCompare(oldLeaf, newLeaf).Equivalent;
case ClusterOpts oldCluster when newValue is ClusterOpts newCluster:
return JsonEquivalent(oldCluster, newCluster);
case WebsocketOpts oldWebsocket when newValue is WebsocketOpts newWebsocket:
return JsonEquivalent(oldWebsocket, newWebsocket);
case MqttOpts oldMqtt when newValue is MqttOpts newMqtt:
return JsonEquivalent(oldMqtt, newMqtt);
default:
return JsonEquivalent(oldValue, newValue);
}
}
private static bool JsonEquivalent(object left, object right)
{
try
{
var leftJson = JsonSerializer.Serialize(left);
var rightJson = JsonSerializer.Serialize(right);
return string.Equals(leftJson, rightJson, StringComparison.Ordinal);
}
catch
{
return Equals(left, right);
}
}
private static T CloneByJson<T>(T value)
{
var json = JsonSerializer.Serialize(value);
var clone = JsonSerializer.Deserialize<T>(json);
return clone ?? throw new InvalidOperationException($"Failed to clone {typeof(T).Name} for reload comparison.");
}
private static bool GatewayEquivalentForReload(GatewayOpts oldValue, GatewayOpts newValue)
{
var oldGateway = CloneByJson(oldValue);
var newGateway = CloneByJson(newValue);
oldGateway.TlsConfig = null;
oldGateway.TlsConfigOpts = null;
newGateway.TlsConfig = null;
newGateway.TlsConfigOpts = null;
oldGateway.Gateways = ConfigReloader.CopyRemoteGWConfigsWithoutTLSConfig(oldGateway.Gateways) ?? [];
newGateway.Gateways = ConfigReloader.CopyRemoteGWConfigsWithoutTLSConfig(newGateway.Gateways) ?? [];
return JsonEquivalent(oldGateway, newGateway);
}
private static (bool Equivalent, bool TlsFirstChanged, bool CompressionChanged, bool DisabledChanged)
LeafNodeReloadCompare(LeafNodeOpts oldValue, LeafNodeOpts newValue)
{
var oldLeaf = CloneByJson(oldValue);
var newLeaf = CloneByJson(newValue);
oldLeaf.TlsConfig = null;
oldLeaf.TlsConfigOpts = null;
newLeaf.TlsConfig = null;
newLeaf.TlsConfigOpts = null;
var tlsFirstChanged =
oldLeaf.TlsHandshakeFirst != newLeaf.TlsHandshakeFirst ||
oldLeaf.TlsHandshakeFirstFallback != newLeaf.TlsHandshakeFirstFallback;
if (tlsFirstChanged)
{
oldLeaf.TlsHandshakeFirst = false;
newLeaf.TlsHandshakeFirst = false;
oldLeaf.TlsHandshakeFirstFallback = default;
newLeaf.TlsHandshakeFirstFallback = default;
}
else if (oldLeaf.Remotes.Count == newLeaf.Remotes.Count)
{
for (var i = 0; i < oldLeaf.Remotes.Count; i++)
{
if (oldLeaf.Remotes[i].TlsHandshakeFirst != newLeaf.Remotes[i].TlsHandshakeFirst)
{
tlsFirstChanged = true;
break;
}
}
}
var compressionChanged = !CompressOptsEqual(oldLeaf.Compression, newLeaf.Compression);
if (compressionChanged)
{
oldLeaf.Compression = new CompressionOpts();
newLeaf.Compression = new CompressionOpts();
}
else if (oldLeaf.Remotes.Count == newLeaf.Remotes.Count)
{
for (var i = 0; i < oldLeaf.Remotes.Count; i++)
{
if (!CompressOptsEqual(oldLeaf.Remotes[i].Compression, newLeaf.Remotes[i].Compression))
{
compressionChanged = true;
break;
}
}
}
var disabledChanged = false;
if (oldLeaf.Remotes.Count == newLeaf.Remotes.Count)
{
for (var i = 0; i < oldLeaf.Remotes.Count; i++)
{
if (oldLeaf.Remotes[i].Disabled != newLeaf.Remotes[i].Disabled)
{
disabledChanged = true;
break;
}
}
}
oldLeaf.Remotes = ConfigReloader.CopyRemoteLNConfigForReloadCompare(oldLeaf.Remotes) ?? [];
newLeaf.Remotes = ConfigReloader.CopyRemoteLNConfigForReloadCompare(newLeaf.Remotes) ?? [];
if (LeafRemotesEquivalent(oldLeaf.Remotes, newLeaf.Remotes))
{
oldLeaf.Remotes = [];
newLeaf.Remotes = [];
}
if (LeafUsersEquivalent(oldLeaf.Users, newLeaf.Users))
{
oldLeaf.Users = null;
newLeaf.Users = null;
}
return (JsonEquivalent(oldLeaf, newLeaf), tlsFirstChanged, compressionChanged, disabledChanged);
}
private static bool LeafUsersEquivalent(List<User>? oldUsers, List<User>? newUsers)
{
if (oldUsers == null && newUsers == null)
return true;
if (oldUsers == null || newUsers == null)
return false;
if (oldUsers.Count != newUsers.Count)
return false;
var oldByUsername = oldUsers.ToDictionary(static user => user.Username, StringComparer.Ordinal);
var newByUsername = newUsers.ToDictionary(static user => user.Username, StringComparer.Ordinal);
foreach (var (username, oldUser) in oldByUsername)
{
if (!newByUsername.TryGetValue(username, out var newUser))
return false;
var oldAccountName = oldUser.Account?.Name ?? string.Empty;
var newAccountName = newUser.Account?.Name ?? string.Empty;
if (!string.Equals(oldUser.Password, newUser.Password, StringComparison.Ordinal) ||
!string.Equals(oldAccountName, newAccountName, StringComparison.Ordinal))
{
return false;
}
}
return true;
}
private static bool LeafRemotesEquivalent(IReadOnlyList<RemoteLeafOpts> oldRemotes, IReadOnlyList<RemoteLeafOpts> newRemotes)
{
if (oldRemotes.Count != newRemotes.Count)
return false;
var remaining = new List<RemoteLeafOpts>(newRemotes.Count);
foreach (var remote in newRemotes)
remaining.Add(NormalizeLeafRemoteForCompare(remote));
foreach (var oldRemote in oldRemotes)
{
var normalizedOld = NormalizeLeafRemoteForCompare(oldRemote);
var matchIndex = remaining.FindIndex(candidate => JsonEquivalent(normalizedOld, candidate));
if (matchIndex < 0)
return false;
remaining.RemoveAt(matchIndex);
}
return remaining.Count == 0;
}
private static RemoteLeafOpts NormalizeLeafRemoteForCompare(RemoteLeafOpts remote)
{
var normalized = CloneByJson(remote);
if (string.IsNullOrEmpty(normalized.LocalAccount))
normalized.LocalAccount = ServerConstants.DefaultGlobalAccount;
return normalized;
}
private static bool PinnedCertSetEquals(PinnedCertSet? left, PinnedCertSet? right)
{
if (ReferenceEquals(left, right))
return true;
if (left is null || right is null)
return false;
return left.SetEquals(right);
}
private static bool MatchesPinnedCert(ClientConnection client, PinnedCertSet? pinnedCerts)
{
return client.MatchesPinnedCert(pinnedCerts);
}
}
internal sealed class ReloadContext
{
public ReloadContext(RoutePermissions? oldClusterPermissions)
{
OldClusterPermissions = oldClusterPermissions;
}
public RoutePermissions? OldClusterPermissions { get; }
}