feat(monitoring+events): add connz filtering, event payloads, and message trace context (E12+E13+E14)

- Add ConnzHandler with sorting, filtering, pagination, CID lookup, and closed connection ring buffer
- Add full Go events.go parity types (ConnectEventMsg, DisconnectEventMsg, ServerStatsMsg, etc.)
- Add MessageTraceContext for per-message trace propagation with header parsing
- 74 new tests (17 ConnzFilter + 16 EventPayload + 41 MessageTraceContext)
This commit is contained in:
Joseph Doherty
2026-02-24 16:17:21 -05:00
parent 37d3cc29ea
commit 94878d3dcc
10 changed files with 2595 additions and 15 deletions

View File

@@ -5,8 +5,10 @@ namespace NATS.Server.Events;
[JsonSerializable(typeof(ConnectEventMsg))]
[JsonSerializable(typeof(DisconnectEventMsg))]
[JsonSerializable(typeof(AccountNumConns))]
[JsonSerializable(typeof(AccNumConnsReq))]
[JsonSerializable(typeof(ServerStatsMsg))]
[JsonSerializable(typeof(ShutdownEventMsg))]
[JsonSerializable(typeof(LameDuckEventMsg))]
[JsonSerializable(typeof(AuthErrorEventMsg))]
[JsonSerializable(typeof(OcspPeerRejectEventMsg))]
internal partial class EventJsonContext : JsonSerializerContext;

View File

@@ -4,6 +4,7 @@ namespace NATS.Server.Events;
/// <summary>
/// Server identity block embedded in all system events.
/// Go reference: events.go:249-265 ServerInfo struct.
/// </summary>
public sealed class EventServerInfo
{
@@ -29,17 +30,34 @@ public sealed class EventServerInfo
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Version { get; set; }
[JsonPropertyName("tags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string[]? Tags { get; set; }
[JsonPropertyName("metadata")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Metadata { get; set; }
[JsonPropertyName("jetstream")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public bool JetStream { get; set; }
[JsonPropertyName("flags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public ulong Flags { get; set; }
[JsonPropertyName("seq")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public ulong Seq { get; set; }
[JsonPropertyName("tags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Tags { get; set; }
[JsonPropertyName("time")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public DateTime Time { get; set; }
}
/// <summary>
/// Client identity block for connect/disconnect events.
/// Go reference: events.go:308-331 ClientInfo struct.
/// </summary>
public sealed class EventClientInfo
{
@@ -62,6 +80,14 @@ public sealed class EventClientInfo
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Account { get; set; }
[JsonPropertyName("svc")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Service { get; set; }
[JsonPropertyName("user")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? User { get; set; }
[JsonPropertyName("name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Name { get; set; }
@@ -77,8 +103,56 @@ public sealed class EventClientInfo
[JsonPropertyName("rtt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public long RttNanos { get; set; }
[JsonPropertyName("server")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Server { get; set; }
[JsonPropertyName("cluster")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Cluster { get; set; }
[JsonPropertyName("alts")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string[]? Alternates { get; set; }
[JsonPropertyName("jwt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Jwt { get; set; }
[JsonPropertyName("issuer_key")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? IssuerKey { get; set; }
[JsonPropertyName("name_tag")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? NameTag { get; set; }
[JsonPropertyName("tags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string[]? Tags { get; set; }
[JsonPropertyName("kind")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Kind { get; set; }
[JsonPropertyName("client_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ClientType { get; set; }
[JsonPropertyName("client_id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? MqttClient { get; set; }
[JsonPropertyName("nonce")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Nonce { get; set; }
}
/// <summary>
/// Message and byte count stats. Applicable for both sent and received.
/// Go reference: events.go:407-410 MsgBytes, events.go:412-418 DataStats.
/// </summary>
public sealed class DataStats
{
[JsonPropertyName("msgs")]
@@ -86,6 +160,31 @@ public sealed class DataStats
[JsonPropertyName("bytes")]
public long Bytes { get; set; }
[JsonPropertyName("gateways")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public MsgBytesStats? Gateways { get; set; }
[JsonPropertyName("routes")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public MsgBytesStats? Routes { get; set; }
[JsonPropertyName("leafs")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public MsgBytesStats? Leafs { get; set; }
}
/// <summary>
/// Sub-stats for gateway/route/leaf message flow.
/// Go reference: events.go:407-410 MsgBytes.
/// </summary>
public sealed class MsgBytesStats
{
[JsonPropertyName("msgs")]
public long Msgs { get; set; }
[JsonPropertyName("bytes")]
public long Bytes { get; set; }
}
/// <summary>Client connect advisory. Go events.go:155-160.</summary>
@@ -139,7 +238,10 @@ public sealed class DisconnectEventMsg
public string Reason { get; set; } = string.Empty;
}
/// <summary>Account connection count heartbeat. Go events.go:210-214.</summary>
/// <summary>
/// Account connection count heartbeat. Go events.go:210-214, 217-227.
/// Includes the full AccountStat fields from Go.
/// </summary>
public sealed class AccountNumConns
{
public const string EventType = "io.nats.server.advisory.v1.account_connections";
@@ -156,23 +258,125 @@ public sealed class AccountNumConns
[JsonPropertyName("server")]
public EventServerInfo Server { get; set; } = new();
/// <summary>Account identifier. Go AccountStat.Account.</summary>
[JsonPropertyName("acc")]
public string AccountName { get; set; } = string.Empty;
/// <summary>Account display name. Go AccountStat.Name.</summary>
[JsonPropertyName("name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Name { get; set; }
/// <summary>Current active connections. Go AccountStat.Conns.</summary>
[JsonPropertyName("conns")]
public int Connections { get; set; }
[JsonPropertyName("total_conns")]
public long TotalConnections { get; set; }
/// <summary>Active leaf node connections. Go AccountStat.LeafNodes.</summary>
[JsonPropertyName("leafnodes")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public int LeafNodes { get; set; }
[JsonPropertyName("subs")]
public int Subscriptions { get; set; }
/// <summary>Total connections over time. Go AccountStat.TotalConns.</summary>
[JsonPropertyName("total_conns")]
public int TotalConnections { get; set; }
/// <summary>Active subscription count. Go AccountStat.NumSubs.</summary>
[JsonPropertyName("num_subscriptions")]
public uint NumSubscriptions { get; set; }
[JsonPropertyName("sent")]
public DataStats Sent { get; set; } = new();
[JsonPropertyName("received")]
public DataStats Received { get; set; } = new();
/// <summary>Slow consumer count. Go AccountStat.SlowConsumers.</summary>
[JsonPropertyName("slow_consumers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public long SlowConsumers { get; set; }
}
/// <summary>
/// Route statistics for server stats broadcast.
/// Go reference: events.go:390-396 RouteStat.
/// </summary>
public sealed class RouteStat
{
[JsonPropertyName("rid")]
public ulong Id { get; set; }
[JsonPropertyName("name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Name { get; set; }
[JsonPropertyName("sent")]
public DataStats Sent { get; set; } = new();
[JsonPropertyName("received")]
public DataStats Received { get; set; } = new();
[JsonPropertyName("pending")]
public int Pending { get; set; }
}
/// <summary>
/// Gateway statistics for server stats broadcast.
/// Go reference: events.go:399-405 GatewayStat.
/// </summary>
public sealed class GatewayStat
{
[JsonPropertyName("gwid")]
public ulong Id { get; set; }
[JsonPropertyName("name")]
public string Name { get; set; } = "";
[JsonPropertyName("sent")]
public DataStats Sent { get; set; } = new();
[JsonPropertyName("received")]
public DataStats Received { get; set; } = new();
[JsonPropertyName("inbound_connections")]
public int InboundConnections { get; set; }
}
/// <summary>
/// Slow consumer breakdown statistics.
/// Go reference: events.go:377 SlowConsumersStats.
/// </summary>
public sealed class SlowConsumersStats
{
[JsonPropertyName("clients")]
public long Clients { get; set; }
[JsonPropertyName("routes")]
public long Routes { get; set; }
[JsonPropertyName("gateways")]
public long Gateways { get; set; }
[JsonPropertyName("leafs")]
public long Leafs { get; set; }
}
/// <summary>
/// Stale connection breakdown statistics.
/// Go reference: events.go:379 StaleConnectionStats.
/// </summary>
public sealed class StaleConnectionStats
{
[JsonPropertyName("clients")]
public long Clients { get; set; }
[JsonPropertyName("routes")]
public long Routes { get; set; }
[JsonPropertyName("gateways")]
public long Gateways { get; set; }
[JsonPropertyName("leafs")]
public long Leafs { get; set; }
}
/// <summary>Server stats broadcast. Go events.go:150-153.</summary>
@@ -185,6 +389,9 @@ public sealed class ServerStatsMsg
public ServerStatsData Stats { get; set; } = new();
}
/// <summary>
/// Server stats data. Full parity with Go events.go:365-387 ServerStats.
/// </summary>
public sealed class ServerStatsData
{
[JsonPropertyName("start")]
@@ -198,6 +405,10 @@ public sealed class ServerStatsData
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public int Cores { get; set; }
[JsonPropertyName("cpu")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public double Cpu { get; set; }
[JsonPropertyName("connections")]
public int Connections { get; set; }
@@ -211,6 +422,43 @@ public sealed class ServerStatsData
[JsonPropertyName("subscriptions")]
public long Subscriptions { get; set; }
/// <summary>Sent stats (msgs + bytes). Go ServerStats.Sent.</summary>
[JsonPropertyName("sent")]
public DataStats Sent { get; set; } = new();
/// <summary>Received stats (msgs + bytes). Go ServerStats.Received.</summary>
[JsonPropertyName("received")]
public DataStats Received { get; set; } = new();
[JsonPropertyName("slow_consumers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public long SlowConsumers { get; set; }
[JsonPropertyName("slow_consumer_stats")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public SlowConsumersStats? SlowConsumerStats { get; set; }
[JsonPropertyName("stale_connections")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public long StaleConnections { get; set; }
[JsonPropertyName("stale_connection_stats")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public StaleConnectionStats? StaleConnectionStats { get; set; }
[JsonPropertyName("routes")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public RouteStat[]? Routes { get; set; }
[JsonPropertyName("gateways")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public GatewayStat[]? Gateways { get; set; }
[JsonPropertyName("active_servers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public int ActiveServers { get; set; }
// Kept for backward compat — flat counters that mirror Sent/Received.
[JsonPropertyName("in_msgs")]
public long InMsgs { get; set; }
@@ -222,10 +470,6 @@ public sealed class ServerStatsData
[JsonPropertyName("out_bytes")]
public long OutBytes { get; set; }
[JsonPropertyName("slow_consumers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public long SlowConsumers { get; set; }
}
/// <summary>Server shutdown notification.</summary>
@@ -268,3 +512,43 @@ public sealed class AuthErrorEventMsg
[JsonPropertyName("reason")]
public string Reason { get; set; } = string.Empty;
}
/// <summary>
/// OCSP peer rejection advisory.
/// Go reference: events.go:182-188 OCSPPeerRejectEventMsg.
/// </summary>
public sealed class OcspPeerRejectEventMsg
{
public const string EventType = "io.nats.server.advisory.v1.ocsp_peer_reject";
[JsonPropertyName("type")]
public string Type { get; set; } = EventType;
[JsonPropertyName("id")]
public string Id { get; set; } = string.Empty;
[JsonPropertyName("timestamp")]
public DateTime Time { get; set; }
[JsonPropertyName("kind")]
public string Kind { get; set; } = "";
[JsonPropertyName("server")]
public EventServerInfo Server { get; set; } = new();
[JsonPropertyName("reason")]
public string Reason { get; set; } = string.Empty;
}
/// <summary>
/// Account numeric connections request.
/// Go reference: events.go:233-236 accNumConnsReq.
/// </summary>
public sealed class AccNumConnsReq
{
[JsonPropertyName("server")]
public EventServerInfo Server { get; set; } = new();
[JsonPropertyName("acc")]
public string Account { get; set; } = string.Empty;
}

View File

@@ -159,6 +159,16 @@ public sealed class InternalEventSystem : IAsyncDisposable
Connections = _server.ClientCount,
TotalConnections = Interlocked.Read(ref _server.Stats.TotalConnections),
Subscriptions = SystemAccount.SubList.Count,
Sent = new DataStats
{
Msgs = Interlocked.Read(ref _server.Stats.OutMsgs),
Bytes = Interlocked.Read(ref _server.Stats.OutBytes),
},
Received = new DataStats
{
Msgs = Interlocked.Read(ref _server.Stats.InMsgs),
Bytes = Interlocked.Read(ref _server.Stats.InBytes),
},
InMsgs = Interlocked.Read(ref _server.Stats.InMsgs),
OutMsgs = Interlocked.Read(ref _server.Stats.OutMsgs),
InBytes = Interlocked.Read(ref _server.Stats.InBytes),

View File

@@ -0,0 +1,686 @@
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using NATS.Server.Events;
namespace NATS.Server.Internal;
/// <summary>
/// Header constants for NATS message tracing.
/// Go reference: msgtrace.go:28-33
/// </summary>
public static class MsgTraceHeaders
{
public const string TraceDest = "Nats-Trace-Dest";
public const string TraceDestDisabled = "trace disabled";
public const string TraceHop = "Nats-Trace-Hop";
public const string TraceOriginAccount = "Nats-Trace-Origin-Account";
public const string TraceOnly = "Nats-Trace-Only";
public const string TraceParent = "traceparent";
}
/// <summary>
/// Types of message trace events in the MsgTraceEvents list.
/// Go reference: msgtrace.go:54-61
/// </summary>
public static class MsgTraceTypes
{
public const string Ingress = "in";
public const string SubjectMapping = "sm";
public const string StreamExport = "se";
public const string ServiceImport = "si";
public const string JetStream = "js";
public const string Egress = "eg";
}
/// <summary>
/// Error messages used in message trace events.
/// Go reference: msgtrace.go:248-258
/// </summary>
public static class MsgTraceErrors
{
public const string OnlyNoSupport = "Not delivered because remote does not support message tracing";
public const string NoSupport = "Message delivered but remote does not support message tracing so no trace event generated from there";
public const string NoEcho = "Not delivered because of no echo";
public const string PubViolation = "Not delivered because publish denied for this subject";
public const string SubDeny = "Not delivered because subscription denies this subject";
public const string SubClosed = "Not delivered because subscription is closed";
public const string ClientClosed = "Not delivered because client is closed";
public const string AutoSubExceeded = "Not delivered because auto-unsubscribe exceeded";
}
/// <summary>
/// Represents the full trace event document published to the trace destination.
/// Go reference: msgtrace.go:63-68
/// </summary>
public sealed class MsgTraceEvent
{
[JsonPropertyName("server")]
public EventServerInfo Server { get; set; } = new();
[JsonPropertyName("request")]
public MsgTraceRequest Request { get; set; } = new();
[JsonPropertyName("hops")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public int Hops { get; set; }
[JsonPropertyName("events")]
public List<MsgTraceEntry> Events { get; set; } = [];
}
/// <summary>
/// The original request information captured for the trace.
/// Go reference: msgtrace.go:70-74
/// </summary>
public sealed class MsgTraceRequest
{
[JsonPropertyName("header")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string[]>? Header { get; set; }
[JsonPropertyName("msgsize")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public int MsgSize { get; set; }
}
/// <summary>
/// Base class for all trace event entries (ingress, egress, JS, etc.).
/// Go reference: msgtrace.go:83-86
/// </summary>
[JsonDerivedType(typeof(MsgTraceIngress))]
[JsonDerivedType(typeof(MsgTraceSubjectMapping))]
[JsonDerivedType(typeof(MsgTraceStreamExport))]
[JsonDerivedType(typeof(MsgTraceServiceImport))]
[JsonDerivedType(typeof(MsgTraceJetStreamEntry))]
[JsonDerivedType(typeof(MsgTraceEgress))]
public class MsgTraceEntry
{
[JsonPropertyName("type")]
public string Type { get; set; } = "";
[JsonPropertyName("ts")]
public DateTime Timestamp { get; set; } = DateTime.UtcNow;
}
/// <summary>
/// Ingress trace event recorded when a message first enters the server.
/// Go reference: msgtrace.go:88-96
/// </summary>
public sealed class MsgTraceIngress : MsgTraceEntry
{
[JsonPropertyName("kind")]
public int Kind { get; set; }
[JsonPropertyName("cid")]
public ulong Cid { get; set; }
[JsonPropertyName("name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Name { get; set; }
[JsonPropertyName("acc")]
public string Account { get; set; } = "";
[JsonPropertyName("subj")]
public string Subject { get; set; } = "";
[JsonPropertyName("error")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Error { get; set; }
}
/// <summary>
/// Subject mapping trace event.
/// Go reference: msgtrace.go:98-101
/// </summary>
public sealed class MsgTraceSubjectMapping : MsgTraceEntry
{
[JsonPropertyName("to")]
public string MappedTo { get; set; } = "";
}
/// <summary>
/// Stream export trace event.
/// Go reference: msgtrace.go:103-107
/// </summary>
public sealed class MsgTraceStreamExport : MsgTraceEntry
{
[JsonPropertyName("acc")]
public string Account { get; set; } = "";
[JsonPropertyName("to")]
public string To { get; set; } = "";
}
/// <summary>
/// Service import trace event.
/// Go reference: msgtrace.go:109-114
/// </summary>
public sealed class MsgTraceServiceImport : MsgTraceEntry
{
[JsonPropertyName("acc")]
public string Account { get; set; } = "";
[JsonPropertyName("from")]
public string From { get; set; } = "";
[JsonPropertyName("to")]
public string To { get; set; } = "";
}
/// <summary>
/// JetStream trace event.
/// Go reference: msgtrace.go:116-122
/// </summary>
public sealed class MsgTraceJetStreamEntry : MsgTraceEntry
{
[JsonPropertyName("stream")]
public string Stream { get; set; } = "";
[JsonPropertyName("subject")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Subject { get; set; }
[JsonPropertyName("nointerest")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public bool NoInterest { get; set; }
[JsonPropertyName("error")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Error { get; set; }
}
/// <summary>
/// Egress trace event recorded for each delivery target.
/// Go reference: msgtrace.go:124-138
/// </summary>
public sealed class MsgTraceEgress : MsgTraceEntry
{
[JsonPropertyName("kind")]
public int Kind { get; set; }
[JsonPropertyName("cid")]
public ulong Cid { get; set; }
[JsonPropertyName("name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Name { get; set; }
[JsonPropertyName("hop")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Hop { get; set; }
[JsonPropertyName("acc")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Account { get; set; }
[JsonPropertyName("sub")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Subscription { get; set; }
[JsonPropertyName("queue")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Queue { get; set; }
[JsonPropertyName("error")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Error { get; set; }
}
/// <summary>
/// Manages trace state as a message traverses the delivery pipeline.
/// Collects trace events and publishes the complete trace to the destination subject.
/// Go reference: msgtrace.go:260-273
/// </summary>
public sealed class MsgTraceContext
{
/// <summary>Kind constant for CLIENT connections.</summary>
public const int KindClient = 0;
/// <summary>Kind constant for ROUTER connections.</summary>
public const int KindRouter = 1;
/// <summary>Kind constant for GATEWAY connections.</summary>
public const int KindGateway = 2;
/// <summary>Kind constant for LEAF connections.</summary>
public const int KindLeaf = 3;
private int _ready;
private MsgTraceJetStreamEntry? _js;
/// <summary>
/// The destination subject where the trace event will be published.
/// </summary>
public string Destination { get; }
/// <summary>
/// The accumulated trace event with all recorded entries.
/// </summary>
public MsgTraceEvent Event { get; }
/// <summary>
/// Current hop identifier for this server.
/// </summary>
public string Hop { get; private set; } = "";
/// <summary>
/// Next hop identifier set before forwarding to routes/gateways/leafs.
/// </summary>
public string NextHop { get; private set; } = "";
/// <summary>
/// Whether to only trace the message without actually delivering it.
/// Go reference: msgtrace.go:271
/// </summary>
public bool TraceOnly { get; }
/// <summary>
/// Whether this trace context is active (non-null destination).
/// </summary>
public bool IsActive => !string.IsNullOrEmpty(Destination);
/// <summary>
/// The account to use when publishing the trace event.
/// </summary>
public string? AccountName { get; }
/// <summary>
/// Callback to publish the trace event. Set by the server.
/// </summary>
public Action<string, string?, object?>? PublishCallback { get; set; }
private MsgTraceContext(string destination, MsgTraceEvent evt, bool traceOnly, string? accountName, string hop)
{
Destination = destination;
Event = evt;
TraceOnly = traceOnly;
AccountName = accountName;
Hop = hop;
}
/// <summary>
/// Creates a new trace context from inbound message headers.
/// Parses Nats-Trace-Dest, Nats-Trace-Only, and Nats-Trace-Hop headers.
/// Go reference: msgtrace.go:332-492
/// </summary>
public static MsgTraceContext? Create(
ReadOnlyMemory<byte> headers,
ulong clientId,
string? clientName,
string accountName,
string subject,
int msgSize,
int clientKind = KindClient)
{
if (headers.Length == 0)
return null;
var parsedHeaders = ParseTraceHeaders(headers.Span);
if (parsedHeaders == null || parsedHeaders.Count == 0)
return null;
// Check for disabled trace
if (parsedHeaders.TryGetValue(MsgTraceHeaders.TraceDest, out var destValues)
&& destValues.Length > 0
&& destValues[0] == MsgTraceHeaders.TraceDestDisabled)
{
return null;
}
var dest = destValues?.Length > 0 ? destValues[0] : null;
if (string.IsNullOrEmpty(dest))
return null;
// Parse trace-only flag
bool traceOnly = false;
if (parsedHeaders.TryGetValue(MsgTraceHeaders.TraceOnly, out var onlyValues) && onlyValues.Length > 0)
{
var val = onlyValues[0].ToLowerInvariant();
traceOnly = val is "1" or "true" or "on";
}
// Parse hop from non-CLIENT connections
string hop = "";
if (clientKind != KindClient
&& parsedHeaders.TryGetValue(MsgTraceHeaders.TraceHop, out var hopValues)
&& hopValues.Length > 0)
{
hop = hopValues[0];
}
// Build ingress event
var evt = new MsgTraceEvent
{
Request = new MsgTraceRequest
{
Header = parsedHeaders,
MsgSize = msgSize,
},
Events =
[
new MsgTraceIngress
{
Type = MsgTraceTypes.Ingress,
Timestamp = DateTime.UtcNow,
Kind = clientKind,
Cid = clientId,
Name = clientName,
Account = accountName,
Subject = subject,
},
],
};
return new MsgTraceContext(dest, evt, traceOnly, accountName, hop);
}
/// <summary>
/// Sets an error on the ingress event.
/// Go reference: msgtrace.go:657-661
/// </summary>
public void SetIngressError(string error)
{
if (Event.Events.Count > 0 && Event.Events[0] is MsgTraceIngress ingress)
{
ingress.Error = error;
}
}
/// <summary>
/// Adds a subject mapping trace event.
/// Go reference: msgtrace.go:663-674
/// </summary>
public void AddSubjectMappingEvent(string mappedTo)
{
Event.Events.Add(new MsgTraceSubjectMapping
{
Type = MsgTraceTypes.SubjectMapping,
Timestamp = DateTime.UtcNow,
MappedTo = mappedTo,
});
}
/// <summary>
/// Adds an egress trace event for a delivery target.
/// Go reference: msgtrace.go:676-711
/// </summary>
public void AddEgressEvent(ulong clientId, string? clientName, int clientKind,
string? subscriptionSubject = null, string? queue = null, string? account = null, string? error = null)
{
var egress = new MsgTraceEgress
{
Type = MsgTraceTypes.Egress,
Timestamp = DateTime.UtcNow,
Kind = clientKind,
Cid = clientId,
Name = clientName,
Hop = string.IsNullOrEmpty(NextHop) ? null : NextHop,
Error = error,
};
NextHop = "";
// Set subscription and queue for CLIENT connections
if (clientKind == KindClient)
{
egress.Subscription = subscriptionSubject;
egress.Queue = queue;
}
// Set account if different from ingress account
if ((clientKind == KindClient || clientKind == KindLeaf) && account != null)
{
if (Event.Events.Count > 0 && Event.Events[0] is MsgTraceIngress ingress && account != ingress.Account)
{
egress.Account = account;
}
}
Event.Events.Add(egress);
}
/// <summary>
/// Adds a stream export trace event.
/// Go reference: msgtrace.go:713-728
/// </summary>
public void AddStreamExportEvent(string accountName, string to)
{
Event.Events.Add(new MsgTraceStreamExport
{
Type = MsgTraceTypes.StreamExport,
Timestamp = DateTime.UtcNow,
Account = accountName,
To = to,
});
}
/// <summary>
/// Adds a service import trace event.
/// Go reference: msgtrace.go:730-743
/// </summary>
public void AddServiceImportEvent(string accountName, string from, string to)
{
Event.Events.Add(new MsgTraceServiceImport
{
Type = MsgTraceTypes.ServiceImport,
Timestamp = DateTime.UtcNow,
Account = accountName,
From = from,
To = to,
});
}
/// <summary>
/// Adds a JetStream trace event for stream storage.
/// Go reference: msgtrace.go:745-757
/// </summary>
public void AddJetStreamEvent(string streamName)
{
_js = new MsgTraceJetStreamEntry
{
Type = MsgTraceTypes.JetStream,
Timestamp = DateTime.UtcNow,
Stream = streamName,
};
Event.Events.Add(_js);
}
/// <summary>
/// Updates the JetStream trace event with subject and interest info.
/// Go reference: msgtrace.go:759-772
/// </summary>
public void UpdateJetStreamEvent(string subject, bool noInterest)
{
if (_js == null) return;
_js.Subject = subject;
_js.NoInterest = noInterest;
_js.Timestamp = DateTime.UtcNow;
}
/// <summary>
/// Sets the hop header for forwarding to routes/gateways/leafs.
/// Increments the hop counter and builds the next hop id.
/// Go reference: msgtrace.go:646-655
/// </summary>
public void SetHopHeader()
{
Event.Hops++;
NextHop = string.IsNullOrEmpty(Hop)
? Event.Hops.ToString()
: $"{Hop}.{Event.Hops}";
}
/// <summary>
/// Sends the accumulated trace event from the JetStream path.
/// Delegates to SendEvent for the two-phase ready logic.
/// Go reference: msgtrace.go:774-786
/// </summary>
public void SendEventFromJetStream(string? error = null)
{
if (_js == null) return;
if (error != null) _js.Error = error;
SendEvent();
}
/// <summary>
/// Sends the accumulated trace event to the destination subject.
/// For non-JetStream paths, sends immediately. For JetStream paths,
/// uses a two-phase ready check: both the message delivery path and
/// the JetStream storage path must call SendEvent before the event
/// is actually published.
/// Go reference: msgtrace.go:788-799
/// </summary>
public void SendEvent()
{
if (_js != null)
{
var ready = Interlocked.Increment(ref _ready) == 2;
if (!ready) return;
}
PublishCallback?.Invoke(Destination, null, Event);
}
/// <summary>
/// Parses NATS headers looking for trace-related headers.
/// Returns null if no trace headers found.
/// Go reference: msgtrace.go:509-591
/// </summary>
internal static Dictionary<string, string[]>? ParseTraceHeaders(ReadOnlySpan<byte> hdr)
{
// Must start with NATS/1.0 header line
var hdrLine = "NATS/1.0 "u8;
if (hdr.Length < hdrLine.Length || !hdr[..hdrLine.Length].SequenceEqual(hdrLine))
{
// Also try NATS/1.0\r\n (status line without status code)
var hdrLine2 = "NATS/1.0\r\n"u8;
if (hdr.Length < hdrLine2.Length || !hdr[..hdrLine2.Length].SequenceEqual(hdrLine2))
return null;
}
bool traceDestFound = false;
bool traceParentFound = false;
var keys = new List<string>();
var vals = new List<string>();
// Skip the first line (status line)
int i = 0;
var crlf = "\r\n"u8;
var firstCrlf = hdr.IndexOf(crlf);
if (firstCrlf < 0) return null;
i = firstCrlf + 2;
while (i < hdr.Length)
{
// Find the colon delimiter
int colonIdx = -1;
for (int j = i; j < hdr.Length; j++)
{
if (hdr[j] == (byte)':')
{
colonIdx = j;
break;
}
if (hdr[j] == (byte)'\r' || hdr[j] == (byte)'\n')
break;
}
if (colonIdx < 0)
{
// Skip to next line
var nextCrlf = hdr[i..].IndexOf(crlf);
if (nextCrlf < 0) break;
i += nextCrlf + 2;
continue;
}
var keySpan = hdr[i..colonIdx];
i = colonIdx + 1;
// Skip leading whitespace in value
while (i < hdr.Length && (hdr[i] == (byte)' ' || hdr[i] == (byte)'\t'))
i++;
// Find end of value (CRLF)
int valStart = i;
var valCrlf = hdr[valStart..].IndexOf(crlf);
if (valCrlf < 0) break;
int valEnd = valStart + valCrlf;
// Trim trailing whitespace
while (valEnd > valStart && (hdr[valEnd - 1] == (byte)' ' || hdr[valEnd - 1] == (byte)'\t'))
valEnd--;
var valSpan = hdr[valStart..valEnd];
if (keySpan.Length > 0 && valSpan.Length > 0)
{
var key = Encoding.ASCII.GetString(keySpan);
var val = Encoding.ASCII.GetString(valSpan);
// Check for trace-dest header
if (!traceDestFound && key == MsgTraceHeaders.TraceDest)
{
if (val == MsgTraceHeaders.TraceDestDisabled)
return null; // Tracing explicitly disabled
traceDestFound = true;
}
// Check for traceparent header (case-insensitive)
else if (!traceParentFound && key.Equals(MsgTraceHeaders.TraceParent, StringComparison.OrdinalIgnoreCase))
{
// Parse W3C trace context: version-traceid-parentid-flags
var parts = val.Split('-');
if (parts.Length == 4 && parts[3].Length == 2)
{
if (int.TryParse(parts[3], System.Globalization.NumberStyles.HexNumber, null, out var flags)
&& (flags & 0x1) == 0x1)
{
traceParentFound = true;
}
}
}
keys.Add(key);
vals.Add(val);
}
i = valStart + valCrlf + 2;
}
if (!traceDestFound && !traceParentFound)
return null;
// Build the header map
var map = new Dictionary<string, string[]>(keys.Count);
for (int k = 0; k < keys.Count; k++)
{
if (map.TryGetValue(keys[k], out var existing))
{
var newArr = new string[existing.Length + 1];
existing.CopyTo(newArr, 0);
newArr[^1] = vals[k];
map[keys[k]] = newArr;
}
else
{
map[keys[k]] = [vals[k]];
}
}
return map;
}
}
/// <summary>
/// JSON serialization context for message trace types.
/// </summary>
[JsonSerializable(typeof(MsgTraceEvent))]
[JsonSerializable(typeof(MsgTraceRequest))]
[JsonSerializable(typeof(MsgTraceEntry))]
[JsonSerializable(typeof(MsgTraceIngress))]
[JsonSerializable(typeof(MsgTraceSubjectMapping))]
[JsonSerializable(typeof(MsgTraceStreamExport))]
[JsonSerializable(typeof(MsgTraceServiceImport))]
[JsonSerializable(typeof(MsgTraceJetStreamEntry))]
[JsonSerializable(typeof(MsgTraceEgress))]
internal partial class MsgTraceJsonContext : JsonSerializerContext;

View File

@@ -218,6 +218,18 @@ public sealed class ConnzOptions
public string MqttClient { get; set; } = "";
/// <summary>
/// When non-zero, returns only the connection with this CID.
/// Go reference: monitor.go ConnzOptions.CID.
/// </summary>
public ulong Cid { get; set; }
/// <summary>
/// Whether to include authorized user info.
/// Go reference: monitor.go ConnzOptions.Username.
/// </summary>
public bool Auth { get; set; }
public int Offset { get; set; }
public int Limit { get; set; } = 1024;

View File

@@ -16,6 +16,13 @@ public sealed class ConnzHandler(NatsServer server)
var connInfos = new List<ConnInfo>();
// If a specific CID is requested, search for that single connection
// Go reference: monitor.go Connz() — CID fast path
if (opts.Cid > 0)
{
return HandleSingleCid(opts, now);
}
// Collect open connections
if (opts.State is ConnState.Open or ConnState.All)
{
@@ -23,7 +30,7 @@ public sealed class ConnzHandler(NatsServer server)
connInfos.AddRange(clients.Select(c => BuildConnInfo(c, now, opts)));
}
// Collect closed connections
// Collect closed connections from the ring buffer
if (opts.State is ConnState.Closed or ConnState.All)
{
connInfos.AddRange(server.GetClosedClients().Select(c => BuildClosedConnInfo(c, now, opts)));
@@ -81,6 +88,59 @@ public sealed class ConnzHandler(NatsServer server)
};
}
/// <summary>
/// Handles a request for a single connection by CID.
/// Go reference: monitor.go Connz() — CID-specific path.
/// </summary>
private Connz HandleSingleCid(ConnzOptions opts, DateTime now)
{
// Search open connections first
var client = server.GetClients().FirstOrDefault(c => c.Id == opts.Cid);
if (client != null)
{
var info = BuildConnInfo(client, now, opts);
return new Connz
{
Id = server.ServerId,
Now = now,
NumConns = 1,
Total = 1,
Offset = 0,
Limit = 1,
Conns = [info],
};
}
// Search closed connections ring buffer
var closed = server.GetClosedClients().FirstOrDefault(c => c.Cid == opts.Cid);
if (closed != null)
{
var info = BuildClosedConnInfo(closed, now, opts);
return new Connz
{
Id = server.ServerId,
Now = now,
NumConns = 1,
Total = 1,
Offset = 0,
Limit = 1,
Conns = [info],
};
}
// Not found — return empty result
return new Connz
{
Id = server.ServerId,
Now = now,
NumConns = 0,
Total = 0,
Offset = 0,
Limit = 0,
Conns = [],
};
}
private static ConnInfo BuildConnInfo(NatsClient client, DateTime now, ConnzOptions opts)
{
var info = new ConnInfo
@@ -228,6 +288,12 @@ public sealed class ConnzHandler(NatsServer server)
if (q.TryGetValue("limit", out var limit) && int.TryParse(limit, out var l))
opts.Limit = l;
if (q.TryGetValue("cid", out var cid) && ulong.TryParse(cid, out var cidValue))
opts.Cid = cidValue;
if (q.TryGetValue("auth", out var auth))
opts.Auth = auth.ToString().ToLowerInvariant() is "1" or "true";
if (q.TryGetValue("mqtt_client", out var mqttClient))
opts.MqttClient = mqttClient.ToString();
@@ -243,10 +309,13 @@ public sealed class ConnzHandler(NatsServer server)
private static bool MatchesSubjectFilter(ConnInfo info, string filterSubject)
{
if (info.Subs.Any(s => SubjectMatch.MatchLiteral(s, filterSubject)))
// Go reference: monitor.go — matchLiteral(testSub, string(sub.subject))
// The filter subject is the literal, the subscription subject is the pattern
// (subscriptions may contain wildcards like orders.> that match the filter orders.new)
if (info.Subs.Any(s => SubjectMatch.MatchLiteral(filterSubject, s)))
return true;
return info.SubsDetail.Any(s => SubjectMatch.MatchLiteral(s.Subject, filterSubject));
return info.SubsDetail.Any(s => SubjectMatch.MatchLiteral(filterSubject, s.Subject));
}
private static string FormatRtt(TimeSpan rtt)