feat(monitoring+events): add connz filtering, event payloads, and message trace context (E12+E13+E14)

- Add ConnzHandler with sorting, filtering, pagination, CID lookup, and closed connection ring buffer - Add full Go events.go parity types (ConnectEventMsg, DisconnectEventMsg, ServerStatsMsg, etc.) - Add MessageTraceContext for per-message trace propagation with header parsing - 74 new tests (17 ConnzFilter + 16 EventPayload + 41 MessageTraceContext)
2026-02-24 16:17:21 -05:00
parent 37d3cc29ea
commit 94878d3dcc
10 changed files with 2595 additions and 15 deletions
--- a/src/NATS.Server/Events/EventJsonContext.cs
+++ b/src/NATS.Server/Events/EventJsonContext.cs
@@ -5,8 +5,10 @@ namespace NATS.Server.Events;
 [JsonSerializable(typeof(ConnectEventMsg))]
 [JsonSerializable(typeof(DisconnectEventMsg))]
 [JsonSerializable(typeof(AccountNumConns))]
+[JsonSerializable(typeof(AccNumConnsReq))]
 [JsonSerializable(typeof(ServerStatsMsg))]
 [JsonSerializable(typeof(ShutdownEventMsg))]
 [JsonSerializable(typeof(LameDuckEventMsg))]
 [JsonSerializable(typeof(AuthErrorEventMsg))]
+[JsonSerializable(typeof(OcspPeerRejectEventMsg))]
 internal partial class EventJsonContext : JsonSerializerContext;
--- a/src/NATS.Server/Events/EventTypes.cs
+++ b/src/NATS.Server/Events/EventTypes.cs
@@ -4,6 +4,7 @@ namespace NATS.Server.Events;

 /// <summary>
 /// Server identity block embedded in all system events.
+/// Go reference: events.go:249-265 ServerInfo struct.
 /// </summary>
 public sealed class EventServerInfo
 {
@@ -29,17 +30,34 @@ public sealed class EventServerInfo
    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
    public string? Version { get; set; }

+    [JsonPropertyName("tags")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string[]? Tags { get; set; }
+
+    [JsonPropertyName("metadata")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public Dictionary<string, string>? Metadata { get; set; }
+
+    [JsonPropertyName("jetstream")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public bool JetStream { get; set; }
+
+    [JsonPropertyName("flags")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public ulong Flags { get; set; }
+
    [JsonPropertyName("seq")]
    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
    public ulong Seq { get; set; }

-    [JsonPropertyName("tags")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public Dictionary<string, string>? Tags { get; set; }
+    [JsonPropertyName("time")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public DateTime Time { get; set; }
 }

 /// <summary>
 /// Client identity block for connect/disconnect events.
+/// Go reference: events.go:308-331 ClientInfo struct.
 /// </summary>
 public sealed class EventClientInfo
 {
@@ -62,6 +80,14 @@ public sealed class EventClientInfo
    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
    public string? Account { get; set; }

+    [JsonPropertyName("svc")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Service { get; set; }
+
+    [JsonPropertyName("user")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? User { get; set; }
+
    [JsonPropertyName("name")]
    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
    public string? Name { get; set; }
@@ -77,8 +103,56 @@ public sealed class EventClientInfo
    [JsonPropertyName("rtt")]
    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
    public long RttNanos { get; set; }
+
+    [JsonPropertyName("server")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Server { get; set; }
+
+    [JsonPropertyName("cluster")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Cluster { get; set; }
+
+    [JsonPropertyName("alts")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string[]? Alternates { get; set; }
+
+    [JsonPropertyName("jwt")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Jwt { get; set; }
+
+    [JsonPropertyName("issuer_key")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? IssuerKey { get; set; }
+
+    [JsonPropertyName("name_tag")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? NameTag { get; set; }
+
+    [JsonPropertyName("tags")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string[]? Tags { get; set; }
+
+    [JsonPropertyName("kind")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Kind { get; set; }
+
+    [JsonPropertyName("client_type")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ClientType { get; set; }
+
+    [JsonPropertyName("client_id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? MqttClient { get; set; }
+
+    [JsonPropertyName("nonce")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Nonce { get; set; }
 }

+/// <summary>
+/// Message and byte count stats. Applicable for both sent and received.
+/// Go reference: events.go:407-410 MsgBytes, events.go:412-418 DataStats.
+/// </summary>
 public sealed class DataStats
 {
    [JsonPropertyName("msgs")]
@@ -86,6 +160,31 @@ public sealed class DataStats

    [JsonPropertyName("bytes")]
    public long Bytes { get; set; }
+
+    [JsonPropertyName("gateways")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public MsgBytesStats? Gateways { get; set; }
+
+    [JsonPropertyName("routes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public MsgBytesStats? Routes { get; set; }
+
+    [JsonPropertyName("leafs")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public MsgBytesStats? Leafs { get; set; }
+}
+
+/// <summary>
+/// Sub-stats for gateway/route/leaf message flow.
+/// Go reference: events.go:407-410 MsgBytes.
+/// </summary>
+public sealed class MsgBytesStats
+{
+    [JsonPropertyName("msgs")]
+    public long Msgs { get; set; }
+
+    [JsonPropertyName("bytes")]
+    public long Bytes { get; set; }
 }

 /// <summary>Client connect advisory. Go events.go:155-160.</summary>
@@ -139,7 +238,10 @@ public sealed class DisconnectEventMsg
    public string Reason { get; set; } = string.Empty;
 }

-/// <summary>Account connection count heartbeat. Go events.go:210-214.</summary>
+/// <summary>
+/// Account connection count heartbeat. Go events.go:210-214, 217-227.
+/// Includes the full AccountStat fields from Go.
+/// </summary>
 public sealed class AccountNumConns
 {
    public const string EventType = "io.nats.server.advisory.v1.account_connections";
@@ -156,23 +258,125 @@ public sealed class AccountNumConns
    [JsonPropertyName("server")]
    public EventServerInfo Server { get; set; } = new();

+    /// <summary>Account identifier. Go AccountStat.Account.</summary>
    [JsonPropertyName("acc")]
    public string AccountName { get; set; } = string.Empty;

+    /// <summary>Account display name. Go AccountStat.Name.</summary>
+    [JsonPropertyName("name")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Name { get; set; }
+
+    /// <summary>Current active connections. Go AccountStat.Conns.</summary>
    [JsonPropertyName("conns")]
    public int Connections { get; set; }

-    [JsonPropertyName("total_conns")]
-    public long TotalConnections { get; set; }
+    /// <summary>Active leaf node connections. Go AccountStat.LeafNodes.</summary>
+    [JsonPropertyName("leafnodes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public int LeafNodes { get; set; }

-    [JsonPropertyName("subs")]
-    public int Subscriptions { get; set; }
+    /// <summary>Total connections over time. Go AccountStat.TotalConns.</summary>
+    [JsonPropertyName("total_conns")]
+    public int TotalConnections { get; set; }
+
+    /// <summary>Active subscription count. Go AccountStat.NumSubs.</summary>
+    [JsonPropertyName("num_subscriptions")]
+    public uint NumSubscriptions { get; set; }

    [JsonPropertyName("sent")]
    public DataStats Sent { get; set; } = new();

    [JsonPropertyName("received")]
    public DataStats Received { get; set; } = new();
+
+    /// <summary>Slow consumer count. Go AccountStat.SlowConsumers.</summary>
+    [JsonPropertyName("slow_consumers")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public long SlowConsumers { get; set; }
+}
+
+/// <summary>
+/// Route statistics for server stats broadcast.
+/// Go reference: events.go:390-396 RouteStat.
+/// </summary>
+public sealed class RouteStat
+{
+    [JsonPropertyName("rid")]
+    public ulong Id { get; set; }
+
+    [JsonPropertyName("name")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Name { get; set; }
+
+    [JsonPropertyName("sent")]
+    public DataStats Sent { get; set; } = new();
+
+    [JsonPropertyName("received")]
+    public DataStats Received { get; set; } = new();
+
+    [JsonPropertyName("pending")]
+    public int Pending { get; set; }
+}
+
+/// <summary>
+/// Gateway statistics for server stats broadcast.
+/// Go reference: events.go:399-405 GatewayStat.
+/// </summary>
+public sealed class GatewayStat
+{
+    [JsonPropertyName("gwid")]
+    public ulong Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; } = "";
+
+    [JsonPropertyName("sent")]
+    public DataStats Sent { get; set; } = new();
+
+    [JsonPropertyName("received")]
+    public DataStats Received { get; set; } = new();
+
+    [JsonPropertyName("inbound_connections")]
+    public int InboundConnections { get; set; }
+}
+
+/// <summary>
+/// Slow consumer breakdown statistics.
+/// Go reference: events.go:377 SlowConsumersStats.
+/// </summary>
+public sealed class SlowConsumersStats
+{
+    [JsonPropertyName("clients")]
+    public long Clients { get; set; }
+
+    [JsonPropertyName("routes")]
+    public long Routes { get; set; }
+
+    [JsonPropertyName("gateways")]
+    public long Gateways { get; set; }
+
+    [JsonPropertyName("leafs")]
+    public long Leafs { get; set; }
+}
+
+/// <summary>
+/// Stale connection breakdown statistics.
+/// Go reference: events.go:379 StaleConnectionStats.
+/// </summary>
+public sealed class StaleConnectionStats
+{
+    [JsonPropertyName("clients")]
+    public long Clients { get; set; }
+
+    [JsonPropertyName("routes")]
+    public long Routes { get; set; }
+
+    [JsonPropertyName("gateways")]
+    public long Gateways { get; set; }
+
+    [JsonPropertyName("leafs")]
+    public long Leafs { get; set; }
 }

 /// <summary>Server stats broadcast. Go events.go:150-153.</summary>
@@ -185,6 +389,9 @@ public sealed class ServerStatsMsg
    public ServerStatsData Stats { get; set; } = new();
 }

+/// <summary>
+/// Server stats data. Full parity with Go events.go:365-387 ServerStats.
+/// </summary>
 public sealed class ServerStatsData
 {
    [JsonPropertyName("start")]
@@ -198,6 +405,10 @@ public sealed class ServerStatsData
    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
    public int Cores { get; set; }

+    [JsonPropertyName("cpu")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public double Cpu { get; set; }
+
    [JsonPropertyName("connections")]
    public int Connections { get; set; }

@@ -211,6 +422,43 @@ public sealed class ServerStatsData
    [JsonPropertyName("subscriptions")]
    public long Subscriptions { get; set; }

+    /// <summary>Sent stats (msgs + bytes). Go ServerStats.Sent.</summary>
+    [JsonPropertyName("sent")]
+    public DataStats Sent { get; set; } = new();
+
+    /// <summary>Received stats (msgs + bytes). Go ServerStats.Received.</summary>
+    [JsonPropertyName("received")]
+    public DataStats Received { get; set; } = new();
+
+    [JsonPropertyName("slow_consumers")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public long SlowConsumers { get; set; }
+
+    [JsonPropertyName("slow_consumer_stats")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public SlowConsumersStats? SlowConsumerStats { get; set; }
+
+    [JsonPropertyName("stale_connections")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public long StaleConnections { get; set; }
+
+    [JsonPropertyName("stale_connection_stats")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public StaleConnectionStats? StaleConnectionStats { get; set; }
+
+    [JsonPropertyName("routes")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public RouteStat[]? Routes { get; set; }
+
+    [JsonPropertyName("gateways")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public GatewayStat[]? Gateways { get; set; }
+
+    [JsonPropertyName("active_servers")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+    public int ActiveServers { get; set; }
+
+    // Kept for backward compat — flat counters that mirror Sent/Received.
    [JsonPropertyName("in_msgs")]
    public long InMsgs { get; set; }

@@ -222,10 +470,6 @@ public sealed class ServerStatsData

    [JsonPropertyName("out_bytes")]
    public long OutBytes { get; set; }
-
-    [JsonPropertyName("slow_consumers")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-    public long SlowConsumers { get; set; }
 }

 /// <summary>Server shutdown notification.</summary>
@@ -268,3 +512,43 @@ public sealed class AuthErrorEventMsg
    [JsonPropertyName("reason")]
    public string Reason { get; set; } = string.Empty;
 }
+
+/// <summary>
+/// OCSP peer rejection advisory.
+/// Go reference: events.go:182-188 OCSPPeerRejectEventMsg.
+/// </summary>
+public sealed class OcspPeerRejectEventMsg
+{
+    public const string EventType = "io.nats.server.advisory.v1.ocsp_peer_reject";
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = EventType;
+
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = string.Empty;
+
+    [JsonPropertyName("timestamp")]
+    public DateTime Time { get; set; }
+
+    [JsonPropertyName("kind")]
+    public string Kind { get; set; } = "";
+
+    [JsonPropertyName("server")]
+    public EventServerInfo Server { get; set; } = new();
+
+    [JsonPropertyName("reason")]
+    public string Reason { get; set; } = string.Empty;
+}
+
+/// <summary>
+/// Account numeric connections request.
+/// Go reference: events.go:233-236 accNumConnsReq.
+/// </summary>
+public sealed class AccNumConnsReq
+{
+    [JsonPropertyName("server")]
+    public EventServerInfo Server { get; set; } = new();
+
+    [JsonPropertyName("acc")]
+    public string Account { get; set; } = string.Empty;
+}
--- a/src/NATS.Server/Events/InternalEventSystem.cs
+++ b/src/NATS.Server/Events/InternalEventSystem.cs
@@ -159,6 +159,16 @@ public sealed class InternalEventSystem : IAsyncDisposable
                Connections = _server.ClientCount,
                TotalConnections = Interlocked.Read(ref _server.Stats.TotalConnections),
                Subscriptions = SystemAccount.SubList.Count,
+                Sent = new DataStats
+                {
+                    Msgs = Interlocked.Read(ref _server.Stats.OutMsgs),
+                    Bytes = Interlocked.Read(ref _server.Stats.OutBytes),
+                },
+                Received = new DataStats
+                {
+                    Msgs = Interlocked.Read(ref _server.Stats.InMsgs),
+                    Bytes = Interlocked.Read(ref _server.Stats.InBytes),
+                },
                InMsgs = Interlocked.Read(ref _server.Stats.InMsgs),
                OutMsgs = Interlocked.Read(ref _server.Stats.OutMsgs),
                InBytes = Interlocked.Read(ref _server.Stats.InBytes),