Fix E2E test gaps and add comprehensive E2E + parity test suites

- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status

src/NATS.Server/JetStream/Api/Handlers/ConsumerApiHandlers.cs

@@ -53,30 +53,57 @@ public static class ConsumerApiHandlers
             : JetStreamApiResponse.NotFound(subject);
     }
 
-    public static JetStreamApiResponse HandleNames(string subject, ConsumerManager consumerManager)
+    public static JetStreamApiResponse HandleNames(string subject, ReadOnlySpan<byte> payload, ConsumerManager consumerManager)
     {
         var stream = ParseStreamSubject(subject, NamesPrefix);
         if (stream == null)
             return JetStreamApiResponse.NotFound(subject);
 
+        var offset = ParseOffset(payload);
+        var all = consumerManager.ListNames(stream);
+        var page = offset >= all.Count ? [] : all.Skip(offset).ToList();
         return new JetStreamApiResponse
         {
-            ConsumerNames = consumerManager.ListNames(stream),
+            ConsumerNames = page,
+            PaginationTotal = all.Count,
+            PaginationOffset = offset,
         };
     }
 
-    public static JetStreamApiResponse HandleList(string subject, ConsumerManager consumerManager)
+    public static JetStreamApiResponse HandleList(string subject, ReadOnlySpan<byte> payload, ConsumerManager consumerManager)
     {
         var stream = ParseStreamSubject(subject, ListPrefix);
         if (stream == null)
             return JetStreamApiResponse.NotFound(subject);
 
+        var offset = ParseOffset(payload);
+        var all = consumerManager.ListConsumerInfos(stream);
+        var page = offset >= all.Count ? [] : all.Skip(offset).ToList();
         return new JetStreamApiResponse
         {
-            ConsumerNames = consumerManager.ListNames(stream),
+            ConsumerInfoList = page,
+            PaginationTotal = all.Count,
+            PaginationOffset = offset,
         };
     }
 
+    private static int ParseOffset(ReadOnlySpan<byte> payload)
+    {
+        if (payload.IsEmpty) return 0;
+        try
+        {
+            using var doc = JsonDocument.Parse(payload.ToArray());
+            if (doc.RootElement.TryGetProperty("offset", out var el) && el.TryGetInt32(out var v))
+                return Math.Max(v, 0);
+        }
+        catch (JsonException ex)
+        {
+            System.Diagnostics.Debug.WriteLine($"Malformed offset payload: {ex.Message}");
+        }
+
+        return 0;
+    }
+
     public static JetStreamApiResponse HandlePause(string subject, ReadOnlySpan<byte> payload, ConsumerManager consumerManager)
     {
         var parsed = ParseSubject(subject, PausePrefix);
@@ -254,15 +281,21 @@ public static class ConsumerApiHandlers
         {
             using var doc = JsonDocument.Parse(payload.ToArray());
             var root = doc.RootElement;
+
+            // The client wraps config in a "config" property (CreateConsumerRequest).
+            // Go reference: consumer.go — CreateConsumerRequest { Config ConsumerConfig `json:"config"` }
+            var configEl = root.TryGetProperty("config", out var nested) ? nested : root;
             var config = new ConsumerConfig();
 
-            if (root.TryGetProperty("durable_name", out var durableEl))
+            if (configEl.TryGetProperty("durable_name", out var durableEl))
                 config.DurableName = durableEl.GetString() ?? string.Empty;
+            else if (configEl.TryGetProperty("name", out var nameEl))
+                config.DurableName = nameEl.GetString() ?? string.Empty;
 
-            if (root.TryGetProperty("filter_subject", out var filterEl))
+            if (configEl.TryGetProperty("filter_subject", out var filterEl))
                 config.FilterSubject = filterEl.GetString();
 
-            if (root.TryGetProperty("filter_subjects", out var filterSubjectsEl) && filterSubjectsEl.ValueKind == JsonValueKind.Array)
+            if (configEl.TryGetProperty("filter_subjects", out var filterSubjectsEl) && filterSubjectsEl.ValueKind == JsonValueKind.Array)
             {
                 foreach (var item in filterSubjectsEl.EnumerateArray())
                 {
@@ -272,41 +305,41 @@ public static class ConsumerApiHandlers
                 }
             }
 
-            if (root.TryGetProperty("ephemeral", out var ephemeralEl) && ephemeralEl.ValueKind == JsonValueKind.True)
+            if (configEl.TryGetProperty("ephemeral", out var ephemeralEl) && ephemeralEl.ValueKind == JsonValueKind.True)
                 config.Ephemeral = true;
 
-            if (root.TryGetProperty("push", out var pushEl) && pushEl.ValueKind == JsonValueKind.True)
+            if (configEl.TryGetProperty("push", out var pushEl) && pushEl.ValueKind == JsonValueKind.True)
                 config.Push = true;
 
-            if (root.TryGetProperty("heartbeat_ms", out var hbEl) && hbEl.TryGetInt32(out var hbMs))
+            if (configEl.TryGetProperty("heartbeat_ms", out var hbEl) && hbEl.TryGetInt32(out var hbMs))
                 config.HeartbeatMs = hbMs;
 
-            if (root.TryGetProperty("ack_wait_ms", out var ackWaitEl) && ackWaitEl.TryGetInt32(out var ackWait))
+            if (configEl.TryGetProperty("ack_wait_ms", out var ackWaitEl) && ackWaitEl.TryGetInt32(out var ackWait))
                 config.AckWaitMs = ackWait;
 
-            if (root.TryGetProperty("max_deliver", out var maxDeliverEl) && maxDeliverEl.TryGetInt32(out var maxDeliver))
+            if (configEl.TryGetProperty("max_deliver", out var maxDeliverEl) && maxDeliverEl.TryGetInt32(out var maxDeliver))
                 config.MaxDeliver = Math.Max(maxDeliver, 0);
 
-            if (root.TryGetProperty("max_ack_pending", out var maxAckPendingEl) && maxAckPendingEl.TryGetInt32(out var maxAckPending))
+            if (configEl.TryGetProperty("max_ack_pending", out var maxAckPendingEl) && maxAckPendingEl.TryGetInt32(out var maxAckPending))
                 config.MaxAckPending = Math.Max(maxAckPending, 0);
 
-            if (root.TryGetProperty("flow_control", out var flowControlEl) && flowControlEl.ValueKind is JsonValueKind.True or JsonValueKind.False)
+            if (configEl.TryGetProperty("flow_control", out var flowControlEl) && flowControlEl.ValueKind is JsonValueKind.True or JsonValueKind.False)
                 config.FlowControl = flowControlEl.GetBoolean();
 
-            if (root.TryGetProperty("rate_limit_bps", out var rateLimitEl) && rateLimitEl.TryGetInt64(out var rateLimit))
+            if (configEl.TryGetProperty("rate_limit_bps", out var rateLimitEl) && rateLimitEl.TryGetInt64(out var rateLimit))
                 config.RateLimitBps = Math.Max(rateLimit, 0);
 
-            if (root.TryGetProperty("opt_start_seq", out var optStartSeqEl) && optStartSeqEl.TryGetUInt64(out var optStartSeq))
+            if (configEl.TryGetProperty("opt_start_seq", out var optStartSeqEl) && optStartSeqEl.TryGetUInt64(out var optStartSeq))
                 config.OptStartSeq = optStartSeq;
 
-            if (root.TryGetProperty("opt_start_time_utc", out var optStartTimeEl)
+            if (configEl.TryGetProperty("opt_start_time_utc", out var optStartTimeEl)
                 && optStartTimeEl.ValueKind == JsonValueKind.String
                 && DateTime.TryParse(optStartTimeEl.GetString(), out var optStartTime))
             {
                 config.OptStartTimeUtc = optStartTime.ToUniversalTime();
             }
 
-            if (root.TryGetProperty("backoff_ms", out var backoffEl) && backoffEl.ValueKind == JsonValueKind.Array)
+            if (configEl.TryGetProperty("backoff_ms", out var backoffEl) && backoffEl.ValueKind == JsonValueKind.Array)
             {
                 foreach (var item in backoffEl.EnumerateArray())
                 {
@@ -315,7 +348,7 @@ public static class ConsumerApiHandlers
                 }
             }
 
-            if (root.TryGetProperty("ack_policy", out var ackPolicyEl))
+            if (configEl.TryGetProperty("ack_policy", out var ackPolicyEl))
             {
                 var ackPolicy = ackPolicyEl.GetString();
                 if (string.Equals(ackPolicy, "explicit", StringComparison.OrdinalIgnoreCase))
@@ -324,7 +357,7 @@ public static class ConsumerApiHandlers
                     config.AckPolicy = AckPolicy.All;
             }
 
-            if (root.TryGetProperty("deliver_policy", out var deliverPolicyEl))
+            if (configEl.TryGetProperty("deliver_policy", out var deliverPolicyEl))
             {
                 var deliver = deliverPolicyEl.GetString();
                 if (string.Equals(deliver, "last", StringComparison.OrdinalIgnoreCase))
@@ -339,7 +372,7 @@ public static class ConsumerApiHandlers
                     config.DeliverPolicy = DeliverPolicy.LastPerSubject;
             }
 
-            if (root.TryGetProperty("replay_policy", out var replayPolicyEl))
+            if (configEl.TryGetProperty("replay_policy", out var replayPolicyEl))
             {
                 var replay = replayPolicyEl.GetString();
                 if (string.Equals(replay, "original", StringComparison.OrdinalIgnoreCase))

src/NATS.Server/JetStream/Api/Handlers/StreamApiHandlers.cs

@@ -102,17 +102,47 @@ public static class StreamApiHandlers
         return JetStreamApiResponse.PurgeResponse((ulong)purged);
     }
 
-    public static JetStreamApiResponse HandleNames(StreamManager streamManager)
+    public static JetStreamApiResponse HandleNames(ReadOnlySpan<byte> payload, StreamManager streamManager)
     {
+        var offset = ParseOffset(payload);
+        var all = streamManager.ListNames();
+        var page = offset >= all.Count ? [] : all.Skip(offset).ToList();
         return new JetStreamApiResponse
         {
-            StreamNames = streamManager.ListNames(),
+            StreamNames = page,
+            PaginationTotal = all.Count,
+            PaginationOffset = offset,
         };
     }
 
-    public static JetStreamApiResponse HandleList(StreamManager streamManager)
+    public static JetStreamApiResponse HandleList(ReadOnlySpan<byte> payload, StreamManager streamManager)
     {
-        return HandleNames(streamManager);
+        var offset = ParseOffset(payload);
+        var all = streamManager.ListStreamInfos();
+        var page = offset >= all.Count ? [] : all.Skip(offset).ToList();
+        return new JetStreamApiResponse
+        {
+            StreamInfoList = page,
+            PaginationTotal = all.Count,
+            PaginationOffset = offset,
+        };
+    }
+
+    private static int ParseOffset(ReadOnlySpan<byte> payload)
+    {
+        if (payload.IsEmpty) return 0;
+        try
+        {
+            using var doc = System.Text.Json.JsonDocument.Parse(payload.ToArray());
+            if (doc.RootElement.TryGetProperty("offset", out var el) && el.TryGetInt32(out var v))
+                return Math.Max(v, 0);
+        }
+        catch (System.Text.Json.JsonException ex)
+        {
+            System.Diagnostics.Debug.WriteLine($"Malformed offset payload: {ex.Message}");
+        }
+
+        return 0;
     }
 
     public static JetStreamApiResponse HandleMessageGet(string subject, ReadOnlySpan<byte> payload, StreamManager streamManager)
@@ -445,7 +475,9 @@ public static class StreamApiHandlers
             if (root.TryGetProperty("max_msgs_per", out var maxMsgsPerEl) && maxMsgsPerEl.TryGetInt32(out var maxMsgsPer))
                 config.MaxMsgsPer = maxMsgsPer;
 
-            if (root.TryGetProperty("max_age_ms", out var maxAgeMsEl) && maxAgeMsEl.TryGetInt32(out var maxAgeMs))
+            if (root.TryGetProperty("max_age", out var maxAgeNsEl) && maxAgeNsEl.TryGetInt64(out var maxAgeNs))
+                config.MaxAge = maxAgeNs;
+            else if (root.TryGetProperty("max_age_ms", out var maxAgeMsEl) && maxAgeMsEl.TryGetInt32(out var maxAgeMs))
                 config.MaxAgeMs = maxAgeMs;
 
             if (root.TryGetProperty("max_msg_size", out var maxMsgSizeEl) && maxMsgSizeEl.TryGetInt32(out var maxMsgSize))

src/NATS.Server/JetStream/Api/JetStreamApiLimits.cs

@@ -0,0 +1,14 @@
+namespace NATS.Server.JetStream.Api;
+
+/// <summary>
+/// JetStream API size and queue limits aligned with Go server constants.
+/// Go reference: server/jetstream_api.go (JSMaxDescriptionLen, JSMaxMetadataLen,
+/// JSMaxNameLen, JSDefaultRequestQueueLimit).
+/// </summary>
+public static class JetStreamApiLimits
+{
+    public const int JSMaxDescriptionLen = 4_096;
+    public const int JSMaxMetadataLen = 128 * 1024;
+    public const int JSMaxNameLen = 255;
+    public const int JSDefaultRequestQueueLimit = 10_000;
+}

src/NATS.Server/JetStream/Api/JetStreamApiResponse.cs

@@ -9,7 +9,9 @@ public sealed class JetStreamApiResponse
     public JetStreamConsumerInfo? ConsumerInfo { get; init; }
     public JetStreamAccountInfo? AccountInfo { get; init; }
     public IReadOnlyList<string>? StreamNames { get; init; }
+    public IReadOnlyList<JetStreamStreamInfo>? StreamInfoList { get; init; }
     public IReadOnlyList<string>? ConsumerNames { get; init; }
+    public IReadOnlyList<JetStreamConsumerInfo>? ConsumerInfoList { get; init; }
     public JetStreamStreamMessage? StreamMessage { get; init; }
     public JetStreamDirectMessage? DirectMessage { get; init; }
     public JetStreamSnapshot? Snapshot { get; init; }
@@ -17,6 +19,17 @@ public sealed class JetStreamApiResponse
     public bool Success { get; init; }
     public ulong Purged { get; init; }
 
+    /// <summary>
+    /// Total count of all items (before pagination). Used by list responses for offset-based pagination.
+    /// Go reference: jetstream_api.go — ApiPaged struct includes Total, Offset, Limit fields.
+    /// </summary>
+    public int PaginationTotal { get; init; }
+
+    /// <summary>
+    /// Requested offset for pagination. Echoed back to client so it can calculate the next page.
+    /// </summary>
+    public int PaginationOffset { get; init; }
+
     /// <summary>
     /// Whether the consumer is currently paused. Populated by pause/resume API responses.
     /// Go reference: server/consumer.go jsConsumerPauseResponse.paused field.
@@ -29,6 +42,123 @@ public sealed class JetStreamApiResponse
     /// </summary>
     public DateTime? PauseUntil { get; init; }
 
+    /// <summary>
+    /// Returns a wire-format object for JSON serialization matching the Go server's
+    /// flat response structure (e.g., config/state at root level for stream responses,
+    /// not nested under a wrapper property).
+    /// </summary>
+    public object ToWireFormat()
+    {
+        if (StreamInfo != null)
+        {
+            if (Error != null)
+                return new { type = "io.nats.jetstream.api.v1.stream_create_response", error = Error };
+            return new
+            {
+                type = "io.nats.jetstream.api.v1.stream_create_response",
+                config = ToWireConfig(StreamInfo.Config),
+                state = ToWireState(StreamInfo.State),
+            };
+        }
+
+        if (ConsumerInfo != null)
+        {
+            if (Error != null)
+                return new { type = "io.nats.jetstream.api.v1.consumer_create_response", error = Error };
+            return new
+            {
+                type = "io.nats.jetstream.api.v1.consumer_create_response",
+                stream_name = ConsumerInfo.StreamName,
+                name = ConsumerInfo.Name,
+                config = ToWireConsumerConfig(ConsumerInfo.Config),
+            };
+        }
+
+        if (Error != null)
+            return new { error = Error };
+
+        if (StreamInfoList != null)
+        {
+            var wireStreams = StreamInfoList.Select(s => new
+            {
+                config = ToWireConfig(s.Config),
+                state = ToWireState(s.State),
+            }).ToList();
+            return new { total = PaginationTotal, offset = PaginationOffset, limit = wireStreams.Count, streams = wireStreams };
+        }
+
+        if (StreamNames != null)
+            return new { total = PaginationTotal, offset = PaginationOffset, limit = StreamNames.Count, streams = StreamNames };
+
+        if (ConsumerInfoList != null)
+        {
+            var wireConsumers = ConsumerInfoList.Select(c => new
+            {
+                stream_name = c.StreamName,
+                name = c.Name,
+                config = ToWireConsumerConfig(c.Config),
+            }).ToList();
+            return new { total = PaginationTotal, offset = PaginationOffset, limit = wireConsumers.Count, consumers = wireConsumers };
+        }
+
+        if (ConsumerNames != null)
+            return new { total = PaginationTotal, offset = PaginationOffset, limit = ConsumerNames.Count, consumers = ConsumerNames };
+
+        if (Purged > 0 || Success)
+            return new { success = Success, purged = Purged };
+
+        return new { success = Success };
+    }
+
+    /// <summary>
+    /// Creates a Go-compatible wire format for StreamConfig.
+    /// Only includes fields the Go server sends, with enums as lowercase strings.
+    /// Go reference: server/stream.go StreamConfig JSON marshaling.
+    /// </summary>
+    private static object ToWireConfig(StreamConfig c) => new
+    {
+        name = c.Name,
+        subjects = c.Subjects,
+        retention = c.Retention.ToString().ToLowerInvariant(),
+        max_consumers = c.MaxConsumers,
+        max_msgs = c.MaxMsgs,
+        max_bytes = c.MaxBytes,
+        max_age = c.MaxAge,
+        max_msgs_per_subject = c.MaxMsgsPer,
+        max_msg_size = c.MaxMsgSize,
+        storage = c.Storage.ToString().ToLowerInvariant(),
+        discard = c.Discard.ToString().ToLowerInvariant(),
+        num_replicas = c.Replicas,
+        duplicate_window = (long)c.DuplicateWindowMs * 1_000_000L,
+        sealed_field = c.Sealed,
+        deny_delete = c.DenyDelete,
+        deny_purge = c.DenyPurge,
+        allow_direct = c.AllowDirect,
+        first_seq = c.FirstSeq,
+    };
+
+    private static object ToWireState(ApiStreamState s) => new
+    {
+        messages = s.Messages,
+        bytes = s.Bytes,
+        first_seq = s.FirstSeq,
+        last_seq = s.LastSeq,
+        consumer_count = 0,
+    };
+
+    private static object ToWireConsumerConfig(ConsumerConfig c) => new
+    {
+        durable_name = string.IsNullOrEmpty(c.DurableName) ? null : c.DurableName,
+        name = string.IsNullOrEmpty(c.DurableName) ? null : c.DurableName,
+        deliver_policy = c.DeliverPolicy.ToString().ToLowerInvariant(),
+        ack_policy = c.AckPolicy.ToString().ToLowerInvariant(),
+        replay_policy = c.ReplayPolicy.ToString().ToLowerInvariant(),
+        ack_wait = (long)c.AckWaitMs * 1_000_000L,
+        max_deliver = c.MaxDeliver,
+        max_ack_pending = c.MaxAckPending,
+        filter_subject = c.FilterSubject,
+    };
+
     public static JetStreamApiResponse NotFound(string subject) => new()
     {
         Error = new JetStreamApiError
@@ -99,6 +229,8 @@ public sealed class JetStreamStreamInfo
 
 public sealed class JetStreamConsumerInfo
 {
+    public string? Name { get; init; }
+    public string? StreamName { get; init; }
     public required ConsumerConfig Config { get; init; }
 }
 

src/NATS.Server/JetStream/Api/JetStreamApiRouter.cs

@@ -249,10 +249,10 @@ public sealed class JetStreamApiRouter
             return StreamApiHandlers.HandleInfo(subject, _streamManager);
 
         if (subject.Equals(JetStreamApiSubjects.StreamNames, StringComparison.Ordinal))
-            return StreamApiHandlers.HandleNames(_streamManager);
+            return StreamApiHandlers.HandleNames(payload, _streamManager);
 
         if (subject.Equals(JetStreamApiSubjects.StreamList, StringComparison.Ordinal))
-            return StreamApiHandlers.HandleList(_streamManager);
+            return StreamApiHandlers.HandleList(payload, _streamManager);
 
         if (subject.StartsWith(JetStreamApiSubjects.StreamUpdate, StringComparison.Ordinal))
             return StreamApiHandlers.HandleUpdate(subject, payload, _streamManager);
@@ -288,10 +288,10 @@ public sealed class JetStreamApiRouter
             return ConsumerApiHandlers.HandleInfo(subject, _consumerManager);
 
         if (subject.StartsWith(JetStreamApiSubjects.ConsumerNames, StringComparison.Ordinal))
-            return ConsumerApiHandlers.HandleNames(subject, _consumerManager);
+            return ConsumerApiHandlers.HandleNames(subject, payload, _consumerManager);
 
         if (subject.StartsWith(JetStreamApiSubjects.ConsumerList, StringComparison.Ordinal))
-            return ConsumerApiHandlers.HandleList(subject, _consumerManager);
+            return ConsumerApiHandlers.HandleList(subject, payload, _consumerManager);
 
         if (subject.StartsWith(JetStreamApiSubjects.ConsumerDelete, StringComparison.Ordinal))
             return ConsumerApiHandlers.HandleDelete(subject, _consumerManager);