Fix E2E test gaps and add comprehensive E2E + parity test suites

- Fix pull consumer fetch: send original stream subject in HMSG (not inbox)
  so NATS client distinguishes data messages from control messages
- Fix MaxAge expiry: add background timer in StreamManager for periodic pruning
- Fix JetStream wire format: Go-compatible anonymous objects with string enums,
  proper offset-based pagination for stream/consumer list APIs
- Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream)
- Add ~1000 parity tests across all subsystems (gaps closure)
- Update gap inventory docs to reflect implementation status
This commit is contained in:
Joseph Doherty
2026-03-12 14:09:23 -04:00
parent 79c1ee8776
commit c30e67a69d
226 changed files with 17801 additions and 709 deletions

View File

@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Text;
using NATS.Server.Auth;
using NATS.Server.JetStream.Api;
using NATS.Server.JetStream.Cluster;
@@ -7,11 +8,12 @@ using NATS.Server.JetStream.Models;
using NATS.Server.JetStream.Publish;
using NATS.Server.JetStream.Snapshots;
using NATS.Server.JetStream.Storage;
using NATS.Server.JetStream.Validation;
using NATS.Server.Subscriptions;
namespace NATS.Server.JetStream;
public sealed class StreamManager
public sealed class StreamManager : IDisposable
{
private readonly Account? _account;
private readonly ConsumerManager? _consumerManager;
@@ -25,12 +27,52 @@ public sealed class StreamManager
private readonly ConcurrentDictionary<string, List<SourceCoordinator>> _sourcesByOrigin =
new(StringComparer.Ordinal);
private readonly StreamSnapshotService _snapshotService = new();
private readonly CancellationTokenSource _expiryTimerCts = new();
private Task? _expiryTimerTask;
public StreamManager(JetStreamMetaGroup? metaGroup = null, Account? account = null, ConsumerManager? consumerManager = null)
{
_metaGroup = metaGroup;
_account = account;
_consumerManager = consumerManager;
_expiryTimerTask = RunExpiryTimerAsync(_expiryTimerCts.Token);
}
public void Dispose()
{
_expiryTimerCts.Cancel();
_expiryTimerCts.Dispose();
}
/// <summary>
/// Periodically prunes expired messages from streams with MaxAge configured.
/// Go reference: stream.go — expireMsgs runs on a timer (checkMaxAge interval).
/// </summary>
private async Task RunExpiryTimerAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1));
while (!ct.IsCancellationRequested)
{
var ticked = false;
try
{
ticked = await timer.WaitForNextTickAsync(ct);
}
catch (OperationCanceledException)
{
return; // Shutdown requested via Dispose — exit the timer loop
}
if (!ticked)
return;
var nowUtc = DateTime.UtcNow;
foreach (var stream in _streams.Values)
{
if (stream.Config.MaxAgeMs > 0)
PruneExpiredMessages(stream, nowUtc);
}
}
}
public IReadOnlyCollection<string> StreamNames => _streams.Keys.ToArray();
@@ -39,10 +81,31 @@ public sealed class StreamManager
public IReadOnlyList<string> ListNames()
=> [.. _streams.Keys.OrderBy(x => x, StringComparer.Ordinal)];
public IReadOnlyList<JetStreamStreamInfo> ListStreamInfos()
{
return _streams.OrderBy(kv => kv.Key, StringComparer.Ordinal)
.Select(kv =>
{
var state = kv.Value.Store.GetStateAsync(default).GetAwaiter().GetResult();
return new JetStreamStreamInfo
{
Config = kv.Value.Config,
State = state,
};
})
.ToList();
}
public JetStreamApiResponse CreateOrUpdate(StreamConfig config)
{
if (string.IsNullOrWhiteSpace(config.Name))
return JetStreamApiResponse.ErrorResponse(400, "stream name required");
if (!JetStreamConfigValidator.IsValidName(config.Name))
return JetStreamApiResponse.ErrorResponse(400, "invalid stream name");
if (Encoding.UTF8.GetByteCount(config.Description) > JetStreamApiLimits.JSMaxDescriptionLen)
return JetStreamApiResponse.ErrorResponse(400, "stream description is too long");
if (!JetStreamConfigValidator.IsMetadataWithinLimit(config.Metadata))
return JetStreamApiResponse.ErrorResponse(400, "stream metadata exceeds maximum size");
var normalized = NormalizeConfig(config);
@@ -302,6 +365,8 @@ public sealed class StreamManager
if (stream == null)
return null;
if (stream.Config.MaxMsgSize > 0 && payload.Length > stream.Config.MaxMsgSize)
{
return new PubAck