Fix E2E test gaps and add comprehensive E2E + parity test suites
- Fix pull consumer fetch: send original stream subject in HMSG (not inbox) so NATS client distinguishes data messages from control messages - Fix MaxAge expiry: add background timer in StreamManager for periodic pruning - Fix JetStream wire format: Go-compatible anonymous objects with string enums, proper offset-based pagination for stream/consumer list APIs - Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream) - Add ~1000 parity tests across all subsystems (gaps closure) - Update gap inventory docs to reflect implementation status
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
using NATS.Server.Auth;
|
||||
using NATS.Server.JetStream.Api;
|
||||
using NATS.Server.JetStream.Cluster;
|
||||
@@ -7,11 +8,12 @@ using NATS.Server.JetStream.Models;
|
||||
using NATS.Server.JetStream.Publish;
|
||||
using NATS.Server.JetStream.Snapshots;
|
||||
using NATS.Server.JetStream.Storage;
|
||||
using NATS.Server.JetStream.Validation;
|
||||
using NATS.Server.Subscriptions;
|
||||
|
||||
namespace NATS.Server.JetStream;
|
||||
|
||||
public sealed class StreamManager
|
||||
public sealed class StreamManager : IDisposable
|
||||
{
|
||||
private readonly Account? _account;
|
||||
private readonly ConsumerManager? _consumerManager;
|
||||
@@ -25,12 +27,52 @@ public sealed class StreamManager
|
||||
private readonly ConcurrentDictionary<string, List<SourceCoordinator>> _sourcesByOrigin =
|
||||
new(StringComparer.Ordinal);
|
||||
private readonly StreamSnapshotService _snapshotService = new();
|
||||
private readonly CancellationTokenSource _expiryTimerCts = new();
|
||||
private Task? _expiryTimerTask;
|
||||
|
||||
public StreamManager(JetStreamMetaGroup? metaGroup = null, Account? account = null, ConsumerManager? consumerManager = null)
|
||||
{
|
||||
_metaGroup = metaGroup;
|
||||
_account = account;
|
||||
_consumerManager = consumerManager;
|
||||
_expiryTimerTask = RunExpiryTimerAsync(_expiryTimerCts.Token);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_expiryTimerCts.Cancel();
|
||||
_expiryTimerCts.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Periodically prunes expired messages from streams with MaxAge configured.
|
||||
/// Go reference: stream.go — expireMsgs runs on a timer (checkMaxAge interval).
|
||||
/// </summary>
|
||||
private async Task RunExpiryTimerAsync(CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1));
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var ticked = false;
|
||||
try
|
||||
{
|
||||
ticked = await timer.WaitForNextTickAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return; // Shutdown requested via Dispose — exit the timer loop
|
||||
}
|
||||
|
||||
if (!ticked)
|
||||
return;
|
||||
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
foreach (var stream in _streams.Values)
|
||||
{
|
||||
if (stream.Config.MaxAgeMs > 0)
|
||||
PruneExpiredMessages(stream, nowUtc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<string> StreamNames => _streams.Keys.ToArray();
|
||||
@@ -39,10 +81,31 @@ public sealed class StreamManager
|
||||
public IReadOnlyList<string> ListNames()
|
||||
=> [.. _streams.Keys.OrderBy(x => x, StringComparer.Ordinal)];
|
||||
|
||||
public IReadOnlyList<JetStreamStreamInfo> ListStreamInfos()
|
||||
{
|
||||
return _streams.OrderBy(kv => kv.Key, StringComparer.Ordinal)
|
||||
.Select(kv =>
|
||||
{
|
||||
var state = kv.Value.Store.GetStateAsync(default).GetAwaiter().GetResult();
|
||||
return new JetStreamStreamInfo
|
||||
{
|
||||
Config = kv.Value.Config,
|
||||
State = state,
|
||||
};
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public JetStreamApiResponse CreateOrUpdate(StreamConfig config)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(config.Name))
|
||||
return JetStreamApiResponse.ErrorResponse(400, "stream name required");
|
||||
if (!JetStreamConfigValidator.IsValidName(config.Name))
|
||||
return JetStreamApiResponse.ErrorResponse(400, "invalid stream name");
|
||||
|
||||
if (Encoding.UTF8.GetByteCount(config.Description) > JetStreamApiLimits.JSMaxDescriptionLen)
|
||||
return JetStreamApiResponse.ErrorResponse(400, "stream description is too long");
|
||||
|
||||
if (!JetStreamConfigValidator.IsMetadataWithinLimit(config.Metadata))
|
||||
return JetStreamApiResponse.ErrorResponse(400, "stream metadata exceeds maximum size");
|
||||
|
||||
var normalized = NormalizeConfig(config);
|
||||
|
||||
@@ -302,6 +365,8 @@ public sealed class StreamManager
|
||||
if (stream == null)
|
||||
return null;
|
||||
|
||||
|
||||
|
||||
if (stream.Config.MaxMsgSize > 0 && payload.Length > stream.Config.MaxMsgSize)
|
||||
{
|
||||
return new PubAck
|
||||
|
||||
Reference in New Issue
Block a user