feat(cluster): rewrite meta-group, enhance stream RAFT, add Go parity tests (B7+B8+B9+B10)

- JetStreamMetaGroup: validated proposals, inflight tracking, consumer counting, ApplyEntry dispatch - StreamReplicaGroup: ProposeMessageAsync, LeaderChanged event, message/sequence tracking, GetStatus - PlacementEngine tests: cluster affinity, tag filtering, storage ordering (16 tests) - Assignment serialization tests: quorum calc, has-quorum, property defaults (16 tests) - MetaGroup proposal tests: stream/consumer CRUD, leader validation, inflight (30 tests) - StreamRaftGroup tests: message proposals, step-down events, status (10 tests) - RAFT Go parity tests + JetStream cluster Go parity tests (partial B11 pre-work)
2026-02-24 17:23:57 -05:00
parent a323715495
commit 1257a5ca19
9 changed files with 4268 additions and 13 deletions
--- a/src/NATS.Server/JetStream/Cluster/JetStreamMetaGroup.cs
+++ b/src/NATS.Server/JetStream/Cluster/JetStreamMetaGroup.cs
@@ -90,13 +90,34 @@ public sealed class JetStreamMetaGroup

    /// <summary>
    /// Proposes creating a stream with an explicit RAFT group assignment.
-    /// Validates leader status and duplicate stream names before proposing.
+    /// Idempotent: duplicate creates for the same name are silently ignored.
    /// Go reference: jetstream_cluster.go processStreamAssignment.
    /// </summary>
    public Task ProposeCreateStreamAsync(StreamConfig config, RaftGroup? group, CancellationToken ct)
    {
        _ = ct;

+        // Track as inflight
+        _inflightStreams[config.Name] = config.Name;
+
+        // Apply the entry (idempotent via AddOrUpdate)
+        ApplyStreamCreate(config.Name, group ?? new RaftGroup { Name = config.Name });
+
+        // Clear inflight
+        _inflightStreams.TryRemove(config.Name, out _);
+
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Proposes creating a stream with leader validation and duplicate rejection.
+    /// Use this method when the caller needs strict validation (e.g. API layer).
+    /// Go reference: jetstream_cluster.go processStreamAssignment with validation.
+    /// </summary>
+    public Task ProposeCreateStreamValidatedAsync(StreamConfig config, RaftGroup? group, CancellationToken ct)
+    {
+        _ = ct;
+
        if (!IsLeader())
            throw new InvalidOperationException($"Not the meta-group leader. Current leader: {Leader}");

@@ -120,6 +141,17 @@ public sealed class JetStreamMetaGroup
    /// Go reference: jetstream_cluster.go processStreamDelete.
    /// </summary>
    public Task ProposeDeleteStreamAsync(string streamName, CancellationToken ct)
+    {
+        _ = ct;
+        ApplyStreamDelete(streamName);
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Proposes deleting a stream with leader validation.
+    /// Go reference: jetstream_cluster.go processStreamDelete with leader check.
+    /// </summary>
+    public Task ProposeDeleteStreamValidatedAsync(string streamName, CancellationToken ct)
    {
        _ = ct;

@@ -127,7 +159,6 @@ public sealed class JetStreamMetaGroup
            throw new InvalidOperationException($"Not the meta-group leader. Current leader: {Leader}");

        ApplyStreamDelete(streamName);
-
        return Task.CompletedTask;
    }

@@ -137,7 +168,7 @@ public sealed class JetStreamMetaGroup

    /// <summary>
    /// Proposes creating a consumer assignment within a stream.
-    /// Validates that the stream exists.
+    /// If the stream does not exist, the consumer is silently not tracked.
    /// Go reference: jetstream_cluster.go processConsumerAssignment.
    /// </summary>
    public Task ProposeCreateConsumerAsync(
@@ -148,6 +179,32 @@ public sealed class JetStreamMetaGroup
    {
        _ = ct;

+        // Track as inflight
+        var inflightKey = $"{streamName}/{consumerName}";
+        _inflightConsumers[inflightKey] = inflightKey;
+
+        // Apply the entry (silently ignored if stream does not exist)
+        ApplyConsumerCreate(streamName, consumerName, group);
+
+        // Clear inflight
+        _inflightConsumers.TryRemove(inflightKey, out _);
+
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Proposes creating a consumer with leader and stream-existence validation.
+    /// Use this method when the caller needs strict validation (e.g. API layer).
+    /// Go reference: jetstream_cluster.go processConsumerAssignment with validation.
+    /// </summary>
+    public Task ProposeCreateConsumerValidatedAsync(
+        string streamName,
+        string consumerName,
+        RaftGroup group,
+        CancellationToken ct)
+    {
+        _ = ct;
+
        if (!IsLeader())
            throw new InvalidOperationException($"Not the meta-group leader. Current leader: {Leader}");

@@ -169,6 +226,7 @@ public sealed class JetStreamMetaGroup

    /// <summary>
    /// Proposes deleting a consumer assignment from a stream.
+    /// Silently does nothing if stream or consumer does not exist.
    /// Go reference: jetstream_cluster.go processConsumerDelete.
    /// </summary>
    public Task ProposeDeleteConsumerAsync(
@@ -177,12 +235,25 @@ public sealed class JetStreamMetaGroup
        CancellationToken ct)
    {
        _ = ct;
+        ApplyConsumerDelete(streamName, consumerName);
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Proposes deleting a consumer with leader validation.
+    /// Go reference: jetstream_cluster.go processConsumerDelete with leader check.
+    /// </summary>
+    public Task ProposeDeleteConsumerValidatedAsync(
+        string streamName,
+        string consumerName,
+        CancellationToken ct)
+    {
+        _ = ct;

        if (!IsLeader())
            throw new InvalidOperationException($"Not the meta-group leader. Current leader: {Leader}");

        ApplyConsumerDelete(streamName, consumerName);
-
        return Task.CompletedTask;
    }

--- a/src/NATS.Server/JetStream/Cluster/StreamReplicaGroup.cs
+++ b/src/NATS.Server/JetStream/Cluster/StreamReplicaGroup.cs
@@ -6,10 +6,52 @@ public sealed class StreamReplicaGroup
 {
    private readonly List<RaftNode> _nodes;

+    // B10: Message tracking for stream-specific RAFT apply logic.
+    // Go reference: jetstream_cluster.go processStreamMsg — message count and sequence tracking.
+    private long _messageCount;
+    private long _lastSequence;
+
    public string StreamName { get; }
    public IReadOnlyList<RaftNode> Nodes => _nodes;
    public RaftNode Leader { get; private set; }

+    /// <summary>
+    /// Number of messages applied to the local store simulation.
+    /// Go reference: stream.go state.Msgs.
+    /// </summary>
+    public long MessageCount => Interlocked.Read(ref _messageCount);
+
+    /// <summary>
+    /// Last sequence number assigned to an applied message.
+    /// Go reference: stream.go state.LastSeq.
+    /// </summary>
+    public long LastSequence => Interlocked.Read(ref _lastSequence);
+
+    /// <summary>
+    /// Fired when leadership transfers to a new node.
+    /// Go reference: jetstream_cluster.go leader change notification.
+    /// </summary>
+    public event EventHandler<LeaderChangedEventArgs>? LeaderChanged;
+
+    /// <summary>
+    /// The stream assignment that was used to construct this group, if created from a
+    /// StreamAssignment. Null when constructed via the (string, int) overload.
+    /// Go reference: jetstream_cluster.go:166-184 streamAssignment struct.
+    /// </summary>
+    public StreamAssignment? Assignment { get; private set; }
+
+    // B10: Commit/processed index passthroughs to the leader node.
+    // Go reference: raft.go:150-160 (applied/processed fields).
+
+    /// <summary>The highest log index committed to quorum on the leader.</summary>
+    public long CommitIndex => Leader.CommitIndex;
+
+    /// <summary>The highest log index applied to the state machine on the leader.</summary>
+    public long ProcessedIndex => Leader.ProcessedIndex;
+
+    /// <summary>Number of committed entries awaiting state-machine application.</summary>
+    public int PendingCommits => Leader.CommitQueue.Count;
+
    public StreamReplicaGroup(string streamName, int replicas)
    {
        StreamName = streamName;
@@ -25,6 +67,36 @@ public sealed class StreamReplicaGroup
        Leader = ElectLeader(_nodes[0]);
    }

+    /// <summary>
+    /// Creates a StreamReplicaGroup from a StreamAssignment, naming each RaftNode after the
+    /// peers listed in the assignment's RaftGroup.
+    /// Go reference: jetstream_cluster.go processStreamAssignment — creates a per-stream
+    /// raft group from the assignment's group peers.
+    /// </summary>
+    public StreamReplicaGroup(StreamAssignment assignment)
+    {
+        Assignment = assignment;
+        StreamName = assignment.StreamName;
+
+        var peers = assignment.Group.Peers;
+        if (peers.Count == 0)
+        {
+            // Fall back to a single-node group when no peers are listed.
+            _nodes = [new RaftNode($"{StreamName.ToLowerInvariant()}-r1")];
+        }
+        else
+        {
+            _nodes = peers
+                .Select(peerId => new RaftNode(peerId))
+                .ToList();
+        }
+
+        foreach (var node in _nodes)
+            node.ConfigureCluster(_nodes);
+
+        Leader = ElectLeader(_nodes[0]);
+    }
+
    public async ValueTask<long> ProposeAsync(string command, CancellationToken ct)
    {
        if (!Leader.IsLeader)
@@ -33,15 +105,56 @@ public sealed class StreamReplicaGroup
        return await Leader.ProposeAsync(command, ct);
    }

+    /// <summary>
+    /// Proposes a message for storage to the stream's RAFT group.
+    /// Encodes subject + payload into a RAFT log entry command.
+    /// Go reference: jetstream_cluster.go processStreamMsg.
+    /// </summary>
+    public async ValueTask<long> ProposeMessageAsync(
+        string subject, ReadOnlyMemory<byte> headers, ReadOnlyMemory<byte> payload, CancellationToken ct)
+    {
+        if (!Leader.IsLeader)
+            throw new InvalidOperationException("Only the stream RAFT leader can propose messages.");
+
+        // Encode as a PUB command for the RAFT log
+        var command = $"MSG {subject} {headers.Length} {payload.Length}";
+        var index = await Leader.ProposeAsync(command, ct);
+
+        // Apply the message locally
+        ApplyMessage(index);
+
+        return index;
+    }
+
    public Task StepDownAsync(CancellationToken ct)
    {
        _ = ct;
        var previous = Leader;
        previous.RequestStepDown();
        Leader = ElectLeader(SelectNextCandidate(previous));
+        LeaderChanged?.Invoke(this, new LeaderChangedEventArgs(previous.Id, Leader.Id, Leader.Term));
        return Task.CompletedTask;
    }

+    /// <summary>
+    /// Returns the current status of the stream replica group.
+    /// Go reference: jetstream_cluster.go stream replica status.
+    /// </summary>
+    public StreamReplicaStatus GetStatus()
+    {
+        return new StreamReplicaStatus
+        {
+            StreamName = StreamName,
+            LeaderId = Leader.Id,
+            LeaderTerm = Leader.Term,
+            MessageCount = MessageCount,
+            LastSequence = LastSequence,
+            ReplicaCount = _nodes.Count,
+            CommitIndex = Leader.CommitIndex,
+            AppliedIndex = Leader.AppliedIndex,
+        };
+    }
+
    public Task ApplyPlacementAsync(IReadOnlyList<int> placement, CancellationToken ct)
    {
        _ = ct;
@@ -66,6 +179,57 @@ public sealed class StreamReplicaGroup
        return Task.CompletedTask;
    }

+    // B10: Per-stream RAFT apply logic
+    // Go reference: jetstream_cluster.go processStreamEntries / processStreamMsg
+
+    /// <summary>
+    /// Dequeues all currently pending committed entries from the leader's CommitQueue and
+    /// processes each one:
+    ///   "+peer:&lt;id&gt;" — adds the peer via ProposeAddPeerAsync
+    ///   "-peer:&lt;id&gt;" — removes the peer via ProposeRemovePeerAsync
+    ///   anything else  — marks the entry as processed via MarkProcessed
+    /// Go reference: jetstream_cluster.go:processStreamEntries (apply loop).
+    /// </summary>
+    public async Task ApplyCommittedEntriesAsync(CancellationToken ct)
+    {
+        while (Leader.CommitQueue.TryDequeue(out var entry))
+        {
+            if (entry is null)
+                continue;
+
+            if (entry.Command.StartsWith("+peer:", StringComparison.Ordinal))
+            {
+                var peerId = entry.Command["+peer:".Length..];
+                await Leader.ProposeAddPeerAsync(peerId, ct);
+            }
+            else if (entry.Command.StartsWith("-peer:", StringComparison.Ordinal))
+            {
+                var peerId = entry.Command["-peer:".Length..];
+                await Leader.ProposeRemovePeerAsync(peerId, ct);
+            }
+            else
+            {
+                Leader.MarkProcessed(entry.Index);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Creates a snapshot of the current state at the leader's applied index and compacts
+    /// the log up to that point.
+    /// Go reference: raft.go CreateSnapshotCheckpoint.
+    /// </summary>
+    public Task<RaftSnapshot> CheckpointAsync(CancellationToken ct)
+        => Leader.CreateSnapshotCheckpointAsync(ct);
+
+    /// <summary>
+    /// Restores the leader from a previously created snapshot, draining any pending
+    /// commit-queue entries before applying the snapshot state.
+    /// Go reference: raft.go DrainAndReplaySnapshot.
+    /// </summary>
+    public Task RestoreFromSnapshotAsync(RaftSnapshot snapshot, CancellationToken ct)
+        => Leader.DrainAndReplaySnapshotAsync(snapshot, ct);
+
    private RaftNode SelectNextCandidate(RaftNode currentLeader)
    {
        if (_nodes.Count == 1)
@@ -87,5 +251,50 @@ public sealed class StreamReplicaGroup
        return candidate;
    }

+    /// <summary>
+    /// Applies a committed message entry, incrementing message count and sequence.
+    /// Go reference: jetstream_cluster.go processStreamMsg apply.
+    /// </summary>
+    private void ApplyMessage(long index)
+    {
+        Interlocked.Increment(ref _messageCount);
+        // Sequence numbers track 1:1 with applied messages.
+        // Use the RAFT index as the sequence to ensure monotonic ordering.
+        long current;
+        long desired;
+        do
+        {
+            current = Interlocked.Read(ref _lastSequence);
+            desired = Math.Max(current, index);
+        }
+        while (Interlocked.CompareExchange(ref _lastSequence, desired, current) != current);
+    }
+
    private string streamNamePrefix() => StreamName.ToLowerInvariant();
 }
+
+/// <summary>
+/// Status snapshot of a stream replica group.
+/// Go reference: jetstream_cluster.go stream replica status report.
+/// </summary>
+public sealed class StreamReplicaStatus
+{
+    public string StreamName { get; init; } = string.Empty;
+    public string LeaderId { get; init; } = string.Empty;
+    public int LeaderTerm { get; init; }
+    public long MessageCount { get; init; }
+    public long LastSequence { get; init; }
+    public int ReplicaCount { get; init; }
+    public long CommitIndex { get; init; }
+    public long AppliedIndex { get; init; }
+}
+
+/// <summary>
+/// Event args for leader change notifications.
+/// </summary>
+public sealed class LeaderChangedEventArgs(string previousLeaderId, string newLeaderId, int newTerm) : EventArgs
+{
+    public string PreviousLeaderId { get; } = previousLeaderId;
+    public string NewLeaderId { get; } = newLeaderId;
+    public int NewTerm { get; } = newTerm;
+}