dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add entry application pipeline for meta and stream RAFT groups (Gap 2.7)

Browse Source 
Extend ApplyEntry in JetStreamMetaGroup with PeerAdd/PeerRemove dispatch
to the existing Task 12 peer management methods. Add StreamMsgOp (Store,
Remove, Purge) and ConsumerOp (Ack, Nak, Deliver, Term, Progress) enums
plus ApplyStreamMsgOp and ApplyConsumerEntry methods to StreamReplicaGroup.
Extend ApplyCommittedEntriesAsync to parse smsg:/centry: command prefixes
and route to the new apply methods. Add MetaEntryType.PeerAdd/PeerRemove
enum values. 35 new tests in EntryApplicationTests.cs, all passing.
This commit is contained in:
Joseph Doherty
2026-02-25 08:38:21 -05:00
parent f1f7bfbb51
commit 0f8f34afaa
3 changed files with 842 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
src/NATS.Server/JetStream/Cluster/JetStreamMetaGroup.cs
View File

@@ -22,6 +22,10 @@ public sealed class JetStreamMetaGroup
private readonly ConcurrentDictionary<string, StreamAssignment> _assignments =
new(StringComparer.Ordinal);
// Known peers in this cluster — used by ProcessAddPeer / ProcessRemovePeer.
// Go reference: jetstream_cluster.go peer tracking in jetStreamCluster.
private readonly HashSet<string> _knownPeers = new(StringComparer.Ordinal);
// Account-scoped inflight proposal tracking -- entries proposed but not yet committed.
// Go reference: jetstream_cluster.go inflight tracking for proposals (jetstream_cluster.go:1193-1278).
// Outer key: account name. Inner key: stream name → InflightInfo.
@@ -596,6 +600,12 @@ public sealed class JetStreamMetaGroup
throw new ArgumentNullException(nameof(streamName), "Stream name required for consumer operations.");
ApplyConsumerDelete(streamName, name);
break;
case MetaEntryType.PeerAdd:
ProcessAddPeer(name);
break;
case MetaEntryType.PeerRemove:
ProcessRemovePeer(name);
break;
}
}
@@ -688,6 +698,163 @@ public sealed class JetStreamMetaGroup
ProcessLeaderChange(isLeader: false);
}
// ---------------------------------------------------------------
// Peer management
// Go reference: jetstream_cluster.go:2290-2439 processAddPeer, processRemovePeer,
// removePeerFromStreamLocked, remapStreamAssignment.
// ---------------------------------------------------------------
/// <summary>
/// Registers a peer as known to this meta-group.
/// Go reference: jetstream_cluster.go peer tracking in jetStreamCluster.
/// </summary>
public void AddKnownPeer(string peerId)
{
lock (_knownPeers)
_knownPeers.Add(peerId);
}
/// <summary>
/// Removes a peer from the known-peers set.
/// Go reference: jetstream_cluster.go peer removal tracking.
/// </summary>
public void RemoveKnownPeer(string peerId)
{
lock (_knownPeers)
_knownPeers.Remove(peerId);
}
/// <summary>
/// Returns a snapshot of the currently known peers.
/// </summary>
public IReadOnlyList<string> GetKnownPeers()
{
lock (_knownPeers)
return [.. _knownPeers];
}
/// <summary>
/// Processes the arrival of a new peer. Only the meta-group leader acts.
/// Finds under-replicated streams (Group.Peers.Count &lt; desired replica count)
/// and adds the new peer to their RaftGroup, triggering re-replication.
/// Go reference: jetstream_cluster.go:2290 processAddPeer.
/// </summary>
public void ProcessAddPeer(string peerId)
{
// Always register the new peer.
AddKnownPeer(peerId);
// Only the meta-leader re-assigns streams.
if (!IsLeader())
return;
foreach (var sa in _assignments.Values)
{
var group = sa.Group;
if (group.Peers.Contains(peerId))
continue;
// Desired replicas is derived from the peer list size at creation time.
// In this model, the Group records the desired size via its original Peers count.
// An under-replicated group is one that is missing peers.
// Go reference: jetstream_cluster.go:2284 sa.missingPeers() — len(Peers) < Config.Replicas.
// Here, DesiredReplicas is stored on the group; if absent we skip (already at desired).
if (!sa.Group.HasDesiredReplicas)
continue;
if (group.Peers.Count < sa.Group.DesiredReplicas)
{
// Add the new peer to restore the desired replica count.
group.Peers.Add(peerId);
}
}
}
/// <summary>
/// Processes the removal of a peer. Only the meta-group leader acts.
/// Finds all streams that had the removed peer in their RaftGroup and
/// triggers reassignment away from that peer.
/// Go reference: jetstream_cluster.go:2342 processRemovePeer.
/// </summary>
public void ProcessRemovePeer(string peerId)
{
// Always remove from known set.
RemoveKnownPeer(peerId);
// Only the meta-leader re-assigns streams.
if (!IsLeader())
return;
foreach (var sa in _assignments.Values)
{
if (sa.Group.Peers.Contains(peerId))
RemovePeerFromStream(sa.StreamName, peerId);
}
}
/// <summary>
/// Removes a specific peer from a stream's RaftGroup and remaps to a replacement
/// drawn from the known peers if possible.
/// Returns true if a replacement peer was found; false if the peer list was merely shrunk.
/// Go reference: jetstream_cluster.go:2403 removePeerFromStreamLocked.
/// </summary>
public bool RemovePeerFromStream(string streamName, string peerId)
{
if (!_assignments.TryGetValue(streamName, out var sa))
return false;
if (!sa.Group.Peers.Contains(peerId))
return false;
IReadOnlyList<string> available;
lock (_knownPeers)
available = [.. _knownPeers];
return RemapStreamAssignment(sa, available, peerId);
}
/// <summary>
/// Reassigns a stream's replica group to a new peer set, replacing the removed peer
/// with one drawn from <paramref name="availablePeers"/>.
/// Retains all current peers except <paramref name="removePeer"/>, then fills the
/// vacancy from the available pool.
/// Returns true when a replacement peer was placed; false if the group was merely shrunk.
/// Go reference: jetstream_cluster.go:7077 remapStreamAssignment.
/// </summary>
public bool RemapStreamAssignment(StreamAssignment assignment, IReadOnlyList<string> availablePeers, string removePeer)
{
var group = assignment.Group;
var currentPeers = group.Peers;
// Peers to retain (all except the removed peer).
var retain = currentPeers.Where(p => p != removePeer).ToList();
// Candidates: available peers not already in the retained set and not the removed peer.
var retainSet = new HashSet<string>(retain, StringComparer.Ordinal);
var candidates = availablePeers
.Where(p => !retainSet.Contains(p) && p != removePeer)
.ToList();
if (candidates.Count > 0)
{
// Pick the first available replacement.
retain.Add(candidates[0]);
currentPeers.Clear();
foreach (var p in retain)
currentPeers.Add(p);
return true;
}
// No replacement available — just shrink the group if R>1.
// Go reference: jetstream_cluster.go:7098-7110 R1 fallback, bare removal for R>1.
if (currentPeers.Count > 1)
{
currentPeers.Remove(removePeer);
}
return false;
}
// ---------------------------------------------------------------
// Internal apply methods
// ---------------------------------------------------------------
@@ -755,6 +922,16 @@ public enum MetaEntryType
StreamDelete,
ConsumerCreate,
ConsumerDelete,
/// <summary>
/// A peer joined the cluster; triggers re-replication of under-replicated streams.
/// Go reference: jetstream_cluster.go processAddPeer.
/// </summary>
PeerAdd,
/// <summary>
/// A peer left the cluster; triggers reassignment away from the removed peer.
/// Go reference: jetstream_cluster.go processRemovePeer.
/// </summary>
PeerRemove,
}
public sealed class MetaGroupState

169
src/NATS.Server/JetStream/Cluster/StreamReplicaGroup.cs
View File

@@ -11,6 +11,15 @@ public sealed class StreamReplicaGroup
private long _messageCount;
private long _lastSequence;
// Track ack/nak/deliver counts for consumer entry apply.
// Go reference: jetstream_cluster.go processConsumerEntries.
private long _ackCount;
private long _nakCount;
private long _deliverCount;
// Last consumer op applied (used for diagnostics / unknown-op logging).
private string _lastUnknownCommand = string.Empty;
public string StreamName { get; }
public IReadOnlyList<RaftNode> Nodes => _nodes;
public RaftNode Leader { get; private set; }
@@ -27,6 +36,30 @@ public sealed class StreamReplicaGroup
/// </summary>
public long LastSequence => Interlocked.Read(ref _lastSequence);
/// <summary>
/// Number of consumer acknowledgement ops applied.
/// Go reference: jetstream_cluster.go processConsumerEntries — ack tracking.
/// </summary>
public long AckCount => Interlocked.Read(ref _ackCount);
/// <summary>
/// Number of consumer negative-acknowledgement (nak) ops applied.
/// Go reference: jetstream_cluster.go processConsumerEntries — nak tracking.
/// </summary>
public long NakCount => Interlocked.Read(ref _nakCount);
/// <summary>
/// Number of consumer deliver ops applied.
/// Go reference: jetstream_cluster.go processConsumerEntries — deliver tracking.
/// </summary>
public long DeliverCount => Interlocked.Read(ref _deliverCount);
/// <summary>
/// The last command string that did not match any known entry type.
/// Used to verify graceful unknown-entry handling.
/// </summary>
public string LastUnknownCommand => _lastUnknownCommand;
/// <summary>
/// Fired when leadership transfers to a new node.
/// Go reference: jetstream_cluster.go leader change notification.
@@ -187,6 +220,8 @@ public sealed class StreamReplicaGroup
/// processes each one:
/// "+peer:&lt;id&gt;" — adds the peer via ProposeAddPeerAsync
/// "-peer:&lt;id&gt;" — removes the peer via ProposeRemovePeerAsync
/// "smsg:&lt;op&gt;[,...]" — dispatches a stream message op (store/remove/purge)
/// "centry:&lt;op&gt;" — dispatches a consumer op (ack/nak/deliver/term/progress)
/// anything else — marks the entry as processed via MarkProcessed
/// Go reference: jetstream_cluster.go:processStreamEntries (apply loop).
/// </summary>
@@ -207,13 +242,115 @@ public sealed class StreamReplicaGroup
var peerId = entry.Command["-peer:".Length..];
await Leader.ProposeRemovePeerAsync(peerId, ct);
}
else if (entry.Command.StartsWith("smsg:", StringComparison.Ordinal))
{
var opStr = entry.Command["smsg:".Length..];
if (TryParseStreamMsgOp(opStr, out var msgOp))
ApplyStreamMsgOp(msgOp, entry.Index);
else
_lastUnknownCommand = entry.Command;
Leader.MarkProcessed(entry.Index);
}
else if (entry.Command.StartsWith("centry:", StringComparison.Ordinal))
{
var opStr = entry.Command["centry:".Length..];
if (TryParseConsumerOp(opStr, out var consumerOp))
ApplyConsumerEntry(consumerOp);
else
_lastUnknownCommand = entry.Command;
Leader.MarkProcessed(entry.Index);
}
else
{
_lastUnknownCommand = entry.Command;
Leader.MarkProcessed(entry.Index);
}
}
}
/// <summary>
/// Applies a stream-level message operation (Store, Remove, Purge) to the local state.
/// Go reference: jetstream_cluster.go:2474-4261 processStreamEntries — per-message ops.
/// </summary>
public void ApplyStreamMsgOp(StreamMsgOp op, long index = 0)
{
switch (op)
{
case StreamMsgOp.Store:
// Increment message count and track the sequence.
ApplyMessage(index > 0 ? index : Interlocked.Read(ref _messageCount) + 1);
break;
case StreamMsgOp.Remove:
// Decrement message count; clamp to zero.
long current;
do
{
current = Interlocked.Read(ref _messageCount);
if (current <= 0)
return;
}
while (Interlocked.CompareExchange(ref _messageCount, current - 1, current) != current);
break;
case StreamMsgOp.Purge:
// Clear all messages: reset count and last sequence.
Interlocked.Exchange(ref _messageCount, 0);
Interlocked.Exchange(ref _lastSequence, 0);
break;
}
}
/// <summary>
/// Applies a consumer state entry (Ack, Nak, Deliver, Term, Progress).
/// Go reference: jetstream_cluster.go processConsumerEntries.
/// </summary>
public void ApplyConsumerEntry(ConsumerOp op)
{
switch (op)
{
case ConsumerOp.Ack:
Interlocked.Increment(ref _ackCount);
break;
case ConsumerOp.Nak:
Interlocked.Increment(ref _nakCount);
break;
case ConsumerOp.Deliver:
Interlocked.Increment(ref _deliverCount);
break;
// Term and Progress are no-ops in the model but are valid ops.
case ConsumerOp.Term:
case ConsumerOp.Progress:
break;
}
}
private static bool TryParseStreamMsgOp(string s, out StreamMsgOp op)
{
op = s switch
{
"store" => StreamMsgOp.Store,
"remove" => StreamMsgOp.Remove,
"purge" => StreamMsgOp.Purge,
_ => (StreamMsgOp)(-1),
};
return (int)op >= 0;
}
private static bool TryParseConsumerOp(string s, out ConsumerOp op)
{
op = s switch
{
"ack" => ConsumerOp.Ack,
"nak" => ConsumerOp.Nak,
"deliver" => ConsumerOp.Deliver,
"term" => ConsumerOp.Term,
"progress" => ConsumerOp.Progress,
_ => (ConsumerOp)(-1),
};
return (int)op >= 0;
}
/// <summary>
/// Creates a snapshot of the current state at the leader's applied index and compacts
/// the log up to that point.
@@ -298,3 +435,35 @@ public sealed class LeaderChangedEventArgs(string previousLeaderId, string newLe
public string NewLeaderId { get; } = newLeaderId;
public int NewTerm { get; } = newTerm;
}
/// <summary>
/// Message-level operation types applied to a stream's RAFT group.
/// Go reference: jetstream_cluster.go:2474-4261 processStreamEntries — op constants.
/// </summary>
public enum StreamMsgOp
{
/// <summary>Store a new message; increments the message count and advances the sequence.</summary>
Store,
/// <summary>Remove a message by sequence; decrements the message count.</summary>
Remove,
/// <summary>Purge all messages; resets message count and last sequence to zero.</summary>
Purge,
}
/// <summary>
/// Consumer-state operation types applied to a consumer's RAFT group.
/// Go reference: jetstream_cluster.go processConsumerEntries — op constants.
/// </summary>
public enum ConsumerOp
{
/// <summary>Consumer acknowledged a message.</summary>
Ack,
/// <summary>Consumer negatively acknowledged a message (redelivery scheduled).</summary>
Nak,
/// <summary>A message was delivered to the consumer.</summary>
Deliver,
/// <summary>Consumer terminated a message (no redelivery).</summary>
Term,
/// <summary>Consumer reported progress on a slow-ack message.</summary>
Progress,
}