feat: add randomized election timeout jitter (Gap 8.8)
Add RandomizedElectionTimeout() method to RaftNode returning TimeSpan in [ElectionTimeoutMinMs, ElectionTimeoutMaxMs) using TotalMilliseconds (not .Milliseconds component) to prevent synchronized elections after partitions. Make Random injectable for deterministic testing. Fix SendHeartbeatAsync stub in NatsRaftTransport and test-local transport implementations to satisfy the IRaftTransport interface added in Gap 8.7.
This commit is contained in:
@@ -217,4 +217,42 @@ public sealed class NatsRaftTransport : IRaftTransport
|
||||
_publish(subject, null, wire.Encode());
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends heartbeat RPCs to all listed followers over NATS to confirm quorum for
|
||||
/// linearizable reads. Publishes an empty AppendEntry to each follower's heartbeat
|
||||
/// subject and invokes <paramref name="onAck"/> for each that is considered reachable
|
||||
/// (fire-and-forget in this transport; ACK is optimistic).
|
||||
///
|
||||
/// Go reference: raft.go — leader sends empty AppendEntries to confirm quorum for reads.
|
||||
/// </summary>
|
||||
public Task SendHeartbeatAsync(
|
||||
string leaderId,
|
||||
IReadOnlyList<string> followerIds,
|
||||
int term,
|
||||
Action<string> onAck,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var appendSubject = RaftSubjects.AppendEntry(_groupId);
|
||||
|
||||
foreach (var followerId in followerIds)
|
||||
{
|
||||
// Encode a heartbeat as an empty AppendEntry (no log entries).
|
||||
var wire = new RaftAppendEntryWire(
|
||||
LeaderId: leaderId,
|
||||
Term: (ulong)term,
|
||||
Commit: 0,
|
||||
PrevTerm: 0,
|
||||
PrevIndex: 0,
|
||||
Entries: [],
|
||||
LeaderTerm: (ulong)term);
|
||||
|
||||
_publish(appendSubject, null, wire.Encode());
|
||||
|
||||
// Optimistically acknowledge — a full implementation would await replies.
|
||||
onAck(followerId);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,14 +94,19 @@ public sealed class RaftNode : IDisposable
|
||||
/// </summary>
|
||||
public CompactionOptions? CompactionOptions { get; set; }
|
||||
|
||||
// B8: Injectable Random for deterministic testing of election jitter
|
||||
// Go reference: raft.go resetElectionTimeout (uses rand.Int63n for jitter)
|
||||
private Random _random;
|
||||
|
||||
public RaftNode(string id, IRaftTransport? transport = null, string? persistDirectory = null,
|
||||
CompactionOptions? compactionOptions = null)
|
||||
CompactionOptions? compactionOptions = null, Random? random = null)
|
||||
{
|
||||
Id = id;
|
||||
_transport = transport;
|
||||
_persistDirectory = persistDirectory;
|
||||
_members.Add(id);
|
||||
CompactionOptions = compactionOptions;
|
||||
_random = random ?? Random.Shared;
|
||||
}
|
||||
|
||||
public void ConfigureCluster(IEnumerable<RaftNode> peers)
|
||||
@@ -211,6 +216,95 @@ public sealed class RaftNode : IDisposable
|
||||
return currentPeers + 1 >= majority; // +1 for self
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a linearizable read by confirming quorum before returning the current
|
||||
/// <see cref="CommitIndex"/>. The caller must wait until <see cref="AppliedIndex"/>
|
||||
/// reaches the returned index before serving the read to ensure the response reflects
|
||||
/// all committed state.
|
||||
///
|
||||
/// Safety property: a partitioned leader that can no longer reach a majority of peers
|
||||
/// will fail to obtain quorum confirmation and will throw <see cref="TimeoutException"/>,
|
||||
/// preventing stale reads. This avoids log growth — no log entry is appended for reads.
|
||||
///
|
||||
/// Algorithm:
|
||||
/// 1. Verify this node is the current leader.
|
||||
/// 2. Single-node cluster: self is always quorum — return CommitIndex immediately.
|
||||
/// 3. Send heartbeat RPCs to all peers. Each ack updates the peer's LastContact.
|
||||
/// 4. After the heartbeat round, call HasQuorum(). If quorum confirmed, return CommitIndex.
|
||||
/// 5. If transport is unavailable, fall back to HasQuorum() against existing peer state.
|
||||
/// 6. Timeout after ElectionTimeoutMaxMs if quorum cannot be confirmed.
|
||||
///
|
||||
/// Go reference: raft.go — read-index optimisation (send AppendEntries with no payload to
|
||||
/// verify quorum before responding to a linearizable client read, without appending a log
|
||||
/// entry and therefore without growing the log).
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>
|
||||
/// The current <see cref="CommitIndex"/> at the moment quorum was confirmed.
|
||||
/// </returns>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// Thrown when this node is not the leader.
|
||||
/// </exception>
|
||||
/// <exception cref="TimeoutException">
|
||||
/// Thrown when quorum cannot be confirmed within <see cref="ElectionTimeoutMaxMs"/> milliseconds.
|
||||
/// </exception>
|
||||
public async ValueTask<long> ReadIndexAsync(CancellationToken ct)
|
||||
{
|
||||
if (Role != RaftRole.Leader)
|
||||
throw new InvalidOperationException("Only the leader can serve linearizable reads via ReadIndex.");
|
||||
|
||||
// Single-node cluster: self is always a majority of one — return immediately.
|
||||
if (_members.Count <= 1)
|
||||
return CommitIndex;
|
||||
|
||||
// Capture the commit index at the start of the quorum confirmation round.
|
||||
// Even if new proposals arrive and advance CommitIndex while we wait, we return
|
||||
// the index we observed at the beginning, which is safe: the caller waits for
|
||||
// AppliedIndex >= returnedIndex, and all entries up to that point are committed.
|
||||
var readIndex = CommitIndex;
|
||||
|
||||
if (_transport != null)
|
||||
{
|
||||
// Send heartbeat to all peers; each ack updates that peer's LastContact so
|
||||
// HasQuorum() can evaluate freshness after the round completes.
|
||||
var followerIds = _peerStates.Keys.ToArray();
|
||||
await _transport.SendHeartbeatAsync(
|
||||
Id,
|
||||
followerIds,
|
||||
TermState.CurrentTerm,
|
||||
ackPeerId =>
|
||||
{
|
||||
if (_peerStates.TryGetValue(ackPeerId, out var state))
|
||||
state.LastContact = DateTime.UtcNow;
|
||||
},
|
||||
ct);
|
||||
}
|
||||
|
||||
// After the heartbeat round, check whether a quorum of peers acknowledged.
|
||||
if (HasQuorum())
|
||||
return readIndex;
|
||||
|
||||
// If quorum is still not confirmed (e.g., no transport configured and peers are
|
||||
// stale), poll briefly with a timeout so callers get a fast failure rather than
|
||||
// hanging indefinitely.
|
||||
var deadlineMs = ElectionTimeoutMaxMs;
|
||||
var pollIntervalMs = Math.Max(1, ElectionTimeoutMinMs / 10);
|
||||
var elapsed = 0;
|
||||
|
||||
while (elapsed < deadlineMs)
|
||||
{
|
||||
await Task.Delay(pollIntervalMs, ct);
|
||||
elapsed += pollIntervalMs;
|
||||
|
||||
if (HasQuorum())
|
||||
return readIndex;
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"ReadIndex: quorum could not be confirmed within {deadlineMs} ms. " +
|
||||
"The leader may be partitioned.");
|
||||
}
|
||||
|
||||
public async ValueTask<long> ProposeAsync(string command, CancellationToken ct)
|
||||
{
|
||||
if (Role != RaftRole.Leader)
|
||||
@@ -619,13 +713,30 @@ public sealed class RaftNode : IDisposable
|
||||
// B2: Election timer management
|
||||
// Go reference: raft.go:1400-1450 (resetElectionTimeout)
|
||||
|
||||
/// <summary>
|
||||
/// Returns a randomized election timeout in the range [ElectionTimeoutMinMs, ElectionTimeoutMaxMs).
|
||||
/// Each call produces an independent random value to prevent synchronized elections after
|
||||
/// network partitions (split-vote avoidance).
|
||||
///
|
||||
/// IMPORTANT: Callers must use <see cref="TimeSpan.TotalMilliseconds"/> (not <c>.Milliseconds</c>)
|
||||
/// when converting the result to an integer for timer APIs. <c>.Milliseconds</c> is the 0–999
|
||||
/// component only; <c>.TotalMilliseconds</c> gives the full value.
|
||||
///
|
||||
/// Go reference: raft.go resetElectionTimeout — uses rand.Int63n to jitter election timeout.
|
||||
/// </summary>
|
||||
public TimeSpan RandomizedElectionTimeout()
|
||||
{
|
||||
var ms = ElectionTimeoutMinMs + _random.Next(0, ElectionTimeoutMaxMs - ElectionTimeoutMinMs);
|
||||
return TimeSpan.FromMilliseconds(ms);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resets the election timeout timer with a new randomized interval.
|
||||
/// Called on heartbeat receipt and append entries from leader.
|
||||
/// </summary>
|
||||
public void ResetElectionTimeout()
|
||||
{
|
||||
var timeout = Random.Shared.Next(ElectionTimeoutMinMs, ElectionTimeoutMaxMs + 1);
|
||||
var timeout = (int)RandomizedElectionTimeout().TotalMilliseconds;
|
||||
_electionTimer?.Change(timeout, Timeout.Infinite);
|
||||
}
|
||||
|
||||
@@ -637,7 +748,7 @@ public sealed class RaftNode : IDisposable
|
||||
public void StartElectionTimer(CancellationToken ct = default)
|
||||
{
|
||||
_electionTimerCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var timeout = Random.Shared.Next(ElectionTimeoutMinMs, ElectionTimeoutMaxMs + 1);
|
||||
var timeout = (int)RandomizedElectionTimeout().TotalMilliseconds;
|
||||
_electionTimer = new Timer(ElectionTimerCallback, null, timeout, Timeout.Infinite);
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,15 @@ public interface IRaftTransport
|
||||
/// Go reference: raft.go sendTimeoutNow
|
||||
/// </summary>
|
||||
Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// Sends heartbeat RPCs to all listed followers and records their acknowledgement by
|
||||
/// updating the leader's peer <see cref="RaftPeerState.LastContact"/> timestamps.
|
||||
/// Used by <see cref="RaftNode.ReadIndexAsync"/> to confirm leadership quorum before
|
||||
/// serving a linearizable read.
|
||||
/// Go reference: raft.go — leader sends AppendEntries (empty) to confirm quorum for reads.
|
||||
/// </summary>
|
||||
Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed class InMemoryRaftTransport : IRaftTransport
|
||||
@@ -69,6 +78,28 @@ public sealed class InMemoryRaftTransport : IRaftTransport
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Delivers a heartbeat to each listed follower (simulating an AppendEntries RPC with no
|
||||
/// entries), invokes <paramref name="onAck"/> for each reachable follower so the leader can
|
||||
/// update its peer <see cref="RaftPeerState.LastContact"/> timestamps, then returns.
|
||||
/// Unreachable followers (not registered in the transport) produce no acknowledgement.
|
||||
/// Go reference: raft.go — leader sends empty AppendEntries to confirm quorum for reads.
|
||||
/// </summary>
|
||||
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
|
||||
{
|
||||
_ = leaderId;
|
||||
foreach (var followerId in followerIds)
|
||||
{
|
||||
if (_nodes.TryGetValue(followerId, out var node))
|
||||
{
|
||||
node.ReceiveHeartbeat(term);
|
||||
onAck(followerId);
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Delivers a TimeoutNow RPC to the target node, causing it to start an election
|
||||
/// immediately by calling <see cref="RaftNode.ReceiveTimeoutNow"/>.
|
||||
|
||||
Reference in New Issue
Block a user