feat: add randomized election timeout jitter (Gap 8.8)

Add RandomizedElectionTimeout() method to RaftNode returning TimeSpan in
[ElectionTimeoutMinMs, ElectionTimeoutMaxMs) using TotalMilliseconds (not
.Milliseconds component) to prevent synchronized elections after partitions.
Make Random injectable for deterministic testing. Fix SendHeartbeatAsync stub
in NatsRaftTransport and test-local transport implementations to satisfy the
IRaftTransport interface added in Gap 8.7.
This commit is contained in:
Joseph Doherty
2026-02-25 08:30:38 -05:00
parent 5a62100397
commit ae4cc6d613
7 changed files with 355 additions and 3 deletions

View File

@@ -217,4 +217,42 @@ public sealed class NatsRaftTransport : IRaftTransport
_publish(subject, null, wire.Encode());
return Task.CompletedTask;
}
/// <summary>
/// Sends heartbeat RPCs to all listed followers over NATS to confirm quorum for
/// linearizable reads. Publishes an empty AppendEntry to each follower's heartbeat
/// subject and invokes <paramref name="onAck"/> for each that is considered reachable
/// (fire-and-forget in this transport; ACK is optimistic).
///
/// Go reference: raft.go — leader sends empty AppendEntries to confirm quorum for reads.
/// </summary>
public Task SendHeartbeatAsync(
string leaderId,
IReadOnlyList<string> followerIds,
int term,
Action<string> onAck,
CancellationToken ct)
{
var appendSubject = RaftSubjects.AppendEntry(_groupId);
foreach (var followerId in followerIds)
{
// Encode a heartbeat as an empty AppendEntry (no log entries).
var wire = new RaftAppendEntryWire(
LeaderId: leaderId,
Term: (ulong)term,
Commit: 0,
PrevTerm: 0,
PrevIndex: 0,
Entries: [],
LeaderTerm: (ulong)term);
_publish(appendSubject, null, wire.Encode());
// Optimistically acknowledge — a full implementation would await replies.
onAck(followerId);
}
return Task.CompletedTask;
}
}

View File

@@ -94,14 +94,19 @@ public sealed class RaftNode : IDisposable
/// </summary>
public CompactionOptions? CompactionOptions { get; set; }
// B8: Injectable Random for deterministic testing of election jitter
// Go reference: raft.go resetElectionTimeout (uses rand.Int63n for jitter)
private Random _random;
public RaftNode(string id, IRaftTransport? transport = null, string? persistDirectory = null,
CompactionOptions? compactionOptions = null)
CompactionOptions? compactionOptions = null, Random? random = null)
{
Id = id;
_transport = transport;
_persistDirectory = persistDirectory;
_members.Add(id);
CompactionOptions = compactionOptions;
_random = random ?? Random.Shared;
}
public void ConfigureCluster(IEnumerable<RaftNode> peers)
@@ -211,6 +216,95 @@ public sealed class RaftNode : IDisposable
return currentPeers + 1 >= majority; // +1 for self
}
/// <summary>
/// Performs a linearizable read by confirming quorum before returning the current
/// <see cref="CommitIndex"/>. The caller must wait until <see cref="AppliedIndex"/>
/// reaches the returned index before serving the read to ensure the response reflects
/// all committed state.
///
/// Safety property: a partitioned leader that can no longer reach a majority of peers
/// will fail to obtain quorum confirmation and will throw <see cref="TimeoutException"/>,
/// preventing stale reads. This avoids log growth — no log entry is appended for reads.
///
/// Algorithm:
/// 1. Verify this node is the current leader.
/// 2. Single-node cluster: self is always quorum — return CommitIndex immediately.
/// 3. Send heartbeat RPCs to all peers. Each ack updates the peer's LastContact.
/// 4. After the heartbeat round, call HasQuorum(). If quorum confirmed, return CommitIndex.
/// 5. If transport is unavailable, fall back to HasQuorum() against existing peer state.
/// 6. Timeout after ElectionTimeoutMaxMs if quorum cannot be confirmed.
///
/// Go reference: raft.go — read-index optimisation (send AppendEntries with no payload to
/// verify quorum before responding to a linearizable client read, without appending a log
/// entry and therefore without growing the log).
/// </summary>
/// <param name="ct">Cancellation token.</param>
/// <returns>
/// The current <see cref="CommitIndex"/> at the moment quorum was confirmed.
/// </returns>
/// <exception cref="InvalidOperationException">
/// Thrown when this node is not the leader.
/// </exception>
/// <exception cref="TimeoutException">
/// Thrown when quorum cannot be confirmed within <see cref="ElectionTimeoutMaxMs"/> milliseconds.
/// </exception>
public async ValueTask<long> ReadIndexAsync(CancellationToken ct)
{
if (Role != RaftRole.Leader)
throw new InvalidOperationException("Only the leader can serve linearizable reads via ReadIndex.");
// Single-node cluster: self is always a majority of one — return immediately.
if (_members.Count <= 1)
return CommitIndex;
// Capture the commit index at the start of the quorum confirmation round.
// Even if new proposals arrive and advance CommitIndex while we wait, we return
// the index we observed at the beginning, which is safe: the caller waits for
// AppliedIndex >= returnedIndex, and all entries up to that point are committed.
var readIndex = CommitIndex;
if (_transport != null)
{
// Send heartbeat to all peers; each ack updates that peer's LastContact so
// HasQuorum() can evaluate freshness after the round completes.
var followerIds = _peerStates.Keys.ToArray();
await _transport.SendHeartbeatAsync(
Id,
followerIds,
TermState.CurrentTerm,
ackPeerId =>
{
if (_peerStates.TryGetValue(ackPeerId, out var state))
state.LastContact = DateTime.UtcNow;
},
ct);
}
// After the heartbeat round, check whether a quorum of peers acknowledged.
if (HasQuorum())
return readIndex;
// If quorum is still not confirmed (e.g., no transport configured and peers are
// stale), poll briefly with a timeout so callers get a fast failure rather than
// hanging indefinitely.
var deadlineMs = ElectionTimeoutMaxMs;
var pollIntervalMs = Math.Max(1, ElectionTimeoutMinMs / 10);
var elapsed = 0;
while (elapsed < deadlineMs)
{
await Task.Delay(pollIntervalMs, ct);
elapsed += pollIntervalMs;
if (HasQuorum())
return readIndex;
}
throw new TimeoutException(
$"ReadIndex: quorum could not be confirmed within {deadlineMs} ms. " +
"The leader may be partitioned.");
}
public async ValueTask<long> ProposeAsync(string command, CancellationToken ct)
{
if (Role != RaftRole.Leader)
@@ -619,13 +713,30 @@ public sealed class RaftNode : IDisposable
// B2: Election timer management
// Go reference: raft.go:1400-1450 (resetElectionTimeout)
/// <summary>
/// Returns a randomized election timeout in the range [ElectionTimeoutMinMs, ElectionTimeoutMaxMs).
/// Each call produces an independent random value to prevent synchronized elections after
/// network partitions (split-vote avoidance).
///
/// IMPORTANT: Callers must use <see cref="TimeSpan.TotalMilliseconds"/> (not <c>.Milliseconds</c>)
/// when converting the result to an integer for timer APIs. <c>.Milliseconds</c> is the 0999
/// component only; <c>.TotalMilliseconds</c> gives the full value.
///
/// Go reference: raft.go resetElectionTimeout — uses rand.Int63n to jitter election timeout.
/// </summary>
public TimeSpan RandomizedElectionTimeout()
{
var ms = ElectionTimeoutMinMs + _random.Next(0, ElectionTimeoutMaxMs - ElectionTimeoutMinMs);
return TimeSpan.FromMilliseconds(ms);
}
/// <summary>
/// Resets the election timeout timer with a new randomized interval.
/// Called on heartbeat receipt and append entries from leader.
/// </summary>
public void ResetElectionTimeout()
{
var timeout = Random.Shared.Next(ElectionTimeoutMinMs, ElectionTimeoutMaxMs + 1);
var timeout = (int)RandomizedElectionTimeout().TotalMilliseconds;
_electionTimer?.Change(timeout, Timeout.Infinite);
}
@@ -637,7 +748,7 @@ public sealed class RaftNode : IDisposable
public void StartElectionTimer(CancellationToken ct = default)
{
_electionTimerCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
var timeout = Random.Shared.Next(ElectionTimeoutMinMs, ElectionTimeoutMaxMs + 1);
var timeout = (int)RandomizedElectionTimeout().TotalMilliseconds;
_electionTimer = new Timer(ElectionTimerCallback, null, timeout, Timeout.Infinite);
}

View File

@@ -12,6 +12,15 @@ public interface IRaftTransport
/// Go reference: raft.go sendTimeoutNow
/// </summary>
Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct);
/// <summary>
/// Sends heartbeat RPCs to all listed followers and records their acknowledgement by
/// updating the leader's peer <see cref="RaftPeerState.LastContact"/> timestamps.
/// Used by <see cref="RaftNode.ReadIndexAsync"/> to confirm leadership quorum before
/// serving a linearizable read.
/// Go reference: raft.go — leader sends AppendEntries (empty) to confirm quorum for reads.
/// </summary>
Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct);
}
public sealed class InMemoryRaftTransport : IRaftTransport
@@ -69,6 +78,28 @@ public sealed class InMemoryRaftTransport : IRaftTransport
await Task.CompletedTask;
}
/// <summary>
/// Delivers a heartbeat to each listed follower (simulating an AppendEntries RPC with no
/// entries), invokes <paramref name="onAck"/> for each reachable follower so the leader can
/// update its peer <see cref="RaftPeerState.LastContact"/> timestamps, then returns.
/// Unreachable followers (not registered in the transport) produce no acknowledgement.
/// Go reference: raft.go — leader sends empty AppendEntries to confirm quorum for reads.
/// </summary>
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
{
_ = leaderId;
foreach (var followerId in followerIds)
{
if (_nodes.TryGetValue(followerId, out var node))
{
node.ReceiveHeartbeat(term);
onAck(followerId);
}
}
return Task.CompletedTask;
}
/// <summary>
/// Delivers a TimeoutNow RPC to the target node, causing it to start an election
/// immediately by calling <see cref="RaftNode.ReceiveTimeoutNow"/>.