dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add randomized election timeout jitter (Gap 8.8)

Browse Source 
Add RandomizedElectionTimeout() method to RaftNode returning TimeSpan in
[ElectionTimeoutMinMs, ElectionTimeoutMaxMs) using TotalMilliseconds (not
.Milliseconds component) to prevent synchronized elections after partitions.
Make Random injectable for deterministic testing. Fix SendHeartbeatAsync stub
in NatsRaftTransport and test-local transport implementations to satisfy the
IRaftTransport interface added in Gap 8.7.
This commit is contained in:
Joseph Doherty
2026-02-25 08:30:38 -05:00
parent 5a62100397
commit ae4cc6d613
7 changed files with 355 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
tests/NATS.Server.Tests/Raft/RaftElectionJitterTests.cs Normal file
View File

@@ -0,0 +1,157 @@
using NATS.Server.Raft;
namespace NATS.Server.Tests.Raft;
/// <summary>
/// Tests for randomized election timeout jitter in RaftNode.
/// Jitter prevents synchronized elections after network partitions (split-vote avoidance).
/// Go reference: raft.go resetElectionTimeout — uses rand.Int63n to jitter election timeout.
/// </summary>
public class RaftElectionJitterTests : IDisposable
{
private readonly List<RaftNode> _nodesToDispose = [];
public void Dispose()
{
foreach (var node in _nodesToDispose)
node.Dispose();
}
private RaftNode CreateTrackedNode(string id, Random? random = null)
{
var node = new RaftNode(id, random: random);
_nodesToDispose.Add(node);
return node;
}
[Fact]
public void RandomizedElectionTimeout_within_range()
{
// Go reference: raft.go resetElectionTimeout — timeout is always within [min, max).
var node = CreateTrackedNode("n1");
node.ElectionTimeoutMinMs = 150;
node.ElectionTimeoutMaxMs = 300;
for (var i = 0; i < 100; i++)
{
var timeout = node.RandomizedElectionTimeout();
timeout.TotalMilliseconds.ShouldBeGreaterThanOrEqualTo(150);
timeout.TotalMilliseconds.ShouldBeLessThan(300);
}
}
[Fact]
public void RandomizedElectionTimeout_varies()
{
// Multiple calls must produce at least 2 distinct values in 10 samples,
// confirming that the timeout is not fixed.
// Go reference: raft.go resetElectionTimeout — jitter ensures nodes don't all
// timeout simultaneously after a partition.
var node = CreateTrackedNode("n1");
node.ElectionTimeoutMinMs = 150;
node.ElectionTimeoutMaxMs = 300;
var samples = Enumerable.Range(0, 10)
.Select(_ => node.RandomizedElectionTimeout().TotalMilliseconds)
.ToList();
var distinct = samples.Distinct().Count();
distinct.ShouldBeGreaterThanOrEqualTo(2);
}
[Fact]
public void RandomizedElectionTimeout_uses_total_milliseconds()
{
// Verifies that TotalMilliseconds (not .Milliseconds component) gives the full
// value. For timeouts >= 1000 ms, .Milliseconds would return only the sub-second
// component (0-999), while TotalMilliseconds returns the complete value.
// This test uses a range that straddles 1000 ms to expose the bug if present.
// Go reference: raft.go resetElectionTimeout uses full duration, not sub-second part.
var node = CreateTrackedNode("n1");
node.ElectionTimeoutMinMs = 1200;
node.ElectionTimeoutMaxMs = 1500;
for (var i = 0; i < 50; i++)
{
var timeout = node.RandomizedElectionTimeout();
// TotalMilliseconds must be in [1200, 1500)
timeout.TotalMilliseconds.ShouldBeGreaterThanOrEqualTo(1200);
timeout.TotalMilliseconds.ShouldBeLessThan(1500);
// The .Milliseconds property would return only the 0-999 sub-second
// component. If the implementation incorrectly used .Milliseconds,
// these values would be wrong (< 1200). Verify TotalMilliseconds is >= 1200.
((int)timeout.TotalMilliseconds).ShouldBeGreaterThanOrEqualTo(1200);
}
}
[Fact]
public void ResetElectionTimeout_randomizes_each_time()
{
// Successive calls to ResetElectionTimeout should obtain fresh random intervals.
// We observe the timer change indirectly by injecting a deterministic Random
// and confirming it gets called on each reset.
// Go reference: raft.go resetElectionTimeout — called on every heartbeat and
// leader append to re-arm with a fresh random deadline.
var callCount = 0;
var deterministicRandom = new CountingRandom(() => callCount++);
var node = CreateTrackedNode("n1", deterministicRandom);
node.ElectionTimeoutMinMs = 150;
node.ElectionTimeoutMaxMs = 300;
node.StartElectionTimer();
var countAfterStart = callCount;
node.ResetElectionTimeout();
node.ResetElectionTimeout();
node.ResetElectionTimeout();
// Each ResetElectionTimeout + StartElectionTimer must have called Next() at least once each
callCount.ShouldBeGreaterThan(countAfterStart);
node.StopElectionTimer();
}
[Fact]
public void Different_nodes_get_different_timeouts()
{
// Two nodes created with the default shared Random should produce at least some
// distinct timeout values across multiple samples (probabilistic).
// Go reference: raft.go — each server picks an independent random timeout so
// they do not all call elections at exactly the same moment.
var node1 = CreateTrackedNode("n1");
var node2 = CreateTrackedNode("n2");
node1.ElectionTimeoutMinMs = node2.ElectionTimeoutMinMs = 150;
node1.ElectionTimeoutMaxMs = node2.ElectionTimeoutMaxMs = 300;
var samples1 = Enumerable.Range(0, 20)
.Select(_ => node1.RandomizedElectionTimeout().TotalMilliseconds)
.ToList();
var samples2 = Enumerable.Range(0, 20)
.Select(_ => node2.RandomizedElectionTimeout().TotalMilliseconds)
.ToList();
// The combined set of 40 values must contain at least 2 distinct values
// (if both nodes returned the exact same value every time, that would
// indicate no jitter at all).
var allValues = samples1.Concat(samples2).Distinct().Count();
allValues.ShouldBeGreaterThanOrEqualTo(2);
}
/// <summary>
/// A <see cref="Random"/> subclass that invokes a callback on every call to
/// <see cref="Next(int, int)"/>, allowing tests to count how many times the
/// RaftNode requests a new random value.
/// </summary>
private sealed class CountingRandom(Action onNext) : Random
{
public override int Next(int minValue, int maxValue)
{
onNext();
return base.Next(minValue, maxValue);
}
}
}

6
tests/NATS.Server.Tests/Raft/RaftLeadershipTransferTests.cs
View File

@@ -332,6 +332,9 @@ file sealed class BlockingTimeoutNowTransport : IRaftTransport
await using var reg = ct.Register(() => tcs.TrySetCanceled(ct));
await tcs.Task;
}
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}
/// <summary>
@@ -408,4 +411,7 @@ file sealed class VoteGrantingTransport : IRaftTransport
return Task.CompletedTask;
}
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}

6
tests/NATS.Server.Tests/Raft/RaftLogReplicationTests.cs
View File

@@ -555,6 +555,9 @@ public class RaftLogReplicationTests
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
=> Task.CompletedTask;
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}
// -- Helper transport that succeeds for first follower, fails for rest --
@@ -596,5 +599,8 @@ public class RaftLogReplicationTests
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
=> Task.CompletedTask;
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}
}

3
tests/NATS.Server.Tests/Raft/RaftStrictConsensusRuntimeTests.cs
View File

@@ -43,5 +43,8 @@ public class RaftStrictConsensusRuntimeTests
public Task SendTimeoutNowAsync(string leaderId, string targetId, ulong term, CancellationToken ct)
=> Task.CompletedTask;
public Task SendHeartbeatAsync(string leaderId, IReadOnlyList<string> followerIds, int term, Action<string> onAck, CancellationToken ct)
=> Task.CompletedTask;
}
}