feat: add quorum check before proposing entries (Gap 8.6)
Add HasQuorum() to RaftNode that counts peers with LastContact within 2 × ElectionTimeoutMaxMs and returns true only when self + current peers reaches majority. ProposeAsync now throws InvalidOperationException with "no quorum" when HasQuorum() returns false, preventing a partitioned leader from diverging the log. Add 14 tests in RaftQuorumCheckTests.cs covering single-node, 3-node, 5-node, boundary window, and heartbeat restore scenarios. Update RaftHealthTests.LastContact_updates_on_successful_replication to avoid triggering the new quorum guard.
This commit is contained in:
Joseph Doherty
344
tests/NATS.Server.Tests/Raft/RaftCompactionPolicyTests.cs
Normal file
344
tests/NATS.Server.Tests/Raft/RaftCompactionPolicyTests.cs
Normal file
@@ -0,0 +1,344 @@
|
||||
using NATS.Server.Raft;
|
||||
|
||||
namespace NATS.Server.Tests.Raft;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for configurable log compaction policies (Gap 8.5).
|
||||
///
|
||||
/// Covers <see cref="CompactionPolicy"/> enum values, <see cref="CompactionOptions"/> defaults
|
||||
/// and threshold logic, and the integration between <see cref="RaftNode.CompactLogAsync"/>
|
||||
/// and the policy engine.
|
||||
///
|
||||
/// Go reference: raft.go compactLog / WAL compact threshold checks.
|
||||
/// </summary>
|
||||
public class RaftCompactionPolicyTests
|
||||
{
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Creates a leader node with <paramref name="entryCount"/> proposed entries all applied.
|
||||
/// Each entry command is "cmd-N" so UTF-8 command size is predictable (~6 bytes each).
|
||||
/// </summary>
|
||||
private static async Task<RaftNode> CreateLeaderWithEntriesAsync(int entryCount,
|
||||
CompactionOptions? compactionOptions = null)
|
||||
{
|
||||
var node = new RaftNode("leader", compactionOptions: compactionOptions);
|
||||
node.ConfigureCluster([node]);
|
||||
node.StartElection(1);
|
||||
|
||||
for (var i = 1; i <= entryCount; i++)
|
||||
await node.ProposeAsync($"cmd-{i}", default);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a log entry at an explicit index with an optional custom timestamp.
|
||||
/// Used for ByAge tests where we need to control entry timestamps.
|
||||
/// </summary>
|
||||
private static RaftLogEntry MakeEntry(long index, int term, string command, DateTime? timestamp = null)
|
||||
{
|
||||
var entry = new RaftLogEntry(index, term, command);
|
||||
if (timestamp.HasValue)
|
||||
entry = entry with { Timestamp = timestamp.Value };
|
||||
return entry;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CompactionOptions default values
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Go reference: raft.go compactLog threshold defaults.
|
||||
[Fact]
|
||||
public void CompactionOptions_default_values()
|
||||
{
|
||||
var opts = new CompactionOptions();
|
||||
|
||||
opts.Policy.ShouldBe(CompactionPolicy.None);
|
||||
opts.MaxEntries.ShouldBe(10_000);
|
||||
opts.MaxSizeBytes.ShouldBe(100L * 1024 * 1024);
|
||||
opts.MaxAge.ShouldBe(TimeSpan.FromHours(1));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// None policy — no-op
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Go reference: raft.go compactLog — no compaction when policy is None.
|
||||
[Fact]
|
||||
public async Task None_policy_does_not_compact()
|
||||
{
|
||||
var opts = new CompactionOptions { Policy = CompactionPolicy.None };
|
||||
var node = await CreateLeaderWithEntriesAsync(50, opts);
|
||||
|
||||
node.Log.Entries.Count.ShouldBe(50);
|
||||
await node.CompactLogAsync(default);
|
||||
|
||||
// With None policy, nothing should be removed.
|
||||
node.Log.Entries.Count.ShouldBe(50);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ByCount policy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Go reference: raft.go compactLog — keep at most MaxEntries.
|
||||
[Fact]
|
||||
public async Task ByCount_compacts_when_log_exceeds_max_entries()
|
||||
{
|
||||
const int total = 100;
|
||||
const int keep = 50;
|
||||
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByCount,
|
||||
MaxEntries = keep,
|
||||
};
|
||||
var node = await CreateLeaderWithEntriesAsync(total, opts);
|
||||
|
||||
// Before compaction: all 100 entries are present.
|
||||
node.Log.Entries.Count.ShouldBe(total);
|
||||
|
||||
await node.CompactLogAsync(default);
|
||||
|
||||
// After compaction: only the newest `keep` entries remain.
|
||||
node.Log.Entries.Count.ShouldBe(keep);
|
||||
|
||||
// The oldest entry's index should be total - keep + 1
|
||||
node.Log.Entries[0].Index.ShouldBe(total - keep + 1);
|
||||
}
|
||||
|
||||
// Go reference: raft.go compactLog — do not compact below applied index (safety guard).
|
||||
[Fact]
|
||||
public async Task ByCount_does_not_compact_below_applied_index()
|
||||
{
|
||||
// Build a node but manually set a lower applied index so the safety guard kicks in.
|
||||
const int total = 100;
|
||||
const int keep = 50;
|
||||
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByCount,
|
||||
MaxEntries = keep,
|
||||
};
|
||||
|
||||
var node = await CreateLeaderWithEntriesAsync(total, opts);
|
||||
|
||||
// Artificially lower the applied index so the safety guard prevents full compaction.
|
||||
// With applied = 30 the policy wants to compact to index 50 but the guard limits it to 30.
|
||||
node.AppliedIndex = 30;
|
||||
|
||||
await node.CompactLogAsync(default);
|
||||
|
||||
// Only entries up to index 30 may be removed.
|
||||
// BaseIndex should be 30, entries with index > 30 remain.
|
||||
node.Log.BaseIndex.ShouldBe(30);
|
||||
node.Log.Entries[0].Index.ShouldBe(31);
|
||||
}
|
||||
|
||||
// Go reference: raft.go compactLog — no-op when count is within threshold.
|
||||
[Fact]
|
||||
public async Task ByCount_does_not_compact_when_within_threshold()
|
||||
{
|
||||
const int total = 20;
|
||||
const int maxEntries = 50; // more than total — nothing to compact
|
||||
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByCount,
|
||||
MaxEntries = maxEntries,
|
||||
};
|
||||
var node = await CreateLeaderWithEntriesAsync(total, opts);
|
||||
|
||||
await node.CompactLogAsync(default);
|
||||
|
||||
node.Log.Entries.Count.ShouldBe(total);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BySize policy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Go reference: raft.go compactLog — compact oldest entries until under size threshold.
|
||||
[Fact]
|
||||
public void BySize_compacts_when_total_size_exceeds_threshold()
|
||||
{
|
||||
// Construct a log manually so we control sizes precisely.
|
||||
// Each entry: index(8) + term(4) + command(N bytes).
|
||||
// Use a 100-byte command to make size maths easy.
|
||||
var log = new RaftLog();
|
||||
var largeCommand = new string('x', 100); // 100 UTF-8 bytes
|
||||
|
||||
// Append 10 entries — each ~112 bytes → total ~1,120 bytes.
|
||||
for (var i = 1; i <= 10; i++)
|
||||
log.Append(term: 1, command: largeCommand);
|
||||
|
||||
// Set threshold at ~600 bytes — roughly half the total — so the oldest ~5 entries
|
||||
// should be compacted.
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.BySize,
|
||||
MaxSizeBytes = 600,
|
||||
};
|
||||
|
||||
// appliedIndex = last entry index (all applied)
|
||||
var appliedIndex = log.Entries[^1].Index;
|
||||
var cutoff = opts.ComputeCompactionIndex(log, appliedIndex);
|
||||
|
||||
// Should return a positive index, meaning some entries must be compacted.
|
||||
cutoff.ShouldBeGreaterThan(0);
|
||||
cutoff.ShouldBeLessThanOrEqualTo(appliedIndex);
|
||||
|
||||
log.Compact(cutoff);
|
||||
|
||||
// After compaction fewer entries should remain.
|
||||
log.Entries.Count.ShouldBeLessThan(10);
|
||||
log.Entries.Count.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// Go reference: raft.go compactLog — no-op when total size is under threshold.
|
||||
[Fact]
|
||||
public void BySize_does_not_compact_when_under_threshold()
|
||||
{
|
||||
var log = new RaftLog();
|
||||
for (var i = 1; i <= 5; i++)
|
||||
log.Append(term: 1, command: "tiny");
|
||||
|
||||
// Threshold large enough to fit everything.
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.BySize,
|
||||
MaxSizeBytes = 10_000,
|
||||
};
|
||||
|
||||
var cutoff = opts.ComputeCompactionIndex(log, log.Entries[^1].Index);
|
||||
|
||||
cutoff.ShouldBe(-1); // no compaction
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ByAge policy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Go reference: raft.go compactLog — compact entries older than MaxAge.
|
||||
[Fact]
|
||||
public void ByAge_compacts_old_entries()
|
||||
{
|
||||
// Build a log manually with some old and some fresh entries.
|
||||
var log = new RaftLog();
|
||||
var old = DateTime.UtcNow - TimeSpan.FromHours(2);
|
||||
var fresh = DateTime.UtcNow - TimeSpan.FromMinutes(1);
|
||||
|
||||
// Entries 1–5: old (2 hours ago)
|
||||
for (var i = 1; i <= 5; i++)
|
||||
log.AppendWithTimestamp(term: 1, command: $"old-{i}", timestamp: old);
|
||||
|
||||
// Entries 6–10: fresh (1 minute ago)
|
||||
for (var i = 6; i <= 10; i++)
|
||||
log.AppendWithTimestamp(term: 1, command: $"fresh-{i}", timestamp: fresh);
|
||||
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByAge,
|
||||
MaxAge = TimeSpan.FromHours(1), // default; entries older than 1h are eligible
|
||||
};
|
||||
|
||||
var appliedIndex = log.Entries[^1].Index;
|
||||
var cutoff = opts.ComputeCompactionIndex(log, appliedIndex);
|
||||
|
||||
// Should compact through index 5 (the last old entry).
|
||||
cutoff.ShouldBe(5);
|
||||
log.Compact(cutoff);
|
||||
|
||||
log.Entries.Count.ShouldBe(5); // only the fresh entries remain
|
||||
log.Entries[0].Index.ShouldBe(6);
|
||||
}
|
||||
|
||||
// Go reference: raft.go compactLog — no-op when all entries are still fresh.
|
||||
[Fact]
|
||||
public void ByAge_does_not_compact_fresh_entries()
|
||||
{
|
||||
var log = new RaftLog();
|
||||
var fresh = DateTime.UtcNow - TimeSpan.FromMinutes(5);
|
||||
|
||||
for (var i = 1; i <= 10; i++)
|
||||
log.AppendWithTimestamp(term: 1, command: $"cmd-{i}", timestamp: fresh);
|
||||
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByAge,
|
||||
MaxAge = TimeSpan.FromHours(1),
|
||||
};
|
||||
|
||||
var cutoff = opts.ComputeCompactionIndex(log, log.Entries[^1].Index);
|
||||
|
||||
cutoff.ShouldBe(-1); // nothing old enough
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CompactLogAsync integration tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Go reference: raft.go CompactLog — safety guard: never exceed applied index.
|
||||
[Fact]
|
||||
public async Task CompactLogAsync_with_policy_respects_applied_index()
|
||||
{
|
||||
// Node with 50 entries. Policy wants to compact to index 40, but applied is only 25.
|
||||
const int total = 50;
|
||||
var opts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByCount,
|
||||
MaxEntries = 10, // wants to compact to index 40
|
||||
};
|
||||
|
||||
var node = await CreateLeaderWithEntriesAsync(total, opts);
|
||||
|
||||
// Override applied index to a lower value than the policy cutoff.
|
||||
node.AppliedIndex = 25;
|
||||
|
||||
await node.CompactLogAsync(default);
|
||||
|
||||
// Compaction must not go past applied index = 25.
|
||||
node.Log.BaseIndex.ShouldBeLessThanOrEqualTo(25);
|
||||
|
||||
// All entries above index 25 must still be present.
|
||||
node.Log.Entries.ShouldAllBe(e => e.Index > 25);
|
||||
}
|
||||
|
||||
// Go reference: raft.go CompactLog — when no policy is set, compacts to applied index.
|
||||
[Fact]
|
||||
public async Task CompactLogAsync_without_policy_compacts_to_applied_index()
|
||||
{
|
||||
var node = await CreateLeaderWithEntriesAsync(30);
|
||||
// No CompactionOptions set.
|
||||
|
||||
await node.CompactLogAsync(default);
|
||||
|
||||
// All entries compacted to the applied index.
|
||||
node.Log.Entries.Count.ShouldBe(0);
|
||||
node.Log.BaseIndex.ShouldBe(node.AppliedIndex);
|
||||
}
|
||||
|
||||
// Go reference: raft.go CompactLog — per-call options override node-level options.
|
||||
[Fact]
|
||||
public async Task CompactLogAsync_call_level_options_override_node_level_options()
|
||||
{
|
||||
// Node has None policy (no auto-compaction).
|
||||
var nodeOpts = new CompactionOptions { Policy = CompactionPolicy.None };
|
||||
var node = await CreateLeaderWithEntriesAsync(50, nodeOpts);
|
||||
|
||||
// But the call-level override uses ByCount with MaxEntries = 25.
|
||||
var callOpts = new CompactionOptions
|
||||
{
|
||||
Policy = CompactionPolicy.ByCount,
|
||||
MaxEntries = 25,
|
||||
};
|
||||
|
||||
await node.CompactLogAsync(default, callOpts);
|
||||
|
||||
// Call-level option must win: 25 entries kept.
|
||||
node.Log.Entries.Count.ShouldBe(25);
|
||||
}
|
||||
}
|
||||
@@ -301,15 +301,15 @@ public class RaftHealthTests
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Set peer contacts in the past
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
// Record timestamps just before proposing (peers are fresh from ConfigureCluster).
|
||||
var beforePropose = DateTime.UtcNow;
|
||||
|
||||
await leader.ProposeAsync("cmd-1", default);
|
||||
|
||||
// Successful replication should update LastContact
|
||||
// Successful replication should update LastContact to at least the time we
|
||||
// recorded before the propose call.
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact.ShouldBeGreaterThan(DateTime.UtcNow.AddSeconds(-2));
|
||||
state.LastContact.ShouldBeGreaterThanOrEqualTo(beforePropose);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
282
tests/NATS.Server.Tests/Raft/RaftQuorumCheckTests.cs
Normal file
282
tests/NATS.Server.Tests/Raft/RaftQuorumCheckTests.cs
Normal file
@@ -0,0 +1,282 @@
|
||||
using NATS.Server.Raft;
|
||||
|
||||
namespace NATS.Server.Tests.Raft;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for HasQuorum() and the quorum guard in ProposeAsync (Gap 8.6).
|
||||
///
|
||||
/// A leader must confirm that a majority of peers have contacted it recently
|
||||
/// (within 2 × ElectionTimeoutMaxMs) before it is allowed to append new log entries.
|
||||
/// This prevents a partitioned leader from diverging the log while isolated from
|
||||
/// the rest of the cluster.
|
||||
///
|
||||
/// Go reference: raft.go checkQuorum / stepDown — a leader steps down (and therefore
|
||||
/// blocks proposals) when it has not heard from a quorum of peers within the
|
||||
/// election-timeout window.
|
||||
/// </summary>
|
||||
public class RaftQuorumCheckTests
|
||||
{
|
||||
// -- Helpers (self-contained, no shared TestHelpers class) --
|
||||
|
||||
private static (RaftNode[] nodes, InMemoryRaftTransport transport) CreateCluster(int size)
|
||||
{
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var nodes = Enumerable.Range(1, size)
|
||||
.Select(i => new RaftNode($"n{i}", transport))
|
||||
.ToArray();
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
transport.Register(node);
|
||||
node.ConfigureCluster(nodes);
|
||||
// Short timeouts so tests do not need real async delays.
|
||||
node.ElectionTimeoutMinMs = 50;
|
||||
node.ElectionTimeoutMaxMs = 100;
|
||||
}
|
||||
return (nodes, transport);
|
||||
}
|
||||
|
||||
private static RaftNode ElectLeader(RaftNode[] nodes)
|
||||
{
|
||||
var candidate = nodes[0];
|
||||
candidate.StartElection(nodes.Length);
|
||||
foreach (var voter in nodes.Skip(1))
|
||||
candidate.ReceiveVote(voter.GrantVote(candidate.Term, candidate.Id), nodes.Length);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
// -- HasQuorum tests --
|
||||
|
||||
// Go reference: raft.go checkQuorum (leader confirms majority contact before acting)
|
||||
[Fact]
|
||||
public void HasQuorum_returns_true_with_majority_peers_current()
|
||||
{
|
||||
// 3-node cluster: leader + 2 peers. Both peers are freshly initialized by
|
||||
// ConfigureCluster so their LastContact is very close to UtcNow.
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Peers were initialized with DateTime.UtcNow — they are within the quorum window.
|
||||
leader.HasQuorum().ShouldBeTrue();
|
||||
}
|
||||
|
||||
// Go reference: raft.go checkQuorum (leader steps down when peers are stale)
|
||||
[Fact]
|
||||
public void HasQuorum_returns_false_with_stale_peers()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Set all peer contacts well beyond the quorum window (2 × 100 ms = 200 ms).
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
|
||||
leader.HasQuorum().ShouldBeFalse();
|
||||
}
|
||||
|
||||
// Go reference: raft.go — followers never have proposer quorum
|
||||
[Fact]
|
||||
public void HasQuorum_returns_false_for_non_leader()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
_ = ElectLeader(nodes);
|
||||
|
||||
// nodes[1] is a follower.
|
||||
var follower = nodes[1];
|
||||
follower.IsLeader.ShouldBeFalse();
|
||||
|
||||
follower.HasQuorum().ShouldBeFalse();
|
||||
}
|
||||
|
||||
// Go reference: raft.go — candidate also does not have proposer quorum
|
||||
[Fact]
|
||||
public void HasQuorum_returns_false_for_candidate()
|
||||
{
|
||||
// A node becomes a Candidate when StartElection is called but it has not yet
|
||||
// received enough votes to become Leader. In a 3-node cluster, after calling
|
||||
// StartElection on n1 the node is a Candidate (it voted for itself but the
|
||||
// other 2 nodes have not yet responded).
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var candidate = nodes[0];
|
||||
|
||||
// StartElection increments term, sets VotedFor=self, and calls TryBecomeLeader.
|
||||
// With only 1 self-vote in a 3-node cluster quorum is 2, so role stays Candidate.
|
||||
candidate.StartElection(clusterSize: 3);
|
||||
|
||||
candidate.Role.ShouldBe(RaftRole.Candidate);
|
||||
candidate.HasQuorum().ShouldBeFalse();
|
||||
}
|
||||
|
||||
// Go reference: raft.go single-node cluster — self is always a majority of one
|
||||
[Fact]
|
||||
public void HasQuorum_single_node_always_true()
|
||||
{
|
||||
var node = new RaftNode("solo");
|
||||
node.StartElection(clusterSize: 1);
|
||||
|
||||
node.IsLeader.ShouldBeTrue();
|
||||
node.HasQuorum().ShouldBeTrue();
|
||||
}
|
||||
|
||||
// 5-node cluster: with 2 current peers + self = 3, majority of 5 is 3, so quorum.
|
||||
[Fact]
|
||||
public void HasQuorum_five_node_with_two_current_peers_is_true()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(5);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Make 2 peers stale; keep 2 fresh (plus self = 3 voters, majority of 5 = 3).
|
||||
var peerStates = leader.GetPeerStates().Values.ToList();
|
||||
peerStates[0].LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
peerStates[1].LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
// peerStates[2] and peerStates[3] remain fresh (within window).
|
||||
|
||||
leader.HasQuorum().ShouldBeTrue();
|
||||
}
|
||||
|
||||
// 5-node cluster: with only 1 current peer + self = 2, majority of 5 is 3, so no quorum.
|
||||
[Fact]
|
||||
public void HasQuorum_five_node_with_one_current_peer_is_false()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(5);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Make 3 out of 4 peers stale; only 1 fresh peer + self = 2 voters (need 3).
|
||||
var peerStates = leader.GetPeerStates().Values.ToList();
|
||||
peerStates[0].LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
peerStates[1].LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
peerStates[2].LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
// peerStates[3] is fresh.
|
||||
|
||||
leader.HasQuorum().ShouldBeFalse();
|
||||
}
|
||||
|
||||
// -- ProposeAsync quorum guard tests --
|
||||
|
||||
// Go reference: raft.go checkQuorum — leader rejects proposals when quorum lost
|
||||
[Fact]
|
||||
public async Task ProposeAsync_throws_when_no_quorum()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Make all peers stale to break quorum.
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => leader.ProposeAsync("cmd", CancellationToken.None).AsTask());
|
||||
|
||||
ex.Message.ShouldContain("no quorum");
|
||||
}
|
||||
|
||||
// Go reference: raft.go normal proposal path when quorum is confirmed
|
||||
[Fact]
|
||||
public async Task ProposeAsync_succeeds_with_quorum()
|
||||
{
|
||||
// Peers are initialized with fresh LastContact by ConfigureCluster, so quorum holds.
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
var index = await leader.ProposeAsync("cmd-ok", CancellationToken.None);
|
||||
|
||||
index.ShouldBeGreaterThan(0);
|
||||
leader.AppliedIndex.ShouldBe(index);
|
||||
}
|
||||
|
||||
// After a heartbeat round, peers are fresh and quorum is restored.
|
||||
[Fact]
|
||||
public async Task ProposeAsync_succeeds_after_heartbeat_restores_quorum()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Make all peers stale.
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
|
||||
// Proposal should fail with no quorum.
|
||||
await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => leader.ProposeAsync("should-fail", CancellationToken.None).AsTask());
|
||||
|
||||
// Simulate heartbeat responses updating LastContact on the leader.
|
||||
foreach (var peer in nodes.Skip(1))
|
||||
leader.GetPeerStates()[peer.Id].LastContact = DateTime.UtcNow;
|
||||
|
||||
// Quorum is restored; proposal should now succeed.
|
||||
var index = await leader.ProposeAsync("after-heartbeat", CancellationToken.None);
|
||||
index.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// -- Heartbeat updates LastContact --
|
||||
|
||||
// Go reference: raft.go processHeartbeat — updates peer last-contact on a valid heartbeat
|
||||
[Fact]
|
||||
public void Heartbeat_updates_last_contact()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var node = nodes[0];
|
||||
|
||||
var peerStates = node.GetPeerStates();
|
||||
var oldTime = DateTime.UtcNow.AddMinutes(-5);
|
||||
peerStates["n2"].LastContact = oldTime;
|
||||
|
||||
node.ReceiveHeartbeat(term: 1, fromPeerId: "n2");
|
||||
|
||||
peerStates["n2"].LastContact.ShouldBeGreaterThan(oldTime);
|
||||
}
|
||||
|
||||
// Heartbeats from the leader to the cluster restore the leader's quorum tracking.
|
||||
[Fact]
|
||||
public void Heartbeat_from_peer_restores_peer_freshness_for_quorum()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// Simulate network partition: mark all peers stale.
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMinutes(-5);
|
||||
|
||||
leader.HasQuorum().ShouldBeFalse();
|
||||
|
||||
// Leader receives heartbeat ACK from n2 (simulating that n2 is still reachable).
|
||||
// In a real RAFT loop the leader sends AppendEntries and processes the response;
|
||||
// here we simulate the response side by directly updating LastContact via ReceiveHeartbeat.
|
||||
// Note: ReceiveHeartbeat is called on a follower when it receives from the leader, not
|
||||
// on the leader itself. We instead update LastContact directly to simulate the leader
|
||||
// processing an AppendEntries response.
|
||||
leader.GetPeerStates()["n2"].LastContact = DateTime.UtcNow;
|
||||
|
||||
// 1 current peer + self = 2 voters; majority of 3 = 2, so quorum is restored.
|
||||
leader.HasQuorum().ShouldBeTrue();
|
||||
}
|
||||
|
||||
// -- Quorum window boundary tests --
|
||||
|
||||
[Fact]
|
||||
public void HasQuorum_peer_just_within_window_counts_as_current()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// ElectionTimeoutMaxMs = 100; window = 2 × 100 = 200 ms.
|
||||
// Set LastContact to 150 ms ago — just inside the 200 ms window.
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMilliseconds(-150);
|
||||
|
||||
leader.HasQuorum().ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HasQuorum_peer_just_outside_window_is_stale()
|
||||
{
|
||||
var (nodes, _) = CreateCluster(3);
|
||||
var leader = ElectLeader(nodes);
|
||||
|
||||
// ElectionTimeoutMaxMs = 100; window = 2 × 100 = 200 ms.
|
||||
// Set LastContact to 500 ms ago — comfortably outside the 200 ms window.
|
||||
foreach (var (_, state) in leader.GetPeerStates())
|
||||
state.LastContact = DateTime.UtcNow.AddMilliseconds(-500);
|
||||
|
||||
leader.HasQuorum().ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user