feat: add cluster split handling (Gap 13.5)
Add RemoveRoute, RemoveAllRoutesExcept, RegisterRoute (internal), and DetectClusterSplit to RouteManager with ClusterSplitResult record, plus 10 tests covering partition detection and route removal behavior.
This commit is contained in:
Joseph Doherty
284
tests/NATS.Server.Tests/Routes/ClusterSplitTests.cs
Normal file
284
tests/NATS.Server.Tests/Routes/ClusterSplitTests.cs
Normal file
@@ -0,0 +1,284 @@
|
||||
// Reference: golang/nats-server/server/route.go:3085 — removeAllRoutesExcept
|
||||
// Reference: golang/nats-server/server/route.go:3113 — removeRoute
|
||||
// Tests for cluster split detection and route partition handling.
|
||||
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
using NATS.Server.Routes;
|
||||
|
||||
namespace NATS.Server.Tests.Routes;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="RouteManager.RemoveRoute"/>,
|
||||
/// <see cref="RouteManager.RemoveAllRoutesExcept"/>, and
|
||||
/// <see cref="RouteManager.DetectClusterSplit"/>.
|
||||
/// Go reference: server/route.go removeRoute (3113), removeAllRoutesExcept (3085).
|
||||
/// </summary>
|
||||
public class ClusterSplitTests
|
||||
{
|
||||
// -- Helpers --
|
||||
|
||||
private static RouteManager CreateManager(string serverId = "local-server")
|
||||
=> new(
|
||||
new ClusterOptions { Host = "127.0.0.1", Port = 0 },
|
||||
new ServerStats(),
|
||||
serverId,
|
||||
_ => { },
|
||||
_ => { },
|
||||
NullLogger<RouteManager>.Instance);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a connected socket pair (listener + client) so that
|
||||
/// <see cref="RouteConnection"/> can build its internal <see cref="NetworkStream"/>.
|
||||
/// Returns both the RouteConnection (which owns the server-side socket) and the
|
||||
/// client-side socket (kept alive for the duration of the test).
|
||||
/// </summary>
|
||||
private static (RouteConnection Route, Socket ClientSocket) CreateConnectedRoute()
|
||||
{
|
||||
var listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
listener.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
listener.Listen(1);
|
||||
var port = ((IPEndPoint)listener.LocalEndPoint!).Port;
|
||||
|
||||
var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
client.Connect(IPAddress.Loopback, port);
|
||||
var server = listener.Accept();
|
||||
listener.Dispose();
|
||||
|
||||
return (new RouteConnection(server), client);
|
||||
}
|
||||
|
||||
// -- RemoveRoute tests --
|
||||
|
||||
[Fact]
|
||||
public void RemoveRoute_ExistingRoute_ReturnsTrue()
|
||||
{
|
||||
// Go ref: route.go removeRoute — returns true when a route was found.
|
||||
var manager = CreateManager();
|
||||
var (conn, clientSocket) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("server-A", conn);
|
||||
manager.RouteCount.ShouldBe(1);
|
||||
|
||||
var removed = manager.RemoveRoute("server-A");
|
||||
|
||||
removed.ShouldBeTrue();
|
||||
manager.RouteCount.ShouldBe(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientSocket.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveRoute_NonExistent_ReturnsFalse()
|
||||
{
|
||||
// Go ref: route.go removeRoute — returns false when server ID is unknown.
|
||||
var manager = CreateManager();
|
||||
|
||||
var removed = manager.RemoveRoute("no-such-server");
|
||||
|
||||
removed.ShouldBeFalse();
|
||||
manager.RouteCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveRoute_RemovesFromConnectedIds()
|
||||
{
|
||||
// Go ref: route.go removeRoute — must update the connectedServerIds set
|
||||
// so topology snapshots no longer list the removed peer.
|
||||
var manager = CreateManager();
|
||||
var (conn, clientSocket) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("server-B", conn);
|
||||
|
||||
var snapshotBefore = manager.BuildTopologySnapshot();
|
||||
snapshotBefore.ConnectedServerIds.ShouldContain("server-B");
|
||||
|
||||
manager.RemoveRoute("server-B");
|
||||
|
||||
var snapshotAfter = manager.BuildTopologySnapshot();
|
||||
snapshotAfter.ConnectedServerIds.ShouldNotContain("server-B");
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientSocket.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
// -- RemoveAllRoutesExcept tests --
|
||||
|
||||
[Fact]
|
||||
public void RemoveAllRoutesExcept_KeepsSpecified()
|
||||
{
|
||||
// Go ref: route.go removeAllRoutesExcept — only removes routes whose
|
||||
// server ID is not in the keep set.
|
||||
var manager = CreateManager();
|
||||
var (connA, clientA) = CreateConnectedRoute();
|
||||
var (connB, clientB) = CreateConnectedRoute();
|
||||
var (connC, clientC) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("server-A", connA);
|
||||
manager.RegisterRoute("server-B", connB);
|
||||
manager.RegisterRoute("server-C", connC);
|
||||
manager.RouteCount.ShouldBe(3);
|
||||
|
||||
var removed = manager.RemoveAllRoutesExcept(new HashSet<string> { "server-A", "server-B" });
|
||||
|
||||
removed.ShouldBe(1);
|
||||
manager.RouteCount.ShouldBe(2);
|
||||
|
||||
var snapshot = manager.BuildTopologySnapshot();
|
||||
snapshot.ConnectedServerIds.ShouldContain("server-A");
|
||||
snapshot.ConnectedServerIds.ShouldContain("server-B");
|
||||
snapshot.ConnectedServerIds.ShouldNotContain("server-C");
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientA.Dispose();
|
||||
clientB.Dispose();
|
||||
clientC.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveAllRoutesExcept_RemovesAll_WhenEmptyKeepSet()
|
||||
{
|
||||
// Go ref: route.go removeAllRoutesExcept — empty keep set removes every route.
|
||||
var manager = CreateManager();
|
||||
var (connA, clientA) = CreateConnectedRoute();
|
||||
var (connB, clientB) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("server-A", connA);
|
||||
manager.RegisterRoute("server-B", connB);
|
||||
manager.RouteCount.ShouldBe(2);
|
||||
|
||||
var removed = manager.RemoveAllRoutesExcept(new HashSet<string>());
|
||||
|
||||
removed.ShouldBe(2);
|
||||
manager.RouteCount.ShouldBe(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientA.Dispose();
|
||||
clientB.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveAllRoutesExcept_NoRoutes_ReturnsZero()
|
||||
{
|
||||
// Go ref: route.go removeAllRoutesExcept — no-op when no routes exist.
|
||||
var manager = CreateManager();
|
||||
|
||||
var removed = manager.RemoveAllRoutesExcept(new HashSet<string> { "server-A" });
|
||||
|
||||
removed.ShouldBe(0);
|
||||
manager.RouteCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
// -- DetectClusterSplit tests --
|
||||
|
||||
[Fact]
|
||||
public void DetectClusterSplit_AllPresent_NotSplit()
|
||||
{
|
||||
// Go ref: cluster split detection — no missing peers means no partition.
|
||||
var manager = CreateManager();
|
||||
var (connA, clientA) = CreateConnectedRoute();
|
||||
var (connB, clientB) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("peer-1", connA);
|
||||
manager.RegisterRoute("peer-2", connB);
|
||||
|
||||
var result = manager.DetectClusterSplit(new HashSet<string> { "peer-1", "peer-2" });
|
||||
|
||||
result.IsSplit.ShouldBeFalse();
|
||||
result.MissingPeers.ShouldBeEmpty();
|
||||
result.UnexpectedPeers.ShouldBeEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientA.Dispose();
|
||||
clientB.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DetectClusterSplit_MissingPeers_IsSplit()
|
||||
{
|
||||
// Go ref: cluster split detection — missing expected peers indicates partition.
|
||||
var manager = CreateManager();
|
||||
var (connA, clientA) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("peer-1", connA);
|
||||
// peer-2 and peer-3 are expected but not connected.
|
||||
|
||||
var result = manager.DetectClusterSplit(new HashSet<string> { "peer-1", "peer-2", "peer-3" });
|
||||
|
||||
result.IsSplit.ShouldBeTrue();
|
||||
result.MissingPeers.Count.ShouldBe(2);
|
||||
result.MissingPeers.ShouldContain("peer-2");
|
||||
result.MissingPeers.ShouldContain("peer-3");
|
||||
result.UnexpectedPeers.ShouldBeEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientA.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DetectClusterSplit_UnexpectedPeers_Listed()
|
||||
{
|
||||
// Go ref: unexpected connected peers are enumerated (extras don't cause IsSplit).
|
||||
var manager = CreateManager();
|
||||
var (connA, clientA) = CreateConnectedRoute();
|
||||
var (connX, clientX) = CreateConnectedRoute();
|
||||
|
||||
try
|
||||
{
|
||||
manager.RegisterRoute("peer-1", connA);
|
||||
manager.RegisterRoute("unexpected-peer", connX);
|
||||
|
||||
var result = manager.DetectClusterSplit(new HashSet<string> { "peer-1" });
|
||||
|
||||
result.IsSplit.ShouldBeFalse();
|
||||
result.MissingPeers.ShouldBeEmpty();
|
||||
result.UnexpectedPeers.Count.ShouldBe(1);
|
||||
result.UnexpectedPeers.ShouldContain("unexpected-peer");
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientA.Dispose();
|
||||
clientX.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DetectClusterSplit_NoPeers_NoExpected_NotSplit()
|
||||
{
|
||||
// Go ref: cluster split detection — empty state with no expectations is healthy.
|
||||
var manager = CreateManager();
|
||||
|
||||
var result = manager.DetectClusterSplit(new HashSet<string>());
|
||||
|
||||
result.IsSplit.ShouldBeFalse();
|
||||
result.MissingPeers.ShouldBeEmpty();
|
||||
result.UnexpectedPeers.ShouldBeEmpty();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user