fix(mesh): gate follows the endpoint not the mode + review fixes
The known-cluster gate published only under Mode=Grpc, but the interceptor it feeds is registered on every admin node and 1B maps CentralCommandService by LISTENER PORT — neither consults the mode. A dark-switch rollback to ClusterClient or Dps would therefore have left that endpoint serving with the gate permanently inactive, silently restoring both the timing oracle and the pre-auth secret-store/audit flood, as a side effect of a rollback nobody would connect to authentication. The gate is central's cluster inventory, not transport state, so it is now decoupled from the mode entirely: - SetKnownClusters + InvalidateAll run on EVERY successful contact load in EVERY mode, and the contact-refresh timer starts in all three. Under Dps this is a ConfigDb query every ContactRefreshSeconds that did not run before; central always has a ConfigDb, and one indexed read a minute is the cheaper half of the trade. - The GATE set is built from ALL ClusterNode rows. The eligibility filter moved out of the query's WHERE and into the loop, so the FAN set is byte-for-byte what it was (enabled && !maintenance) while the gate widens. This also dissolves the maintenance coupling flagged in 1A.2: a cluster whose nodes are all disabled or in maintenance can still ack, and refusing it would make maintenance mode an authentication switch. Central still refuses clusters it has never heard of. - HandleContactsLoaded keeps a documented order — targets adopted, gate published, cache dropped — now pinned by test. Guards: PreStart mirrors the null-dialer error for a null PSK provider (silent absence is the worst outcome there — nothing malfunctions, so nothing is investigated, while the gate never publishes and keys never rotate); a failed contact load now says out loud that the gate is stale and, before any load has succeeded, inactive; MeshGrpcCommandDialer clamps a non-positive DispatchTimeoutSeconds to 10 with a construction Warning, because the validator for that key is driver-role-scoped and a pure-admin central reads it unvalidated; the dialer's local PSK refusal was shortened so its remediation survives the actor's 200-char truncation of remote-authored status detail; the portless-node warn-once set is rebuilt each refresh, so a node that regains then loses a port warns again — the regression is the occurrence worth seeing. Docs/pins: dialer disposal comment no longer claims one-way; the two interceptor-test comments claiming Matches has a default implementation corrected (it deliberately has none); new pins for per-refresh invalidation, the SetKnownClusters-before-InvalidateAll order, and one channel per endpoint via a new CachedChannelCount. The_known_cluster_set_is_not_published_by_the_other_two_modes is replaced — its premise is obsolete — by three facts: every mode publishes, gate set is a strict superset of the fan set, and the handler ordering. ControlPlane.Tests 230 -> 236, Host.Tests 92, Cluster.Tests 184, 0 warnings.
This commit is contained in:
Joseph Doherty
+76
-9
@@ -1,6 +1,8 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
@@ -41,6 +43,7 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
=> new(
|
||||
psk,
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = dispatchTimeoutSeconds }),
|
||||
NullLogger<MeshGrpcCommandDialer>.Instance,
|
||||
_ => client);
|
||||
|
||||
[Fact]
|
||||
@@ -136,17 +139,25 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_zero_dispatch_timeout_leaves_the_dial_bounded_only_by_the_callers_token()
|
||||
public async Task A_non_positive_dispatch_timeout_is_clamped_rather_than_honoured()
|
||||
{
|
||||
// The validator requires > 0 under Mode=Grpc, so this is the misconfigured/other-mode shape.
|
||||
// It must not be read as "expire immediately" — that would drop every command instantly.
|
||||
var client = new RecordingClient();
|
||||
var dialer = CreateDialer(
|
||||
new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 0);
|
||||
// Zero would mean an UNBOUNDED call — the exact failure the deadline exists to prevent — and
|
||||
// MeshTransportOptionsValidator's rule for this key is scoped to driver-role nodes, so a
|
||||
// pure-admin central reaches this code with the value unvalidated. Clamp loudly.
|
||||
var client = new RecordingClient { HangUntilCancelled = true };
|
||||
var logger = new RecordingLogger();
|
||||
var dialer = new MeshGrpcCommandDialer(
|
||||
new FakePskProvider(("SITE-A", "psk-site-a")),
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 0 }),
|
||||
logger,
|
||||
_ => client);
|
||||
|
||||
await dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None);
|
||||
logger.Warnings.ShouldContain(w => w.Contains("DispatchTimeoutSeconds"));
|
||||
|
||||
client.Calls.ShouldBe(1);
|
||||
// And the clamp is real, not just logged: the dial must still expire on its own.
|
||||
await Should.ThrowAsync<OperationCanceledException>(
|
||||
() => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromSeconds(20)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -169,7 +180,8 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
// and Dispose must then clean up the channel the dial created.
|
||||
var dialer = new MeshGrpcCommandDialer(
|
||||
new FakePskProvider(("SITE-A", "psk-site-a")),
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }));
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }),
|
||||
NullLogger<MeshGrpcCommandDialer>.Instance);
|
||||
|
||||
await Should.ThrowAsync<RpcException>(
|
||||
() => dialer.DispatchAsync(
|
||||
@@ -178,6 +190,38 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
Should.NotThrow(dialer.Dispose);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Two_dials_to_one_endpoint_share_a_single_channel()
|
||||
{
|
||||
// The cache is the whole defence against a socket per call: a deploy dispatch fans across
|
||||
// every node in the fleet, and it repeats on every redeploy. A channel per dial exhausts
|
||||
// ports on central long before anything else notices.
|
||||
var dialer = new MeshGrpcCommandDialer(
|
||||
new FakePskProvider(("SITE-A", "psk-site-a")),
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }),
|
||||
NullLogger<MeshGrpcCommandDialer>.Instance);
|
||||
|
||||
// Nothing is listening, so both dials fail — irrelevant here: the channel is constructed
|
||||
// before the call is placed, which is exactly what is being counted.
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
await Should.ThrowAsync<RpcException>(
|
||||
() => dialer.DispatchAsync(
|
||||
"http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None));
|
||||
}
|
||||
|
||||
dialer.CachedChannelCount.ShouldBe(1);
|
||||
|
||||
// A second endpoint is a second channel — the cache keys on the endpoint, it does not
|
||||
// collapse every node onto one connection.
|
||||
await Should.ThrowAsync<RpcException>(
|
||||
() => dialer.DispatchAsync(
|
||||
"http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None));
|
||||
|
||||
dialer.CachedChannelCount.ShouldBe(2);
|
||||
Should.NotThrow(dialer.Dispose);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void It_activates_from_the_container_despite_the_optional_client_factory_parameter()
|
||||
{
|
||||
@@ -186,6 +230,7 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
// constructor — and the failure is a startup crash on the admin node, invisible to every
|
||||
// other test here because they all construct the dialer by hand.
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddSingleton<IMeshPskProvider>(new FakePskProvider(("SITE-A", "psk-site-a")));
|
||||
services.AddSingleton<IOptions<MeshTransportOptions>>(
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 10 }));
|
||||
@@ -249,6 +294,28 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Captures Warning-level messages so a construction-time check is assertable.</summary>
|
||||
private sealed class RecordingLogger : ILogger<MeshGrpcCommandDialer>
|
||||
{
|
||||
/// <summary>Every warning logged, formatted.</summary>
|
||||
public List<string> Warnings { get; } = [];
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel,
|
||||
EventId eventId,
|
||||
TState state,
|
||||
Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
if (logLevel == LogLevel.Warning)
|
||||
Warnings.Add(formatter(state, exception));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>An <see cref="IMeshPskProvider"/> over a fixed key set; unknown clusters fail closed.</summary>
|
||||
private sealed class FakePskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user