fix(controlplane): evict retired mesh channels and refresh PSKs off the request path
- MeshGrpcCommandDialer.RetainOnly (IMeshCommandDialer, default no-op): CentralCommunicationActor hands the dialer the current endpoint set on every contact refresh, so a renamed host or moved port releases its cached GrpcChannel and idle HTTP/2 connection instead of living for the process (R03:1.3). - IMeshPskProvider.RefreshAsync (default = the old InvalidateAll): MeshPskProvider pre-resolves every known cluster's key concurrently, bounded like the sync path, and swaps an immutable snapshot in atomically; a failed read keeps that cluster's previous key. The actor calls it from ContactsLoaded (after SetKnownClusters, off-actor via PipeTo) instead of InvalidateAll, so the first ack per cluster per ContactRefreshSeconds no longer blocks a gRPC thread on a 3 s-bounded store read or gets denied past the bound (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
+41
@@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
||||
/// <summary>A spawned gRPC-mode actor plus the two seams the tests observe it through.</summary>
|
||||
private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk);
|
||||
|
||||
/// <summary>
|
||||
/// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the
|
||||
/// channels of endpoints that left it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain()
|
||||
{
|
||||
var factory = NewInMemoryDbFactory();
|
||||
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false));
|
||||
var rig = SpawnGrpcRig(factory);
|
||||
AwaitContactsLoaded(rig);
|
||||
// The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes.
|
||||
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
||||
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
||||
|
||||
using (var db = factory.CreateDbContext())
|
||||
{
|
||||
db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true;
|
||||
db.SaveChanges();
|
||||
}
|
||||
rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts());
|
||||
|
||||
AwaitCondition(
|
||||
() => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")),
|
||||
TimeSpan.FromSeconds(5));
|
||||
rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" });
|
||||
}
|
||||
|
||||
private GrpcRig SpawnGrpcRig(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
||||
RecordingDialer? dialer = null,
|
||||
@@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
||||
: Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Every endpoint set handed to <see cref="RetainOnly"/>, in order.</summary>
|
||||
public IReadOnlyList<IReadOnlyCollection<string>> RetainCalls
|
||||
{
|
||||
get { lock (_gate) return _retainCalls.ToList(); }
|
||||
}
|
||||
|
||||
private readonly List<IReadOnlyCollection<string>> _retainCalls = [];
|
||||
|
||||
public void RetainOnly(IReadOnlyCollection<string> nodeEndpoints)
|
||||
{
|
||||
lock (_gate) _retainCalls.Add(nodeEndpoints.ToList());
|
||||
}
|
||||
|
||||
/// <summary>One recorded dial.</summary>
|
||||
public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId);
|
||||
}
|
||||
|
||||
+29
@@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests
|
||||
Should.NotThrow(dialer.Dispose);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is
|
||||
/// released on the next refresh instead of living, with its idle connection, for the life of central.
|
||||
/// Positive control: a dial to that endpoint afterwards rebuilds a channel.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set()
|
||||
{
|
||||
var dialer = new MeshGrpcCommandDialer(
|
||||
new FakePskProvider(("SITE-A", "psk-site-a")),
|
||||
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }),
|
||||
NullLogger<MeshGrpcCommandDialer>.Instance);
|
||||
foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" })
|
||||
{
|
||||
await Should.ThrowAsync<RpcException>(
|
||||
() => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None));
|
||||
}
|
||||
dialer.CachedChannelCount.ShouldBe(2);
|
||||
|
||||
dialer.RetainOnly(new[] { "http://127.0.0.1:1" });
|
||||
|
||||
dialer.CachedChannelCount.ShouldBe(1);
|
||||
|
||||
await Should.ThrowAsync<RpcException>(
|
||||
() => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None));
|
||||
dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel");
|
||||
Should.NotThrow(dialer.Dispose);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void It_activates_from_the_container_despite_the_optional_client_factory_parameter()
|
||||
{
|
||||
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the
|
||||
/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store
|
||||
/// read and was denied past the bound. <see cref="MeshPskProvider.RefreshAsync"/> pre-resolves the
|
||||
/// keys off the request path and swaps them in atomically.
|
||||
/// </summary>
|
||||
public sealed class MeshPskProviderRefreshTests
|
||||
{
|
||||
private static MeshPskProvider Provider(ISecretResolver resolver) =>
|
||||
new(Options.Create(new MeshTransportOptions()), resolver, NullLogger<MeshPskProvider>.Instance);
|
||||
|
||||
[Fact]
|
||||
public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store()
|
||||
{
|
||||
var resolver = new GatedResolver { Value = "k-1" };
|
||||
var provider = Provider(resolver);
|
||||
|
||||
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
resolver.Calls.ShouldBe(1);
|
||||
|
||||
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
|
||||
psk.ShouldBe("k-1");
|
||||
resolver.Calls.ShouldBe(1, "the request path did not consult the store");
|
||||
provider.Matches("SITE-A", "k-1").ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window);
|
||||
/// once it completes the rotated key is served. Positive control on both sides of the swap.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then()
|
||||
{
|
||||
var resolver = new GatedResolver { Value = "old" };
|
||||
var provider = Provider(resolver);
|
||||
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue();
|
||||
before.ShouldBe("old");
|
||||
|
||||
resolver.Value = "new";
|
||||
resolver.Hold();
|
||||
var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue();
|
||||
|
||||
// Mid-refresh: the old key, from memory — the store is NOT re-read on this path.
|
||||
var callsMidRefresh = resolver.Calls;
|
||||
provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue();
|
||||
during.ShouldBe("old");
|
||||
resolver.Calls.ShouldBe(callsMidRefresh);
|
||||
|
||||
resolver.Release();
|
||||
await refresh;
|
||||
provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue();
|
||||
after.ShouldBe("new");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key()
|
||||
{
|
||||
var resolver = new GatedResolver { Value = "k-1" };
|
||||
var provider = Provider(resolver);
|
||||
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
|
||||
resolver.Throw = new InvalidOperationException("store offline");
|
||||
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
|
||||
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
|
||||
psk.ShouldBe("k-1");
|
||||
}
|
||||
|
||||
/// <summary>A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour).</summary>
|
||||
[Fact]
|
||||
public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily()
|
||||
{
|
||||
var resolver = new GatedResolver { Value = "k-b" };
|
||||
var provider = Provider(resolver);
|
||||
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
|
||||
provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue();
|
||||
psk.ShouldBe("k-b");
|
||||
resolver.Calls.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InvalidateAll_drops_refreshed_keys_too()
|
||||
{
|
||||
var resolver = new GatedResolver { Value = "k-1" };
|
||||
var provider = Provider(resolver);
|
||||
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
|
||||
|
||||
provider.InvalidateAll();
|
||||
resolver.Value = "k-2";
|
||||
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
|
||||
psk.ShouldBe("k-2");
|
||||
}
|
||||
|
||||
private sealed class GatedResolver : ISecretResolver
|
||||
{
|
||||
private int _calls;
|
||||
private volatile TaskCompletionSource? _gate;
|
||||
|
||||
public string? Value { get; set; }
|
||||
public Exception? Throw { get; set; }
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
public ManualResetEventSlim Entered { get; } = new();
|
||||
|
||||
public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Release() => _gate?.TrySetResult();
|
||||
|
||||
public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
Entered.Set();
|
||||
if (_gate is { } gate) await gate.Task.WaitAsync(ct);
|
||||
if (Throw is not null) throw Throw;
|
||||
return Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user