fix(controlplane): evict retired mesh channels and refresh PSKs off the request path

- MeshGrpcCommandDialer.RetainOnly (IMeshCommandDialer, default no-op): CentralCommunicationActor
  hands the dialer the current endpoint set on every contact refresh, so a renamed host or moved
  port releases its cached GrpcChannel and idle HTTP/2 connection instead of living for the
  process (R03:1.3).
- IMeshPskProvider.RefreshAsync (default = the old InvalidateAll): MeshPskProvider pre-resolves
  every known cluster's key concurrently, bounded like the sync path, and swaps an immutable
  snapshot in atomically; a failed read keeps that cluster's previous key. The actor calls it from
  ContactsLoaded (after SetKnownClusters, off-actor via PipeTo) instead of InvalidateAll, so the
  first ack per cluster per ContactRefreshSeconds no longer blocks a gRPC thread on a 3 s-bounded
  store read or gets denied past the bound (R03:3.4).

Refs #733

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 10:34:39 -04:00
parent 18352442ab
commit 63714e430a
6 changed files with 404 additions and 2 deletions
@@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
/// <summary>A spawned gRPC-mode actor plus the two seams the tests observe it through.</summary>
private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk);
/// <summary>
/// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the
/// channels of endpoints that left it.
/// </summary>
[Fact]
public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain()
{
var factory = NewInMemoryDbFactory();
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false));
var rig = SpawnGrpcRig(factory);
AwaitContactsLoaded(rig);
// The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes.
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
using (var db = factory.CreateDbContext())
{
db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true;
db.SaveChanges();
}
rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts());
AwaitCondition(
() => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")),
TimeSpan.FromSeconds(5));
rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" });
}
private GrpcRig SpawnGrpcRig(
IDbContextFactory<OtOpcUaConfigDbContext> factory,
RecordingDialer? dialer = null,
@@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
: Task.CompletedTask;
}
/// <summary>Every endpoint set handed to <see cref="RetainOnly"/>, in order.</summary>
public IReadOnlyList<IReadOnlyCollection<string>> RetainCalls
{
get { lock (_gate) return _retainCalls.ToList(); }
}
private readonly List<IReadOnlyCollection<string>> _retainCalls = [];
public void RetainOnly(IReadOnlyCollection<string> nodeEndpoints)
{
lock (_gate) _retainCalls.Add(nodeEndpoints.ToList());
}
/// <summary>One recorded dial.</summary>
public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId);
}
@@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests
Should.NotThrow(dialer.Dispose);
}
/// <summary>
/// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is
/// released on the next refresh instead of living, with its idle connection, for the life of central.
/// Positive control: a dial to that endpoint afterwards rebuilds a channel.
/// </summary>
[Fact]
public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set()
{
var dialer = new MeshGrpcCommandDialer(
new FakePskProvider(("SITE-A", "psk-site-a")),
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }),
NullLogger<MeshGrpcCommandDialer>.Instance);
foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" })
{
await Should.ThrowAsync<RpcException>(
() => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None));
}
dialer.CachedChannelCount.ShouldBe(2);
dialer.RetainOnly(new[] { "http://127.0.0.1:1" });
dialer.CachedChannelCount.ShouldBe(1);
await Should.ThrowAsync<RpcException>(
() => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None));
dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel");
Should.NotThrow(dialer.Dispose);
}
[Fact]
public void It_activates_from_the_container_despite_the_optional_client_factory_parameter()
{
@@ -0,0 +1,129 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
/// <summary>
/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the
/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store
/// read and was denied past the bound. <see cref="MeshPskProvider.RefreshAsync"/> pre-resolves the
/// keys off the request path and swaps them in atomically.
/// </summary>
public sealed class MeshPskProviderRefreshTests
{
private static MeshPskProvider Provider(ISecretResolver resolver) =>
new(Options.Create(new MeshTransportOptions()), resolver, NullLogger<MeshPskProvider>.Instance);
[Fact]
public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
resolver.Calls.ShouldBe(1);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("k-1");
resolver.Calls.ShouldBe(1, "the request path did not consult the store");
provider.Matches("SITE-A", "k-1").ShouldBeTrue();
}
/// <summary>
/// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window);
/// once it completes the rotated key is served. Positive control on both sides of the swap.
/// </summary>
[Fact]
public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then()
{
var resolver = new GatedResolver { Value = "old" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue();
before.ShouldBe("old");
resolver.Value = "new";
resolver.Hold();
var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue();
// Mid-refresh: the old key, from memory — the store is NOT re-read on this path.
var callsMidRefresh = resolver.Calls;
provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue();
during.ShouldBe("old");
resolver.Calls.ShouldBe(callsMidRefresh);
resolver.Release();
await refresh;
provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue();
after.ShouldBe("new");
}
[Fact]
public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
resolver.Throw = new InvalidOperationException("store offline");
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("k-1");
}
/// <summary>A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour).</summary>
[Fact]
public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily()
{
var resolver = new GatedResolver { Value = "k-b" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue();
psk.ShouldBe("k-b");
resolver.Calls.ShouldBe(2);
}
[Fact]
public async Task InvalidateAll_drops_refreshed_keys_too()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.InvalidateAll();
resolver.Value = "k-2";
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("k-2");
}
private sealed class GatedResolver : ISecretResolver
{
private int _calls;
private volatile TaskCompletionSource? _gate;
public string? Value { get; set; }
public Exception? Throw { get; set; }
public int Calls => Volatile.Read(ref _calls);
public ManualResetEventSlim Entered { get; } = new();
public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _gate?.TrySetResult();
public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
{
Interlocked.Increment(ref _calls);
Entered.Set();
if (_gate is { } gate) await gate.Task.WaitAsync(ct);
if (Throw is not null) throw Throw;
return Value;
}
}
}