fix(host): follow an in-place site PSK rotation without a central restart (ScadaBridge#96)
SitePskProvider cached each site's gRPC preshared key forever; only site removal
invalidated it, so rotating SB-GRPC-PSK-{site} broke the central<->site link on
both central nodes until they restarted. The cached key is now dropped when:
- the secret is written through /admin/secrets: central replaces the secrets
library's ISecretCacheInvalidator with SitePskRotationInvalidator, which also
publishes SitePskInvalidated on a new DistributedPubSub topic
(site-psk-invalidation) so the peer central node drops its copy and its
resolver cache entry too (SitePskInvalidationSubscriber);
- a site rejects a central->site call with PermissionDenied/Unauthenticated:
WithSiteCredentials wraps every per-site channel's handler in
PskRejectionRefreshHandler, so the next call re-reads the key (covers
out-of-process writes such as the secret CLI);
- a site->central call presents a token that does not match the cached key:
CentralControlAuthInterceptor invalidates and re-reads once before refusing;
- ScadaBridge:Communication:SitePsks reloads.
Invalidation is case-insensitive because SecretName lowercases names. Rotation
runbook (topology-guide), Component-Communication/Host and the #85 cache audit
(D2) updated.
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -7,8 +7,10 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
@@ -267,4 +269,87 @@ public class ControlPlaneAuthEndToEndTests : IAsyncLifetime
|
||||
var service = _host.Services.GetRequiredService<EchoSiteStreamService>();
|
||||
Assert.Equal("site-a", service.LastSiteHeader);
|
||||
}
|
||||
// --- Gitea #96: a site's rejection drops central's cached key ---------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Central's real <see cref="SitePskProvider"/> over a store that served a stale key first and
|
||||
/// the rotated (site-accepted) key after: the in-place rotation scenario.
|
||||
/// </summary>
|
||||
private static (SitePskProvider Provider, ISecretResolver Resolver) RotatedProvider()
|
||||
{
|
||||
var resolver = Substitute.For<ISecretResolver>();
|
||||
resolver.GetAsync(Arg.Any<SecretName>(), Arg.Any<CancellationToken>())
|
||||
.Returns("the-pre-rotation-key", SiteKey);
|
||||
var options = Substitute.For<IOptionsMonitor<CommunicationOptions>>();
|
||||
options.CurrentValue.Returns(new CommunicationOptions());
|
||||
return (new SitePskProvider(resolver, options, NullLogger<SitePskProvider>.Instance), resolver);
|
||||
}
|
||||
|
||||
private GrpcChannel Channel(ISitePskProvider provider) =>
|
||||
GrpcChannel.ForAddress(_server.BaseAddress, new GrpcChannelOptions { HttpHandler = _server.CreateHandler() }
|
||||
.WithMessageLimits().WithSiteCredentials(provider, "site-a"));
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that after a site rejects the cached (pre-rotation) key on a unary call, the next
|
||||
/// call re-reads the key and is accepted — no restart needed.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RejectedUnaryCall_DropsTheCachedKey_SoTheNextCallUsesTheRotatedOne()
|
||||
{
|
||||
var (provider, _) = RotatedProvider();
|
||||
using var channel = Channel(provider);
|
||||
var client = new SiteStreamService.SiteStreamServiceClient(channel);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await client.PullAuditEventsAsync(new PullAuditEventsRequest()));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
|
||||
var reply = await client.PullAuditEventsAsync(new PullAuditEventsRequest { BatchSize = 1 });
|
||||
Assert.False(reply.MoreAvailable);
|
||||
}
|
||||
|
||||
/// <summary>Verifies the same refresh for a rejected streaming call.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RejectedStreamingCall_DropsTheCachedKey_SoTheNextCallUsesTheRotatedOne()
|
||||
{
|
||||
var (provider, _) = RotatedProvider();
|
||||
using var channel = Channel(provider);
|
||||
var client = new SiteStreamService.SiteStreamServiceClient(channel);
|
||||
|
||||
using (var rejected = client.SubscribeInstance(new InstanceStreamRequest { CorrelationId = "c1" }))
|
||||
{
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await rejected.ResponseStream.MoveNext(CancellationToken.None));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
using var accepted = client.SubscribeInstance(new InstanceStreamRequest { CorrelationId = "c2" });
|
||||
Assert.True(await accepted.ResponseStream.MoveNext(CancellationToken.None));
|
||||
Assert.Equal("c2", accepted.ResponseStream.Current.CorrelationId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Negative control: accepted calls never drop the cache, so the store is read once no
|
||||
/// matter how many calls succeed.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AcceptedCalls_KeepTheCachedKey()
|
||||
{
|
||||
var resolver = Substitute.For<ISecretResolver>();
|
||||
resolver.GetAsync(Arg.Any<SecretName>(), Arg.Any<CancellationToken>()).Returns(SiteKey);
|
||||
var options = Substitute.For<IOptionsMonitor<CommunicationOptions>>();
|
||||
options.CurrentValue.Returns(new CommunicationOptions());
|
||||
var provider = new SitePskProvider(resolver, options, NullLogger<SitePskProvider>.Instance);
|
||||
using var channel = Channel(provider);
|
||||
var client = new SiteStreamService.SiteStreamServiceClient(channel);
|
||||
|
||||
await client.PullAuditEventsAsync(new PullAuditEventsRequest());
|
||||
await client.PullAuditEventsAsync(new PullAuditEventsRequest());
|
||||
await client.PullAuditEventsAsync(new PullAuditEventsRequest());
|
||||
|
||||
await resolver.Received(1).GetAsync(Arg.Any<SecretName>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user