fix(host): follow an in-place site PSK rotation without a central restart (ScadaBridge#96)
SitePskProvider cached each site's gRPC preshared key forever; only site removal
invalidated it, so rotating SB-GRPC-PSK-{site} broke the central<->site link on
both central nodes until they restarted. The cached key is now dropped when:
- the secret is written through /admin/secrets: central replaces the secrets
library's ISecretCacheInvalidator with SitePskRotationInvalidator, which also
publishes SitePskInvalidated on a new DistributedPubSub topic
(site-psk-invalidation) so the peer central node drops its copy and its
resolver cache entry too (SitePskInvalidationSubscriber);
- a site rejects a central->site call with PermissionDenied/Unauthenticated:
WithSiteCredentials wraps every per-site channel's handler in
PskRejectionRefreshHandler, so the next call re-reads the key (covers
out-of-process writes such as the secret CLI);
- a site->central call presents a token that does not match the cached key:
CentralControlAuthInterceptor invalidates and re-reads once before refusing;
- ScadaBridge:Communication:SitePsks reloads.
Invalidation is case-insensitive because SecretName lowercases names. Rotation
runbook (topology-guide), Component-Communication/Host and the #85 cache audit
(D2) updated.
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,168 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.Secrets;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// Central's <see cref="ISecretCacheInvalidator"/>: forwards every eviction to the secret
|
||||
/// resolver's cache as before, and when the evicted secret is a site's gRPC preshared key
|
||||
/// (<c>SB-GRPC-PSK-{siteId}</c>) also drops that site's entry from <see cref="ISitePskProvider"/>
|
||||
/// and tells the peer central node to do the same (Gitea #96).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why the invalidator seam.</b> Every <c>ZB.MOM.WW.Secrets</c> write path — the
|
||||
/// <c>/admin/secrets</c> editor and delete modal — calls <see cref="ISecretCacheInvalidator"/>
|
||||
/// right after a rotate or delete. Hooking that seam catches an in-place PSK rotation at the
|
||||
/// moment it is written, without ScadaBridge owning a write path of its own.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why the peer needs telling.</b> Both central nodes cache the key in their own
|
||||
/// <see cref="SitePskProvider"/> (and their own resolver cache), and the write only ran on one of
|
||||
/// them. The peer is told over the central <c>DistributedPubSub</c> mediator on
|
||||
/// <see cref="Topic"/> — the same mediator the peer health replication uses. A node receives its
|
||||
/// own broadcast too; applying it twice is harmless, so there is no origin filtering.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Writes made outside this process (the <c>secret</c> CLI, a direct SQL write) never reach this
|
||||
/// seam; the site's rejection of the stale key covers those (see
|
||||
/// <c>ControlPlaneCredentials.WithSiteCredentials</c>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator
|
||||
{
|
||||
/// <summary>The <c>DistributedPubSub</c> topic the central nodes exchange PSK invalidations on.</summary>
|
||||
public const string Topic = "site-psk-invalidation";
|
||||
|
||||
private readonly ISecretCacheInvalidator _inner;
|
||||
private readonly ISitePskProvider _pskProvider;
|
||||
private readonly ILogger<SitePskRotationInvalidator> _logger;
|
||||
private volatile IActorRef? _mediator;
|
||||
|
||||
/// <summary>Creates the invalidator.</summary>
|
||||
/// <param name="inner">The library's own invalidator (the secret resolver's cache).</param>
|
||||
/// <param name="pskProvider">Central's per-site PSK cache.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public SitePskRotationInvalidator(
|
||||
ISecretCacheInvalidator inner,
|
||||
ISitePskProvider pskProvider,
|
||||
ILogger<SitePskRotationInvalidator> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
ArgumentNullException.ThrowIfNull(pskProvider);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_inner = inner;
|
||||
_pskProvider = pskProvider;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Supplies the <c>DistributedPubSub</c> mediator once the actor system exists. Until then
|
||||
/// (and on a host without one) evictions stay local.
|
||||
/// </summary>
|
||||
/// <param name="mediator">The mediator to publish peer invalidations to.</param>
|
||||
public void SetMediator(IActorRef mediator) => _mediator = mediator;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Invalidate(SecretName name)
|
||||
{
|
||||
_inner.Invalidate(name);
|
||||
|
||||
var siteId = SitePskProvider.SiteIdFromSecretName(name.Value);
|
||||
if (siteId is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_pskProvider.Invalidate(siteId);
|
||||
_logger.LogInformation(
|
||||
"gRPC preshared key secret '{SecretName}' was written; dropped the cached key for site "
|
||||
+ "{SiteId} and notified the peer central node.", name.Value, siteId);
|
||||
_mediator?.Tell(new Publish(Topic, new SitePskInvalidated(siteId)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies an invalidation received from a central node (the peer, or this node's own
|
||||
/// broadcast): evicts the resolver's cached secret and the PSK cache entry. Never re-publishes.
|
||||
/// </summary>
|
||||
/// <param name="siteId">The site whose key was rotated.</param>
|
||||
public void ApplyPeerInvalidation(string siteId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(siteId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_inner.Invalidate(new SecretName(SitePskProvider.SecretNameFor(siteId)));
|
||||
_pskProvider.Invalidate(siteId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Peer message: the gRPC preshared key of <paramref name="SiteId"/> was rotated on a central node.
|
||||
/// </summary>
|
||||
/// <param name="SiteId">The site whose key changed.</param>
|
||||
public sealed record SitePskInvalidated(string SiteId);
|
||||
|
||||
/// <summary>
|
||||
/// Central actor that subscribes to <see cref="SitePskRotationInvalidator.Topic"/> and applies
|
||||
/// each <see cref="SitePskInvalidated"/> to this node's caches.
|
||||
/// </summary>
|
||||
public sealed class SitePskInvalidationSubscriber : ReceiveActor
|
||||
{
|
||||
private readonly SitePskRotationInvalidator _invalidator;
|
||||
|
||||
/// <summary>Creates the subscriber.</summary>
|
||||
/// <param name="invalidator">This node's invalidator.</param>
|
||||
public SitePskInvalidationSubscriber(SitePskRotationInvalidator invalidator)
|
||||
{
|
||||
_invalidator = invalidator;
|
||||
Receive<SitePskInvalidated>(m => _invalidator.ApplyPeerInvalidation(m.SiteId));
|
||||
Receive<SubscribeAck>(_ => { });
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
try
|
||||
{
|
||||
DistributedPubSub.Get(Context.System).Mediator.Tell(
|
||||
new Subscribe(SitePskRotationInvalidator.Topic, Self));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Non-clustered host (TestKit): peer invalidation is simply unavailable.
|
||||
Context.GetLogger().Debug("DistributedPubSub not available — peer PSK invalidation disabled: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>DI wiring for <see cref="SitePskRotationInvalidator"/>.</summary>
|
||||
public static class SitePskRotationServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Replaces the secrets library's <see cref="ISecretCacheInvalidator"/> with
|
||||
/// <see cref="SitePskRotationInvalidator"/> wrapping the library's resolver. Central only;
|
||||
/// call after <c>AddScadaBridgeSecrets</c> and after <see cref="ISitePskProvider"/> is
|
||||
/// registered.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <returns>The same collection.</returns>
|
||||
public static IServiceCollection AddSitePskRotationInvalidation(this IServiceCollection services)
|
||||
{
|
||||
services.TryAddSingleton(sp => new SitePskRotationInvalidator(
|
||||
sp.GetRequiredService<DefaultSecretResolver>(),
|
||||
sp.GetRequiredService<ISitePskProvider>(),
|
||||
sp.GetRequiredService<ILogger<SitePskRotationInvalidator>>()));
|
||||
services.Replace(ServiceDescriptor.Singleton<ISecretCacheInvalidator>(
|
||||
sp => sp.GetRequiredService<SitePskRotationInvalidator>()));
|
||||
return services;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user