fix(host): follow an in-place site PSK rotation without a central restart (ScadaBridge#96)

SitePskProvider cached each site's gRPC preshared key forever; only site removal
invalidated it, so rotating SB-GRPC-PSK-{site} broke the central<->site link on
both central nodes until they restarted. The cached key is now dropped when:

- the secret is written through /admin/secrets: central replaces the secrets
  library's ISecretCacheInvalidator with SitePskRotationInvalidator, which also
  publishes SitePskInvalidated on a new DistributedPubSub topic
  (site-psk-invalidation) so the peer central node drops its copy and its
  resolver cache entry too (SitePskInvalidationSubscriber);
- a site rejects a central->site call with PermissionDenied/Unauthenticated:
  WithSiteCredentials wraps every per-site channel's handler in
  PskRejectionRefreshHandler, so the next call re-reads the key (covers
  out-of-process writes such as the secret CLI);
- a site->central call presents a token that does not match the cached key:
  CentralControlAuthInterceptor invalidates and re-reads once before refusing;
- ScadaBridge:Communication:SitePsks reloads.

Invalidation is case-insensitive because SecretName lowercases names. Rotation
runbook (topology-guide), Component-Communication/Host and the #85 cache audit
(D2) updated.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/ISitePskProvider.cs

@@ -110,6 +110,12 @@ public static class ControlPlaneCredentials
     /// A null provider leaves the options untouched — the shape used by test-only and
     /// default constructors that never dial a gated endpoint.
     /// </summary>
+    /// <remarks>
+    /// Also wraps the channel's HTTP handler in <see cref="PskRejectionRefreshHandler"/>, so a
+    /// call the peer rejects as unauthenticated drops the cached key and the NEXT call re-reads
+    /// it (Gitea #96). The rejected call itself is not retried; every caller already retries
+    /// (heartbeats, stream reconnects, the next pull tick).
+    /// </remarks>
     /// <param name="options">Channel options being built.</param>
     /// <param name="provider">Key provider, or null to leave the channel unauthenticated.</param>
     /// <param name="siteId">The site this channel talks to.</param>
@@ -127,6 +133,78 @@ public static class ControlPlaneCredentials
         options.Credentials = ChannelCredentials.Create(
             ChannelCredentials.Insecure, ForSite(provider, siteId));
         options.UnsafeUseInsecureChannelCallCredentials = true;
+        if (options.HttpHandler is not PskRejectionRefreshHandler)
+        {
+            // Null means "let GrpcChannel build its default", which is this same handler shape.
+            options.HttpHandler = new PskRejectionRefreshHandler(
+                provider,
+                siteId,
+                options.HttpHandler ?? new SocketsHttpHandler { EnableMultipleHttp2Connections = true });
+        }
+
         return options;
     }
 }
+
+/// <summary>
+/// Drops a site's cached preshared key when the peer rejects a call as unauthenticated, so a key
+/// rotated in place is re-read on the next call instead of being presented until a restart
+/// (Gitea #96).
+/// </summary>
+/// <remarks>
+/// <para>
+/// <b>Why an HTTP handler.</b> Every per-site channel is built through
+/// <see cref="ControlPlaneCredentials.WithSiteCredentials"/>, which can see the channel's HTTP
+/// handler but not its call invoker. The control-plane interceptors reject before the service
+/// runs, which gRPC sends as a trailers-only response — the <c>grpc-status</c> rides the response
+/// headers, where this handler can read it without touching the body. That covers unary and
+/// streaming calls alike.
+/// </para>
+/// <para>
+/// <b>No throttle.</b> An invalidation only costs the next call a re-read, which the secret
+/// resolver serves from its own short-TTL cache; a site with a genuinely wrong key costs at most
+/// one store read per resolver TTL.
+/// </para>
+/// </remarks>
+public sealed class PskRejectionRefreshHandler : DelegatingHandler
+{
+    private readonly ISitePskProvider _provider;
+    private readonly string _siteId;
+
+    /// <summary>Creates the handler.</summary>
+    /// <param name="provider">The key cache to invalidate.</param>
+    /// <param name="siteId">The site the channel talks to.</param>
+    /// <param name="innerHandler">The transport handler.</param>
+    public PskRejectionRefreshHandler(
+        ISitePskProvider provider, string siteId, HttpMessageHandler innerHandler)
+        : base(innerHandler)
+    {
+        ArgumentNullException.ThrowIfNull(provider);
+        ArgumentException.ThrowIfNullOrWhiteSpace(siteId);
+        _provider = provider;
+        _siteId = siteId;
+    }
+
+    /// <inheritdoc />
+    protected override async Task<HttpResponseMessage> SendAsync(
+        HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
+        if (IsAuthRejection(response))
+        {
+            _provider.Invalidate(_siteId);
+        }
+
+        return response;
+    }
+
+    /// <summary>
+    /// True when the response is a trailers-only gRPC <c>PermissionDenied</c> (7) or
+    /// <c>Unauthenticated</c> (16) — the statuses the control-plane interceptors reject with.
+    /// </summary>
+    /// <param name="response">The HTTP response.</param>
+    /// <returns>Whether the call was rejected as unauthenticated.</returns>
+    internal static bool IsAuthRejection(HttpResponseMessage response) =>
+        response.Headers.TryGetValues("grpc-status", out var values)
+        && values.FirstOrDefault() is "7" or "16";
+}

src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs

@@ -463,6 +463,18 @@ akka {{
             "CentralControlGrpcService readiness set (service bound: {Bound})",
             centralControlGrpc is not null);
 
+        // Peer half of site-PSK rotation invalidation (Gitea #96): subscribe to the topic, and give
+        // the invalidator the mediator so a rotation written on this node reaches the other one.
+        var pskRotation = _serviceProvider.GetService<SitePskRotationInvalidator>();
+        if (pskRotation is not null)
+        {
+            _actorSystem!.ActorOf(
+                Props.Create(() => new SitePskInvalidationSubscriber(pskRotation)),
+                "site-psk-invalidation");
+            pskRotation.SetMediator(
+                Akka.Cluster.Tools.PublishSubscribe.DistributedPubSub.Get(_actorSystem!).Mediator);
+        }
+
         // Wire up the CommunicationService with the actor reference
         var commService = _serviceProvider.GetService<CommunicationService>();
         commService?.SetCommunicationActor(centralCommActor);

src/ZB.MOM.WW.ScadaBridge.Host/CentralControlAuthInterceptor.cs

@@ -158,10 +158,35 @@ public sealed class CentralControlAuthInterceptor : Interceptor
             throw Denied("missing site identity header");
         }
 
-        string expected;
+        var expected = await ResolveExpectedAsync(siteId, context).ConfigureAwait(false);
+        var presented = ExtractBearerToken(context.RequestHeaders);
+
+        if (presented is not null && !FixedTimeEquals(presented, expected))
+        {
+            // Gitea #96: the cached key may predate an in-place rotation the site already runs
+            // on. Drop it and re-read once; the secret resolver's own short-TTL cache bounds what
+            // a stream of wrong tokens can cost the store.
+            _pskProvider.Invalidate(siteId);
+            expected = await ResolveExpectedAsync(siteId, context).ConfigureAwait(false);
+        }
+
+        if (presented is null || !FixedTimeEquals(presented, expected))
+        {
+            _logger.LogWarning(
+                "Rejected a central control-plane call to {Method} from site {SiteId}: {Reason}.",
+                context.Method, siteId,
+                presented is null ? "no bearer token presented" : "bearer token did not match");
+            throw Denied("control plane authentication failed");
+        }
+
+        CentralControlSiteIdentity.SetVerified(context, siteId);
+    }
+
+    private async Task<string> ResolveExpectedAsync(string siteId, ServerCallContext context)
+    {
         try
         {
-            expected = await _pskProvider.GetAsync(siteId, context.CancellationToken)
+            return await _pskProvider.GetAsync(siteId, context.CancellationToken)
                 .ConfigureAwait(false);
         }
         catch (OperationCanceledException)
@@ -178,18 +203,6 @@ public sealed class CentralControlAuthInterceptor : Interceptor
                 context.Method, siteId);
             throw Denied("no key configured for the presented site");
         }
-
-        var presented = ExtractBearerToken(context.RequestHeaders);
-        if (presented is null || !FixedTimeEquals(presented, expected))
-        {
-            _logger.LogWarning(
-                "Rejected a central control-plane call to {Method} from site {SiteId}: {Reason}.",
-                context.Method, siteId,
-                presented is null ? "no bearer token presented" : "bearer token did not match");
-            throw Denied("control plane authentication failed");
-        }
-
-        CentralControlSiteIdentity.SetVerified(context, siteId);
     }
 
     private static RpcException Denied(string reason)

src/ZB.MOM.WW.ScadaBridge.Host/Program.cs

@@ -411,6 +411,10 @@ try
         // options validation) and why the connection string can never itself be a ${secret:}
         // reference.
         builder.Services.AddScadaBridgeSecrets(builder.Configuration, SecretsNodeRole.Central);
+        // An in-place rotation of a site's SB-GRPC-PSK-{siteId} through /admin/secrets must drop
+        // central's cached copy of that key on BOTH central nodes (Gitea #96). Wraps the library's
+        // ISecretCacheInvalidator; the peer half is wired in AkkaHostedService.RegisterCentralActors.
+        builder.Services.AddSitePskRotationInvalidation();
         // Secrets UI authorization: adds the named policies secrets:manage + secrets:reveal
         // (role-based) consumed by the /admin/secrets page. AddSecretsAuthorization only ADDS
         // these two policies via Configure<AuthorizationOptions> — it composes additively with

src/ZB.MOM.WW.ScadaBridge.Host/SitePskProvider.cs

@@ -22,19 +22,31 @@ namespace ZB.MOM.WW.ScadaBridge.Host;
 /// connection already uses for its per-connection API keys.
 /// </para>
 /// <para>
-/// <b>Caching.</b> Successful resolves are cached indefinitely; a site's key changes only on
-/// rotation, and rotation restarts the pair. Failures are deliberately NOT cached, so a key
-/// seeded after the first (failed) dial is picked up on the next attempt instead of requiring a
-/// restart. <see cref="Invalidate"/> drops an entry when a site is removed or a key is rotated
-/// in place.
+/// <b>Caching.</b> Successful resolves are cached until something says the key may have changed.
+/// Failures are deliberately NOT cached, so a key seeded after the first (failed) dial is picked
+/// up on the next attempt instead of requiring a restart. <see cref="Invalidate"/> drops an entry,
+/// and four things call it (Gitea #96 — before that only site removal did, so an in-place
+/// rotation left both central nodes dialling with the old key until they restarted):
 /// </para>
+/// <list type="bullet">
+/// <item>site removal (the gRPC client paths);</item>
+/// <item>a write to <c>SB-GRPC-PSK-{siteId}</c> through this node's secret write path
+/// (<c>/admin/secrets</c>) — <see cref="SitePskRotationInvalidator"/>, which also tells the peer
+/// central node over <c>DistributedPubSub</c>;</item>
+/// <item>a site rejecting a call with <c>PermissionDenied</c>/<c>Unauthenticated</c>
+/// (<c>ControlPlaneCredentials.WithSiteCredentials</c>), so a key rotated out of band (the
+/// <c>secret</c> CLI, a direct SQL write) is re-read on the next call — after the secret
+/// resolver's own <c>Secrets:ResolveCacheTtl</c> at most;</item>
+/// <item>a reload of <c>ScadaBridge:Communication:SitePsks</c>, which clears the whole cache.</item>
+/// </list>
+/// <para>
 /// <para>
 /// <b>Fail-closed.</b> A key absent from BOTH sources throws, and the dial that needed it fails.
 /// It never degrades to an unauthenticated call. The message names the exact config key and the
 /// exact secret name, so the fix is one setting or one <c>secret</c> CLI seed.
 /// </para>
 /// </remarks>
-public sealed class SitePskProvider : ISitePskProvider
+public sealed class SitePskProvider : ISitePskProvider, IDisposable
 {
     /// <summary>Prefix of the per-site secret name; the site identifier is appended verbatim.</summary>
     public const string SecretNamePrefix = "SB-GRPC-PSK-";
@@ -43,6 +55,7 @@ public sealed class SitePskProvider : ISitePskProvider
     private readonly ISecretResolver _resolver;
     private readonly IOptionsMonitor<CommunicationOptions> _options;
     private readonly ILogger<SitePskProvider> _logger;
+    private readonly IDisposable? _optionsChange;
 
     /// <summary>Creates the provider over the host's secret resolver.</summary>
     /// <param name="resolver">Runtime secret resolver from the host container.</param>
@@ -64,6 +77,11 @@ public sealed class SitePskProvider : ISitePskProvider
         _resolver = resolver;
         _options = options;
         _logger = logger;
+
+        // A reloaded SitePsks map may carry a rotated key for any site; drop everything so the
+        // next dial to each site re-reads its source. Re-reading is cheap (a dictionary lookup,
+        // or the secret resolver's own cache), so clearing all is simpler than diffing.
+        _optionsChange = options.OnChange((_, _) => _cache.Clear());
     }
 
     /// <summary>Builds the secret name for a site.</summary>
@@ -142,13 +160,42 @@ public sealed class SitePskProvider : ISitePskProvider
     }
 
     /// <inheritdoc />
+    /// <remarks>
+    /// Case-insensitive: the rotation hook derives the site id from a <c>SecretName</c>, which the
+    /// secrets library lowercases, and a secret name addresses the same secret in any case.
+    /// </remarks>
     public void Invalidate(string siteId)
     {
-        if (!string.IsNullOrWhiteSpace(siteId))
+        if (string.IsNullOrWhiteSpace(siteId))
         {
-            _cache.TryRemove(siteId, out _);
+            return;
+        }
+
+        foreach (var key in _cache.Keys)
+        {
+            if (string.Equals(key, siteId, StringComparison.OrdinalIgnoreCase))
+            {
+                _cache.TryRemove(key, out _);
+            }
         }
     }
+
+    /// <summary>
+    /// Returns the site identifier a secret name is the PSK of, or <c>null</c> when the name is
+    /// not a <see cref="SecretNamePrefix"/> name. The prefix matches in any case, because the
+    /// secrets library stores names lowercased.
+    /// </summary>
+    /// <param name="secretName">A secret name.</param>
+    /// <returns>The site identifier, or <c>null</c>.</returns>
+    public static string? SiteIdFromSecretName(string secretName) =>
+        secretName is not null
+        && secretName.Length > SecretNamePrefix.Length
+        && secretName.StartsWith(SecretNamePrefix, StringComparison.OrdinalIgnoreCase)
+            ? secretName[SecretNamePrefix.Length..]
+            : null;
+
+    /// <inheritdoc />
+    public void Dispose() => _optionsChange?.Dispose();
 }
 
 /// <summary>

src/ZB.MOM.WW.ScadaBridge.Host/SitePskRotationInvalidator.cs

@@ -0,0 +1,168 @@
+using Akka.Actor;
+using Akka.Cluster.Tools.PublishSubscribe;
+using Akka.Event;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
+using ZB.MOM.WW.Secrets;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.ScadaBridge.Host;
+
+/// <summary>
+/// Central's <see cref="ISecretCacheInvalidator"/>: forwards every eviction to the secret
+/// resolver's cache as before, and when the evicted secret is a site's gRPC preshared key
+/// (<c>SB-GRPC-PSK-{siteId}</c>) also drops that site's entry from <see cref="ISitePskProvider"/>
+/// and tells the peer central node to do the same (Gitea #96).
+/// </summary>
+/// <remarks>
+/// <para>
+/// <b>Why the invalidator seam.</b> Every <c>ZB.MOM.WW.Secrets</c> write path — the
+/// <c>/admin/secrets</c> editor and delete modal — calls <see cref="ISecretCacheInvalidator"/>
+/// right after a rotate or delete. Hooking that seam catches an in-place PSK rotation at the
+/// moment it is written, without ScadaBridge owning a write path of its own.
+/// </para>
+/// <para>
+/// <b>Why the peer needs telling.</b> Both central nodes cache the key in their own
+/// <see cref="SitePskProvider"/> (and their own resolver cache), and the write only ran on one of
+/// them. The peer is told over the central <c>DistributedPubSub</c> mediator on
+/// <see cref="Topic"/> — the same mediator the peer health replication uses. A node receives its
+/// own broadcast too; applying it twice is harmless, so there is no origin filtering.
+/// </para>
+/// <para>
+/// Writes made outside this process (the <c>secret</c> CLI, a direct SQL write) never reach this
+/// seam; the site's rejection of the stale key covers those (see
+/// <c>ControlPlaneCredentials.WithSiteCredentials</c>).
+/// </para>
+/// </remarks>
+public sealed class SitePskRotationInvalidator : ISecretCacheInvalidator
+{
+    /// <summary>The <c>DistributedPubSub</c> topic the central nodes exchange PSK invalidations on.</summary>
+    public const string Topic = "site-psk-invalidation";
+
+    private readonly ISecretCacheInvalidator _inner;
+    private readonly ISitePskProvider _pskProvider;
+    private readonly ILogger<SitePskRotationInvalidator> _logger;
+    private volatile IActorRef? _mediator;
+
+    /// <summary>Creates the invalidator.</summary>
+    /// <param name="inner">The library's own invalidator (the secret resolver's cache).</param>
+    /// <param name="pskProvider">Central's per-site PSK cache.</param>
+    /// <param name="logger">Logger.</param>
+    public SitePskRotationInvalidator(
+        ISecretCacheInvalidator inner,
+        ISitePskProvider pskProvider,
+        ILogger<SitePskRotationInvalidator> logger)
+    {
+        ArgumentNullException.ThrowIfNull(inner);
+        ArgumentNullException.ThrowIfNull(pskProvider);
+        ArgumentNullException.ThrowIfNull(logger);
+        _inner = inner;
+        _pskProvider = pskProvider;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Supplies the <c>DistributedPubSub</c> mediator once the actor system exists. Until then
+    /// (and on a host without one) evictions stay local.
+    /// </summary>
+    /// <param name="mediator">The mediator to publish peer invalidations to.</param>
+    public void SetMediator(IActorRef mediator) => _mediator = mediator;
+
+    /// <inheritdoc />
+    public void Invalidate(SecretName name)
+    {
+        _inner.Invalidate(name);
+
+        var siteId = SitePskProvider.SiteIdFromSecretName(name.Value);
+        if (siteId is null)
+        {
+            return;
+        }
+
+        _pskProvider.Invalidate(siteId);
+        _logger.LogInformation(
+            "gRPC preshared key secret '{SecretName}' was written; dropped the cached key for site "
+            + "{SiteId} and notified the peer central node.", name.Value, siteId);
+        _mediator?.Tell(new Publish(Topic, new SitePskInvalidated(siteId)));
+    }
+
+    /// <summary>
+    /// Applies an invalidation received from a central node (the peer, or this node's own
+    /// broadcast): evicts the resolver's cached secret and the PSK cache entry. Never re-publishes.
+    /// </summary>
+    /// <param name="siteId">The site whose key was rotated.</param>
+    public void ApplyPeerInvalidation(string siteId)
+    {
+        if (string.IsNullOrWhiteSpace(siteId))
+        {
+            return;
+        }
+
+        _inner.Invalidate(new SecretName(SitePskProvider.SecretNameFor(siteId)));
+        _pskProvider.Invalidate(siteId);
+    }
+}
+
+/// <summary>
+/// Peer message: the gRPC preshared key of <paramref name="SiteId"/> was rotated on a central node.
+/// </summary>
+/// <param name="SiteId">The site whose key changed.</param>
+public sealed record SitePskInvalidated(string SiteId);
+
+/// <summary>
+/// Central actor that subscribes to <see cref="SitePskRotationInvalidator.Topic"/> and applies
+/// each <see cref="SitePskInvalidated"/> to this node's caches.
+/// </summary>
+public sealed class SitePskInvalidationSubscriber : ReceiveActor
+{
+    private readonly SitePskRotationInvalidator _invalidator;
+
+    /// <summary>Creates the subscriber.</summary>
+    /// <param name="invalidator">This node's invalidator.</param>
+    public SitePskInvalidationSubscriber(SitePskRotationInvalidator invalidator)
+    {
+        _invalidator = invalidator;
+        Receive<SitePskInvalidated>(m => _invalidator.ApplyPeerInvalidation(m.SiteId));
+        Receive<SubscribeAck>(_ => { });
+    }
+
+    /// <inheritdoc />
+    protected override void PreStart()
+    {
+        try
+        {
+            DistributedPubSub.Get(Context.System).Mediator.Tell(
+                new Subscribe(SitePskRotationInvalidator.Topic, Self));
+        }
+        catch (Exception ex)
+        {
+            // Non-clustered host (TestKit): peer invalidation is simply unavailable.
+            Context.GetLogger().Debug("DistributedPubSub not available — peer PSK invalidation disabled: {0}", ex.Message);
+        }
+    }
+}
+
+/// <summary>DI wiring for <see cref="SitePskRotationInvalidator"/>.</summary>
+public static class SitePskRotationServiceCollectionExtensions
+{
+    /// <summary>
+    /// Replaces the secrets library's <see cref="ISecretCacheInvalidator"/> with
+    /// <see cref="SitePskRotationInvalidator"/> wrapping the library's resolver. Central only;
+    /// call after <c>AddScadaBridgeSecrets</c> and after <see cref="ISitePskProvider"/> is
+    /// registered.
+    /// </summary>
+    /// <param name="services">The service collection.</param>
+    /// <returns>The same collection.</returns>
+    public static IServiceCollection AddSitePskRotationInvalidation(this IServiceCollection services)
+    {
+        services.TryAddSingleton(sp => new SitePskRotationInvalidator(
+            sp.GetRequiredService<DefaultSecretResolver>(),
+            sp.GetRequiredService<ISitePskProvider>(),
+            sp.GetRequiredService<ILogger<SitePskRotationInvalidator>>()));
+        services.Replace(ServiceDescriptor.Singleton<ISecretCacheInvalidator>(
+            sp => sp.GetRequiredService<SitePskRotationInvalidator>()));
+        return services;
+    }
+}