feat(mesh): per-cluster PSK provider + central interceptor

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs

@@ -0,0 +1,202 @@
+using System.Collections.Concurrent;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
+
+/// <summary>
+///     Central's per-cluster pre-shared key lookup for the gRPC command plane. A node holds exactly
+///     one key (<c>MeshTransport:GrpcPsk</c>, its own cluster's); central talks to every cluster, so
+///     it needs one key per <c>Cluster</c> — both to authenticate itself when dialling a node and to
+///     verify an inbound deployment ack.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Fail-closed by contract.</b> Every failure — unknown cluster, absent secret, blank
+///         value, a store that throws — is reported as "no key" (<c>false</c>), and the caller
+///         denies the inbound call or skips the outbound dial. There is no arm on which a caller may
+///         proceed unauthenticated; a missing key must cost delivery, not authentication.
+///     </para>
+///     <para>
+///         The <c>Invalidate</c> hooks exist because central's cluster set is DB-dynamic. They are
+///         deliberately unconsumed at Phase 0 — Phase 1A wires them to the contact refresh, which is
+///         the point at which central learns a cluster's membership (and so possibly its key) has
+///         changed.
+///     </para>
+/// </remarks>
+public interface IMeshPskProvider
+{
+    /// <summary>Gets the pre-shared key for one application <c>Cluster</c>.</summary>
+    /// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
+    /// <param name="psk">The key, or <c>null</c> when none could be established.</param>
+    /// <returns><c>true</c> when a usable key was found; otherwise <c>false</c>.</returns>
+    bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk);
+
+    /// <summary>Drops any cached key for one cluster, so the next lookup re-reads the store.</summary>
+    /// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
+    void Invalidate(string clusterId);
+
+    /// <summary>Drops every cached key.</summary>
+    void InvalidateAll();
+}
+
+/// <inheritdoc />
+public sealed class MeshPskProvider : IMeshPskProvider
+{
+    /// <summary>
+    ///     Prefix of the secret holding a cluster's key — the full name is
+    ///     <c>OT-MESH-PSK-{clusterId}</c>. The same name a node's
+    ///     <c>${secret:OT-MESH-PSK-&lt;clusterId&gt;}</c> reference expands, so central and the node
+    ///     read one row rather than two conventions that drift.
+    /// </summary>
+    public const string SecretNamePrefix = "OT-MESH-PSK-";
+
+    private readonly IOptions<MeshTransportOptions> _options;
+    private readonly ISecretResolver _secretResolver;
+    private readonly ILogger<MeshPskProvider> _logger;
+
+    /// <summary>
+    ///     Cached store resolutions, keyed by cluster id. Holds <b>only</b> values that came from the
+    ///     Secrets store: the config map is re-read on every lookup (it is an in-memory dictionary),
+    ///     which is what keeps the override unconditional rather than merely first-past-the-post.
+    /// </summary>
+    /// <remarks>
+    ///     <see cref="Lazy{T}"/> under <see cref="LazyThreadSafetyMode.ExecutionAndPublication"/> so a
+    ///     burst of concurrent inbound acks for one cluster resolves the secret once instead of
+    ///     stampeding the store. Negatives are never retained — see <see cref="TryGetPsk"/>.
+    /// </remarks>
+    private readonly ConcurrentDictionary<string, Lazy<string?>> _cache =
+        new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>Creates the provider.</summary>
+    /// <param name="options">Mesh transport options; <c>ClusterPsks</c> is the override map.</param>
+    /// <param name="secretResolver">The shared secret store read seam.</param>
+    /// <param name="logger">Logger for lookup-failure diagnostics.</param>
+    public MeshPskProvider(
+        IOptions<MeshTransportOptions> options,
+        ISecretResolver secretResolver,
+        ILogger<MeshPskProvider> logger)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        ArgumentNullException.ThrowIfNull(secretResolver);
+        ArgumentNullException.ThrowIfNull(logger);
+
+        _options = options;
+        _secretResolver = secretResolver;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
+    {
+        psk = null;
+
+        if (string.IsNullOrWhiteSpace(clusterId))
+        {
+            _logger.LogWarning(
+                "Mesh PSK lookup was asked for a blank cluster id; refusing rather than resolving a "
+                + "key that would belong to no cluster.");
+            return false;
+        }
+
+        // Config map first, on every call. Blank counts as absent: an empty
+        // MeshTransport__ClusterPsks__<id> is an unset environment variable, not an operator
+        // deciding this cluster has an empty key.
+        if (_options.Value.ClusterPsks.TryGetValue(clusterId, out var configured)
+            && !string.IsNullOrWhiteSpace(configured))
+        {
+            psk = configured;
+            return true;
+        }
+
+        var entry = _cache.GetOrAdd(
+            clusterId,
+            static (id, self) => new Lazy<string?>(
+                () => self.ResolveFromStore(id), LazyThreadSafetyMode.ExecutionAndPublication),
+            this);
+
+        var resolved = entry.Value;
+        if (string.IsNullOrWhiteSpace(resolved))
+        {
+            // Never retain a negative. A store that was briefly unreachable would otherwise stay
+            // "no key" for this cluster until something called Invalidate — and at Phase 0 nothing
+            // does, so a transient fault would present as a permanent, silent command outage.
+            _cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
+            return false;
+        }
+
+        psk = resolved;
+        return true;
+    }
+
+    /// <inheritdoc />
+    public void Invalidate(string clusterId)
+    {
+        if (string.IsNullOrWhiteSpace(clusterId))
+            return;
+
+        _cache.TryRemove(clusterId, out _);
+    }
+
+    /// <inheritdoc />
+    public void InvalidateAll() => _cache.Clear();
+
+    /// <summary>
+    ///     Reads <c>OT-MESH-PSK-{clusterId}</c> from the Secrets store, converting every failure mode
+    ///     into <c>null</c>.
+    /// </summary>
+    /// <param name="clusterId">The application <c>Cluster</c> id.</param>
+    /// <returns>The stored key, or <c>null</c> when it could not be established.</returns>
+    /// <remarks>
+    ///     <para>
+    ///         Blocking on the async resolver is deliberate. The consumer is a gRPC server
+    ///         interceptor, whose <c>Authorize</c> is synchronous in all four sibling interceptors,
+    ///         and making this seam async would push that shape onto every one of them. The block
+    ///         happens at most once per cluster per process — every later lookup is a dictionary hit
+    ///         — and the store is a node-local SQLite file, not a network call.
+    ///     </para>
+    ///     <para>
+    ///         The catch is broad on purpose: this is an authentication boundary, and the set of
+    ///         exceptions a secret store can raise (missing KEK, corrupt row, IO) is neither closed
+    ///         nor interesting here. Anything other than a usable key is "no key".
+    ///     </para>
+    /// </remarks>
+    private string? ResolveFromStore(string clusterId)
+    {
+        SecretName name;
+        try
+        {
+            name = new SecretName(SecretNamePrefix + clusterId);
+        }
+        catch (ArgumentException ex)
+        {
+            // Cluster ids are operator-authored and may carry characters SecretName's allow-list
+            // rejects. Log the id — it is not sensitive — and never the key.
+            _logger.LogWarning(
+                ex,
+                "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
+                + "resolved from the store. Supply it through MeshTransport:ClusterPsks instead, or "
+                + "rename the cluster to [a-z0-9._/-].",
+                clusterId);
+            return null;
+        }
+
+        try
+        {
+            return _secretResolver.GetAsync(name, CancellationToken.None).GetAwaiter().GetResult();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(
+                ex,
+                "Failed to resolve the mesh PSK for cluster {ClusterId} from secret {SecretName}. "
+                + "Commands to that cluster and acks from it are refused until it resolves.",
+                clusterId,
+                name.Value);
+            return null;
+        }
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ZB.MOM.WW.OtOpcUa.ControlPlane.csproj

@@ -19,6 +19,10 @@
              Google.Protobuf + Grpc.Core.Api flow transitively from Commons (the generated client). -->
         <PackageReference Include="Grpc.Net.Client"/>
         <PackageReference Include="ZB.MOM.WW.Audit"/>
+        <!-- ClusterClient→gRPC: MeshPskProvider resolves OT-MESH-PSK-{clusterId} from the shared
+             secret store when MeshTransport:ClusterPsks carries no override. Abstractions only —
+             the concrete store is the host's choice (SecretsRegistration), not the control plane's. -->
+        <PackageReference Include="ZB.MOM.WW.Secrets.Abstractions"/>
         <!-- For ClusterActiveNode — the shared "oldest Up member of role X" primitive. The control
              plane and the /health/active tier must give the SAME answer for which node is in charge,
              or the tier reports one node active while the Primary-gated data plane runs on another;

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs

@@ -0,0 +1,199 @@
+using System.Security.Cryptography;
+using System.Text;
+using Grpc.Core;
+using Grpc.Core.Interceptors;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+/// <summary>
+///     Gates the central-side command endpoint (ClusterClient→gRPC migration). The
+///     <c>CentralCommandService.ReportApply</c> RPC is how a node tells central its deployment
+///     applied; the coordinator counts those acks, so anything able to reach central's command port
+///     could otherwise forge a fleet-wide "everything applied". This interceptor is the ONLY inbound
+///     auth on that endpoint.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Scoped by method path.</b> Only calls under
+///         <c>/mesh_transport.v1.CentralCommandService/</c> are gated; every other method on the
+///         shared gRPC pipeline (config-serve, and on a fused node sync + telemetry + the node
+///         command service) passes through untouched, so this shares one <c>AddGrpc</c> registration
+///         with its siblings.
+///     </para>
+///     <para>
+///         <b>Per-cluster key, not one server key.</b> Unlike the four sibling interceptors there is
+///         no single expected token: central talks to N site clusters, each with its own PSK. The
+///         caller names its cluster in <c>x-otopcua-cluster</c> and the bearer is verified against
+///         <i>that</i> cluster's key via <see cref="IMeshPskProvider"/>. The header is therefore a
+///         claim the caller must then prove, not a credential — choosing a different cluster only
+///         changes which key it is graded against. A single fleet-wide key would let one compromised
+///         site node forge acks on behalf of every other site.
+///     </para>
+///     <para>
+///         <b>Fail-closed.</b> A missing or blank cluster header, and a cluster whose key cannot be
+///         established (unseeded, blank, or a secret store that is down), are both denied. There is
+///         deliberately no "try every known key" fallback: it would restore exactly the fleet-wide
+///         key this design exists to avoid.
+///     </para>
+///     <para>
+///         Comparison is <see cref="CryptographicOperations.FixedTimeEquals"/> over UTF-8 bytes, so a
+///         wrong key cannot be recovered byte-by-byte from response timing. Length differences are
+///         unavoidably observable and are not sensitive.
+///     </para>
+///     <para>
+///         <b>All three denials return the same status text</b>, unlike the sibling interceptors,
+///         which distinguish "no key configured" from "authentication failed". Here that distinction
+///         would be an enumeration oracle: the caller supplies the cluster id, so a distinguishable
+///         "no key for that cluster" reply would let an unauthenticated caller map which clusters
+///         central holds keys for. The operator-facing detail goes to the log instead.
+///     </para>
+///     <para>
+///         <b>Exactly one public constructor.</b> <c>Grpc.AspNetCore</c> silently stops invoking an
+///         interceptor that has more than one public ctor — see
+///         <c>CentralCommandAuthInterceptorTests.HasExactlyOnePublicConstructor</c>.
+///     </para>
+/// </remarks>
+public sealed class CentralCommandAuthInterceptor : Interceptor
+{
+    private const string ServicePrefix = "/mesh_transport.v1.CentralCommandService/";
+    private const string AuthorizationHeader = "authorization";
+    private const string ClusterHeader = "x-otopcua-cluster";
+    private const string BearerPrefix = "Bearer ";
+
+    private readonly IMeshPskProvider _pskProvider;
+    private readonly ILogger<CentralCommandAuthInterceptor> _logger;
+
+    /// <summary>Creates the interceptor.</summary>
+    /// <param name="pskProvider">Supplies the expected bearer token for a given cluster.</param>
+    /// <param name="logger">Logger for denial diagnostics.</param>
+    public CentralCommandAuthInterceptor(
+        IMeshPskProvider pskProvider,
+        ILogger<CentralCommandAuthInterceptor> logger)
+    {
+        ArgumentNullException.ThrowIfNull(pskProvider);
+        ArgumentNullException.ThrowIfNull(logger);
+
+        _pskProvider = pskProvider;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public override Task<TResponse> UnaryServerHandler<TRequest, TResponse>(
+        TRequest request,
+        ServerCallContext context,
+        UnaryServerMethod<TRequest, TResponse> continuation)
+    {
+        Authorize(context);
+        return continuation(request, context);
+    }
+
+    /// <inheritdoc />
+    public override Task DuplexStreamingServerHandler<TRequest, TResponse>(
+        IAsyncStreamReader<TRequest> requestStream,
+        IServerStreamWriter<TResponse> responseStream,
+        ServerCallContext context,
+        DuplexStreamingServerMethod<TRequest, TResponse> continuation)
+    {
+        Authorize(context);
+        return continuation(requestStream, responseStream, context);
+    }
+
+    /// <inheritdoc />
+    public override Task<TResponse> ClientStreamingServerHandler<TRequest, TResponse>(
+        IAsyncStreamReader<TRequest> requestStream,
+        ServerCallContext context,
+        ClientStreamingServerMethod<TRequest, TResponse> continuation)
+    {
+        Authorize(context);
+        return continuation(requestStream, context);
+    }
+
+    /// <inheritdoc />
+    public override Task ServerStreamingServerHandler<TRequest, TResponse>(
+        TRequest request,
+        IServerStreamWriter<TResponse> responseStream,
+        ServerCallContext context,
+        ServerStreamingServerMethod<TRequest, TResponse> continuation)
+    {
+        Authorize(context);
+        return continuation(request, responseStream, context);
+    }
+
+    /// <summary>
+    ///     Throws <see cref="RpcException"/> with <see cref="StatusCode.PermissionDenied"/> if this is
+    ///     a central-command call that does not name a cluster, names one with no resolvable key, or
+    ///     does not carry that cluster's bearer token. Non-command calls return immediately.
+    /// </summary>
+    private void Authorize(ServerCallContext context)
+    {
+        if (!context.Method.StartsWith(ServicePrefix, StringComparison.Ordinal))
+            return;
+
+        var clusterId = ExtractHeader(context.RequestHeaders, ClusterHeader);
+        if (string.IsNullOrWhiteSpace(clusterId))
+        {
+            _logger.LogWarning(
+                "Rejected a central command call to {Method}: no {Header} header, so there is no "
+                + "cluster whose pre-shared key the caller could be graded against.",
+                context.Method,
+                ClusterHeader);
+            throw new RpcException(new Status(
+                StatusCode.PermissionDenied,
+                "Central command authentication failed."));
+        }
+
+        if (!_pskProvider.TryGetPsk(clusterId, out var expected))
+        {
+            _logger.LogWarning(
+                "Rejected a central command call to {Method} claiming cluster {ClusterId}: no "
+                + "pre-shared key could be established for it. Seed the secret "
+                + "OT-MESH-PSK-<clusterId> on this node, or set "
+                + "MeshTransport:ClusterPsks:<clusterId> on a rig.",
+                context.Method,
+                clusterId);
+            throw new RpcException(new Status(
+                StatusCode.PermissionDenied,
+                "Central command authentication failed."));
+        }
+
+        var presented = ExtractBearerToken(context.RequestHeaders);
+        if (presented is null || !FixedTimeEquals(presented, expected))
+        {
+            _logger.LogWarning(
+                "Rejected a central command call to {Method} claiming cluster {ClusterId}: {Reason}.",
+                context.Method,
+                clusterId,
+                presented is null ? "no bearer token presented" : "bearer token did not match");
+            throw new RpcException(new Status(
+                StatusCode.PermissionDenied,
+                "Central command authentication failed."));
+        }
+    }
+
+    private static string? ExtractBearerToken(Metadata headers)
+    {
+        var value = ExtractHeader(headers, AuthorizationHeader);
+        return value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase)
+            ? value[BearerPrefix.Length..]
+            : null;
+    }
+
+    private static string? ExtractHeader(Metadata headers, string name)
+    {
+        // gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
+        // Metadata in a test behaves the same as a real request.
+        foreach (var entry in headers)
+        {
+            if (string.Equals(entry.Key, name, StringComparison.OrdinalIgnoreCase))
+                return entry.Value;
+        }
+
+        return null;
+    }
+
+    private static bool FixedTimeEquals(string presented, string expected)
+        => CryptographicOperations.FixedTimeEquals(
+            Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -18,6 +18,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Configuration.Services;
 using ZB.MOM.WW.OtOpcUa.ControlPlane;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Commons.Engines;
@@ -526,6 +527,13 @@ if (hasAdmin)
     // hosted service has built the system).
     builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
     builder.Services.AddSingleton<IManualFailoverService, ManualFailoverService>();
+    // Per-cluster mesh PSKs (ClusterClient→gRPC). Admin-only, and singleton because its whole job is
+    // to hold a cache: central talks to N clusters and would otherwise re-read the secret store on
+    // every inbound ack. A node needs nothing of this — it holds exactly its own MeshTransport:GrpcPsk
+    // — so registering it fleet-wide would give a driver-only node a lookup it can never use.
+    // Depends on ISecretResolver, which AddOtOpcUaSecrets registers unconditionally below (DI
+    // registration order is irrelevant; resolution order is not).
+    builder.Services.AddSingleton<IMeshPskProvider, MeshPskProvider>();
 }
 
 // Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI.
@@ -548,11 +556,14 @@ builder.Services.AddOtOpcUaObservability(builder.Configuration);
 // Registered whenever either role is present so an admin-only node still serves config and a
 // driver-only node still gates sync + telemetry.
 //
-// MeshCommandAuthInterceptor (ClusterClient→gRPC migration) joins on the same terms: it gates the
+// The two mesh-command interceptors (ClusterClient→gRPC migration) join on the same terms:
-// node-side NodeCommandService and is registered NOW, a pure pass-through until Phase 1 maps that
+// MeshCommandAuthInterceptor gates the node-side NodeCommandService (driver-role, one key — this
-// service — an interceptor whose service is unmapped never sees a matching path. Registering it
+// node's own), CentralCommandAuthInterceptor gates the central-side CentralCommandService
-// alongside the service it gates would put a security gate's wiring in the same commit as the write
+// (admin-role, one key per cluster via IMeshPskProvider). Both are registered NOW and are pure
-// surface it protects; doing it here means the gate cannot be forgotten when that surface lights up.
+// pass-throughs until Phase 1 maps those services — an interceptor whose service is unmapped never
+// sees a matching path. Registering them alongside the services they gate would put a security
+// gate's wiring in the same commit as the write surface it protects; doing it here means the gate
+// cannot be forgotten when that surface lights up.
 if (hasDriver || hasAdmin)
 {
     builder.Services.AddGrpc(o =>
@@ -564,7 +575,10 @@ if (hasDriver || hasAdmin)
             o.Interceptors.Add<MeshCommandAuthInterceptor>();
         }
         if (hasAdmin)
+        {
             o.Interceptors.Add<ConfigServeAuthInterceptor>();
+            o.Interceptors.Add<CentralCommandAuthInterceptor>();
+        }
     });
 }
 

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs

@@ -0,0 +1,337 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
+
+/// <summary>
+///     ClusterClient→gRPC Phase 0 — central's per-cluster PSK lookup. Central talks to N site
+///     clusters, each with its own key, so every outbound dial and every inbound ack verification
+///     goes through here.
+/// </summary>
+/// <remarks>
+///     The tests that matter are the adversarial ones: the store being down, the key being absent
+///     or blank, and the cache handing back a value the store no longer agrees with. Every one of
+///     those must end in "no key" — the caller then denies or skips, never proceeds unauthenticated.
+/// </remarks>
+public sealed class MeshPskProviderTests
+{
+    private static MeshPskProvider CreateProvider(
+        ISecretResolver resolver,
+        Dictionary<string, string>? clusterPsks = null)
+    {
+        var options = new MeshTransportOptions();
+        if (clusterPsks is not null)
+        {
+            foreach (var (key, value) in clusterPsks)
+                options.ClusterPsks[key] = value;
+        }
+
+        return new MeshPskProvider(
+            Options.Create(options), resolver, NullLogger<MeshPskProvider>.Instance);
+    }
+
+    [Fact]
+    public void ConfigMapEntry_IsReturned_WithoutConsultingTheSecretStore()
+    {
+        var resolver = new FakeSecretResolver();
+        var provider = CreateProvider(
+            resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+
+        psk.ShouldBe("rig-psk-site-a");
+        resolver.CallCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public void ConfigMapEntry_WinsOverTheSecretStore()
+    {
+        // The map exists so a rig can supply literals without standing up a KEK. If the store also
+        // holds a key for that cluster the map must still win, or a stale store row would silently
+        // beat the value the operator just set.
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(
+            resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+
+        psk.ShouldBe("rig-psk-site-a");
+        resolver.CallCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public void ConfigMapLookup_IsCaseInsensitive()
+    {
+        // MeshTransport__ClusterPsks__site-a and __SITE-A are the same cluster. The map's comparer
+        // (set on MeshTransportOptions) is what makes that true; this pins the pairing.
+        var resolver = new FakeSecretResolver();
+        var provider = CreateProvider(
+            resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
+
+        provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();
+
+        psk.ShouldBe("rig-psk-site-a");
+    }
+
+    [Fact]
+    public void BlankConfigMapEntry_FallsThroughToTheSecretStore()
+    {
+        // A blank map entry is an empty environment variable, not a decision. Treat it as absent
+        // rather than as an empty key that an empty bearer would satisfy.
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(
+            resolver, new Dictionary<string, string> { ["SITE-A"] = "   " });
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+
+        psk.ShouldBe("stored-psk");
+        resolver.CallCount.ShouldBe(1);
+    }
+
+    [Fact]
+    public void SecretStoreValue_IsReturned_UnderTheConventionalName()
+    {
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+
+        psk.ShouldBe("stored-psk");
+        // SecretName normalizes to lowercase — the same normalization ${secret:OT-MESH-PSK-SITE-A}
+        // goes through in the pre-host expander, so both paths address one row.
+        resolver.LastName.ShouldBe("ot-mesh-psk-site-a");
+    }
+
+    [Fact]
+    public void UnknownCluster_FailsClosed()
+    {
+        // Absent secret + no map entry. The caller denies; it must never fall back to "no auth".
+        var resolver = new FakeSecretResolver { Value = null };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();
+
+        psk.ShouldBeNull();
+    }
+
+    [Fact]
+    public void BlankSecretValue_FailsClosed()
+    {
+        // A row that decrypts to whitespace is a broken seed, not a key. Accepting it would make
+        // "Bearer    " a valid credential for the whole cluster.
+        var resolver = new FakeSecretResolver { Value = "   " };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
+
+        psk.ShouldBeNull();
+    }
+
+    [Fact]
+    public void ResolverThrow_FailsClosed()
+    {
+        // The store is a local SQLite file with an env KEK; a missing KEK throws rather than
+        // returning null. Any failure is "no key", never an exception escaping into the gRPC gate.
+        var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
+
+        psk.ShouldBeNull();
+    }
+
+    [Fact]
+    public void ResolverThrow_IsNotCached_SoRecoveryNeedsNoInvalidate()
+    {
+        // Caching a negative would turn a transient store fault into a permanent outage for that
+        // cluster, recoverable only by a restart or a hook nothing calls yet.
+        var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
+
+        resolver.Throw = null;
+        resolver.Value = "stored-psk";
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+        psk.ShouldBe("stored-psk");
+    }
+
+    [Fact]
+    public void ResolvedValue_IsCached_SoTheStoreIsHitOnce()
+    {
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
+        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
+        provider.TryGetPsk("site-a", out _).ShouldBeTrue();
+
+        resolver.CallCount.ShouldBe(1);
+    }
+
+    [Fact]
+    public void Cache_ServesTheStaleValue_UntilInvalidated()
+    {
+        // Both directions matter: stale-until-invalidated is the whole reason Invalidate exists, and
+        // an invalidate that did not actually re-read would be indistinguishable from a no-op.
+        var resolver = new FakeSecretResolver { Value = "first-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out var first).ShouldBeTrue();
+        first.ShouldBe("first-psk");
+
+        resolver.Value = "rotated-psk";
+
+        provider.TryGetPsk("SITE-A", out var stale).ShouldBeTrue();
+        stale.ShouldBe("first-psk");
+
+        provider.Invalidate("SITE-A");
+
+        provider.TryGetPsk("SITE-A", out var fresh).ShouldBeTrue();
+        fresh.ShouldBe("rotated-psk");
+    }
+
+    [Fact]
+    public void Invalidate_IsCaseInsensitive_AndScopedToOneCluster()
+    {
+        var resolver = new FakeSecretResolver { Value = "first-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
+        provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
+
+        resolver.Value = "rotated-psk";
+        provider.Invalidate("site-a");
+
+        provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
+        a.ShouldBe("rotated-psk");
+
+        provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
+        b.ShouldBe("first-psk");
+    }
+
+    [Fact]
+    public void Invalidate_OnACacheMiss_IsANoOp()
+    {
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(resolver);
+
+        Should.NotThrow(() => provider.Invalidate("never-seen"));
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+        psk.ShouldBe("stored-psk");
+    }
+
+    [Fact]
+    public void InvalidateAll_DropsEveryCluster()
+    {
+        var resolver = new FakeSecretResolver { Value = "first-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
+        provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
+
+        resolver.Value = "rotated-psk";
+        provider.InvalidateAll();
+
+        provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
+        provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
+
+        a.ShouldBe("rotated-psk");
+        b.ShouldBe("rotated-psk");
+    }
+
+    [Theory]
+    [InlineData("")]
+    [InlineData("   ")]
+    public void BlankClusterId_FailsClosed(string clusterId)
+    {
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk(clusterId, out var psk).ShouldBeFalse();
+
+        psk.ShouldBeNull();
+        resolver.CallCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public void ClusterIdThatCannotFormASecretName_FailsClosed()
+    {
+        // Cluster ids are operator-authored; SecretName's allow-list is [a-z0-9._/-]. A '$' throws
+        // out of the SecretName ctor, and that throw must become "no key", not a 500 in the gate.
+        var resolver = new FakeSecretResolver { Value = "stored-psk" };
+        var provider = CreateProvider(resolver);
+
+        provider.TryGetPsk("SITE$A", out var psk).ShouldBeFalse();
+
+        psk.ShouldBeNull();
+        resolver.CallCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public async Task ConcurrentCallers_ResolveOnce_AndAllSeeTheSameValue()
+    {
+        // The gRPC gate calls this from every inbound ack, on arbitrary thread-pool threads. A
+        // non-atomic fill would stampede the store and could hand two callers different keys.
+        var gate = new SemaphoreSlim(0, 1);
+        var resolver = new FakeSecretResolver { Value = "stored-psk", Gate = gate };
+        var provider = CreateProvider(resolver);
+
+        var callers = Enumerable.Range(0, 16).Select(_ => Task.Run(() =>
+        {
+            provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+            return psk;
+        })).ToArray();
+
+        gate.Release();
+        var results = await Task.WhenAll(callers);
+
+        results.ShouldAllBe(r => r == "stored-psk");
+        resolver.CallCount.ShouldBe(1);
+    }
+
+    /// <summary>
+    ///     An <see cref="ISecretResolver"/> whose answer the test controls between calls, so cache
+    ///     staleness and invalidation are observable without a real store.
+    /// </summary>
+    private sealed class FakeSecretResolver : ISecretResolver
+    {
+        private int _callCount;
+
+        /// <summary>What <see cref="GetAsync"/> returns when <see cref="Throw"/> is null.</summary>
+        public string? Value { get; set; }
+
+        /// <summary>When set, <see cref="GetAsync"/> throws it instead of returning.</summary>
+        public Exception? Throw { get; set; }
+
+        /// <summary>When set, the first call waits on it — used to force a concurrent stampede.</summary>
+        public SemaphoreSlim? Gate { get; set; }
+
+        /// <summary>How many times the store was consulted.</summary>
+        public int CallCount => Volatile.Read(ref _callCount);
+
+        /// <summary>The name of the most recently requested secret.</summary>
+        public string? LastName { get; private set; }
+
+        public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
+        {
+            LastName = name.Value;
+            Interlocked.Increment(ref _callCount);
+
+            if (Gate is not null)
+                await Gate.WaitAsync(ct).ConfigureAwait(false);
+
+            if (Throw is not null)
+                throw Throw;
+
+            return Value;
+        }
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs

@@ -0,0 +1,239 @@
+using System.Diagnostics.CodeAnalysis;
+using Grpc.Core;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
+using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
+
+/// <summary>
+///     ClusterClient→gRPC Phase 0 — the central-side ack endpoint's inbound gate. Mapping
+///     <c>CentralCommandService</c> onto the pipeline is a Phase 1 task; this test covers only the
+///     interceptor's own authorization behavior.
+/// </summary>
+/// <remarks>
+///     Differs from <see cref="MeshCommandAuthInterceptor"/> in the one way central differs from a
+///     node: there is no single expected key. The caller names its cluster in
+///     <c>x-otopcua-cluster</c>, the bearer is verified against <i>that</i> cluster's PSK, and a
+///     cluster with no resolvable key is denied — so a caller cannot pick a cluster to be graded
+///     against, only claim one it must then prove.
+/// </remarks>
+public sealed class CentralCommandAuthInterceptorTests
+{
+    private const string ReportApplyMethod = "/mesh_transport.v1.CentralCommandService/ReportApply";
+    private const string OtherMethod = "/deployment_artifact.v1.DeploymentArtifactService/Fetch";
+
+    private static CentralCommandAuthInterceptor CreateInterceptor(params (string Cluster, string Psk)[] keys)
+        => new(new FakeMeshPskProvider(keys), NullLogger<CentralCommandAuthInterceptor>.Instance);
+
+    private static ServerCallContext CreateContext(
+        string method, string? authorizationHeader, string? clusterHeader)
+    {
+        var headers = new Metadata();
+        if (authorizationHeader is not null)
+            headers.Add("authorization", authorizationHeader);
+        if (clusterHeader is not null)
+            headers.Add("x-otopcua-cluster", clusterHeader);
+
+        return new FakeServerCallContext(method, headers);
+    }
+
+    /// <summary>Invokes the interceptor's unary path (how <c>ReportApply</c> actually runs).</summary>
+    private static Task Invoke(CentralCommandAuthInterceptor interceptor, ServerCallContext context)
+        => interceptor.UnaryServerHandler<string, string>(
+            "request", context, (_, _) => Task.FromResult("ok"));
+
+    [Fact]
+    public async Task NonCentralCommandMethod_PassesThrough_EvenWithNoKeysAtAll()
+    {
+        // The interceptor shares the AddGrpc pipeline with config-serve (and, on a fused node, sync
+        // and telemetry), so it sees every call. It must be scoped strictly to its own service.
+        var interceptor = CreateInterceptor();
+        var context = CreateContext(OtherMethod, authorizationHeader: null, clusterHeader: null);
+
+        await Invoke(interceptor, context); // no throw
+    }
+
+    [Fact]
+    public async Task ReportApply_WithCorrectBearerForItsCluster_PassesThrough()
+    {
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", "SITE-A");
+
+        await Invoke(interceptor, context); // no throw
+    }
+
+    [Fact]
+    public async Task ReportApply_WithNoClusterHeader_IsDenied()
+    {
+        // Without the cluster there is no key to grade against. Falling back to "try them all" would
+        // make one compromised site's key valid for forging every other site's acks.
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", clusterHeader: null);
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ReportApply_WithBlankClusterHeader_IsDenied()
+    {
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", "   ");
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ReportApply_ForAClusterWithNoResolvableKey_IsDenied()
+    {
+        // Fail-closed on the provider's "no key" — an unseeded or unreachable key must cost the ack,
+        // never authentication.
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer anything-at-all", "SITE-Z");
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ReportApply_WithAnotherClustersBearer_IsDenied()
+    {
+        // The isolation proof: SITE-B's key does not authenticate a call claiming to be SITE-A.
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"), ("SITE-B", "psk-site-b"));
+        var context = CreateContext(ReportApplyMethod, "Bearer psk-site-b", "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ReportApply_WithNoBearerToken_IsDenied()
+    {
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, authorizationHeader: null, clusterHeader: "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ReportApply_WithMalformedAuthorizationHeader_IsDenied()
+    {
+        // No "Bearer " prefix at all — must be treated the same as a missing header.
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "psk-site-a", "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ReportApply_TokenComparison_IsNotAPrefixMatch()
+    {
+        // A prefix/StartsWith comparison would accept a truncated key and make the secret
+        // recoverable one character at a time.
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer psk-site-", "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ServerStreamingPath_IsAlsoGated()
+    {
+        // ReportApply is unary today, but gating only that path would leave any future streaming
+        // method on the same service open. Gate every handler shape.
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() =>
+            interceptor.ServerStreamingServerHandler<string, string>(
+                "request", responseStream: null!, context, (_, _, _) => Task.CompletedTask));
+
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task ClientStreamingPath_IsAlsoGated()
+    {
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() =>
+            interceptor.ClientStreamingServerHandler<string, string>(
+                requestStream: null!, context, (_, _) => Task.FromResult("ok")));
+
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public async Task DuplexStreamingPath_IsAlsoGated()
+    {
+        var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
+        var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
+
+        var ex = await Should.ThrowAsync<RpcException>(() =>
+            interceptor.DuplexStreamingServerHandler<string, string>(
+                requestStream: null!, responseStream: null!, context, (_, _, _) => Task.CompletedTask));
+
+        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
+    }
+
+    [Fact]
+    public void HasExactlyOnePublicConstructor()
+    {
+        // Grpc.AspNetCore silently stops invoking an interceptor with more than one public ctor —
+        // this pins the shape so a well-meaning overload doesn't quietly disable auth in production.
+        var publicCtors = typeof(CentralCommandAuthInterceptor).GetConstructors();
+
+        publicCtors.Length.ShouldBe(1);
+    }
+
+    /// <summary>An <see cref="IMeshPskProvider"/> over a fixed key set; unknown clusters fail closed.</summary>
+    private sealed class FakeMeshPskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider
+    {
+        private readonly Dictionary<string, string> _keys =
+            keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase);
+
+        public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
+            => _keys.TryGetValue(clusterId, out psk);
+
+        public void Invalidate(string clusterId) { }
+
+        public void InvalidateAll() { }
+    }
+
+    /// <summary>
+    ///     Minimal <see cref="ServerCallContext"/> carrying just a method name and request headers —
+    ///     the only two things the interceptor reads. Hand-rolled because
+    ///     <c>Grpc.Core.Testing.TestServerCallContext</c> ships only in the retired native package.
+    /// </summary>
+    private sealed class FakeServerCallContext(string method, Metadata requestHeaders)
+        : ServerCallContext
+    {
+        protected override string MethodCore => method;
+        protected override string HostCore => "localhost";
+        protected override string PeerCore => "ipv4:127.0.0.1:12345";
+        protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
+        protected override Metadata RequestHeadersCore => requestHeaders;
+        protected override CancellationToken CancellationTokenCore => CancellationToken.None;
+        protected override Metadata ResponseTrailersCore { get; } = [];
+        protected override Status StatusCore { get; set; }
+        protected override WriteOptions? WriteOptionsCore { get; set; }
+
+        protected override AuthContext AuthContextCore { get; } =
+            new(null, new Dictionary<string, List<AuthProperty>>());
+
+        protected override ContextPropagationToken CreatePropagationTokenCore(
+            ContextPropagationOptions? options)
+            => throw new NotSupportedException();
+
+        protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
+            => Task.CompletedTask;
+    }
+}