dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mesh): per-cluster PSK provider + central interceptor

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-06 03:43:05 -04:00
parent fbda015c6e
commit 68a416e4fc
6 changed files with 1000 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs
+202
View File
@@ -0,0 +1,202 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
/// <summary>
/// Central's per-cluster pre-shared key lookup for the gRPC command plane. A node holds exactly
/// one key (<c>MeshTransport:GrpcPsk</c>, its own cluster's); central talks to every cluster, so
/// it needs one key per <c>Cluster</c> — both to authenticate itself when dialling a node and to
/// verify an inbound deployment ack.
/// </summary>
/// <remarks>
/// <para>
/// <b>Fail-closed by contract.</b> Every failure — unknown cluster, absent secret, blank
/// value, a store that throws — is reported as "no key" (<c>false</c>), and the caller
/// denies the inbound call or skips the outbound dial. There is no arm on which a caller may
/// proceed unauthenticated; a missing key must cost delivery, not authentication.
/// </para>
/// <para>
/// The <c>Invalidate</c> hooks exist because central's cluster set is DB-dynamic. They are
/// deliberately unconsumed at Phase 0 — Phase 1A wires them to the contact refresh, which is
/// the point at which central learns a cluster's membership (and so possibly its key) has
/// changed.
/// </para>
/// </remarks>
public interface IMeshPskProvider
{
/// <summary>Gets the pre-shared key for one application <c>Cluster</c>.</summary>
/// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
/// <param name="psk">The key, or <c>null</c> when none could be established.</param>
/// <returns><c>true</c> when a usable key was found; otherwise <c>false</c>.</returns>
bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk);
/// <summary>Drops any cached key for one cluster, so the next lookup re-reads the store.</summary>
/// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
void Invalidate(string clusterId);
/// <summary>Drops every cached key.</summary>
void InvalidateAll();
}
/// <inheritdoc />
public sealed class MeshPskProvider : IMeshPskProvider
{
/// <summary>
/// Prefix of the secret holding a cluster's key — the full name is
/// <c>OT-MESH-PSK-{clusterId}</c>. The same name a node's
/// <c>${secret:OT-MESH-PSK-&lt;clusterId&gt;}</c> reference expands, so central and the node
/// read one row rather than two conventions that drift.
/// </summary>
public const string SecretNamePrefix = "OT-MESH-PSK-";
private readonly IOptions<MeshTransportOptions> _options;
private readonly ISecretResolver _secretResolver;
private readonly ILogger<MeshPskProvider> _logger;
/// <summary>
/// Cached store resolutions, keyed by cluster id. Holds <b>only</b> values that came from the
/// Secrets store: the config map is re-read on every lookup (it is an in-memory dictionary),
/// which is what keeps the override unconditional rather than merely first-past-the-post.
/// </summary>
/// <remarks>
/// <see cref="Lazy{T}"/> under <see cref="LazyThreadSafetyMode.ExecutionAndPublication"/> so a
/// burst of concurrent inbound acks for one cluster resolves the secret once instead of
/// stampeding the store. Negatives are never retained — see <see cref="TryGetPsk"/>.
/// </remarks>
private readonly ConcurrentDictionary<string, Lazy<string?>> _cache =
new(StringComparer.OrdinalIgnoreCase);
/// <summary>Creates the provider.</summary>
/// <param name="options">Mesh transport options; <c>ClusterPsks</c> is the override map.</param>
/// <param name="secretResolver">The shared secret store read seam.</param>
/// <param name="logger">Logger for lookup-failure diagnostics.</param>
public MeshPskProvider(
IOptions<MeshTransportOptions> options,
ISecretResolver secretResolver,
ILogger<MeshPskProvider> logger)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(secretResolver);
ArgumentNullException.ThrowIfNull(logger);
_options = options;
_secretResolver = secretResolver;
_logger = logger;
}
/// <inheritdoc />
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
{
psk = null;
if (string.IsNullOrWhiteSpace(clusterId))
{
_logger.LogWarning(
"Mesh PSK lookup was asked for a blank cluster id; refusing rather than resolving a "
+ "key that would belong to no cluster.");
return false;
}
// Config map first, on every call. Blank counts as absent: an empty
// MeshTransport__ClusterPsks__<id> is an unset environment variable, not an operator
// deciding this cluster has an empty key.
if (_options.Value.ClusterPsks.TryGetValue(clusterId, out var configured)
&& !string.IsNullOrWhiteSpace(configured))
{
psk = configured;
return true;
}
var entry = _cache.GetOrAdd(
clusterId,
static (id, self) => new Lazy<string?>(
() => self.ResolveFromStore(id), LazyThreadSafetyMode.ExecutionAndPublication),
this);
var resolved = entry.Value;
if (string.IsNullOrWhiteSpace(resolved))
{
// Never retain a negative. A store that was briefly unreachable would otherwise stay
// "no key" for this cluster until something called Invalidate — and at Phase 0 nothing
// does, so a transient fault would present as a permanent, silent command outage.
_cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
return false;
}
psk = resolved;
return true;
}
/// <inheritdoc />
public void Invalidate(string clusterId)
{
if (string.IsNullOrWhiteSpace(clusterId))
return;
_cache.TryRemove(clusterId, out _);
}
/// <inheritdoc />
public void InvalidateAll() => _cache.Clear();
/// <summary>
/// Reads <c>OT-MESH-PSK-{clusterId}</c> from the Secrets store, converting every failure mode
/// into <c>null</c>.
/// </summary>
/// <param name="clusterId">The application <c>Cluster</c> id.</param>
/// <returns>The stored key, or <c>null</c> when it could not be established.</returns>
/// <remarks>
/// <para>
/// Blocking on the async resolver is deliberate. The consumer is a gRPC server
/// interceptor, whose <c>Authorize</c> is synchronous in all four sibling interceptors,
/// and making this seam async would push that shape onto every one of them. The block
/// happens at most once per cluster per process — every later lookup is a dictionary hit
/// — and the store is a node-local SQLite file, not a network call.
/// </para>
/// <para>
/// The catch is broad on purpose: this is an authentication boundary, and the set of
/// exceptions a secret store can raise (missing KEK, corrupt row, IO) is neither closed
/// nor interesting here. Anything other than a usable key is "no key".
/// </para>
/// </remarks>
private string? ResolveFromStore(string clusterId)
{
SecretName name;
try
{
name = new SecretName(SecretNamePrefix + clusterId);
}
catch (ArgumentException ex)
{
// Cluster ids are operator-authored and may carry characters SecretName's allow-list
// rejects. Log the id — it is not sensitive — and never the key.
_logger.LogWarning(
ex,
"Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
+ "resolved from the store. Supply it through MeshTransport:ClusterPsks instead, or "
+ "rename the cluster to [a-z0-9._/-].",
clusterId);
return null;
}
try
{
return _secretResolver.GetAsync(name, CancellationToken.None).GetAwaiter().GetResult();
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to resolve the mesh PSK for cluster {ClusterId} from secret {SecretName}. "
+ "Commands to that cluster and acks from it are refused until it resolves.",
clusterId,
name.Value);
return null;
}
}
}
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ZB.MOM.WW.OtOpcUa.ControlPlane.csproj
+4
View File
@@ -19,6 +19,10 @@
Google.Protobuf + Grpc.Core.Api flow transitively from Commons (the generated client). -->
<PackageReference Include="Grpc.Net.Client"/>
<PackageReference Include="ZB.MOM.WW.Audit"/>
<!-- ClusterClient→gRPC: MeshPskProvider resolves OT-MESH-PSK-{clusterId} from the shared
secret store when MeshTransport:ClusterPsks carries no override. Abstractions only —
the concrete store is the host's choice (SecretsRegistration), not the control plane's. -->
<PackageReference Include="ZB.MOM.WW.Secrets.Abstractions"/>
<!-- For ClusterActiveNode — the shared "oldest Up member of role X" primitive. The control
plane and the /health/active tier must give the SAME answer for which node is in charge,
or the tier reports one node active while the Primary-gated data plane runs on another;
src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs
+199
View File
@@ -0,0 +1,199 @@
using System.Security.Cryptography;
using System.Text;
using Grpc.Core;
using Grpc.Core.Interceptors;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// Gates the central-side command endpoint (ClusterClient→gRPC migration). The
/// <c>CentralCommandService.ReportApply</c> RPC is how a node tells central its deployment
/// applied; the coordinator counts those acks, so anything able to reach central's command port
/// could otherwise forge a fleet-wide "everything applied". This interceptor is the ONLY inbound
/// auth on that endpoint.
/// </summary>
/// <remarks>
/// <para>
/// <b>Scoped by method path.</b> Only calls under
/// <c>/mesh_transport.v1.CentralCommandService/</c> are gated; every other method on the
/// shared gRPC pipeline (config-serve, and on a fused node sync + telemetry + the node
/// command service) passes through untouched, so this shares one <c>AddGrpc</c> registration
/// with its siblings.
/// </para>
/// <para>
/// <b>Per-cluster key, not one server key.</b> Unlike the four sibling interceptors there is
/// no single expected token: central talks to N site clusters, each with its own PSK. The
/// caller names its cluster in <c>x-otopcua-cluster</c> and the bearer is verified against
/// <i>that</i> cluster's key via <see cref="IMeshPskProvider"/>. The header is therefore a
/// claim the caller must then prove, not a credential — choosing a different cluster only
/// changes which key it is graded against. A single fleet-wide key would let one compromised
/// site node forge acks on behalf of every other site.
/// </para>
/// <para>
/// <b>Fail-closed.</b> A missing or blank cluster header, and a cluster whose key cannot be
/// established (unseeded, blank, or a secret store that is down), are both denied. There is
/// deliberately no "try every known key" fallback: it would restore exactly the fleet-wide
/// key this design exists to avoid.
/// </para>
/// <para>
/// Comparison is <see cref="CryptographicOperations.FixedTimeEquals"/> over UTF-8 bytes, so a
/// wrong key cannot be recovered byte-by-byte from response timing. Length differences are
/// unavoidably observable and are not sensitive.
/// </para>
/// <para>
/// <b>All three denials return the same status text</b>, unlike the sibling interceptors,
/// which distinguish "no key configured" from "authentication failed". Here that distinction
/// would be an enumeration oracle: the caller supplies the cluster id, so a distinguishable
/// "no key for that cluster" reply would let an unauthenticated caller map which clusters
/// central holds keys for. The operator-facing detail goes to the log instead.
/// </para>
/// <para>
/// <b>Exactly one public constructor.</b> <c>Grpc.AspNetCore</c> silently stops invoking an
/// interceptor that has more than one public ctor — see
/// <c>CentralCommandAuthInterceptorTests.HasExactlyOnePublicConstructor</c>.
/// </para>
/// </remarks>
public sealed class CentralCommandAuthInterceptor : Interceptor
{
private const string ServicePrefix = "/mesh_transport.v1.CentralCommandService/";
private const string AuthorizationHeader = "authorization";
private const string ClusterHeader = "x-otopcua-cluster";
private const string BearerPrefix = "Bearer ";
private readonly IMeshPskProvider _pskProvider;
private readonly ILogger<CentralCommandAuthInterceptor> _logger;
/// <summary>Creates the interceptor.</summary>
/// <param name="pskProvider">Supplies the expected bearer token for a given cluster.</param>
/// <param name="logger">Logger for denial diagnostics.</param>
public CentralCommandAuthInterceptor(
IMeshPskProvider pskProvider,
ILogger<CentralCommandAuthInterceptor> logger)
{
ArgumentNullException.ThrowIfNull(pskProvider);
ArgumentNullException.ThrowIfNull(logger);
_pskProvider = pskProvider;
_logger = logger;
}
/// <inheritdoc />
public override Task<TResponse> UnaryServerHandler<TRequest, TResponse>(
TRequest request,
ServerCallContext context,
UnaryServerMethod<TRequest, TResponse> continuation)
{
Authorize(context);
return continuation(request, context);
}
/// <inheritdoc />
public override Task DuplexStreamingServerHandler<TRequest, TResponse>(
IAsyncStreamReader<TRequest> requestStream,
IServerStreamWriter<TResponse> responseStream,
ServerCallContext context,
DuplexStreamingServerMethod<TRequest, TResponse> continuation)
{
Authorize(context);
return continuation(requestStream, responseStream, context);
}
/// <inheritdoc />
public override Task<TResponse> ClientStreamingServerHandler<TRequest, TResponse>(
IAsyncStreamReader<TRequest> requestStream,
ServerCallContext context,
ClientStreamingServerMethod<TRequest, TResponse> continuation)
{
Authorize(context);
return continuation(requestStream, context);
}
/// <inheritdoc />
public override Task ServerStreamingServerHandler<TRequest, TResponse>(
TRequest request,
IServerStreamWriter<TResponse> responseStream,
ServerCallContext context,
ServerStreamingServerMethod<TRequest, TResponse> continuation)
{
Authorize(context);
return continuation(request, responseStream, context);
}
/// <summary>
/// Throws <see cref="RpcException"/> with <see cref="StatusCode.PermissionDenied"/> if this is
/// a central-command call that does not name a cluster, names one with no resolvable key, or
/// does not carry that cluster's bearer token. Non-command calls return immediately.
/// </summary>
private void Authorize(ServerCallContext context)
{
if (!context.Method.StartsWith(ServicePrefix, StringComparison.Ordinal))
return;
var clusterId = ExtractHeader(context.RequestHeaders, ClusterHeader);
if (string.IsNullOrWhiteSpace(clusterId))
{
_logger.LogWarning(
"Rejected a central command call to {Method}: no {Header} header, so there is no "
+ "cluster whose pre-shared key the caller could be graded against.",
context.Method,
ClusterHeader);
throw new RpcException(new Status(
StatusCode.PermissionDenied,
"Central command authentication failed."));
}
if (!_pskProvider.TryGetPsk(clusterId, out var expected))
{
_logger.LogWarning(
"Rejected a central command call to {Method} claiming cluster {ClusterId}: no "
+ "pre-shared key could be established for it. Seed the secret "
+ "OT-MESH-PSK-<clusterId> on this node, or set "
+ "MeshTransport:ClusterPsks:<clusterId> on a rig.",
context.Method,
clusterId);
throw new RpcException(new Status(
StatusCode.PermissionDenied,
"Central command authentication failed."));
}
var presented = ExtractBearerToken(context.RequestHeaders);
if (presented is null || !FixedTimeEquals(presented, expected))
{
_logger.LogWarning(
"Rejected a central command call to {Method} claiming cluster {ClusterId}: {Reason}.",
context.Method,
clusterId,
presented is null ? "no bearer token presented" : "bearer token did not match");
throw new RpcException(new Status(
StatusCode.PermissionDenied,
"Central command authentication failed."));
}
}
private static string? ExtractBearerToken(Metadata headers)
{
var value = ExtractHeader(headers, AuthorizationHeader);
return value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase)
? value[BearerPrefix.Length..]
: null;
}
private static string? ExtractHeader(Metadata headers, string name)
{
// gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
// Metadata in a test behaves the same as a real request.
foreach (var entry in headers)
{
if (string.Equals(entry.Key, name, StringComparison.OrdinalIgnoreCase))
return entry.Value;
}
return null;
}
private static bool FixedTimeEquals(string presented, string expected)
=> CryptographicOperations.FixedTimeEquals(
Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
}
src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs
+19 -5
View File
@@ -18,6 +18,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Services;
using ZB.MOM.WW.OtOpcUa.ControlPlane;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
@@ -526,6 +527,13 @@ if (hasAdmin)
// hosted service has built the system).
builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
builder.Services.AddSingleton<IManualFailoverService, ManualFailoverService>();
// Per-cluster mesh PSKs (ClusterClient→gRPC). Admin-only, and singleton because its whole job is
// to hold a cache: central talks to N clusters and would otherwise re-read the secret store on
// every inbound ack. A node needs nothing of this — it holds exactly its own MeshTransport:GrpcPsk
// — so registering it fleet-wide would give a driver-only node a lookup it can never use.
// Depends on ISecretResolver, which AddOtOpcUaSecrets registers unconditionally below (DI
// registration order is irrelevant; resolution order is not).
builder.Services.AddSingleton<IMeshPskProvider, MeshPskProvider>();
}
// Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI.
@@ -548,11 +556,14 @@ builder.Services.AddOtOpcUaObservability(builder.Configuration);
// Registered whenever either role is present so an admin-only node still serves config and a
// driver-only node still gates sync + telemetry.
//
// MeshCommandAuthInterceptor (ClusterClient→gRPC migration) joins on the same terms: it gates the
// node-side NodeCommandService and is registered NOW, a pure pass-through until Phase 1 maps that
// service — an interceptor whose service is unmapped never sees a matching path. Registering it
// alongside the service it gates would put a security gate's wiring in the same commit as the write
// surface it protects; doing it here means the gate cannot be forgotten when that surface lights up.
// The two mesh-command interceptors (ClusterClient→gRPC migration) join on the same terms:
// MeshCommandAuthInterceptor gates the node-side NodeCommandService (driver-role, one key — this
// node's own), CentralCommandAuthInterceptor gates the central-side CentralCommandService
// (admin-role, one key per cluster via IMeshPskProvider). Both are registered NOW and are pure
// pass-throughs until Phase 1 maps those services — an interceptor whose service is unmapped never
// sees a matching path. Registering them alongside the services they gate would put a security
// gate's wiring in the same commit as the write surface it protects; doing it here means the gate
// cannot be forgotten when that surface lights up.
if (hasDriver || hasAdmin)
{
builder.Services.AddGrpc(o =>
@@ -564,7 +575,10 @@ if (hasDriver || hasAdmin)
o.Interceptors.Add<MeshCommandAuthInterceptor>();
}
if (hasAdmin)
{
o.Interceptors.Add<ConfigServeAuthInterceptor>();
o.Interceptors.Add<CentralCommandAuthInterceptor>();
}
});
}
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs
+337
View File
@@ -0,0 +1,337 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
/// <summary>
/// ClusterClient→gRPC Phase 0 — central's per-cluster PSK lookup. Central talks to N site
/// clusters, each with its own key, so every outbound dial and every inbound ack verification
/// goes through here.
/// </summary>
/// <remarks>
/// The tests that matter are the adversarial ones: the store being down, the key being absent
/// or blank, and the cache handing back a value the store no longer agrees with. Every one of
/// those must end in "no key" — the caller then denies or skips, never proceeds unauthenticated.
/// </remarks>
public sealed class MeshPskProviderTests
{
private static MeshPskProvider CreateProvider(
ISecretResolver resolver,
Dictionary<string, string>? clusterPsks = null)
{
var options = new MeshTransportOptions();
if (clusterPsks is not null)
{
foreach (var (key, value) in clusterPsks)
options.ClusterPsks[key] = value;
}
return new MeshPskProvider(
Options.Create(options), resolver, NullLogger<MeshPskProvider>.Instance);
}
[Fact]
public void ConfigMapEntry_IsReturned_WithoutConsultingTheSecretStore()
{
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ConfigMapEntry_WinsOverTheSecretStore()
{
// The map exists so a rig can supply literals without standing up a KEK. If the store also
// holds a key for that cluster the map must still win, or a stale store row would silently
// beat the value the operator just set.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ConfigMapLookup_IsCaseInsensitive()
{
// MeshTransport__ClusterPsks__site-a and __SITE-A are the same cluster. The map's comparer
// (set on MeshTransportOptions) is what makes that true; this pins the pairing.
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
}
[Fact]
public void BlankConfigMapEntry_FallsThroughToTheSecretStore()
{
// A blank map entry is an empty environment variable, not a decision. Treat it as absent
// rather than as an empty key that an empty bearer would satisfy.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = " " });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void SecretStoreValue_IsReturned_UnderTheConventionalName()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
// SecretName normalizes to lowercase — the same normalization ${secret:OT-MESH-PSK-SITE-A}
// goes through in the pre-host expander, so both paths address one row.
resolver.LastName.ShouldBe("ot-mesh-psk-site-a");
}
[Fact]
public void UnknownCluster_FailsClosed()
{
// Absent secret + no map entry. The caller denies; it must never fall back to "no auth".
var resolver = new FakeSecretResolver { Value = null };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void BlankSecretValue_FailsClosed()
{
// A row that decrypts to whitespace is a broken seed, not a key. Accepting it would make
// "Bearer " a valid credential for the whole cluster.
var resolver = new FakeSecretResolver { Value = " " };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void ResolverThrow_FailsClosed()
{
// The store is a local SQLite file with an env KEK; a missing KEK throws rather than
// returning null. Any failure is "no key", never an exception escaping into the gRPC gate.
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void ResolverThrow_IsNotCached_SoRecoveryNeedsNoInvalidate()
{
// Caching a negative would turn a transient store fault into a permanent outage for that
// cluster, recoverable only by a restart or a hook nothing calls yet.
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
resolver.Throw = null;
resolver.Value = "stored-psk";
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void ResolvedValue_IsCached_SoTheStoreIsHitOnce()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("site-a", out _).ShouldBeTrue();
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void Cache_ServesTheStaleValue_UntilInvalidated()
{
// Both directions matter: stale-until-invalidated is the whole reason Invalidate exists, and
// an invalidate that did not actually re-read would be indistinguishable from a no-op.
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var first).ShouldBeTrue();
first.ShouldBe("first-psk");
resolver.Value = "rotated-psk";
provider.TryGetPsk("SITE-A", out var stale).ShouldBeTrue();
stale.ShouldBe("first-psk");
provider.Invalidate("SITE-A");
provider.TryGetPsk("SITE-A", out var fresh).ShouldBeTrue();
fresh.ShouldBe("rotated-psk");
}
[Fact]
public void Invalidate_IsCaseInsensitive_AndScopedToOneCluster()
{
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
resolver.Value = "rotated-psk";
provider.Invalidate("site-a");
provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
a.ShouldBe("rotated-psk");
provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
b.ShouldBe("first-psk");
}
[Fact]
public void Invalidate_OnACacheMiss_IsANoOp()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
Should.NotThrow(() => provider.Invalidate("never-seen"));
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void InvalidateAll_DropsEveryCluster()
{
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
resolver.Value = "rotated-psk";
provider.InvalidateAll();
provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
a.ShouldBe("rotated-psk");
b.ShouldBe("rotated-psk");
}
[Theory]
[InlineData("")]
[InlineData(" ")]
public void BlankClusterId_FailsClosed(string clusterId)
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk(clusterId, out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ClusterIdThatCannotFormASecretName_FailsClosed()
{
// Cluster ids are operator-authored; SecretName's allow-list is [a-z0-9._/-]. A '$' throws
// out of the SecretName ctor, and that throw must become "no key", not a 500 in the gate.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE$A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact]
public async Task ConcurrentCallers_ResolveOnce_AndAllSeeTheSameValue()
{
// The gRPC gate calls this from every inbound ack, on arbitrary thread-pool threads. A
// non-atomic fill would stampede the store and could hand two callers different keys.
var gate = new SemaphoreSlim(0, 1);
var resolver = new FakeSecretResolver { Value = "stored-psk", Gate = gate };
var provider = CreateProvider(resolver);
var callers = Enumerable.Range(0, 16).Select(_ => Task.Run(() =>
{
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
return psk;
})).ToArray();
gate.Release();
var results = await Task.WhenAll(callers);
results.ShouldAllBe(r => r == "stored-psk");
resolver.CallCount.ShouldBe(1);
}
/// <summary>
/// An <see cref="ISecretResolver"/> whose answer the test controls between calls, so cache
/// staleness and invalidation are observable without a real store.
/// </summary>
private sealed class FakeSecretResolver : ISecretResolver
{
private int _callCount;
/// <summary>What <see cref="GetAsync"/> returns when <see cref="Throw"/> is null.</summary>
public string? Value { get; set; }
/// <summary>When set, <see cref="GetAsync"/> throws it instead of returning.</summary>
public Exception? Throw { get; set; }
/// <summary>When set, the first call waits on it — used to force a concurrent stampede.</summary>
public SemaphoreSlim? Gate { get; set; }
/// <summary>How many times the store was consulted.</summary>
public int CallCount => Volatile.Read(ref _callCount);
/// <summary>The name of the most recently requested secret.</summary>
public string? LastName { get; private set; }
public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
{
LastName = name.Value;
Interlocked.Increment(ref _callCount);
if (Gate is not null)
await Gate.WaitAsync(ct).ConfigureAwait(false);
if (Throw is not null)
throw Throw;
return Value;
}
}
}
tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs
+239
View File
@@ -0,0 +1,239 @@
using System.Diagnostics.CodeAnalysis;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
/// <summary>
/// ClusterClient→gRPC Phase 0 — the central-side ack endpoint's inbound gate. Mapping
/// <c>CentralCommandService</c> onto the pipeline is a Phase 1 task; this test covers only the
/// interceptor's own authorization behavior.
/// </summary>
/// <remarks>
/// Differs from <see cref="MeshCommandAuthInterceptor"/> in the one way central differs from a
/// node: there is no single expected key. The caller names its cluster in
/// <c>x-otopcua-cluster</c>, the bearer is verified against <i>that</i> cluster's PSK, and a
/// cluster with no resolvable key is denied — so a caller cannot pick a cluster to be graded
/// against, only claim one it must then prove.
/// </remarks>
public sealed class CentralCommandAuthInterceptorTests
{
private const string ReportApplyMethod = "/mesh_transport.v1.CentralCommandService/ReportApply";
private const string OtherMethod = "/deployment_artifact.v1.DeploymentArtifactService/Fetch";
private static CentralCommandAuthInterceptor CreateInterceptor(params (string Cluster, string Psk)[] keys)
=> new(new FakeMeshPskProvider(keys), NullLogger<CentralCommandAuthInterceptor>.Instance);
private static ServerCallContext CreateContext(
string method, string? authorizationHeader, string? clusterHeader)
{
var headers = new Metadata();
if (authorizationHeader is not null)
headers.Add("authorization", authorizationHeader);
if (clusterHeader is not null)
headers.Add("x-otopcua-cluster", clusterHeader);
return new FakeServerCallContext(method, headers);
}
/// <summary>Invokes the interceptor's unary path (how <c>ReportApply</c> actually runs).</summary>
private static Task Invoke(CentralCommandAuthInterceptor interceptor, ServerCallContext context)
=> interceptor.UnaryServerHandler<string, string>(
"request", context, (_, _) => Task.FromResult("ok"));
[Fact]
public async Task NonCentralCommandMethod_PassesThrough_EvenWithNoKeysAtAll()
{
// The interceptor shares the AddGrpc pipeline with config-serve (and, on a fused node, sync
// and telemetry), so it sees every call. It must be scoped strictly to its own service.
var interceptor = CreateInterceptor();
var context = CreateContext(OtherMethod, authorizationHeader: null, clusterHeader: null);
await Invoke(interceptor, context); // no throw
}
[Fact]
public async Task ReportApply_WithCorrectBearerForItsCluster_PassesThrough()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", "SITE-A");
await Invoke(interceptor, context); // no throw
}
[Fact]
public async Task ReportApply_WithNoClusterHeader_IsDenied()
{
// Without the cluster there is no key to grade against. Falling back to "try them all" would
// make one compromised site's key valid for forging every other site's acks.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", clusterHeader: null);
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithBlankClusterHeader_IsDenied()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", " ");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_ForAClusterWithNoResolvableKey_IsDenied()
{
// Fail-closed on the provider's "no key" — an unseeded or unreachable key must cost the ack,
// never authentication.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer anything-at-all", "SITE-Z");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithAnotherClustersBearer_IsDenied()
{
// The isolation proof: SITE-B's key does not authenticate a call claiming to be SITE-A.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"), ("SITE-B", "psk-site-b"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-b", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithNoBearerToken_IsDenied()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, authorizationHeader: null, clusterHeader: "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithMalformedAuthorizationHeader_IsDenied()
{
// No "Bearer " prefix at all — must be treated the same as a missing header.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "psk-site-a", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_TokenComparison_IsNotAPrefixMatch()
{
// A prefix/StartsWith comparison would accept a truncated key and make the secret
// recoverable one character at a time.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ServerStreamingPath_IsAlsoGated()
{
// ReportApply is unary today, but gating only that path would leave any future streaming
// method on the same service open. Gate every handler shape.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() =>
interceptor.ServerStreamingServerHandler<string, string>(
"request", responseStream: null!, context, (_, _, _) => Task.CompletedTask));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ClientStreamingPath_IsAlsoGated()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() =>
interceptor.ClientStreamingServerHandler<string, string>(
requestStream: null!, context, (_, _) => Task.FromResult("ok")));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task DuplexStreamingPath_IsAlsoGated()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() =>
interceptor.DuplexStreamingServerHandler<string, string>(
requestStream: null!, responseStream: null!, context, (_, _, _) => Task.CompletedTask));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public void HasExactlyOnePublicConstructor()
{
// Grpc.AspNetCore silently stops invoking an interceptor with more than one public ctor —
// this pins the shape so a well-meaning overload doesn't quietly disable auth in production.
var publicCtors = typeof(CentralCommandAuthInterceptor).GetConstructors();
publicCtors.Length.ShouldBe(1);
}
/// <summary>An <see cref="IMeshPskProvider"/> over a fixed key set; unknown clusters fail closed.</summary>
private sealed class FakeMeshPskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider
{
private readonly Dictionary<string, string> _keys =
keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase);
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
=> _keys.TryGetValue(clusterId, out psk);
public void Invalidate(string clusterId) { }
public void InvalidateAll() { }
}
/// <summary>
/// Minimal <see cref="ServerCallContext"/> carrying just a method name and request headers —
/// the only two things the interceptor reads. Hand-rolled because
/// <c>Grpc.Core.Testing.TestServerCallContext</c> ships only in the retired native package.
/// </summary>
private sealed class FakeServerCallContext(string method, Metadata requestHeaders)
: ServerCallContext
{
protected override string MethodCore => method;
protected override string HostCore => "localhost";
protected override string PeerCore => "ipv4:127.0.0.1:12345";
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
protected override Metadata RequestHeadersCore => requestHeaders;
protected override CancellationToken CancellationTokenCore => CancellationToken.None;
protected override Metadata ResponseTrailersCore { get; } = [];
protected override Status StatusCore { get; set; }
protected override WriteOptions? WriteOptionsCore { get; set; }
protected override AuthContext AuthContextCore { get; } =
new(null, new Dictionary<string, List<AuthProperty>>());
protected override ContextPropagationToken CreatePropagationTokenCore(
ContextPropagationOptions? options)
=> throw new NotSupportedException();
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
=> Task.CompletedTask;
}
}