dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mesh): per-cluster PSK provider + central interceptor

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-06 03:43:05 -04:00
parent fbda015c6e
commit 68a416e4fc
6 changed files with 1000 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs
+337
View File
@@ -0,0 +1,337 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
/// <summary>
/// ClusterClient→gRPC Phase 0 — central's per-cluster PSK lookup. Central talks to N site
/// clusters, each with its own key, so every outbound dial and every inbound ack verification
/// goes through here.
/// </summary>
/// <remarks>
/// The tests that matter are the adversarial ones: the store being down, the key being absent
/// or blank, and the cache handing back a value the store no longer agrees with. Every one of
/// those must end in "no key" — the caller then denies or skips, never proceeds unauthenticated.
/// </remarks>
public sealed class MeshPskProviderTests
{
private static MeshPskProvider CreateProvider(
ISecretResolver resolver,
Dictionary<string, string>? clusterPsks = null)
{
var options = new MeshTransportOptions();
if (clusterPsks is not null)
{
foreach (var (key, value) in clusterPsks)
options.ClusterPsks[key] = value;
}
return new MeshPskProvider(
Options.Create(options), resolver, NullLogger<MeshPskProvider>.Instance);
}
[Fact]
public void ConfigMapEntry_IsReturned_WithoutConsultingTheSecretStore()
{
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ConfigMapEntry_WinsOverTheSecretStore()
{
// The map exists so a rig can supply literals without standing up a KEK. If the store also
// holds a key for that cluster the map must still win, or a stale store row would silently
// beat the value the operator just set.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ConfigMapLookup_IsCaseInsensitive()
{
// MeshTransport__ClusterPsks__site-a and __SITE-A are the same cluster. The map's comparer
// (set on MeshTransportOptions) is what makes that true; this pins the pairing.
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
}
[Fact]
public void BlankConfigMapEntry_FallsThroughToTheSecretStore()
{
// A blank map entry is an empty environment variable, not a decision. Treat it as absent
// rather than as an empty key that an empty bearer would satisfy.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(
resolver, new Dictionary<string, string> { ["SITE-A"] = " " });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void SecretStoreValue_IsReturned_UnderTheConventionalName()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
// SecretName normalizes to lowercase — the same normalization ${secret:OT-MESH-PSK-SITE-A}
// goes through in the pre-host expander, so both paths address one row.
resolver.LastName.ShouldBe("ot-mesh-psk-site-a");
}
[Fact]
public void UnknownCluster_FailsClosed()
{
// Absent secret + no map entry. The caller denies; it must never fall back to "no auth".
var resolver = new FakeSecretResolver { Value = null };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void BlankSecretValue_FailsClosed()
{
// A row that decrypts to whitespace is a broken seed, not a key. Accepting it would make
// "Bearer " a valid credential for the whole cluster.
var resolver = new FakeSecretResolver { Value = " " };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void ResolverThrow_FailsClosed()
{
// The store is a local SQLite file with an env KEK; a missing KEK throws rather than
// returning null. Any failure is "no key", never an exception escaping into the gRPC gate.
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void ResolverThrow_IsNotCached_SoRecoveryNeedsNoInvalidate()
{
// Caching a negative would turn a transient store fault into a permanent outage for that
// cluster, recoverable only by a restart or a hook nothing calls yet.
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
resolver.Throw = null;
resolver.Value = "stored-psk";
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void ResolvedValue_IsCached_SoTheStoreIsHitOnce()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("site-a", out _).ShouldBeTrue();
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void Cache_ServesTheStaleValue_UntilInvalidated()
{
// Both directions matter: stale-until-invalidated is the whole reason Invalidate exists, and
// an invalidate that did not actually re-read would be indistinguishable from a no-op.
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var first).ShouldBeTrue();
first.ShouldBe("first-psk");
resolver.Value = "rotated-psk";
provider.TryGetPsk("SITE-A", out var stale).ShouldBeTrue();
stale.ShouldBe("first-psk");
provider.Invalidate("SITE-A");
provider.TryGetPsk("SITE-A", out var fresh).ShouldBeTrue();
fresh.ShouldBe("rotated-psk");
}
[Fact]
public void Invalidate_IsCaseInsensitive_AndScopedToOneCluster()
{
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
resolver.Value = "rotated-psk";
provider.Invalidate("site-a");
provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
a.ShouldBe("rotated-psk");
provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
b.ShouldBe("first-psk");
}
[Fact]
public void Invalidate_OnACacheMiss_IsANoOp()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
Should.NotThrow(() => provider.Invalidate("never-seen"));
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void InvalidateAll_DropsEveryCluster()
{
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
resolver.Value = "rotated-psk";
provider.InvalidateAll();
provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
a.ShouldBe("rotated-psk");
b.ShouldBe("rotated-psk");
}
[Theory]
[InlineData("")]
[InlineData(" ")]
public void BlankClusterId_FailsClosed(string clusterId)
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk(clusterId, out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ClusterIdThatCannotFormASecretName_FailsClosed()
{
// Cluster ids are operator-authored; SecretName's allow-list is [a-z0-9._/-]. A '$' throws
// out of the SecretName ctor, and that throw must become "no key", not a 500 in the gate.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE$A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact]
public async Task ConcurrentCallers_ResolveOnce_AndAllSeeTheSameValue()
{
// The gRPC gate calls this from every inbound ack, on arbitrary thread-pool threads. A
// non-atomic fill would stampede the store and could hand two callers different keys.
var gate = new SemaphoreSlim(0, 1);
var resolver = new FakeSecretResolver { Value = "stored-psk", Gate = gate };
var provider = CreateProvider(resolver);
var callers = Enumerable.Range(0, 16).Select(_ => Task.Run(() =>
{
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
return psk;
})).ToArray();
gate.Release();
var results = await Task.WhenAll(callers);
results.ShouldAllBe(r => r == "stored-psk");
resolver.CallCount.ShouldBe(1);
}
/// <summary>
/// An <see cref="ISecretResolver"/> whose answer the test controls between calls, so cache
/// staleness and invalidation are observable without a real store.
/// </summary>
private sealed class FakeSecretResolver : ISecretResolver
{
private int _callCount;
/// <summary>What <see cref="GetAsync"/> returns when <see cref="Throw"/> is null.</summary>
public string? Value { get; set; }
/// <summary>When set, <see cref="GetAsync"/> throws it instead of returning.</summary>
public Exception? Throw { get; set; }
/// <summary>When set, the first call waits on it — used to force a concurrent stampede.</summary>
public SemaphoreSlim? Gate { get; set; }
/// <summary>How many times the store was consulted.</summary>
public int CallCount => Volatile.Read(ref _callCount);
/// <summary>The name of the most recently requested secret.</summary>
public string? LastName { get; private set; }
public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
{
LastName = name.Value;
Interlocked.Increment(ref _callCount);
if (Gate is not null)
await Gate.WaitAsync(ct).ConfigureAwait(false);
if (Throw is not null)
throw Throw;
return Value;
}
}
}
tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs
+239
View File
@@ -0,0 +1,239 @@
using System.Diagnostics.CodeAnalysis;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
/// <summary>
/// ClusterClient→gRPC Phase 0 — the central-side ack endpoint's inbound gate. Mapping
/// <c>CentralCommandService</c> onto the pipeline is a Phase 1 task; this test covers only the
/// interceptor's own authorization behavior.
/// </summary>
/// <remarks>
/// Differs from <see cref="MeshCommandAuthInterceptor"/> in the one way central differs from a
/// node: there is no single expected key. The caller names its cluster in
/// <c>x-otopcua-cluster</c>, the bearer is verified against <i>that</i> cluster's PSK, and a
/// cluster with no resolvable key is denied — so a caller cannot pick a cluster to be graded
/// against, only claim one it must then prove.
/// </remarks>
public sealed class CentralCommandAuthInterceptorTests
{
private const string ReportApplyMethod = "/mesh_transport.v1.CentralCommandService/ReportApply";
private const string OtherMethod = "/deployment_artifact.v1.DeploymentArtifactService/Fetch";
private static CentralCommandAuthInterceptor CreateInterceptor(params (string Cluster, string Psk)[] keys)
=> new(new FakeMeshPskProvider(keys), NullLogger<CentralCommandAuthInterceptor>.Instance);
private static ServerCallContext CreateContext(
string method, string? authorizationHeader, string? clusterHeader)
{
var headers = new Metadata();
if (authorizationHeader is not null)
headers.Add("authorization", authorizationHeader);
if (clusterHeader is not null)
headers.Add("x-otopcua-cluster", clusterHeader);
return new FakeServerCallContext(method, headers);
}
/// <summary>Invokes the interceptor's unary path (how <c>ReportApply</c> actually runs).</summary>
private static Task Invoke(CentralCommandAuthInterceptor interceptor, ServerCallContext context)
=> interceptor.UnaryServerHandler<string, string>(
"request", context, (_, _) => Task.FromResult("ok"));
[Fact]
public async Task NonCentralCommandMethod_PassesThrough_EvenWithNoKeysAtAll()
{
// The interceptor shares the AddGrpc pipeline with config-serve (and, on a fused node, sync
// and telemetry), so it sees every call. It must be scoped strictly to its own service.
var interceptor = CreateInterceptor();
var context = CreateContext(OtherMethod, authorizationHeader: null, clusterHeader: null);
await Invoke(interceptor, context); // no throw
}
[Fact]
public async Task ReportApply_WithCorrectBearerForItsCluster_PassesThrough()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", "SITE-A");
await Invoke(interceptor, context); // no throw
}
[Fact]
public async Task ReportApply_WithNoClusterHeader_IsDenied()
{
// Without the cluster there is no key to grade against. Falling back to "try them all" would
// make one compromised site's key valid for forging every other site's acks.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", clusterHeader: null);
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithBlankClusterHeader_IsDenied()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-a", " ");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_ForAClusterWithNoResolvableKey_IsDenied()
{
// Fail-closed on the provider's "no key" — an unseeded or unreachable key must cost the ack,
// never authentication.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer anything-at-all", "SITE-Z");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithAnotherClustersBearer_IsDenied()
{
// The isolation proof: SITE-B's key does not authenticate a call claiming to be SITE-A.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"), ("SITE-B", "psk-site-b"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-b", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithNoBearerToken_IsDenied()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, authorizationHeader: null, clusterHeader: "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_WithMalformedAuthorizationHeader_IsDenied()
{
// No "Bearer " prefix at all — must be treated the same as a missing header.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "psk-site-a", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ReportApply_TokenComparison_IsNotAPrefixMatch()
{
// A prefix/StartsWith comparison would accept a truncated key and make the secret
// recoverable one character at a time.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer psk-site-", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() => Invoke(interceptor, context));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ServerStreamingPath_IsAlsoGated()
{
// ReportApply is unary today, but gating only that path would leave any future streaming
// method on the same service open. Gate every handler shape.
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() =>
interceptor.ServerStreamingServerHandler<string, string>(
"request", responseStream: null!, context, (_, _, _) => Task.CompletedTask));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task ClientStreamingPath_IsAlsoGated()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() =>
interceptor.ClientStreamingServerHandler<string, string>(
requestStream: null!, context, (_, _) => Task.FromResult("ok")));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public async Task DuplexStreamingPath_IsAlsoGated()
{
var interceptor = CreateInterceptor(("SITE-A", "psk-site-a"));
var context = CreateContext(ReportApplyMethod, "Bearer wrong", "SITE-A");
var ex = await Should.ThrowAsync<RpcException>(() =>
interceptor.DuplexStreamingServerHandler<string, string>(
requestStream: null!, responseStream: null!, context, (_, _, _) => Task.CompletedTask));
ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
}
[Fact]
public void HasExactlyOnePublicConstructor()
{
// Grpc.AspNetCore silently stops invoking an interceptor with more than one public ctor —
// this pins the shape so a well-meaning overload doesn't quietly disable auth in production.
var publicCtors = typeof(CentralCommandAuthInterceptor).GetConstructors();
publicCtors.Length.ShouldBe(1);
}
/// <summary>An <see cref="IMeshPskProvider"/> over a fixed key set; unknown clusters fail closed.</summary>
private sealed class FakeMeshPskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider
{
private readonly Dictionary<string, string> _keys =
keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase);
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
=> _keys.TryGetValue(clusterId, out psk);
public void Invalidate(string clusterId) { }
public void InvalidateAll() { }
}
/// <summary>
/// Minimal <see cref="ServerCallContext"/> carrying just a method name and request headers —
/// the only two things the interceptor reads. Hand-rolled because
/// <c>Grpc.Core.Testing.TestServerCallContext</c> ships only in the retired native package.
/// </summary>
private sealed class FakeServerCallContext(string method, Metadata requestHeaders)
: ServerCallContext
{
protected override string MethodCore => method;
protected override string HostCore => "localhost";
protected override string PeerCore => "ipv4:127.0.0.1:12345";
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
protected override Metadata RequestHeadersCore => requestHeaders;
protected override CancellationToken CancellationTokenCore => CancellationToken.None;
protected override Metadata ResponseTrailersCore { get; } = [];
protected override Status StatusCore { get; set; }
protected override WriteOptions? WriteOptionsCore { get; set; }
protected override AuthContext AuthContextCore { get; } =
new(null, new Dictionary<string, List<AuthProperty>>());
protected override ContextPropagationToken CreatePropagationTokenCore(
ContextPropagationOptions? options)
=> throw new NotSupportedException();
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders)
=> Task.CompletedTask;
}
}