using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
///
/// ClusterClient→gRPC Phase 0 — central's per-cluster PSK lookup. Central talks to N site
/// clusters, each with its own key, so every outbound dial and every inbound ack verification
/// goes through here.
///
///
/// The tests that matter are the adversarial ones: the store being down, the key being absent
/// or blank, and the cache handing back a value the store no longer agrees with. Every one of
/// those must end in "no key" — the caller then denies or skips, never proceeds unauthenticated.
///
public sealed class MeshPskProviderTests
{
private static MeshPskProvider CreateProvider(
ISecretResolver resolver,
Dictionary? clusterPsks = null,
ILogger? logger = null)
{
var options = new MeshTransportOptions();
if (clusterPsks is not null)
{
foreach (var (key, value) in clusterPsks)
options.ClusterPsks[key] = value;
}
return new MeshPskProvider(
Options.Create(options), resolver, logger ?? NullLogger.Instance);
}
[Fact]
public void ConfigMapEntry_IsReturned_WithoutConsultingTheSecretStore()
{
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ConfigMapEntry_WinsOverTheSecretStore()
{
// The map exists so a rig can supply literals without standing up a KEK. If the store also
// holds a key for that cluster the map must still win, or a stale store row would silently
// beat the value the operator just set.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ConfigMapLookup_IsCaseInsensitive()
{
// MeshTransport__ClusterPsks__site-a and __SITE-A are the same cluster. The map's comparer
// (set on MeshTransportOptions) is what makes that true; this pins the pairing.
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = "rig-psk-site-a" });
provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
}
[Fact]
public void BlankConfigMapEntry_FallsThroughToTheSecretStore()
{
// A blank map entry is an empty environment variable, not a decision. Treat it as absent
// rather than as an empty key that an empty bearer would satisfy.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = " " });
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void SecretStoreValue_IsReturned_UnderTheConventionalName()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
// SecretName normalizes to lowercase — the same normalization ${secret:OT-MESH-PSK-SITE-A}
// goes through in the pre-host expander, so both paths address one row.
resolver.LastName.ShouldBe("ot-mesh-psk-site-a");
}
[Fact]
public void UnknownCluster_FailsClosed()
{
// Absent secret + no map entry. The caller denies; it must never fall back to "no auth".
var resolver = new FakeSecretResolver { Value = null };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void BlankSecretValue_FailsClosed()
{
// A row that decrypts to whitespace is a broken seed, not a key. Accepting it would make
// "Bearer " a valid credential for the whole cluster.
var resolver = new FakeSecretResolver { Value = " " };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void ResolverThrow_FailsClosed()
{
// The store is a local SQLite file with an env KEK; a missing KEK throws rather than
// returning null. Any failure is "no key", never an exception escaping into the gRPC gate.
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void ResolverThrow_IsNotCached_SoRecoveryNeedsNoInvalidate()
{
// Caching a negative would turn a transient store fault into a permanent outage for that
// cluster, recoverable only by a restart or a hook nothing calls yet.
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
resolver.Throw = null;
resolver.Value = "stored-psk";
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void ResolvedValue_IsCached_SoTheStoreIsHitOnce()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("site-a", out _).ShouldBeTrue();
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void Cache_ServesTheStaleValue_UntilInvalidated()
{
// Both directions matter: stale-until-invalidated is the whole reason Invalidate exists, and
// an invalidate that did not actually re-read would be indistinguishable from a no-op.
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out var first).ShouldBeTrue();
first.ShouldBe("first-psk");
resolver.Value = "rotated-psk";
provider.TryGetPsk("SITE-A", out var stale).ShouldBeTrue();
stale.ShouldBe("first-psk");
provider.Invalidate("SITE-A");
provider.TryGetPsk("SITE-A", out var fresh).ShouldBeTrue();
fresh.ShouldBe("rotated-psk");
}
[Fact]
public void Invalidate_IsCaseInsensitive_AndScopedToOneCluster()
{
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
resolver.Value = "rotated-psk";
provider.Invalidate("site-a");
provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
a.ShouldBe("rotated-psk");
provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
b.ShouldBe("first-psk");
}
[Fact]
public void Invalidate_OnACacheMiss_IsANoOp()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
Should.NotThrow(() => provider.Invalidate("never-seen"));
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void InvalidateAll_DropsEveryCluster()
{
var resolver = new FakeSecretResolver { Value = "first-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
resolver.Value = "rotated-psk";
provider.InvalidateAll();
provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
a.ShouldBe("rotated-psk");
b.ShouldBe("rotated-psk");
}
[Theory]
[InlineData("")]
[InlineData(" ")]
public void BlankClusterId_FailsClosed(string clusterId)
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk(clusterId, out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void ClusterIdThatCannotFormASecretName_FailsClosed()
{
// Cluster ids are operator-authored; SecretName's allow-list is [a-z0-9._/-]. A '$' throws
// out of the SecretName ctor, and that throw must become "no key", not a 500 in the gate.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE$A", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact(Timeout = 30_000)]
public async Task ConcurrentCallers_ResolveOnce_AndAllSeeTheSameValue()
{
// The gRPC gate calls this from every inbound ack, on arbitrary thread-pool threads. A
// non-atomic fill would stampede the store and could hand two callers different keys.
//
// The shape matters more than the assertion. Callers park BEFORE TryGetPsk and are released
// together, so the race is real rather than accidentally serialized by task scheduling —
// parking them INSIDE the resolver could never work, because the whole point of the Lazy is
// that only one caller ever gets in there. And the resolver's delay RETURNS rather than
// waiting on a gate: a regression that lets a second caller reach the store must drive
// CallCount to 2 (red), never sit on an exhausted semaphore (hang, which reads as a flake).
const int Callers = 16;
using var arrived = new CountdownEvent(Callers);
using var start = new ManualResetEventSlim(false);
var resolver = new FakeSecretResolver { Value = "stored-psk", ResolveDelay = TimeSpan.FromMilliseconds(50) };
var provider = CreateProvider(resolver);
// LongRunning gives each caller a dedicated thread: reaching the arrival gate must not
// depend on thread-pool injection, which is rate-limited and made this flake on a
// contended box (the gate timed out before all 16 pool threads existed).
var callers = Enumerable.Range(0, Callers).Select(_ => Task.Factory.StartNew(() =>
{
arrived.Signal();
start.Wait();
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
return psk;
}, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)).ToArray();
arrived.Wait(TimeSpan.FromSeconds(10)).ShouldBeTrue("all callers should reach the start gate");
start.Set();
var results = await Task.WhenAll(callers);
results.ShouldAllBe(r => r == "stored-psk");
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void APoisonedCacheEntry_IsEvicted_RatherThanDenyingTheClusterForever()
{
// ResolveFromStore swallows what the store throws, but it LOGS while doing so — and a logger
// provider that throws escapes the factory. Lazy under ExecutionAndPublication caches that
// exception and rethrows it on every later read, so an unguarded entry.Value would deny this
// cluster for the life of the process; nothing calls Invalidate until Phase 1A.
var logger = new ThrowingLogger { ThrowOnLog = true };
var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
var provider = CreateProvider(resolver, logger: logger);
// The logger throwing must not escape the gate either — a denial, not a 500.
provider.TryGetPsk("SITE-A", out var first).ShouldBeFalse();
first.ShouldBeNull();
resolver.CallCount.ShouldBe(1);
logger.ThrowOnLog = false;
resolver.Throw = null;
resolver.Value = "stored-psk";
provider.TryGetPsk("SITE-A", out var second).ShouldBeTrue();
second.ShouldBe("stored-psk");
resolver.CallCount.ShouldBe(2); // reached the store again — the entry was not poisoned
}
[Fact]
public void ConfigMapLookup_StaysCaseInsensitive_ThroughRealConfigurationBinding()
{
// Writing into the options object pins only the field initializer's comparer. What reaches
// production is a BOUND instance, and the binder is free to construct its own Dictionary and
// drop the comparer silently — the symptom being a rig lookup that misses on a ClusterId
// spelled the way the DB spells it. Bind for real, then go through the PROVIDER.
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary
{
["MeshTransport:ClusterPsks:SITE-A"] = "rig-psk-site-a",
})
.Build();
var options = new MeshTransportOptions();
configuration.GetSection(MeshTransportOptions.SectionName).Bind(options);
var resolver = new FakeSecretResolver();
var provider = new MeshPskProvider(
Options.Create(options), resolver, NullLogger.Instance);
provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();
psk.ShouldBe("rig-psk-site-a");
resolver.CallCount.ShouldBe(0);
}
// ------------------------------------------------------------------ known-cluster gate
//
// Phase 1A security register item 1. Central rebuilds its cluster set from ClusterNode rows on
// every contact refresh; a cluster id outside that set must be refused in O(1) BEFORE anything
// touches the secret store. Two things ride on that: an unauthenticated caller can no longer
// make central perform a store read (and an audit write) per forged x-otopcua-cluster header,
// and the timing difference between a known cluster (cached) and an unknown one (never cached,
// because negatives are deliberately not retained) stops being remotely measurable.
[Fact]
public void UnknownCluster_WithTheGatePopulated_IsRefused_WithoutTouchingTheStore()
{
// THE assertion of the register item. CallCount == 0 is the whole point: the refusal must
// cost nothing, or the gate is a comment rather than a control.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(["SITE-A", "SITE-B"]);
provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
resolver.CallCount.ShouldBe(0);
}
[Fact]
public void KnownCluster_WithTheGatePopulated_StillResolves()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(["SITE-A"]);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
resolver.CallCount.ShouldBe(1);
}
[Fact]
public void TheKnownClusterSet_IsCaseInsensitive()
{
// ClusterId case is whatever the operator typed into the DB row; the header is whatever the
// node was configured with. A case-sensitive gate would refuse a legitimate cluster.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(["SITE-A"]);
provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void SwappingTheKnownClusterSet_AddsAndRemovesClusters()
{
// The set is rebuilt from the DB every refresh, so both directions happen in production: a
// new site appears, a retired one stops being accepted.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(["SITE-A"]);
provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-B", out _).ShouldBeFalse();
provider.SetKnownClusters(["SITE-B"]);
provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
}
[Fact]
public void AnEmptyKnownClusterSet_LeavesTheGateInactive()
{
// Fail-OPEN, deliberately and narrowly: central publishes the set from its contact refresh,
// and an empty publish means "the DB told me nothing", not "no cluster is legitimate".
// Refusing everything there would make a fleet-wide ack outage the failure mode of an empty
// or unreadable ClusterNode table.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(["SITE-A"]);
provider.SetKnownClusters([]);
provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void AnUnsetKnownClusterSet_LeavesTheGateInactive()
{
// Admin boot: the interceptor is live from the first request, but the first contact refresh
// has not run yet. Gating on a set nobody has published would deny every ack in that window.
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue();
psk.ShouldBe("stored-psk");
}
[Fact]
public void TheGate_IsConsultedBeforeTheConfigMap()
{
// Consequence worth pinning rather than discovering: a ClusterPsks entry for a cluster with
// no enabled ClusterNode row is refused once the gate is populated. That is correct — central
// would neither dial such a cluster nor accept an ack from it — but it is surprising enough
// on a rig to deserve a test that names it.
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-X"] = "rig-psk-site-x" });
provider.SetKnownClusters(["SITE-A"]);
provider.TryGetPsk("SITE-X", out var psk).ShouldBeFalse();
psk.ShouldBeNull();
}
[Fact]
public void SetKnownClusters_TakesACopy_SoALaterMutationOfTheCallersList_CannotWidenTheGate()
{
var clusters = new List { "SITE-A" };
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(clusters);
clusters.Add("SITE-Z");
provider.TryGetPsk("SITE-Z", out _).ShouldBeFalse();
}
// ------------------------------------------------------------------ Matches
[Fact]
public void Matches_AcceptsTheClustersOwnKey()
{
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = "psk-site-a" });
provider.Matches("SITE-A", "psk-site-a").ShouldBeTrue();
}
[Fact]
public void Matches_RejectsAnotherClustersKey()
{
// The isolation property, asserted at the provider rather than only at the interceptor:
// choosing a cluster only changes which key you are graded against.
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver,
new Dictionary { ["SITE-A"] = "psk-site-a", ["SITE-B"] = "psk-site-b" });
provider.Matches("SITE-A", "psk-site-b").ShouldBeFalse();
}
[Fact]
public void Matches_IsNotAPrefixMatch()
{
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = "psk-site-a" });
provider.Matches("SITE-A", "psk-site-").ShouldBeFalse();
provider.Matches("SITE-A", "psk-site-a-and-more").ShouldBeFalse();
}
[Fact]
public void Matches_FailsClosed_WhenTheClusterHasNoKey()
{
var resolver = new FakeSecretResolver { Value = null };
var provider = CreateProvider(resolver);
provider.Matches("SITE-Z", "anything-at-all").ShouldBeFalse();
}
[Theory]
[InlineData("")]
[InlineData(" ")]
public void Matches_RejectsABlankPresentedToken(string presented)
{
// A blank key never reaches the cache (blank resolves fail closed), but a blank PRESENTED
// token must be refused on its own terms rather than relying on that.
var resolver = new FakeSecretResolver();
var provider = CreateProvider(
resolver, new Dictionary { ["SITE-A"] = "psk-site-a" });
provider.Matches("SITE-A", presented).ShouldBeFalse();
}
[Fact]
public void Matches_IsRefusedByTheKnownClusterGate_WithoutTouchingTheStore()
{
var resolver = new FakeSecretResolver { Value = "stored-psk" };
var provider = CreateProvider(resolver);
provider.SetKnownClusters(["SITE-A"]);
provider.Matches("SITE-Z", "stored-psk").ShouldBeFalse();
resolver.CallCount.ShouldBe(0);
}
///
/// An whose answer the test controls between calls, so cache
/// staleness and invalidation are observable without a real store.
///
private sealed class FakeSecretResolver : ISecretResolver
{
private int _callCount;
/// What returns when is null.
public string? Value { get; set; }
/// When set, throws it instead of returning.
public Exception? Throw { get; set; }
///
/// How long each call takes before answering. Widens the window a concurrent stampede
/// would have to exploit — and it RETURNS, so a second entrant shows up as a second call
/// rather than as a hang.
///
public TimeSpan ResolveDelay { get; set; } = TimeSpan.Zero;
/// How many times the store was consulted.
public int CallCount => Volatile.Read(ref _callCount);
/// The name of the most recently requested secret.
public string? LastName { get; private set; }
public async Task GetAsync(SecretName name, CancellationToken ct)
{
LastName = name.Value;
Interlocked.Increment(ref _callCount);
if (ResolveDelay > TimeSpan.Zero)
await Task.Delay(ResolveDelay, ct).ConfigureAwait(false);
if (Throw is not null)
throw Throw;
return Value;
}
}
///
/// An that can be made to throw, standing in for a logger provider
/// that faults — the only way the provider's own catch blocks can throw.
///
private sealed class ThrowingLogger : ILogger
{
/// When true, every call throws.
public bool ThrowOnLog { get; set; }
public IDisposable? BeginScope(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func formatter)
{
if (ThrowOnLog)
throw new InvalidOperationException("logger provider faulted");
}
}
}