using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; using ZB.MOM.WW.Secrets.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; /// /// ClusterClient→gRPC Phase 0 — central's per-cluster PSK lookup. Central talks to N site /// clusters, each with its own key, so every outbound dial and every inbound ack verification /// goes through here. /// /// /// The tests that matter are the adversarial ones: the store being down, the key being absent /// or blank, and the cache handing back a value the store no longer agrees with. Every one of /// those must end in "no key" — the caller then denies or skips, never proceeds unauthenticated. /// public sealed class MeshPskProviderTests { private static MeshPskProvider CreateProvider( ISecretResolver resolver, Dictionary? clusterPsks = null, ILogger? logger = null) { var options = new MeshTransportOptions(); if (clusterPsks is not null) { foreach (var (key, value) in clusterPsks) options.ClusterPsks[key] = value; } return new MeshPskProvider( Options.Create(options), resolver, logger ?? NullLogger.Instance); } [Fact] public void ConfigMapEntry_IsReturned_WithoutConsultingTheSecretStore() { var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "rig-psk-site-a" }); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("rig-psk-site-a"); resolver.CallCount.ShouldBe(0); } [Fact] public void ConfigMapEntry_WinsOverTheSecretStore() { // The map exists so a rig can supply literals without standing up a KEK. If the store also // holds a key for that cluster the map must still win, or a stale store row would silently // beat the value the operator just set. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "rig-psk-site-a" }); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("rig-psk-site-a"); resolver.CallCount.ShouldBe(0); } [Fact] public void ConfigMapLookup_IsCaseInsensitive() { // MeshTransport__ClusterPsks__site-a and __SITE-A are the same cluster. The map's comparer // (set on MeshTransportOptions) is what makes that true; this pins the pairing. var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "rig-psk-site-a" }); provider.TryGetPsk("site-a", out var psk).ShouldBeTrue(); psk.ShouldBe("rig-psk-site-a"); } [Fact] public void BlankConfigMapEntry_FallsThroughToTheSecretStore() { // A blank map entry is an empty environment variable, not a decision. Treat it as absent // rather than as an empty key that an empty bearer would satisfy. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = " " }); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); resolver.CallCount.ShouldBe(1); } [Fact] public void SecretStoreValue_IsReturned_UnderTheConventionalName() { var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); // SecretName normalizes to lowercase — the same normalization ${secret:OT-MESH-PSK-SITE-A} // goes through in the pre-host expander, so both paths address one row. resolver.LastName.ShouldBe("ot-mesh-psk-site-a"); } [Fact] public void UnknownCluster_FailsClosed() { // Absent secret + no map entry. The caller denies; it must never fall back to "no auth". var resolver = new FakeSecretResolver { Value = null }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse(); psk.ShouldBeNull(); } [Fact] public void BlankSecretValue_FailsClosed() { // A row that decrypts to whitespace is a broken seed, not a key. Accepting it would make // "Bearer " a valid credential for the whole cluster. var resolver = new FakeSecretResolver { Value = " " }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse(); psk.ShouldBeNull(); } [Fact] public void ResolverThrow_FailsClosed() { // The store is a local SQLite file with an env KEK; a missing KEK throws rather than // returning null. Any failure is "no key", never an exception escaping into the gRPC gate. var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse(); psk.ShouldBeNull(); } [Fact] public void ResolverThrow_IsNotCached_SoRecoveryNeedsNoInvalidate() { // Caching a negative would turn a transient store fault into a permanent outage for that // cluster, recoverable only by a restart or a hook nothing calls yet. var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); resolver.Throw = null; resolver.Value = "stored-psk"; provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); } [Fact] public void ResolvedValue_IsCached_SoTheStoreIsHitOnce() { var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out _).ShouldBeTrue(); provider.TryGetPsk("SITE-A", out _).ShouldBeTrue(); provider.TryGetPsk("site-a", out _).ShouldBeTrue(); resolver.CallCount.ShouldBe(1); } [Fact] public void Cache_ServesTheStaleValue_UntilInvalidated() { // Both directions matter: stale-until-invalidated is the whole reason Invalidate exists, and // an invalidate that did not actually re-read would be indistinguishable from a no-op. var resolver = new FakeSecretResolver { Value = "first-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out var first).ShouldBeTrue(); first.ShouldBe("first-psk"); resolver.Value = "rotated-psk"; provider.TryGetPsk("SITE-A", out var stale).ShouldBeTrue(); stale.ShouldBe("first-psk"); provider.Invalidate("SITE-A"); provider.TryGetPsk("SITE-A", out var fresh).ShouldBeTrue(); fresh.ShouldBe("rotated-psk"); } [Fact] public void Invalidate_IsCaseInsensitive_AndScopedToOneCluster() { var resolver = new FakeSecretResolver { Value = "first-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out _).ShouldBeTrue(); provider.TryGetPsk("SITE-B", out _).ShouldBeTrue(); resolver.Value = "rotated-psk"; provider.Invalidate("site-a"); provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue(); a.ShouldBe("rotated-psk"); provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue(); b.ShouldBe("first-psk"); } [Fact] public void Invalidate_OnACacheMiss_IsANoOp() { var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); Should.NotThrow(() => provider.Invalidate("never-seen")); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); } [Fact] public void InvalidateAll_DropsEveryCluster() { var resolver = new FakeSecretResolver { Value = "first-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-A", out _).ShouldBeTrue(); provider.TryGetPsk("SITE-B", out _).ShouldBeTrue(); resolver.Value = "rotated-psk"; provider.InvalidateAll(); provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue(); provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue(); a.ShouldBe("rotated-psk"); b.ShouldBe("rotated-psk"); } [Theory] [InlineData("")] [InlineData(" ")] public void BlankClusterId_FailsClosed(string clusterId) { var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk(clusterId, out var psk).ShouldBeFalse(); psk.ShouldBeNull(); resolver.CallCount.ShouldBe(0); } [Fact] public void ClusterIdThatCannotFormASecretName_FailsClosed() { // Cluster ids are operator-authored; SecretName's allow-list is [a-z0-9._/-]. A '$' throws // out of the SecretName ctor, and that throw must become "no key", not a 500 in the gate. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE$A", out var psk).ShouldBeFalse(); psk.ShouldBeNull(); resolver.CallCount.ShouldBe(0); } [Fact(Timeout = 30_000)] public async Task ConcurrentCallers_ResolveOnce_AndAllSeeTheSameValue() { // The gRPC gate calls this from every inbound ack, on arbitrary thread-pool threads. A // non-atomic fill would stampede the store and could hand two callers different keys. // // The shape matters more than the assertion. Callers park BEFORE TryGetPsk and are released // together, so the race is real rather than accidentally serialized by task scheduling — // parking them INSIDE the resolver could never work, because the whole point of the Lazy is // that only one caller ever gets in there. And the resolver's delay RETURNS rather than // waiting on a gate: a regression that lets a second caller reach the store must drive // CallCount to 2 (red), never sit on an exhausted semaphore (hang, which reads as a flake). const int Callers = 16; using var arrived = new CountdownEvent(Callers); using var start = new ManualResetEventSlim(false); var resolver = new FakeSecretResolver { Value = "stored-psk", ResolveDelay = TimeSpan.FromMilliseconds(50) }; var provider = CreateProvider(resolver); // LongRunning gives each caller a dedicated thread: reaching the arrival gate must not // depend on thread-pool injection, which is rate-limited and made this flake on a // contended box (the gate timed out before all 16 pool threads existed). var callers = Enumerable.Range(0, Callers).Select(_ => Task.Factory.StartNew(() => { arrived.Signal(); start.Wait(); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); return psk; }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)).ToArray(); arrived.Wait(TimeSpan.FromSeconds(10)).ShouldBeTrue("all callers should reach the start gate"); start.Set(); var results = await Task.WhenAll(callers); results.ShouldAllBe(r => r == "stored-psk"); resolver.CallCount.ShouldBe(1); } [Fact] public void APoisonedCacheEntry_IsEvicted_RatherThanDenyingTheClusterForever() { // ResolveFromStore swallows what the store throws, but it LOGS while doing so — and a logger // provider that throws escapes the factory. Lazy under ExecutionAndPublication caches that // exception and rethrows it on every later read, so an unguarded entry.Value would deny this // cluster for the life of the process; nothing calls Invalidate until Phase 1A. var logger = new ThrowingLogger { ThrowOnLog = true }; var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") }; var provider = CreateProvider(resolver, logger: logger); // The logger throwing must not escape the gate either — a denial, not a 500. provider.TryGetPsk("SITE-A", out var first).ShouldBeFalse(); first.ShouldBeNull(); resolver.CallCount.ShouldBe(1); logger.ThrowOnLog = false; resolver.Throw = null; resolver.Value = "stored-psk"; provider.TryGetPsk("SITE-A", out var second).ShouldBeTrue(); second.ShouldBe("stored-psk"); resolver.CallCount.ShouldBe(2); // reached the store again — the entry was not poisoned } [Fact] public void ConfigMapLookup_StaysCaseInsensitive_ThroughRealConfigurationBinding() { // Writing into the options object pins only the field initializer's comparer. What reaches // production is a BOUND instance, and the binder is free to construct its own Dictionary and // drop the comparer silently — the symptom being a rig lookup that misses on a ClusterId // spelled the way the DB spells it. Bind for real, then go through the PROVIDER. var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["MeshTransport:ClusterPsks:SITE-A"] = "rig-psk-site-a", }) .Build(); var options = new MeshTransportOptions(); configuration.GetSection(MeshTransportOptions.SectionName).Bind(options); var resolver = new FakeSecretResolver(); var provider = new MeshPskProvider( Options.Create(options), resolver, NullLogger.Instance); provider.TryGetPsk("site-a", out var psk).ShouldBeTrue(); psk.ShouldBe("rig-psk-site-a"); resolver.CallCount.ShouldBe(0); } // ------------------------------------------------------------------ known-cluster gate // // Phase 1A security register item 1. Central rebuilds its cluster set from ClusterNode rows on // every contact refresh; a cluster id outside that set must be refused in O(1) BEFORE anything // touches the secret store. Two things ride on that: an unauthenticated caller can no longer // make central perform a store read (and an audit write) per forged x-otopcua-cluster header, // and the timing difference between a known cluster (cached) and an unknown one (never cached, // because negatives are deliberately not retained) stops being remotely measurable. [Fact] public void UnknownCluster_WithTheGatePopulated_IsRefused_WithoutTouchingTheStore() { // THE assertion of the register item. CallCount == 0 is the whole point: the refusal must // cost nothing, or the gate is a comment rather than a control. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(["SITE-A", "SITE-B"]); provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse(); psk.ShouldBeNull(); resolver.CallCount.ShouldBe(0); } [Fact] public void KnownCluster_WithTheGatePopulated_StillResolves() { var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(["SITE-A"]); provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); resolver.CallCount.ShouldBe(1); } [Fact] public void TheKnownClusterSet_IsCaseInsensitive() { // ClusterId case is whatever the operator typed into the DB row; the header is whatever the // node was configured with. A case-sensitive gate would refuse a legitimate cluster. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(["SITE-A"]); provider.TryGetPsk("site-a", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); } [Fact] public void SwappingTheKnownClusterSet_AddsAndRemovesClusters() { // The set is rebuilt from the DB every refresh, so both directions happen in production: a // new site appears, a retired one stops being accepted. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(["SITE-A"]); provider.TryGetPsk("SITE-A", out _).ShouldBeTrue(); provider.TryGetPsk("SITE-B", out _).ShouldBeFalse(); provider.SetKnownClusters(["SITE-B"]); provider.TryGetPsk("SITE-B", out _).ShouldBeTrue(); provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); } [Fact] public void AnEmptyKnownClusterSet_LeavesTheGateInactive() { // Fail-OPEN, deliberately and narrowly: central publishes the set from its contact refresh, // and an empty publish means "the DB told me nothing", not "no cluster is legitimate". // Refusing everything there would make a fleet-wide ack outage the failure mode of an empty // or unreadable ClusterNode table. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(["SITE-A"]); provider.SetKnownClusters([]); provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); } [Fact] public void AnUnsetKnownClusterSet_LeavesTheGateInactive() { // Admin boot: the interceptor is live from the first request, but the first contact refresh // has not run yet. Gating on a set nobody has published would deny every ack in that window. var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue(); psk.ShouldBe("stored-psk"); } [Fact] public void TheGate_IsConsultedBeforeTheConfigMap() { // Consequence worth pinning rather than discovering: a ClusterPsks entry for a cluster with // no enabled ClusterNode row is refused once the gate is populated. That is correct — central // would neither dial such a cluster nor accept an ack from it — but it is surprising enough // on a rig to deserve a test that names it. var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-X"] = "rig-psk-site-x" }); provider.SetKnownClusters(["SITE-A"]); provider.TryGetPsk("SITE-X", out var psk).ShouldBeFalse(); psk.ShouldBeNull(); } [Fact] public void SetKnownClusters_TakesACopy_SoALaterMutationOfTheCallersList_CannotWidenTheGate() { var clusters = new List { "SITE-A" }; var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(clusters); clusters.Add("SITE-Z"); provider.TryGetPsk("SITE-Z", out _).ShouldBeFalse(); } // ------------------------------------------------------------------ Matches [Fact] public void Matches_AcceptsTheClustersOwnKey() { var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "psk-site-a" }); provider.Matches("SITE-A", "psk-site-a").ShouldBeTrue(); } [Fact] public void Matches_RejectsAnotherClustersKey() { // The isolation property, asserted at the provider rather than only at the interceptor: // choosing a cluster only changes which key you are graded against. var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "psk-site-a", ["SITE-B"] = "psk-site-b" }); provider.Matches("SITE-A", "psk-site-b").ShouldBeFalse(); } [Fact] public void Matches_IsNotAPrefixMatch() { var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "psk-site-a" }); provider.Matches("SITE-A", "psk-site-").ShouldBeFalse(); provider.Matches("SITE-A", "psk-site-a-and-more").ShouldBeFalse(); } [Fact] public void Matches_FailsClosed_WhenTheClusterHasNoKey() { var resolver = new FakeSecretResolver { Value = null }; var provider = CreateProvider(resolver); provider.Matches("SITE-Z", "anything-at-all").ShouldBeFalse(); } [Theory] [InlineData("")] [InlineData(" ")] public void Matches_RejectsABlankPresentedToken(string presented) { // A blank key never reaches the cache (blank resolves fail closed), but a blank PRESENTED // token must be refused on its own terms rather than relying on that. var resolver = new FakeSecretResolver(); var provider = CreateProvider( resolver, new Dictionary { ["SITE-A"] = "psk-site-a" }); provider.Matches("SITE-A", presented).ShouldBeFalse(); } [Fact] public void Matches_IsRefusedByTheKnownClusterGate_WithoutTouchingTheStore() { var resolver = new FakeSecretResolver { Value = "stored-psk" }; var provider = CreateProvider(resolver); provider.SetKnownClusters(["SITE-A"]); provider.Matches("SITE-Z", "stored-psk").ShouldBeFalse(); resolver.CallCount.ShouldBe(0); } /// /// An whose answer the test controls between calls, so cache /// staleness and invalidation are observable without a real store. /// private sealed class FakeSecretResolver : ISecretResolver { private int _callCount; /// What returns when is null. public string? Value { get; set; } /// When set, throws it instead of returning. public Exception? Throw { get; set; } /// /// How long each call takes before answering. Widens the window a concurrent stampede /// would have to exploit — and it RETURNS, so a second entrant shows up as a second call /// rather than as a hang. /// public TimeSpan ResolveDelay { get; set; } = TimeSpan.Zero; /// How many times the store was consulted. public int CallCount => Volatile.Read(ref _callCount); /// The name of the most recently requested secret. public string? LastName { get; private set; } public async Task GetAsync(SecretName name, CancellationToken ct) { LastName = name.Value; Interlocked.Increment(ref _callCount); if (ResolveDelay > TimeSpan.Zero) await Task.Delay(ResolveDelay, ct).ConfigureAwait(false); if (Throw is not null) throw Throw; return Value; } } /// /// An that can be made to throw, standing in for a logger provider /// that faults — the only way the provider's own catch blocks can throw. /// private sealed class ThrowingLogger : ILogger { /// When true, every call throws. public bool ThrowOnLog { get; set; } public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { if (ThrowOnLog) throw new InvalidOperationException("logger provider faulted"); } } }