lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs

using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.Secrets.Abstractions;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;

/// <summary>
///     mesh gRPC command plane, Phase 0 — central's per-cluster PSK lookup. Central talks to N site
///     clusters, each with its own key, so every outbound dial and every inbound ack verification
///     goes through here.
/// </summary>
/// <remarks>
///     The tests that matter are the adversarial ones: the store being down, the key being absent
///     or blank, and the cache handing back a value the store no longer agrees with. Every one of
///     those must end in "no key" — the caller then denies or skips, never proceeds unauthenticated.
/// </remarks>
public sealed class MeshPskProviderTests
{
    private static MeshPskProvider CreateProvider(
        ISecretResolver resolver,
        Dictionary<string, string>? clusterPsks = null,
        ILogger<MeshPskProvider>? logger = null)
    {
        var options = new MeshTransportOptions();
        if (clusterPsks is not null)
        {
            foreach (var (key, value) in clusterPsks)
                options.ClusterPsks[key] = value;
        }

        return new MeshPskProvider(
            Options.Create(options), resolver, logger ?? NullLogger<MeshPskProvider>.Instance);
    }

    [Fact]
    public void ConfigMapEntry_IsReturned_WithoutConsultingTheSecretStore()
    {
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();

        psk.ShouldBe("rig-psk-site-a");
        resolver.CallCount.ShouldBe(0);
    }

    [Fact]
    public void ConfigMapEntry_WinsOverTheSecretStore()
    {
        // The map exists so a rig can supply literals without standing up a KEK. If the store also
        // holds a key for that cluster the map must still win, or a stale store row would silently
        // beat the value the operator just set.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();

        psk.ShouldBe("rig-psk-site-a");
        resolver.CallCount.ShouldBe(0);
    }

    [Fact]
    public void ConfigMapLookup_IsCaseInsensitive()
    {
        // MeshTransport__ClusterPsks__site-a and __SITE-A are the same cluster. The map's comparer
        // (set on MeshTransportOptions) is what makes that true; this pins the pairing.
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "rig-psk-site-a" });

        provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();

        psk.ShouldBe("rig-psk-site-a");
    }

    [Fact]
    public void BlankConfigMapEntry_FallsThroughToTheSecretStore()
    {
        // A blank map entry is an empty environment variable, not a decision. Treat it as absent
        // rather than as an empty key that an empty bearer would satisfy.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "   " });

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();

        psk.ShouldBe("stored-psk");
        resolver.CallCount.ShouldBe(1);
    }

    [Fact]
    public void SecretStoreValue_IsReturned_UnderTheConventionalName()
    {
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();

        psk.ShouldBe("stored-psk");
        // SecretName normalizes to lowercase — the same normalization ${secret:OT-MESH-PSK-SITE-A}
        // goes through in the pre-host expander, so both paths address one row.
        resolver.LastName.ShouldBe("ot-mesh-psk-site-a");
    }

    [Fact]
    public void UnknownCluster_FailsClosed()
    {
        // Absent secret + no map entry. The caller denies; it must never fall back to "no auth".
        var resolver = new FakeSecretResolver { Value = null };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
    }

    [Fact]
    public void BlankSecretValue_FailsClosed()
    {
        // A row that decrypts to whitespace is a broken seed, not a key. Accepting it would make
        // "Bearer    " a valid credential for the whole cluster.
        var resolver = new FakeSecretResolver { Value = "   " };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
    }

    [Fact]
    public void ResolverThrow_FailsClosed()
    {
        // The store is a local SQLite file with an env KEK; a missing KEK throws rather than
        // returning null. Any failure is "no key", never an exception escaping into the gRPC gate.
        var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
    }

    [Fact]
    public void ResolverThrow_IsNotCached_SoRecoveryNeedsNoInvalidate()
    {
        // Caching a negative would turn a transient store fault into a permanent outage for that
        // cluster, recoverable only by a restart or a hook nothing calls yet.
        var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();

        resolver.Throw = null;
        resolver.Value = "stored-psk";

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
        psk.ShouldBe("stored-psk");
    }

    [Fact]
    public void ResolvedValue_IsCached_SoTheStoreIsHitOnce()
    {
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
        provider.TryGetPsk("site-a", out _).ShouldBeTrue();

        resolver.CallCount.ShouldBe(1);
    }

    [Fact]
    public void Cache_ServesTheStaleValue_UntilInvalidated()
    {
        // Both directions matter: stale-until-invalidated is the whole reason Invalidate exists, and
        // an invalidate that did not actually re-read would be indistinguishable from a no-op.
        var resolver = new FakeSecretResolver { Value = "first-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out var first).ShouldBeTrue();
        first.ShouldBe("first-psk");

        resolver.Value = "rotated-psk";

        provider.TryGetPsk("SITE-A", out var stale).ShouldBeTrue();
        stale.ShouldBe("first-psk");

        provider.Invalidate("SITE-A");

        provider.TryGetPsk("SITE-A", out var fresh).ShouldBeTrue();
        fresh.ShouldBe("rotated-psk");
    }

    [Fact]
    public void Invalidate_IsCaseInsensitive_AndScopedToOneCluster()
    {
        var resolver = new FakeSecretResolver { Value = "first-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
        provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();

        resolver.Value = "rotated-psk";
        provider.Invalidate("site-a");

        provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
        a.ShouldBe("rotated-psk");

        provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();
        b.ShouldBe("first-psk");
    }

    [Fact]
    public void Invalidate_OnACacheMiss_IsANoOp()
    {
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        Should.NotThrow(() => provider.Invalidate("never-seen"));

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
        psk.ShouldBe("stored-psk");
    }

    [Fact]
    public void InvalidateAll_DropsEveryCluster()
    {
        var resolver = new FakeSecretResolver { Value = "first-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
        provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();

        resolver.Value = "rotated-psk";
        provider.InvalidateAll();

        provider.TryGetPsk("SITE-A", out var a).ShouldBeTrue();
        provider.TryGetPsk("SITE-B", out var b).ShouldBeTrue();

        a.ShouldBe("rotated-psk");
        b.ShouldBe("rotated-psk");
    }

    [Theory]
    [InlineData("")]
    [InlineData("   ")]
    public void BlankClusterId_FailsClosed(string clusterId)
    {
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk(clusterId, out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
        resolver.CallCount.ShouldBe(0);
    }

    [Fact]
    public void ClusterIdThatCannotFormASecretName_FailsClosed()
    {
        // Cluster ids are operator-authored; SecretName's allow-list is [a-z0-9._/-]. A '$' throws
        // out of the SecretName ctor, and that throw must become "no key", not a 500 in the gate.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE$A", out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
        resolver.CallCount.ShouldBe(0);
    }

    [Fact(Timeout = 30_000)]
    public async Task ConcurrentCallers_ResolveOnce_AndAllSeeTheSameValue()
    {
        // The gRPC gate calls this from every inbound ack, on arbitrary thread-pool threads. A
        // non-atomic fill would stampede the store and could hand two callers different keys.
        //
        // The shape matters more than the assertion. Callers park BEFORE TryGetPsk and are released
        // together, so the race is real rather than accidentally serialized by task scheduling —
        // parking them INSIDE the resolver could never work, because the whole point of the Lazy is
        // that only one caller ever gets in there. And the resolver's delay RETURNS rather than
        // waiting on a gate: a regression that lets a second caller reach the store must drive
        // CallCount to 2 (red), never sit on an exhausted semaphore (hang, which reads as a flake).
        const int Callers = 16;
        using var arrived = new CountdownEvent(Callers);
        using var start = new ManualResetEventSlim(false);
        var resolver = new FakeSecretResolver { Value = "stored-psk", ResolveDelay = TimeSpan.FromMilliseconds(50) };
        var provider = CreateProvider(resolver);

        // LongRunning gives each caller a dedicated thread: reaching the arrival gate must not
        // depend on thread-pool injection, which is rate-limited and made this flake on a
        // contended box (the gate timed out before all 16 pool threads existed).
        var callers = Enumerable.Range(0, Callers).Select(_ => Task.Factory.StartNew(() =>
        {
            arrived.Signal();
            start.Wait();
            provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
            return psk;
        }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)).ToArray();

        arrived.Wait(TimeSpan.FromSeconds(10)).ShouldBeTrue("all callers should reach the start gate");
        start.Set();
        var results = await Task.WhenAll(callers);

        results.ShouldAllBe(r => r == "stored-psk");
        resolver.CallCount.ShouldBe(1);
    }

    [Fact]
    public void APoisonedCacheEntry_IsEvicted_RatherThanDenyingTheClusterForever()
    {
        // ResolveFromStore swallows what the store throws, but it LOGS while doing so — and a logger
        // provider that throws escapes the factory. Lazy under ExecutionAndPublication caches that
        // exception and rethrows it on every later read, so an unguarded entry.Value would deny this
        // cluster for the life of the process; nothing calls Invalidate until Phase 1A.
        var logger = new ThrowingLogger { ThrowOnLog = true };
        var resolver = new FakeSecretResolver { Throw = new InvalidOperationException("KEK unavailable") };
        var provider = CreateProvider(resolver, logger: logger);

        // The logger throwing must not escape the gate either — a denial, not a 500.
        provider.TryGetPsk("SITE-A", out var first).ShouldBeFalse();
        first.ShouldBeNull();
        resolver.CallCount.ShouldBe(1);

        logger.ThrowOnLog = false;
        resolver.Throw = null;
        resolver.Value = "stored-psk";

        provider.TryGetPsk("SITE-A", out var second).ShouldBeTrue();
        second.ShouldBe("stored-psk");
        resolver.CallCount.ShouldBe(2); // reached the store again — the entry was not poisoned
    }

    [Fact]
    public void ConfigMapLookup_StaysCaseInsensitive_ThroughRealConfigurationBinding()
    {
        // Writing into the options object pins only the field initializer's comparer. What reaches
        // production is a BOUND instance, and the binder is free to construct its own Dictionary and
        // drop the comparer silently — the symptom being a rig lookup that misses on a ClusterId
        // spelled the way the DB spells it. Bind for real, then go through the PROVIDER.
        var configuration = new ConfigurationBuilder()
            .AddInMemoryCollection(new Dictionary<string, string?>
            {
                ["MeshTransport:ClusterPsks:SITE-A"] = "rig-psk-site-a",
            })
            .Build();

        var options = new MeshTransportOptions();
        configuration.GetSection(MeshTransportOptions.SectionName).Bind(options);

        var resolver = new FakeSecretResolver();
        var provider = new MeshPskProvider(
            Options.Create(options), resolver, NullLogger<MeshPskProvider>.Instance);

        provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();

        psk.ShouldBe("rig-psk-site-a");
        resolver.CallCount.ShouldBe(0);
    }

    // ------------------------------------------------------------------ known-cluster gate
    //
    // Phase 1A security register item 1. Central rebuilds its cluster set from ClusterNode rows on
    // every contact refresh; a cluster id outside that set must be refused in O(1) BEFORE anything
    // touches the secret store. Two things ride on that: an unauthenticated caller can no longer
    // make central perform a store read (and an audit write) per forged x-otopcua-cluster header,
    // and the timing difference between a known cluster (cached) and an unknown one (never cached,
    // because negatives are deliberately not retained) stops being remotely measurable.

    [Fact]
    public void UnknownCluster_WithTheGatePopulated_IsRefused_WithoutTouchingTheStore()
    {
        // THE assertion of the register item. CallCount == 0 is the whole point: the refusal must
        // cost nothing, or the gate is a comment rather than a control.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);
        provider.SetKnownClusters(["SITE-A", "SITE-B"]);

        provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
        resolver.CallCount.ShouldBe(0);
    }

    [Fact]
    public void KnownCluster_WithTheGatePopulated_StillResolves()
    {
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);
        provider.SetKnownClusters(["SITE-A"]);

        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();

        psk.ShouldBe("stored-psk");
        resolver.CallCount.ShouldBe(1);
    }

    [Fact]
    public void TheKnownClusterSet_IsCaseInsensitive()
    {
        // ClusterId case is whatever the operator typed into the DB row; the header is whatever the
        // node was configured with. A case-sensitive gate would refuse a legitimate cluster.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);
        provider.SetKnownClusters(["SITE-A"]);

        provider.TryGetPsk("site-a", out var psk).ShouldBeTrue();

        psk.ShouldBe("stored-psk");
    }

    [Fact]
    public void SwappingTheKnownClusterSet_AddsAndRemovesClusters()
    {
        // The set is rebuilt from the DB every refresh, so both directions happen in production: a
        // new site appears, a retired one stops being accepted.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        provider.SetKnownClusters(["SITE-A"]);
        provider.TryGetPsk("SITE-A", out _).ShouldBeTrue();
        provider.TryGetPsk("SITE-B", out _).ShouldBeFalse();

        provider.SetKnownClusters(["SITE-B"]);
        provider.TryGetPsk("SITE-B", out _).ShouldBeTrue();
        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
    }

    [Fact]
    public void AnEmptyKnownClusterSet_LeavesTheGateInactive()
    {
        // Fail-OPEN, deliberately and narrowly: central publishes the set from its contact refresh,
        // and an empty publish means "the DB told me nothing", not "no cluster is legitimate".
        // Refusing everything there would make a fleet-wide ack outage the failure mode of an empty
        // or unreadable ClusterNode table.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);
        provider.SetKnownClusters(["SITE-A"]);
        provider.SetKnownClusters([]);

        provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue();

        psk.ShouldBe("stored-psk");
    }

    [Fact]
    public void AnUnsetKnownClusterSet_LeavesTheGateInactive()
    {
        // Admin boot: the interceptor is live from the first request, but the first contact refresh
        // has not run yet. Gating on a set nobody has published would deny every ack in that window.
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);

        provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue();

        psk.ShouldBe("stored-psk");
    }

    [Fact]
    public void TheGate_IsConsultedBeforeTheConfigMap()
    {
        // Consequence worth pinning rather than discovering: a ClusterPsks entry for a cluster with
        // no enabled ClusterNode row is refused once the gate is populated. That is correct — central
        // would neither dial such a cluster nor accept an ack from it — but it is surprising enough
        // on a rig to deserve a test that names it.
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-X"] = "rig-psk-site-x" });
        provider.SetKnownClusters(["SITE-A"]);

        provider.TryGetPsk("SITE-X", out var psk).ShouldBeFalse();

        psk.ShouldBeNull();
    }

    [Fact]
    public void SetKnownClusters_TakesACopy_SoALaterMutationOfTheCallersList_CannotWidenTheGate()
    {
        var clusters = new List<string> { "SITE-A" };
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);
        provider.SetKnownClusters(clusters);

        clusters.Add("SITE-Z");

        provider.TryGetPsk("SITE-Z", out _).ShouldBeFalse();
    }

    // ------------------------------------------------------------------ Matches

    [Fact]
    public void Matches_AcceptsTheClustersOwnKey()
    {
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "psk-site-a" });

        provider.Matches("SITE-A", "psk-site-a").ShouldBeTrue();
    }

    [Fact]
    public void Matches_RejectsAnotherClustersKey()
    {
        // The isolation property, asserted at the provider rather than only at the interceptor:
        // choosing a cluster only changes which key you are graded against.
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver,
            new Dictionary<string, string> { ["SITE-A"] = "psk-site-a", ["SITE-B"] = "psk-site-b" });

        provider.Matches("SITE-A", "psk-site-b").ShouldBeFalse();
    }

    [Fact]
    public void Matches_IsNotAPrefixMatch()
    {
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "psk-site-a" });

        provider.Matches("SITE-A", "psk-site-").ShouldBeFalse();
        provider.Matches("SITE-A", "psk-site-a-and-more").ShouldBeFalse();
    }

    [Fact]
    public void Matches_FailsClosed_WhenTheClusterHasNoKey()
    {
        var resolver = new FakeSecretResolver { Value = null };
        var provider = CreateProvider(resolver);

        provider.Matches("SITE-Z", "anything-at-all").ShouldBeFalse();
    }

    [Theory]
    [InlineData("")]
    [InlineData("   ")]
    public void Matches_RejectsABlankPresentedToken(string presented)
    {
        // A blank key never reaches the cache (blank resolves fail closed), but a blank PRESENTED
        // token must be refused on its own terms rather than relying on that.
        var resolver = new FakeSecretResolver();
        var provider = CreateProvider(
            resolver, new Dictionary<string, string> { ["SITE-A"] = "psk-site-a" });

        provider.Matches("SITE-A", presented).ShouldBeFalse();
    }

    [Fact]
    public void Matches_IsRefusedByTheKnownClusterGate_WithoutTouchingTheStore()
    {
        var resolver = new FakeSecretResolver { Value = "stored-psk" };
        var provider = CreateProvider(resolver);
        provider.SetKnownClusters(["SITE-A"]);

        provider.Matches("SITE-Z", "stored-psk").ShouldBeFalse();

        resolver.CallCount.ShouldBe(0);
    }

    /// <summary>
    ///     An <see cref="ISecretResolver"/> whose answer the test controls between calls, so cache
    ///     staleness and invalidation are observable without a real store.
    /// </summary>
    private sealed class FakeSecretResolver : ISecretResolver
    {
        private int _callCount;

        /// <summary>What <see cref="GetAsync"/> returns when <see cref="Throw"/> is null.</summary>
        public string? Value { get; set; }

        /// <summary>When set, <see cref="GetAsync"/> throws it instead of returning.</summary>
        public Exception? Throw { get; set; }

        /// <summary>
        ///     How long each call takes before answering. Widens the window a concurrent stampede
        ///     would have to exploit — and it RETURNS, so a second entrant shows up as a second call
        ///     rather than as a hang.
        /// </summary>
        public TimeSpan ResolveDelay { get; set; } = TimeSpan.Zero;

        /// <summary>How many times the store was consulted.</summary>
        public int CallCount => Volatile.Read(ref _callCount);

        /// <summary>The name of the most recently requested secret.</summary>
        public string? LastName { get; private set; }

        public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
        {
            LastName = name.Value;
            Interlocked.Increment(ref _callCount);

            if (ResolveDelay > TimeSpan.Zero)
                await Task.Delay(ResolveDelay, ct).ConfigureAwait(false);

            if (Throw is not null)
                throw Throw;

            return Value;
        }
    }

    /// <summary>
    ///     An <see cref="ILogger{T}"/> that can be made to throw, standing in for a logger provider
    ///     that faults — the only way the provider's own catch blocks can throw.
    /// </summary>
    private sealed class ThrowingLogger : ILogger<MeshPskProvider>
    {
        /// <summary>When true, every <see cref="Log"/> call throws.</summary>
        public bool ThrowOnLog { get; set; }

        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;

        public bool IsEnabled(LogLevel logLevel) => true;

        public void Log<TState>(
            LogLevel logLevel,
            EventId eventId,
            TState state,
            Exception? exception,
            Func<TState, Exception?, string> formatter)
        {
            if (ThrowOnLog)
                throw new InvalidOperationException("logger provider faulted");
        }
    }
}