lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs

using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.Secrets.Abstractions;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;

/// <summary>
///     Central's per-cluster pre-shared key lookup for the gRPC command plane. A node holds exactly
///     one key (<c>MeshTransport:GrpcPsk</c>, its own cluster's); central talks to every cluster, so
///     it needs one key per <c>Cluster</c> — both to authenticate itself when dialling a node and to
///     verify an inbound deployment ack.
/// </summary>
/// <remarks>
///     <para>
///         <b>Fail-closed by contract.</b> Every failure — unknown cluster, absent secret, blank
///         value, a store that throws — is reported as "no key" (<c>false</c>), and the caller
///         denies the inbound call or skips the outbound dial. There is no arm on which a caller may
///         proceed unauthenticated; a missing key must cost delivery, not authentication.
///     </para>
///     <para>
///         The <c>Invalidate</c> hooks exist because central's cluster set is DB-dynamic. Phase 1A
///         wires them to <c>CentralCommunicationActor</c>'s contact refresh, which is the point at
///         which central learns a cluster's membership (and so possibly its key) has changed — so a
///         rotated key is picked up within one <c>ContactRefreshSeconds</c> without a restart.
///     </para>
///     <para>
///         <b>The known-cluster gate is a real security control, not an optimization</b> (Phase 1A
///         security register item 1). <see cref="SetKnownClusters"/> publishes the cluster ids
///         central just read from <c>ClusterNode</c>; <see cref="TryGetPsk"/> consults that set
///         first and refuses anything outside it in O(1), touching neither the config map nor the
///         secret store. Without it, an unauthenticated caller can pick an arbitrary
///         <c>x-otopcua-cluster</c> value and make central perform a secret-store read plus an audit
///         write per probe — and, because negatives are deliberately never cached, the cost
///         difference between a known and an unknown cluster is measurable from the wire, which
///         defeats the uniform denial text on the central interceptor.
///     </para>
///     <para>
///         <b><c>Invalidate</c> is not a revocation barrier.</b> It drops the cached entry; it does
///         not wait for, cancel, or supersede a resolve already in flight. A lookup that entered the
///         store before the call can still publish and return the pre-rotation key to its caller, and
///         a lookup racing the eviction can re-cache the old value it had already read. Key rotation
///         is therefore eventually consistent by seconds, not atomic — treat <c>Invalidate</c> as
///         "stop serving this from memory", never as "this key is now dead everywhere".
///     </para>
/// </remarks>
public interface IMeshPskProvider
{
    /// <summary>Gets the pre-shared key for one application <c>Cluster</c>.</summary>
    /// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
    /// <param name="psk">The key, or <c>null</c> when none could be established.</param>
    /// <returns><c>true</c> when a usable key was found; otherwise <c>false</c>.</returns>
    bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk);

    /// <summary>
    ///     Publishes the set of application <c>Cluster</c> ids central currently knows about.
    ///     <see cref="TryGetPsk"/> refuses anything outside it before consulting any source.
    /// </summary>
    /// <param name="clusterIds">
    ///     The known cluster ids, matched case-insensitively. <b>An empty collection deactivates the
    ///     gate</b> rather than refusing everything: central publishes this from its DB contact
    ///     refresh, and "the DB told me nothing" must not become a fleet-wide authentication outage.
    ///     Never having called this leaves the gate inactive for the same reason — the central
    ///     interceptor is live from the first request, before the first refresh has run.
    /// </param>
    /// <remarks>
    ///     Implementations must take a copy: the caller owns its collection and may reuse it. The
    ///     swap must also be atomic with respect to concurrent <see cref="TryGetPsk"/> callers,
    ///     which run on arbitrary gRPC threads.
    /// </remarks>
    void SetKnownClusters(IReadOnlyCollection<string> clusterIds);

    /// <summary>
    ///     Verifies a presented bearer token against one cluster's key, in constant time with
    ///     respect to the key's content.
    /// </summary>
    /// <param name="clusterId">The application <c>Cluster</c> the caller claims to be.</param>
    /// <param name="presented">The bearer token the caller supplied.</param>
    /// <returns><c>true</c> only when a key exists for that cluster and the token equals it.</returns>
    /// <remarks>
    ///     <para>
    ///         <b>This is the only comparison on the verify path</b> (Phase 1A security register
    ///         item 2). Handing the raw key back so a caller can compare it is how <c>==</c>
    ///         eventually appears in an interceptor and compiles; keeping the comparison behind this
    ///         member means the verify side never holds key material at all. <see cref="TryGetPsk"/>
    ///         survives for the <i>dial</i> side, which has to send the key rather than compare it.
    ///     </para>
    ///     <para>
    ///         Declared without a default implementation on purpose: a security member a new
    ///         implementer can inherit silently is a member a new implementer never considers.
    ///     </para>
    ///     <para>
    ///         Length differences remain observable through timing; that is inherent to comparing
    ///         byte arrays of different lengths and is not sensitive. Content is not.
    ///     </para>
    /// </remarks>
    bool Matches(string clusterId, string presented);

    /// <summary>Drops any cached key for one cluster, so the next lookup re-reads the store.</summary>
    /// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
    void Invalidate(string clusterId);

    /// <summary>Drops every cached key.</summary>
    void InvalidateAll();

    /// <summary>
    ///     Re-reads the keys for <paramref name="clusterIds"/> from the store OFF the request path and
    ///     swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep
    ///     serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose
    ///     secret the store reports ABSENT is revoked (no key from then on). Replaces
    ///     "invalidate on every refresh", which made the first inbound ack per cluster per
    ///     <c>ContactRefreshSeconds</c> block a gRPC thread on a synchronous, 3 s-bounded store read.
    /// </summary>
    /// <param name="clusterIds">The clusters central currently knows (the same set given to <see cref="SetKnownClusters"/>).</param>
    /// <param name="ct">Cancels the refresh.</param>
    /// <returns>Completes when the swap has happened.</returns>
    /// <remarks>
    ///     The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup
    ///     re-read — so a fake or a non-caching provider needs nothing more.
    /// </remarks>
    Task RefreshAsync(IReadOnlyCollection<string> clusterIds, CancellationToken ct)
    {
        InvalidateAll();
        return Task.CompletedTask;
    }
}

/// <inheritdoc />
public sealed class MeshPskProvider : IMeshPskProvider
{
    /// <summary>
    ///     Prefix of the secret holding a cluster's key — the full name is
    ///     <c>OT-MESH-PSK-{clusterId}</c>. The same name a node's
    ///     <c>${secret:OT-MESH-PSK-&lt;clusterId&gt;}</c> reference expands, so central and the node
    ///     read one row rather than two conventions that drift.
    /// </summary>
    public const string SecretNamePrefix = "OT-MESH-PSK-";

    /// <summary>
    ///     Ceiling on one store read. Not a tuning knob — a backstop: the read runs under the
    ///     <see cref="Lazy{T}"/> publication lock, so a store that never answers would park the
    ///     calling gRPC thread <i>and</i> every other caller queued behind it for that cluster.
    ///     Expiry fails closed like any other fault.
    /// </summary>
    private static readonly TimeSpan StoreReadTimeout = TimeSpan.FromSeconds(3);

    private readonly IOptions<MeshTransportOptions> _options;
    private readonly ISecretResolver _secretResolver;
    private readonly ILogger<MeshPskProvider> _logger;

    /// <summary>
    ///     Cached store resolutions, keyed by cluster id. Holds <b>only</b> values that came from the
    ///     Secrets store: the config map is re-read on every lookup (it is an in-memory dictionary),
    ///     which is what keeps the override unconditional rather than merely first-past-the-post.
    /// </summary>
    /// <remarks>
    ///     <see cref="Lazy{T}"/> under <see cref="LazyThreadSafetyMode.ExecutionAndPublication"/> so a
    ///     burst of concurrent inbound acks for one cluster resolves the secret once instead of
    ///     stampeding the store. Negatives are never retained — see <see cref="TryGetPsk"/>.
    /// </remarks>
    private readonly ConcurrentDictionary<string, Lazy<string?>> _cache =
        new(StringComparer.OrdinalIgnoreCase);

    /// <summary>
    ///     Keys pre-resolved by <see cref="RefreshAsync"/>, consulted after the config map and before the
    ///     lazy <see cref="_cache"/>. Swapped wholesale (volatile reference to an immutable dictionary),
    ///     never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next
    ///     one. A cluster missing from it — never refreshed, or whose read failed — falls through to the
    ///     lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key".
    /// </summary>
    private volatile IReadOnlyDictionary<string, string> _refreshed =
        new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

    /// <summary>
    ///     The known-cluster gate's membership set, or <see langword="null"/> when the gate is
    ///     inactive (never published, or published empty — see <see cref="SetKnownClusters"/>).
    /// </summary>
    /// <remarks>
    ///     Swapped wholesale, never mutated in place, so a reader either sees the previous set or the
    ///     next one and never a half-built one. <c>volatile</c> is what makes the publish visible to
    ///     the gRPC threads doing the reading; no lock is needed because there is exactly one writer
    ///     (the contact-refresh handler, on the actor's own thread).
    /// </remarks>
    private volatile IReadOnlySet<string>? _knownClusters;

    /// <summary>Creates the provider.</summary>
    /// <param name="options">Mesh transport options; <c>ClusterPsks</c> is the override map.</param>
    /// <param name="secretResolver">The shared secret store read seam.</param>
    /// <param name="logger">Logger for lookup-failure diagnostics.</param>
    public MeshPskProvider(
        IOptions<MeshTransportOptions> options,
        ISecretResolver secretResolver,
        ILogger<MeshPskProvider> logger)
    {
        ArgumentNullException.ThrowIfNull(options);
        ArgumentNullException.ThrowIfNull(secretResolver);
        ArgumentNullException.ThrowIfNull(logger);

        _options = options;
        _secretResolver = secretResolver;
        _logger = logger;
    }

    /// <inheritdoc />
    public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
    {
        psk = null;

        if (string.IsNullOrWhiteSpace(clusterId))
        {
            _logger.LogWarning(
                "Mesh PSK lookup was asked for a blank cluster id; refusing rather than resolving a "
                + "key that would belong to no cluster.");
            return false;
        }

        // The known-cluster gate, ahead of BOTH sources. Deliberately silent: the caller that
        // rejected the call already logs the denial, and a second line here would be a log flood
        // driven by exactly the unauthenticated traffic this gate exists to make cheap.
        //
        // Ahead of the config map too, not just the store. That has a visible consequence — a
        // MeshTransport:ClusterPsks entry for a cluster with no enabled, non-maintenance ClusterNode
        // row stops resolving once central publishes its set — and it is the right one: central
        // would neither dial such a cluster nor accept an ack from it, so serving a key for it only
        // widens what an unauthenticated caller can probe.
        var known = _knownClusters;
        if (known is not null && !known.Contains(clusterId))
            return false;

        // Config map first, on every call. Blank counts as absent: an empty
        // MeshTransport__ClusterPsks__<id> is an unset environment variable, not an operator
        // deciding this cluster has an empty key.
        if (_options.Value.ClusterPsks.TryGetValue(clusterId, out var configured)
            && !string.IsNullOrWhiteSpace(configured))
        {
            psk = configured;
            return true;
        }

        // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all.
        if (_refreshed.TryGetValue(clusterId, out var refreshed))
        {
            psk = refreshed;
            return true;
        }

        var entry = _cache.GetOrAdd(
            clusterId,
            static (id, self) => new Lazy<string?>(
                () => self.ResolveFromStore(id), LazyThreadSafetyMode.ExecutionAndPublication),
            this);

        string? resolved;
        try
        {
            resolved = entry.Value;
        }
        catch
        {
            // ResolveFromStore swallows everything the STORE can throw — but it logs while doing so,
            // and a logger provider that throws escapes the factory. Lazy under
            // ExecutionAndPublication caches that exception and rethrows it on every later read, so
            // without this guard one bad log call would deny the cluster for the life of the process
            // (nothing calls Invalidate until Phase 1A). Evict instance-keyed, same as the negative
            // path below, and fail closed.
            //
            // Deliberately silent: the most likely reason we are here is that logging is what threw,
            // so logging the failure is the one action with a real chance of throwing again — out of
            // an authentication gate, turning a denial into a 500.
            _cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
            return false;
        }

        if (string.IsNullOrWhiteSpace(resolved))
        {
            // Never retain a negative. A store that was briefly unreachable would otherwise stay
            // "no key" for this cluster until something called Invalidate — and at Phase 0 nothing
            // does, so a transient fault would present as a permanent, silent command outage.
            _cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
            return false;
        }

        psk = resolved;
        return true;
    }

    /// <inheritdoc />
    public bool Matches(string clusterId, string presented)
    {
        // Blank is refused on its own terms rather than by relying on the fact that a blank key
        // never survives resolution: this is an authentication gate, and "both sides happened to be
        // empty" must never be a pass.
        if (string.IsNullOrWhiteSpace(presented))
            return false;

        if (!TryGetPsk(clusterId, out var expected))
            return false;

        return CryptographicOperations.FixedTimeEquals(
            Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
    }

    /// <inheritdoc />
    public void SetKnownClusters(IReadOnlyCollection<string> clusterIds)
    {
        ArgumentNullException.ThrowIfNull(clusterIds);

        // Copied, so a caller reusing its list cannot widen the gate after the fact; empty collapses
        // to null so the gate has exactly one "inactive" representation to test for.
        _knownClusters = clusterIds.Count == 0
            ? null
            : clusterIds.ToHashSet(StringComparer.OrdinalIgnoreCase);
    }

    /// <inheritdoc />
    public void Invalidate(string clusterId)
    {
        if (string.IsNullOrWhiteSpace(clusterId))
            return;

        _cache.TryRemove(clusterId, out _);
    }

    /// <inheritdoc />
    public void InvalidateAll()
    {
        _cache.Clear();
        _refreshed = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
    }

    /// <summary>
    ///     Ceiling on concurrent store reads during one <see cref="RefreshAsync"/>. A fleet of N clusters used to
    ///     fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most
    ///     this many at once now.
    /// </summary>
    public const int MaxConcurrentRefreshReads = 8;

    /// <inheritdoc />
    public async Task RefreshAsync(IReadOnlyCollection<string> clusterIds, CancellationToken ct)
    {
        ArgumentNullException.ThrowIfNull(clusterIds);

        var previous = _refreshed;
        var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
        var results = new ConcurrentDictionary<string, StoreRead>(StringComparer.OrdinalIgnoreCase);

        // Bounded fan-out; each read bounded like the synchronous path.
        await Parallel.ForEachAsync(
                ids,
                new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct },
                async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false))
            .ConfigureAwait(false);

        // Three outcomes per cluster (fix round 1 / I8):
        //  - Found     → the new key;
        //  - NotFound  → the store answered that the secret does not exist: a REVOCATION — the cluster gets no
        //                key (and its lazily cached one is evicted below), so a deleted secret stops working
        //                within one refresh instead of living on for the life of the process;
        //  - Failed    → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key —
        //                a transient fault must not open a window in which the cluster has no key at all.
        var next = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
        var answered = new List<string>();
        foreach (var id in ids)
        {
            var read = results[id];
            switch (read.Outcome)
            {
                case StoreReadOutcome.Found:
                    next[id] = read.Value!;
                    answered.Add(id);
                    break;
                case StoreReadOutcome.NotFound:
                    answered.Add(id);
                    break;
                default:
                    if (previous.TryGetValue(id, out var kept)) next[id] = kept;
                    break;
            }
        }

        // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new
        // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too).
        // Entries for clusters whose read failed are left alone.
        _refreshed = next;
        foreach (var id in answered) _cache.TryRemove(id, out _);
    }

    private enum StoreReadOutcome { Found, NotFound, Failed }

    private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value);

    /// <summary>
    ///     The asynchronous twin of <see cref="ResolveFromStore"/>: same name derivation, same bound — awaited
    ///     instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor —
    ///     but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read".
    /// </summary>
    /// <param name="clusterId">The application <c>Cluster</c> id.</param>
    /// <param name="ct">The caller's token, linked under the store-read bound.</param>
    /// <returns>The outcome and, when found, the key.</returns>
    private async Task<StoreRead> ResolveFromStoreAsync(string clusterId, CancellationToken ct)
    {
        SecretName name;
        try
        {
            name = new SecretName(SecretNamePrefix + clusterId);
        }
        catch (ArgumentException ex)
        {
            _logger.LogWarning(
                ex,
                "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
                + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or "
                + "rename the cluster to [a-z0-9._/-].",
                clusterId);
            return new StoreRead(StoreReadOutcome.Failed, null);
        }

        using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
        cts.CancelAfter(StoreReadTimeout);
        try
        {
            var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false);
            if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value);

            _logger.LogWarning(
                "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is "
                + "REVOKED — commands to that cluster and acks from it are refused until it is restored.",
                name.Value,
                clusterId);
            return new StoreRead(StoreReadOutcome.NotFound, null);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(
                ex,
                "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the "
                + "previously resolved key (if any) stays in service until the next refresh.",
                clusterId,
                name.Value);
            return new StoreRead(StoreReadOutcome.Failed, null);
        }
    }

    /// <summary>
    ///     Reads <c>OT-MESH-PSK-{clusterId}</c> from the Secrets store, converting every failure mode
    ///     into <c>null</c>.
    /// </summary>
    /// <param name="clusterId">The application <c>Cluster</c> id.</param>
    /// <returns>The stored key, or <c>null</c> when it could not be established.</returns>
    /// <remarks>
    ///     <para>
    ///         Blocking on the async resolver is deliberate. The consumer is a gRPC server
    ///         interceptor, whose <c>Authorize</c> is synchronous in all four sibling interceptors,
    ///         and making this seam async would push that shape onto every one of them. What makes
    ///         the block acceptable is that it is <b>bounded</b> (<see cref="StoreReadTimeout"/>) and
    ///         happens at most once per cluster per process — every later lookup is a dictionary hit.
    ///         The bound does not assume anything about what backs the store: this code cannot see
    ///         whether the resolver reads a local file or crosses a network, so it refuses to wait
    ///         forever on either.
    ///     </para>
    ///     <para>
    ///         The catch is broad on purpose: this is an authentication boundary, and the set of
    ///         exceptions a secret store can raise (missing KEK, corrupt row, IO) is neither closed
    ///         nor interesting here. Anything other than a usable key is "no key".
    ///     </para>
    /// </remarks>
    private string? ResolveFromStore(string clusterId)
    {
        SecretName name;
        try
        {
            name = new SecretName(SecretNamePrefix + clusterId);
        }
        catch (ArgumentException ex)
        {
            // Cluster ids are operator-authored and may carry characters SecretName's allow-list
            // rejects. Log the id — it is not sensitive — and never the key.
            _logger.LogWarning(
                ex,
                "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
                + "resolved from the store. Supply it through MeshTransport:ClusterPsks instead, or "
                + "rename the cluster to [a-z0-9._/-].",
                clusterId);
            return null;
        }

        using var cts = new CancellationTokenSource(StoreReadTimeout);
        try
        {
            return _secretResolver.GetAsync(name, cts.Token).GetAwaiter().GetResult();
        }
        catch (Exception ex)
        {
            // Covers the timeout too — a cancelled read arrives here as OperationCanceledException
            // and is treated as every other fault is: no key.
            _logger.LogWarning(
                ex,
                "Failed to resolve the mesh PSK for cluster {ClusterId} from secret {SecretName}. "
                + "Commands to that cluster and acks from it are refused until it resolves.",
                clusterId,
                name.Value);
            return null;
        }
    }
}