feat(mesh): per-cluster PSK provider + central interceptor
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,202 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Central's per-cluster pre-shared key lookup for the gRPC command plane. A node holds exactly
|
||||
/// one key (<c>MeshTransport:GrpcPsk</c>, its own cluster's); central talks to every cluster, so
|
||||
/// it needs one key per <c>Cluster</c> — both to authenticate itself when dialling a node and to
|
||||
/// verify an inbound deployment ack.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Fail-closed by contract.</b> Every failure — unknown cluster, absent secret, blank
|
||||
/// value, a store that throws — is reported as "no key" (<c>false</c>), and the caller
|
||||
/// denies the inbound call or skips the outbound dial. There is no arm on which a caller may
|
||||
/// proceed unauthenticated; a missing key must cost delivery, not authentication.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <c>Invalidate</c> hooks exist because central's cluster set is DB-dynamic. They are
|
||||
/// deliberately unconsumed at Phase 0 — Phase 1A wires them to the contact refresh, which is
|
||||
/// the point at which central learns a cluster's membership (and so possibly its key) has
|
||||
/// changed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IMeshPskProvider
|
||||
{
|
||||
/// <summary>Gets the pre-shared key for one application <c>Cluster</c>.</summary>
|
||||
/// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
|
||||
/// <param name="psk">The key, or <c>null</c> when none could be established.</param>
|
||||
/// <returns><c>true</c> when a usable key was found; otherwise <c>false</c>.</returns>
|
||||
bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk);
|
||||
|
||||
/// <summary>Drops any cached key for one cluster, so the next lookup re-reads the store.</summary>
|
||||
/// <param name="clusterId">The application <c>Cluster</c> id, matched case-insensitively.</param>
|
||||
void Invalidate(string clusterId);
|
||||
|
||||
/// <summary>Drops every cached key.</summary>
|
||||
void InvalidateAll();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public sealed class MeshPskProvider : IMeshPskProvider
|
||||
{
|
||||
/// <summary>
|
||||
/// Prefix of the secret holding a cluster's key — the full name is
|
||||
/// <c>OT-MESH-PSK-{clusterId}</c>. The same name a node's
|
||||
/// <c>${secret:OT-MESH-PSK-<clusterId>}</c> reference expands, so central and the node
|
||||
/// read one row rather than two conventions that drift.
|
||||
/// </summary>
|
||||
public const string SecretNamePrefix = "OT-MESH-PSK-";
|
||||
|
||||
private readonly IOptions<MeshTransportOptions> _options;
|
||||
private readonly ISecretResolver _secretResolver;
|
||||
private readonly ILogger<MeshPskProvider> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Cached store resolutions, keyed by cluster id. Holds <b>only</b> values that came from the
|
||||
/// Secrets store: the config map is re-read on every lookup (it is an in-memory dictionary),
|
||||
/// which is what keeps the override unconditional rather than merely first-past-the-post.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <see cref="Lazy{T}"/> under <see cref="LazyThreadSafetyMode.ExecutionAndPublication"/> so a
|
||||
/// burst of concurrent inbound acks for one cluster resolves the secret once instead of
|
||||
/// stampeding the store. Negatives are never retained — see <see cref="TryGetPsk"/>.
|
||||
/// </remarks>
|
||||
private readonly ConcurrentDictionary<string, Lazy<string?>> _cache =
|
||||
new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Creates the provider.</summary>
|
||||
/// <param name="options">Mesh transport options; <c>ClusterPsks</c> is the override map.</param>
|
||||
/// <param name="secretResolver">The shared secret store read seam.</param>
|
||||
/// <param name="logger">Logger for lookup-failure diagnostics.</param>
|
||||
public MeshPskProvider(
|
||||
IOptions<MeshTransportOptions> options,
|
||||
ISecretResolver secretResolver,
|
||||
ILogger<MeshPskProvider> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(secretResolver);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_options = options;
|
||||
_secretResolver = secretResolver;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
|
||||
{
|
||||
psk = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(clusterId))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Mesh PSK lookup was asked for a blank cluster id; refusing rather than resolving a "
|
||||
+ "key that would belong to no cluster.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Config map first, on every call. Blank counts as absent: an empty
|
||||
// MeshTransport__ClusterPsks__<id> is an unset environment variable, not an operator
|
||||
// deciding this cluster has an empty key.
|
||||
if (_options.Value.ClusterPsks.TryGetValue(clusterId, out var configured)
|
||||
&& !string.IsNullOrWhiteSpace(configured))
|
||||
{
|
||||
psk = configured;
|
||||
return true;
|
||||
}
|
||||
|
||||
var entry = _cache.GetOrAdd(
|
||||
clusterId,
|
||||
static (id, self) => new Lazy<string?>(
|
||||
() => self.ResolveFromStore(id), LazyThreadSafetyMode.ExecutionAndPublication),
|
||||
this);
|
||||
|
||||
var resolved = entry.Value;
|
||||
if (string.IsNullOrWhiteSpace(resolved))
|
||||
{
|
||||
// Never retain a negative. A store that was briefly unreachable would otherwise stay
|
||||
// "no key" for this cluster until something called Invalidate — and at Phase 0 nothing
|
||||
// does, so a transient fault would present as a permanent, silent command outage.
|
||||
_cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
|
||||
return false;
|
||||
}
|
||||
|
||||
psk = resolved;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Invalidate(string clusterId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(clusterId))
|
||||
return;
|
||||
|
||||
_cache.TryRemove(clusterId, out _);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void InvalidateAll() => _cache.Clear();
|
||||
|
||||
/// <summary>
|
||||
/// Reads <c>OT-MESH-PSK-{clusterId}</c> from the Secrets store, converting every failure mode
|
||||
/// into <c>null</c>.
|
||||
/// </summary>
|
||||
/// <param name="clusterId">The application <c>Cluster</c> id.</param>
|
||||
/// <returns>The stored key, or <c>null</c> when it could not be established.</returns>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Blocking on the async resolver is deliberate. The consumer is a gRPC server
|
||||
/// interceptor, whose <c>Authorize</c> is synchronous in all four sibling interceptors,
|
||||
/// and making this seam async would push that shape onto every one of them. The block
|
||||
/// happens at most once per cluster per process — every later lookup is a dictionary hit
|
||||
/// — and the store is a node-local SQLite file, not a network call.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The catch is broad on purpose: this is an authentication boundary, and the set of
|
||||
/// exceptions a secret store can raise (missing KEK, corrupt row, IO) is neither closed
|
||||
/// nor interesting here. Anything other than a usable key is "no key".
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private string? ResolveFromStore(string clusterId)
|
||||
{
|
||||
SecretName name;
|
||||
try
|
||||
{
|
||||
name = new SecretName(SecretNamePrefix + clusterId);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
// Cluster ids are operator-authored and may carry characters SecretName's allow-list
|
||||
// rejects. Log the id — it is not sensitive — and never the key.
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
|
||||
+ "resolved from the store. Supply it through MeshTransport:ClusterPsks instead, or "
|
||||
+ "rename the cluster to [a-z0-9._/-].",
|
||||
clusterId);
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return _secretResolver.GetAsync(name, CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to resolve the mesh PSK for cluster {ClusterId} from secret {SecretName}. "
|
||||
+ "Commands to that cluster and acks from it are refused until it resolves.",
|
||||
clusterId,
|
||||
name.Value);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,10 @@
|
||||
Google.Protobuf + Grpc.Core.Api flow transitively from Commons (the generated client). -->
|
||||
<PackageReference Include="Grpc.Net.Client"/>
|
||||
<PackageReference Include="ZB.MOM.WW.Audit"/>
|
||||
<!-- ClusterClient→gRPC: MeshPskProvider resolves OT-MESH-PSK-{clusterId} from the shared
|
||||
secret store when MeshTransport:ClusterPsks carries no override. Abstractions only —
|
||||
the concrete store is the host's choice (SecretsRegistration), not the control plane's. -->
|
||||
<PackageReference Include="ZB.MOM.WW.Secrets.Abstractions"/>
|
||||
<!-- For ClusterActiveNode — the shared "oldest Up member of role X" primitive. The control
|
||||
plane and the /health/active tier must give the SAME answer for which node is in charge,
|
||||
or the tier reports one node active while the Primary-gated data plane runs on another;
|
||||
|
||||
Reference in New Issue
Block a user