dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(mesh): Batch B review fixes — bounded store read, cache poison guard, test hardening

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-06 04:10:03 -04:00
parent 68a416e4fc
commit 24e55d3395
7 changed files with 314 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs
+47 -5
View File
@@ -26,6 +26,14 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
/// the point at which central learns a cluster's membership (and so possibly its key) has
/// changed.
/// </para>
/// <para>
/// <b><c>Invalidate</c> is not a revocation barrier.</b> It drops the cached entry; it does
/// not wait for, cancel, or supersede a resolve already in flight. A lookup that entered the
/// store before the call can still publish and return the pre-rotation key to its caller, and
/// a lookup racing the eviction can re-cache the old value it had already read. Key rotation
/// is therefore eventually consistent by seconds, not atomic — treat <c>Invalidate</c> as
/// "stop serving this from memory", never as "this key is now dead everywhere".
/// </para>
/// </remarks>
public interface IMeshPskProvider
{
@@ -54,6 +62,14 @@ public sealed class MeshPskProvider : IMeshPskProvider
/// </summary>
public const string SecretNamePrefix = "OT-MESH-PSK-";
/// <summary>
/// Ceiling on one store read. Not a tuning knob — a backstop: the read runs under the
/// <see cref="Lazy{T}"/> publication lock, so a store that never answers would park the
/// calling gRPC thread <i>and</i> every other caller queued behind it for that cluster.
/// Expiry fails closed like any other fault.
/// </summary>
private static readonly TimeSpan StoreReadTimeout = TimeSpan.FromSeconds(3);
private readonly IOptions<MeshTransportOptions> _options;
private readonly ISecretResolver _secretResolver;
private readonly ILogger<MeshPskProvider> _logger;
@@ -118,7 +134,27 @@ public sealed class MeshPskProvider : IMeshPskProvider
() => self.ResolveFromStore(id), LazyThreadSafetyMode.ExecutionAndPublication),
this);
var resolved = entry.Value;
string? resolved;
try
{
resolved = entry.Value;
}
catch
{
// ResolveFromStore swallows everything the STORE can throw — but it logs while doing so,
// and a logger provider that throws escapes the factory. Lazy under
// ExecutionAndPublication caches that exception and rethrows it on every later read, so
// without this guard one bad log call would deny the cluster for the life of the process
// (nothing calls Invalidate until Phase 1A). Evict instance-keyed, same as the negative
// path below, and fail closed.
//
// Deliberately silent: the most likely reason we are here is that logging is what threw,
// so logging the failure is the one action with a real chance of throwing again — out of
// an authentication gate, turning a denial into a 500.
_cache.TryRemove(new KeyValuePair<string, Lazy<string?>>(clusterId, entry));
return false;
}
if (string.IsNullOrWhiteSpace(resolved))
{
// Never retain a negative. A store that was briefly unreachable would otherwise stay
@@ -154,9 +190,12 @@ public sealed class MeshPskProvider : IMeshPskProvider
/// <para>
/// Blocking on the async resolver is deliberate. The consumer is a gRPC server
/// interceptor, whose <c>Authorize</c> is synchronous in all four sibling interceptors,
/// and making this seam async would push that shape onto every one of them. The block
/// happens at most once per cluster per process — every later lookup is a dictionary hit
/// — and the store is a node-local SQLite file, not a network call.
/// and making this seam async would push that shape onto every one of them. What makes
/// the block acceptable is that it is <b>bounded</b> (<see cref="StoreReadTimeout"/>) and
/// happens at most once per cluster per process — every later lookup is a dictionary hit.
/// The bound does not assume anything about what backs the store: this code cannot see
/// whether the resolver reads a local file or crosses a network, so it refuses to wait
/// forever on either.
/// </para>
/// <para>
/// The catch is broad on purpose: this is an authentication boundary, and the set of
@@ -184,12 +223,15 @@ public sealed class MeshPskProvider : IMeshPskProvider
return null;
}
using var cts = new CancellationTokenSource(StoreReadTimeout);
try
{
return _secretResolver.GetAsync(name, CancellationToken.None).GetAwaiter().GetResult();
return _secretResolver.GetAsync(name, cts.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
// Covers the timeout too — a cancelled read arrives here as OperationCanceledException
// and is treated as every other fault is: no key.
_logger.LogWarning(
ex,
"Failed to resolve the mesh PSK for cluster {ClusterId} from secret {SecretName}. "
src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs
+14 -28
View File
@@ -45,9 +45,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <para>
/// <b>All three denials return the same status text</b>, unlike the sibling interceptors,
/// which distinguish "no key configured" from "authentication failed". Here that distinction
/// would be an enumeration oracle: the caller supplies the cluster id, so a distinguishable
/// "no key for that cluster" reply would let an unauthenticated caller map which clusters
/// central holds keys for. The operator-facing detail goes to the log instead.
/// would be a <i>content</i> oracle: the caller supplies the cluster id, so a reply that said
/// "no key for that cluster" would let an unauthenticated caller read central's cluster set
/// straight off the wire. The operator-facing detail goes to the log instead.
/// </para>
/// <para>
/// <b>That is not yet full enumeration resistance, and should not be read as such.</b> A
/// timing oracle survives it: an unknown cluster misses the provider's cache and pays a store
/// read on every probe, while a known one is answered from memory after the first. Closing
/// that needs the lookup to be refused before it reaches the store for any cluster central
/// does not know — which is the Phase 1A known-cluster-set gate, where central learns its
/// cluster set from the DB. Until then, uniform text buys content-level indistinguishability
/// only.
/// </para>
/// <para>
/// <b>Exactly one public constructor.</b> <c>Grpc.AspNetCore</c> silently stops invoking an
@@ -58,9 +67,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
public sealed class CentralCommandAuthInterceptor : Interceptor
{
private const string ServicePrefix = "/mesh_transport.v1.CentralCommandService/";
private const string AuthorizationHeader = "authorization";
private const string ClusterHeader = "x-otopcua-cluster";
private const string BearerPrefix = "Bearer ";
private readonly IMeshPskProvider _pskProvider;
private readonly ILogger<CentralCommandAuthInterceptor> _logger;
@@ -131,7 +138,7 @@ public sealed class CentralCommandAuthInterceptor : Interceptor
if (!context.Method.StartsWith(ServicePrefix, StringComparison.Ordinal))
return;
var clusterId = ExtractHeader(context.RequestHeaders, ClusterHeader);
var clusterId = GrpcRequestHeaders.First(context.RequestHeaders, ClusterHeader);
if (string.IsNullOrWhiteSpace(clusterId))
{
_logger.LogWarning(
@@ -158,7 +165,7 @@ public sealed class CentralCommandAuthInterceptor : Interceptor
"Central command authentication failed."));
}
var presented = ExtractBearerToken(context.RequestHeaders);
var presented = GrpcRequestHeaders.BearerToken(context.RequestHeaders);
if (presented is null || !FixedTimeEquals(presented, expected))
{
_logger.LogWarning(
@@ -172,27 +179,6 @@ public sealed class CentralCommandAuthInterceptor : Interceptor
}
}
private static string? ExtractBearerToken(Metadata headers)
{
var value = ExtractHeader(headers, AuthorizationHeader);
return value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase)
? value[BearerPrefix.Length..]
: null;
}
private static string? ExtractHeader(Metadata headers, string name)
{
// gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
// Metadata in a test behaves the same as a real request.
foreach (var entry in headers)
{
if (string.Equals(entry.Key, name, StringComparison.OrdinalIgnoreCase))
return entry.Value;
}
return null;
}
private static bool FixedTimeEquals(string presented, string expected)
=> CryptographicOperations.FixedTimeEquals(
Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));
src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/GrpcRequestHeaders.cs
+70
View File
@@ -0,0 +1,70 @@
using Grpc.Core;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// Request-metadata reads shared by the two mesh-command auth interceptors
/// (<see cref="MeshCommandAuthInterceptor"/>, <see cref="CentralCommandAuthInterceptor"/>).
/// </summary>
/// <remarks>
/// <para>
/// Exists because those two are documented structural clones of each other, and the one place
/// a clone pair can silently diverge is header parsing — where a difference is not a style
/// difference but a different answer to "which of these two <c>authorization</c> headers
/// counts". They had exactly that divergence before this type: one took the first entry, the
/// other scanned for the first <c>Bearer</c>-prefixed entry, so a request carrying
/// <c>authorization: junk</c> followed by <c>authorization: Bearer &lt;psk&gt;</c> was denied
/// by one and accepted by the other.
/// </para>
/// <para>
/// <b>First entry wins, and it must parse.</b> A non-<c>Bearer</c> first entry is a failed
/// authentication, never a reason to keep looking: scanning past it lets a caller smuggle a
/// credential behind a header some intermediary may have inspected, matched or rewritten, and
/// makes which credential is checked depend on ordering nothing guarantees.
/// </para>
/// <para>
/// The three pre-existing sibling interceptors (LocalDb sync, config-serve, telemetry) keep
/// their own private copies deliberately — they are a shipped, live-gated wire contract, and
/// tightening their header handling is a behaviour change to three running endpoints, not a
/// refactor.
/// </para>
/// </remarks>
internal static class GrpcRequestHeaders
{
/// <summary>The bearer-token header, lowercased as gRPC puts it on the wire.</summary>
internal const string Authorization = "authorization";
/// <summary>Scheme prefix expected on the <see cref="Authorization"/> value.</summary>
internal const string BearerPrefix = "Bearer ";
/// <summary>Gets the first value carried under <paramref name="name"/>, or null if absent.</summary>
/// <param name="headers">The call's request metadata.</param>
/// <param name="name">The header name, matched case-insensitively.</param>
/// <returns>The first matching value, or <c>null</c>.</returns>
internal static string? First(Metadata headers, string name)
{
// gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
// Metadata in a test behaves the same as a real request.
foreach (var entry in headers)
{
if (string.Equals(entry.Key, name, StringComparison.OrdinalIgnoreCase))
return entry.Value;
}
return null;
}
/// <summary>
/// Gets the token from the first <c>authorization</c> header, or null when that header is
/// absent or does not carry the <c>Bearer</c> scheme.
/// </summary>
/// <param name="headers">The call's request metadata.</param>
/// <returns>The bearer token, or <c>null</c>.</returns>
internal static string? BearerToken(Metadata headers)
{
var value = First(headers, Authorization);
return value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase)
? value[BearerPrefix.Length..]
: null;
}
}
src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshCommandAuthInterceptor.cs
+7 -20
View File
@@ -43,6 +43,12 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// unavoidably observable and are not sensitive.
/// </para>
/// <para>
/// Header reads go through <see cref="GrpcRequestHeaders"/>, shared with
/// <see cref="CentralCommandAuthInterceptor"/> — the two are clones, and header parsing is
/// the one place a clone pair can diverge into two different answers about which of several
/// <c>authorization</c> headers counts.
/// </para>
/// <para>
/// <b>Exactly one public constructor.</b> <c>Grpc.AspNetCore</c> silently stops invoking an
/// interceptor that has more than one public ctor — see
/// <c>MeshCommandAuthInterceptorTests.HasExactlyOnePublicConstructor</c>.
@@ -51,8 +57,6 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
public sealed class MeshCommandAuthInterceptor : Interceptor
{
private const string ServicePrefix = "/mesh_transport.v1.NodeCommandService/";
private const string AuthorizationHeader = "authorization";
private const string BearerPrefix = "Bearer ";
private readonly IOptions<MeshTransportOptions> _options;
private readonly ILogger<MeshCommandAuthInterceptor> _logger;
@@ -137,7 +141,7 @@ public sealed class MeshCommandAuthInterceptor : Interceptor
"Node command service is not accepting calls: no pre-shared key is configured on this node."));
}
var presented = ExtractBearerToken(context.RequestHeaders);
var presented = GrpcRequestHeaders.BearerToken(context.RequestHeaders);
if (presented is null || !FixedTimeEquals(presented, expected))
{
_logger.LogWarning(
@@ -150,23 +154,6 @@ public sealed class MeshCommandAuthInterceptor : Interceptor
}
}
private static string? ExtractBearerToken(Metadata headers)
{
// gRPC lowercases header keys on the wire; compare case-insensitively anyway so a hand-built
// Metadata in a test behaves the same as a real request.
foreach (var entry in headers)
{
if (!string.Equals(entry.Key, AuthorizationHeader, StringComparison.OrdinalIgnoreCase))
continue;
var value = entry.Value;
if (value is not null && value.StartsWith(BearerPrefix, StringComparison.OrdinalIgnoreCase))
return value[BearerPrefix.Length..];
}
return null;
}
private static bool FixedTimeEquals(string presented, string expected)
=> CryptographicOperations.FixedTimeEquals(
Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected));