feat(secrets): Grpc hub server — SecretsHubService + fail-closed bearer interceptor + DI

The server half of the central-hosted pull-only secrets hub. SecretsHubService
adapts the node's own ISecretStore onto secrets_hub.v1 using exactly two read
members (GetManifestAsync, GetAsync) — the proto exposes no write RPC, so a
follower has no path to put a row into central whatever its credentials allow.

GetSecrets omits a name the hub does not hold and skips one that will not parse
as a SecretName; neither is an error, because both describe a follower acting on
a manifest that was true a moment ago. Failing the call would stall the sweep and
let one bad name deny the follower every other row in the batch. Names are
re-validated server-side (the wire type is a plain string) and de-duplicated on
the normalized value.

DomainToWire mirrors the Akka replicator's SecretRowDto discipline exactly:
round-trip "O" timestamps so a DateTimeOffset's offset and ticks survive the
last-writer-wins equality compare, the content type as its enum MEMBER NAME so an
unknown value is rejected rather than decoded to the zero member, and nullable
fields left absent rather than written as "" so null and empty stay distinct.

SecretsHubAuthInterceptor fails closed: with no token configured every call is
denied and one warning names the exact configuration key — never a token value,
presented or configured, at any level. Denial is uniform (one Status, one detail,
no "why") so a caller gets no oracle. Comparison is constant-time over UTF-8
bytes. All four handler shapes are gated, not just unary, so a future RPC cannot
arrive unauthenticated by omission; more than one Authorization header is refused
as a smuggling shape. The type keeps EXACTLY ONE public constructor — Grpc's
activator silently stops invoking an interceptor it cannot construct
unambiguously, and the failure mode is an auth gate that quietly does not run — so
a test pins the count.

Tests: 50, red-first (compile-red, then a fail-closed mutation verified 5 of them
bite). Full Secrets suite 328 passed / 0 failed / 15 live-gated skips, 0 new
warnings.

Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
This commit is contained in:
Joseph Doherty
2026-08-07 06:09:23 -04:00
parent cc95590196
commit 7fc77f2790
8 changed files with 1290 additions and 0 deletions
@@ -0,0 +1,87 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Routing;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace ZB.MOM.WW.Secrets.Replicator.Grpc.DependencyInjection;
/// <summary>
/// Dependency-injection and endpoint wiring for hosting the central secrets hub.
/// </summary>
/// <remarks>
/// <para>
/// Register this on the <b>central</b> node only. A site node registers the client half instead —
/// the two are separate registrations on purpose, so "central hosts, sites follow" is visible in
/// each role's composition root rather than inferred from configuration.
/// </para>
/// <para>
/// <b>Every node must be configured with the same KEK.</b> The hub serves ciphertext, so a follower
/// whose master key differs cannot decrypt what it pulls; it fails closed on resolve with a
/// <c>kek_id</c> mismatch, which reads like data corruption but is a deployment error.
/// </para>
/// </remarks>
public static class SecretsGrpcHubServerExtensions
{
/// <summary>
/// Binds and validates <see cref="SecretsGrpcHubServerOptions"/>, registers the fail-closed
/// bearer interceptor, and attaches it to <see cref="SecretsHubService"/>.
/// </summary>
/// <remarks>
/// Validation runs here, eagerly, as well as through the options pipeline: a hub with no bearer
/// token should refuse to start rather than start and refuse every follower. The interceptor's
/// deny-all is the second layer, for the case where a host constructs the options some other way.
/// The service requires an <c>ISecretStore</c> in the container — register the store first.
/// </remarks>
/// <param name="services">The service collection to add to.</param>
/// <param name="configuration">The application configuration.</param>
/// <param name="sectionPath">Configuration section holding the hub options.</param>
/// <returns>The same <paramref name="services"/> instance, for chaining.</returns>
/// <exception cref="InvalidOperationException">No bearer token is configured.</exception>
public static IServiceCollection AddZbSecretsGrpcHub(
this IServiceCollection services,
IConfiguration configuration,
string sectionPath = "Secrets:GrpcHub")
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
ArgumentException.ThrowIfNullOrWhiteSpace(sectionPath);
IConfigurationSection section = configuration.GetSection(sectionPath);
services.Configure<SecretsGrpcHubServerOptions>(section);
// SectionPath is internal, so binding cannot set it; stamping it here is what lets the
// startup failure and the runtime warning both name the key this host actually reads.
services.PostConfigure<SecretsGrpcHubServerOptions>(options => options.SectionPath = sectionPath);
var eager = new SecretsGrpcHubServerOptions { SectionPath = sectionPath };
section.Bind(eager);
eager.SectionPath = sectionPath;
eager.Validate();
// Singleton so the "unconfigured" warning is emitted once per process rather than once per
// interceptor instance; the gRPC interceptor activator prefers a registered instance.
services.TryAddSingleton<SecretsHubAuthInterceptor>();
// AddGrpc is safe to call again if the host already called it — it registers with TryAdd.
services.AddGrpc()
.AddServiceOptions<SecretsHubService>(
options => options.Interceptors.Add<SecretsHubAuthInterceptor>());
return services;
}
/// <summary>
/// Maps the secrets hub gRPC endpoint. The host must have set up routing;
/// <see cref="AddZbSecretsGrpcHub"/> supplies the gRPC services and the auth interceptor.
/// </summary>
/// <param name="endpoints">The endpoint route builder.</param>
/// <returns>The same <paramref name="endpoints"/> instance, for chaining.</returns>
public static IEndpointRouteBuilder MapZbSecretsHub(this IEndpointRouteBuilder endpoints)
{
ArgumentNullException.ThrowIfNull(endpoints);
endpoints.MapGrpcService<SecretsHubService>();
return endpoints;
}
}
@@ -0,0 +1,58 @@
namespace ZB.MOM.WW.Secrets.Replicator.Grpc;
/// <summary>
/// Configuration for the central-hosted secrets hub. Bound from a configuration section by
/// <c>AddZbSecretsGrpcHub</c>.
/// </summary>
/// <remarks>
/// <para>
/// The hub serves ciphertext to followers that authenticate with one shared bearer token. That
/// token is the whole of the hub's authorization: there is no per-follower identity, because a
/// follower's authority is uniform (read the manifest, read rows) and the pull-only contract gives
/// it nothing else to be authorized for.
/// </para>
/// <para>
/// It cannot itself be supplied as a <c>${secret:}</c> reference — resolving that reference is what
/// the hub exists to make possible — so it comes from appsettings or the environment, the same way
/// the family's mesh pre-shared keys do.
/// </para>
/// </remarks>
public sealed class SecretsGrpcHubServerOptions
{
/// <summary>The configuration section the hub is bound from unless a host overrides it.</summary>
internal const string DefaultSectionPath = "Secrets:GrpcHub";
/// <summary>
/// The shared bearer token a follower must present. Required — an unset token is not "open
/// access", it is a hub that refuses every call (see <see cref="SecretsHubAuthInterceptor"/>).
/// </summary>
public string BearerToken { get; set; } = string.Empty;
/// <summary>
/// The configuration section this instance was bound from, used so failure messages name the
/// key an operator actually has to set.
/// </summary>
/// <remarks>
/// Deliberately <see langword="internal"/>: configuration binding only writes public settable
/// properties, so a stray <c>SectionPath</c> key in an appsettings file cannot make the hub
/// misreport which key it wants. The registration extension sets it.
/// </remarks>
internal string SectionPath { get; set; } = DefaultSectionPath;
/// <summary>
/// Throws if the options are unusable. Called by the registration extension so a hub with no
/// token fails at startup, rather than starting and rejecting every follower at runtime.
/// </summary>
/// <exception cref="InvalidOperationException">The bearer token is missing or blank.</exception>
public void Validate()
{
if (string.IsNullOrWhiteSpace(BearerToken))
{
// Names the key, never the value — there is no value here, and there never will be one
// in an error message from this type.
throw new InvalidOperationException(
$"The gRPC secrets hub requires a shared bearer token ({SectionPath}:BearerToken). " +
"Until it is set the hub refuses every call.");
}
}
}
@@ -0,0 +1,200 @@
using System.Security.Cryptography;
using System.Text;
using Grpc.Core;
using Grpc.Core.Interceptors;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ZB.MOM.WW.Secrets.Replicator.Grpc;
/// <summary>
/// Gates every call to the secrets hub on a shared bearer token presented as
/// <c>Authorization: Bearer &lt;token&gt;</c>.
/// </summary>
/// <remarks>
/// <para>
/// <b>Fail closed.</b> If no token is configured, every call is denied — an unconfigured hub is a
/// hub that serves nothing, never a hub that serves everyone. The refusal is paired with exactly one
/// warning naming the configuration key an operator has to set.
/// </para>
/// <para>
/// <b>Uniform denial.</b> Every rejection — absent header, wrong scheme, wrong token, unconfigured
/// hub — throws the identical <see cref="StatusCode.Unauthenticated"/> status with the identical
/// detail. A caller learns that it was refused and nothing else; distinguishing "no token" from
/// "wrong token" would hand an attacker a free oracle.
/// </para>
/// <para>
/// <b>No token material is ever logged</b>, presented or configured, at any level.
/// </para>
/// </remarks>
public sealed class SecretsHubAuthInterceptor : Interceptor
{
/// <summary>gRPC normalizes metadata keys to lower case, so the lookup key is lower case too.</summary>
private const string AuthorizationHeaderName = "authorization";
/// <summary>
/// Matched ordinally, including case. RFC 7235 makes the scheme case-insensitive, but this is a
/// machine-to-machine channel with one client implementation, so the stricter rule costs nothing
/// and removes a class of near-miss that would otherwise have to be reasoned about.
/// </summary>
private const string BearerScheme = "Bearer ";
/// <summary>The one detail every denial carries. It says nothing about why.</summary>
private const string DenialDetail = "Unauthenticated.";
private readonly byte[] _expectedTokenUtf8;
private readonly string _bearerTokenConfigurationKey;
private readonly ILogger<SecretsHubAuthInterceptor> _logger;
private int _unconfiguredWarningEmitted;
/// <summary>
/// Creates the interceptor over the hub's configured token.
/// </summary>
/// <remarks>
/// <b>This type must keep EXACTLY ONE public constructor.</b> Grpc.AspNetCore activates an
/// interceptor through <c>ActivatorUtilities</c>, which cannot choose between overloads — an
/// interceptor with more than one public constructor silently stops being invoked, and the
/// failure mode is an authentication gate that quietly does not run. A test pins the count.
/// </remarks>
/// <param name="options">The hub options carrying the shared bearer token.</param>
/// <param name="logger">Diagnostics sink. Never receives token material.</param>
public SecretsHubAuthInterceptor(
IOptions<SecretsGrpcHubServerOptions> options,
ILogger<SecretsHubAuthInterceptor> logger)
{
ArgumentNullException.ThrowIfNull(options);
SecretsGrpcHubServerOptions value = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_bearerTokenConfigurationKey = $"{value.SectionPath}:BearerToken";
_expectedTokenUtf8 = string.IsNullOrWhiteSpace(value.BearerToken)
? []
: Encoding.UTF8.GetBytes(value.BearerToken);
}
/// <inheritdoc/>
public override Task<TResponse> UnaryServerHandler<TRequest, TResponse>(
TRequest request,
ServerCallContext context,
UnaryServerMethod<TRequest, TResponse> continuation)
{
ArgumentNullException.ThrowIfNull(continuation);
Authenticate(context);
return continuation(request, context);
}
/// <inheritdoc/>
public override Task<TResponse> ClientStreamingServerHandler<TRequest, TResponse>(
IAsyncStreamReader<TRequest> requestStream,
ServerCallContext context,
ClientStreamingServerMethod<TRequest, TResponse> continuation)
{
ArgumentNullException.ThrowIfNull(continuation);
Authenticate(context);
return continuation(requestStream, context);
}
/// <inheritdoc/>
public override Task ServerStreamingServerHandler<TRequest, TResponse>(
TRequest request,
IServerStreamWriter<TResponse> responseStream,
ServerCallContext context,
ServerStreamingServerMethod<TRequest, TResponse> continuation)
{
ArgumentNullException.ThrowIfNull(continuation);
Authenticate(context);
return continuation(request, responseStream, context);
}
/// <inheritdoc/>
public override Task DuplexStreamingServerHandler<TRequest, TResponse>(
IAsyncStreamReader<TRequest> requestStream,
IServerStreamWriter<TResponse> responseStream,
ServerCallContext context,
DuplexStreamingServerMethod<TRequest, TResponse> continuation)
{
ArgumentNullException.ThrowIfNull(continuation);
Authenticate(context);
return continuation(requestStream, responseStream, context);
}
/// <summary>
/// Admits the call or throws the one denial. The streaming overloads are gated as well as the
/// unary ones even though the contract is unary-only today, so a future RPC of any shape cannot
/// arrive unauthenticated by omission.
/// </summary>
private void Authenticate(ServerCallContext context)
{
ArgumentNullException.ThrowIfNull(context);
if (_expectedTokenUtf8.Length == 0)
{
WarnUnconfiguredOnce();
throw Denied();
}
Metadata headers = context.RequestHeaders;
// More than one Authorization header is a smuggling shape, not a legitimate request: taking
// "the first one" would let a caller present a second value that some other hop honours.
int authorizationHeaderCount = 0;
string? presented = null;
foreach (Metadata.Entry entry in headers)
{
if (!string.Equals(entry.Key, AuthorizationHeaderName, StringComparison.Ordinal))
{
continue;
}
authorizationHeaderCount++;
presented ??= entry.Value;
}
if (authorizationHeaderCount != 1 || presented is null)
{
throw Denied();
}
if (!presented.StartsWith(BearerScheme, StringComparison.Ordinal))
{
throw Denied();
}
// FixedTimeEquals compares in time independent of the contents, and returns false for a
// length mismatch without comparing at all — so the only thing a caller can time is the
// length of its own token.
byte[] presentedTokenUtf8 = Encoding.UTF8.GetBytes(presented[BearerScheme.Length..]);
if (!CryptographicOperations.FixedTimeEquals(presentedTokenUtf8, _expectedTokenUtf8))
{
throw Denied();
}
}
/// <summary>
/// Builds the single denial every failure path throws. A fresh exception each time (the stack
/// trace is per-call) carrying an identical status.
/// </summary>
private static RpcException Denied() => new(new Status(StatusCode.Unauthenticated, DenialDetail));
/// <summary>
/// Warns once — not once per call — that the hub cannot serve, naming the exact key to set. A
/// per-call warning would bury the one line that matters under the flood it describes.
/// </summary>
private void WarnUnconfiguredOnce()
{
if (Interlocked.Exchange(ref _unconfiguredWarningEmitted, 1) != 0)
{
return;
}
_logger.LogWarning(
"The gRPC secrets hub has no bearer token configured ({BearerTokenConfigurationKey}); " +
"every call is being refused until it is set.",
_bearerTokenConfigurationKey);
}
}
@@ -0,0 +1,131 @@
using Grpc.Core;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.Grpc.V1;
using ZB.MOM.WW.Secrets.Replicator.Grpc.Wire;
namespace ZB.MOM.WW.Secrets.Replicator.Grpc;
/// <summary>
/// The central-hosted secrets hub: adapts the node's own <see cref="ISecretStore"/> onto the
/// <c>secrets_hub.v1</c> contract so site nodes can pull from it.
/// </summary>
/// <remarks>
/// <para>
/// <b>Read-only by construction.</b> It touches exactly two store members —
/// <see cref="ISecretStore.GetManifestAsync"/> and <see cref="ISecretStore.GetAsync"/> — mirroring
/// the read half of <c>ISecretReplicationHub</c>. The proto exposes no write RPC, so there is no
/// path by which a follower can put a row into central, whatever its credentials allow.
/// </para>
/// <para>
/// Everything it returns is ciphertext or metadata about ciphertext. The KEK never travels: a row
/// carries the KEK's identifier and the DEK wrapped by it, and every node in a replicating
/// deployment is configured with the same KEK out of band.
/// </para>
/// </remarks>
public sealed class SecretsHubService : SecretsHub.SecretsHubBase
{
private readonly ISecretStore _store;
private readonly ILogger<SecretsHubService> _logger;
/// <summary>Creates the hub service over the node's secret store.</summary>
/// <param name="store">The store to serve. Only its read members are used.</param>
/// <param name="logger">Diagnostics sink.</param>
public SecretsHubService(ISecretStore store, ILogger<SecretsHubService> logger)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>
/// Returns the hub's full manifest, <b>tombstones included</b> — a follower converges on a
/// delete by seeing the tombstone, so omitting them would make deletes unpropagatable.
/// </summary>
/// <param name="request">The (empty) manifest request.</param>
/// <param name="context">The call context.</param>
/// <returns>Every manifest entry the store holds.</returns>
public override async Task<GetManifestResponse> GetManifest(
GetManifestRequest request, ServerCallContext context)
{
ArgumentNullException.ThrowIfNull(context);
IReadOnlyList<SecretManifestEntry> entries =
await _store.GetManifestAsync(context.CancellationToken).ConfigureAwait(false);
var response = new GetManifestResponse();
foreach (SecretManifestEntry entry in entries)
{
response.Entries.Add(DomainToWire.ToWire(entry));
}
_logger.LogDebug("Secrets hub served a manifest of {EntryCount} entries.", response.Entries.Count);
return response;
}
/// <summary>
/// Fetches the named encrypted rows in bulk.
/// </summary>
/// <remarks>
/// A requested name the hub does not hold is <b>omitted</b> from the response, and an unparseable
/// name is <b>skipped</b> — neither is an error. Both describe the same ordinary situation: the
/// follower is acting on a manifest that was true a moment ago. Failing the whole call would turn
/// a routine race into a stalled sweep, and would let one bad name deny the follower every other
/// row in the batch.
/// </remarks>
/// <param name="request">The names to fetch.</param>
/// <param name="context">The call context.</param>
/// <returns>One row per distinct name the hub holds.</returns>
public override async Task<GetSecretsResponse> GetSecrets(
GetSecretsRequest request, ServerCallContext context)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(context);
var response = new GetSecretsResponse();
var seen = new HashSet<string>(StringComparer.Ordinal);
int skipped = 0;
foreach (string requested in request.Names)
{
SecretName name;
try
{
// Re-validated server-side: the wire type is a plain string, so this is where a
// path-traversing or otherwise malformed name is stopped — before it reaches a store
// that may treat the name as a path segment.
name = new SecretName(requested);
}
catch (ArgumentException)
{
skipped++;
continue;
}
// Normalization happens in the SecretName constructor, so de-duplication has to key off
// the normalized value — otherwise "Db/Central" and "db/central" would each cost a read
// and each return the same row.
if (!seen.Add(name.Value))
{
continue;
}
StoredSecret? row = await _store.GetAsync(name, context.CancellationToken).ConfigureAwait(false);
if (row is null)
{
continue;
}
response.Secrets.Add(DomainToWire.ToWire(row));
}
if (skipped > 0)
{
// The count, never the values: the names are caller-supplied text and this is a log.
_logger.LogWarning(
"Secrets hub skipped {SkippedCount} unparseable name(s) in a GetSecrets request.", skipped);
}
_logger.LogDebug("Secrets hub served {RowCount} row(s).", response.Secrets.Count);
return response;
}
}
@@ -0,0 +1,98 @@
using System.Globalization;
using Google.Protobuf;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replicator.Grpc.V1;
namespace ZB.MOM.WW.Secrets.Replicator.Grpc.Wire;
/// <summary>
/// Projects the domain types onto the <c>secrets_hub.v1</c> wire messages — the hub's outbound
/// half. The inbound half (wire → domain) lives in <c>WireToDomain</c>.
/// </summary>
/// <remarks>
/// <para>
/// The mapping is a deliberate mirror of the Akka replicator's <c>SecretRowDto.FromStoredSecret</c>
/// and follows the same two rules, for the same reasons. Timestamps use the round-trip <c>"O"</c>
/// format so a <see cref="DateTimeOffset"/>'s offset and 100 ns ticks survive intact — last-writer-wins
/// compares these for equality, so "close" is wrong. Content type travels as the enum MEMBER NAME,
/// so an unrecognized value is rejected at the far boundary instead of silently decoding to the
/// zero member.
/// </para>
/// <para>
/// Nullable .NET fields map onto proto3 explicit-presence fields and are left <em>absent</em> when
/// null rather than written as <c>""</c>, so null and empty string stay distinguishable end to end.
/// </para>
/// </remarks>
internal static class DomainToWire
{
/// <summary>The round-trip ("O") format specifier used for every timestamp on this wire.</summary>
private const string RoundTripFormat = "O";
/// <summary>Projects a manifest entry onto its wire message.</summary>
/// <param name="entry">The manifest entry to project.</param>
/// <returns>The wire message.</returns>
public static ManifestEntry ToWire(SecretManifestEntry entry)
{
ArgumentNullException.ThrowIfNull(entry);
return new ManifestEntry
{
Name = entry.Name.Value,
Revision = entry.Revision,
UpdatedUtc = Format(entry.UpdatedUtc),
IsDeleted = entry.IsDeleted,
};
}
/// <summary>Projects an encrypted row onto its wire message.</summary>
/// <param name="row">The encrypted row to project.</param>
/// <returns>The wire message — ciphertext and metadata about ciphertext, never key material.</returns>
public static SecretRow ToWire(StoredSecret row)
{
ArgumentNullException.ThrowIfNull(row);
var wire = new SecretRow
{
Name = row.Name.Value,
ContentType = row.ContentType.ToString(),
Ciphertext = ByteString.CopyFrom(row.Ciphertext),
Nonce = ByteString.CopyFrom(row.Nonce),
Tag = ByteString.CopyFrom(row.Tag),
WrappedDek = ByteString.CopyFrom(row.WrappedDek),
WrapNonce = ByteString.CopyFrom(row.WrapNonce),
WrapTag = ByteString.CopyFrom(row.WrapTag),
KekId = row.KekId,
Revision = row.Revision,
IsDeleted = row.IsDeleted,
CreatedUtc = Format(row.CreatedUtc),
UpdatedUtc = Format(row.UpdatedUtc),
};
// Assigned only when present: a proto3 optional string field rejects null, and writing ""
// instead would erase the distinction between "no description" and "an empty one".
if (row.Description is not null)
{
wire.Description = row.Description;
}
if (row.DeletedUtc is { } deleted)
{
wire.DeletedUtc = Format(deleted);
}
if (row.CreatedBy is not null)
{
wire.CreatedBy = row.CreatedBy;
}
if (row.UpdatedBy is not null)
{
wire.UpdatedBy = row.UpdatedBy;
}
return wire;
}
private static string Format(DateTimeOffset value) =>
value.ToString(RoundTripFormat, CultureInfo.InvariantCulture);
}