fix(host): refuse unknown API-key ids in memory and budget store misses; document the mesh gate's boot window
BoundedApiKeyStore's negative cache was keyed by key id, so a flood of DISTINCT well-formed
otua_<random>_x credentials missed it every time and cost one SQL query per attempt under the
SDK's SessionManager-wide lock, no credential needed (R03:3.3).
- IApiKeyStore.ListKeyIdsAsync (default null = unsupported; ConfigDbApiKeyStore lists every
ApiKeyId): the bounded store keeps a known-id set refreshed every StoreCacheSeconds
(single-flight, failed refresh keeps the previous set) and refuses an unknown id without a store
read ({outcome=unknown_id}).
- A process-wide miss budget (token bucket, Security:ApiKeys:MissBurst=100 /
MissRefillPerSecond=20, ApiKeyStoreGuardOptions) bounds lookups that reach the store and miss
while no set is available; a hit refunds its token. Per-endpoint accounting needs the
OpcUaServer impersonation seam to carry the client address — follow-up.
- docs/security.md: the new knobs, and the known-cluster gate's documented fail-open window at
central boot (R03:4.2) plus the async PSK refresh (R03:3.4).
Refs #733
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration
|
||||
sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
|
||||
sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
|
||||
sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
|
||||
sp.GetRequiredService<TimeProvider>()));
|
||||
sp.GetRequiredService<TimeProvider>(),
|
||||
// R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as
|
||||
// ApiKeyOptions, Host-owned knobs.
|
||||
ApiKeyStoreGuardOptions.Bind(sp.GetService<IConfiguration>())));
|
||||
services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
|
||||
|
||||
// Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// The pre-authentication guards <see cref="BoundedApiKeyStore"/> puts in front of the ConfigDb key
|
||||
/// lookup (R03:3.3). Bound from the same <c>Security:ApiKeys</c> section as <c>ApiKeyOptions</c> but
|
||||
/// owned by the Host, where the store lives.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why.</b> The negative cache was keyed by key id, so a flood of DISTINCT well-formed
|
||||
/// <c>otua_<random>_x</c> credentials missed it every time and cost one indexed SQL query per
|
||||
/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close
|
||||
/// that: a <b>known-id set</b> (every <c>ApiKeyId</c> the ConfigDb holds, refreshed every
|
||||
/// <c>StoreCacheSeconds</c>) rejects an unknown id in memory, and a <b>miss budget</b> (token
|
||||
/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the
|
||||
/// set is unavailable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback
|
||||
/// (the <c>IOpcUaUserAuthenticator</c> seam) does not carry the client's address. Per-endpoint
|
||||
/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ApiKeyStoreGuardOptions
|
||||
{
|
||||
/// <summary>The configuration section the knobs live in (shared with <c>ApiKeyOptions</c>).</summary>
|
||||
public const string SectionName = "Security:ApiKeys";
|
||||
|
||||
/// <summary>
|
||||
/// Whether the known-id set is consulted. <see langword="false"/> restores a store read per unknown
|
||||
/// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact
|
||||
/// store is never wrapped).
|
||||
/// </summary>
|
||||
public bool KnownIdSet { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// How many store MISSES (lookups that reach the inner store and find no key) may burst before
|
||||
/// further lookups for unknown ids are refused without a store read. <c>0</c> disables the budget.
|
||||
/// </summary>
|
||||
public int MissBurst { get; set; } = 100;
|
||||
|
||||
/// <summary>How many miss tokens are restored per second, up to <see cref="MissBurst"/>.</summary>
|
||||
public double MissRefillPerSecond { get; set; } = 20;
|
||||
|
||||
/// <summary>Binds the section, with defaults for anything absent.</summary>
|
||||
/// <param name="configuration">The app configuration; null yields the defaults.</param>
|
||||
/// <returns>The bound options.</returns>
|
||||
public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration)
|
||||
{
|
||||
var options = new ApiKeyStoreGuardOptions();
|
||||
configuration?.GetSection(SectionName).Bind(options);
|
||||
return options;
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,21 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
|
||||
new(StringComparer.Ordinal);
|
||||
private readonly ApiKeyStoreGuardOptions _guard;
|
||||
|
||||
// R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds
|
||||
// (single-flight; a refresh that fails or times out keeps the previous set). Null until the first
|
||||
// successful load, or when the inner store cannot list ids — then the per-id path runs as before.
|
||||
private volatile IReadOnlySet<string>? _knownIds;
|
||||
private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue;
|
||||
private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1);
|
||||
|
||||
// R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards
|
||||
// the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the
|
||||
// configured rate instead of one SQL round trip per attempt under the SessionManager lock.
|
||||
private readonly object _bucketGate = new();
|
||||
private double _missTokens;
|
||||
private DateTimeOffset _missRefilledUtc;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
|
||||
/// <param name="inner">The store being bounded.</param>
|
||||
@@ -66,11 +81,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
/// <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
|
||||
/// detail.
|
||||
/// </param>
|
||||
/// <param name="guard">The pre-authentication guards (R03:3.3); defaults when omitted.</param>
|
||||
public BoundedApiKeyStore(
|
||||
IApiKeyStore inner,
|
||||
IOptions<ApiKeyOptions> options,
|
||||
ILogger<BoundedApiKeyStore> log,
|
||||
TimeProvider time)
|
||||
TimeProvider time,
|
||||
ApiKeyStoreGuardOptions? guard = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
@@ -81,8 +98,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
_options = options;
|
||||
_log = log;
|
||||
_time = time;
|
||||
_guard = guard ?? new ApiKeyStoreGuardOptions();
|
||||
_missTokens = Math.Max(0, _guard.MissBurst);
|
||||
_missRefilledUtc = time.GetUtcNow();
|
||||
}
|
||||
|
||||
/// <summary>The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards.</summary>
|
||||
public IReadOnlySet<string>? KnownKeyIds => _knownIds;
|
||||
|
||||
/// <summary>The store being bounded.</summary>
|
||||
/// <remarks>
|
||||
/// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
|
||||
@@ -115,12 +138,104 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
return cached;
|
||||
}
|
||||
|
||||
// R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked.
|
||||
var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false);
|
||||
if (known is not null && !known.Contains(apiKeyId))
|
||||
{
|
||||
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "unknown_id"));
|
||||
return null;
|
||||
}
|
||||
|
||||
// R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known
|
||||
// to exist (a known id is a real key being verified, never a guess).
|
||||
if (known is null && !TryTakeMissToken())
|
||||
{
|
||||
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "miss_budget"));
|
||||
throw new InvalidOperationException(
|
||||
"ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / "
|
||||
+ "MissRefillPerSecond); refusing this lookup without a store read.");
|
||||
}
|
||||
|
||||
var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
|
||||
.ConfigureAwait(false);
|
||||
if (fresh is not null && known is null) RefundMissToken();
|
||||
if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
|
||||
return fresh;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The known-id set, refreshed when older than <paramref name="ttl"/>. Single-flight: concurrent
|
||||
/// callers during a refresh use the previous set (or, before the first load, the per-id path). A
|
||||
/// refresh that fails or exceeds the deadline logs and keeps the previous set.
|
||||
/// </summary>
|
||||
/// <param name="ttl">The refresh interval (the cache TTL); non-positive disables the set.</param>
|
||||
/// <param name="ct">The caller's token.</param>
|
||||
/// <returns>The set, or null.</returns>
|
||||
private async Task<IReadOnlySet<string>?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct)
|
||||
{
|
||||
if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null;
|
||||
|
||||
var now = _time.GetUtcNow();
|
||||
var current = _knownIds;
|
||||
if (current is not null && now - _knownIdsStampedUtc < ttl) return current;
|
||||
|
||||
if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current;
|
||||
try
|
||||
{
|
||||
// Re-check under the gate: another caller may have refreshed while this one waited.
|
||||
if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed;
|
||||
var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false);
|
||||
if (ids is null)
|
||||
{
|
||||
// Unsupported by the inner store: say so once and stop asking.
|
||||
_guard.KnownIdSet = false;
|
||||
return null;
|
||||
}
|
||||
_knownIds = ids;
|
||||
_knownIdsStampedUtc = _time.GetUtcNow();
|
||||
return ids;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning(ex,
|
||||
"ApiKeyStore: refreshing the known key-id set failed; {Mode}",
|
||||
current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force");
|
||||
return current;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_knownIdsRefresh.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled.</summary>
|
||||
/// <returns><see langword="false"/> when the budget is exhausted.</returns>
|
||||
private bool TryTakeMissToken()
|
||||
{
|
||||
var burst = _guard.MissBurst;
|
||||
if (burst <= 0) return true;
|
||||
lock (_bucketGate)
|
||||
{
|
||||
var now = _time.GetUtcNow();
|
||||
var elapsed = (now - _missRefilledUtc).TotalSeconds;
|
||||
if (elapsed > 0)
|
||||
{
|
||||
_missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond));
|
||||
_missRefilledUtc = now;
|
||||
}
|
||||
if (_missTokens < 1) return false;
|
||||
_missTokens -= 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A lookup that reached the store and HIT was not a miss: its token goes back.</summary>
|
||||
private void RefundMissToken()
|
||||
{
|
||||
if (_guard.MissBurst <= 0) return;
|
||||
lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1);
|
||||
}
|
||||
|
||||
// Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
|
||||
|
||||
@@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore
|
||||
return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
|
||||
{
|
||||
await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false);
|
||||
// One projection over the whole (small) table — the unique index on ApiKeyId makes this an index
|
||||
// scan — so an unknown id can be refused in memory instead of costing a query per attempt.
|
||||
var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false);
|
||||
return ids.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>Loads a principal's grants, confined to this node's cluster.</summary>
|
||||
/// <remarks>
|
||||
/// <b>The full grant set, not just the <c>Cluster</c>-scoped rows</b>. The old
|
||||
|
||||
@@ -35,6 +35,16 @@ public interface IApiKeyStore
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The matching ACL rows; empty when none.</returns>
|
||||
Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// Every <c>ApiKeyId</c> the store holds (enabled or not), for the pre-authentication known-id set
|
||||
/// (R03:3.3); <see langword="null"/> when the store cannot cheaply list them, in which case callers
|
||||
/// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry
|
||||
/// and the secret are still decided by <see cref="FindAsync"/>.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The id set, or null when unsupported.</returns>
|
||||
Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult<IReadOnlySet<string>?>(null);
|
||||
}
|
||||
|
||||
/// <summary>An API key resolved for authentication, with the ACL grants naming its principal.</summary>
|
||||
|
||||
Reference in New Issue
Block a user