Merge branch 'master' into fix/r3-02-stale-good-family

# Conflicts:
#	src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
#	tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs
This commit is contained in:
Joseph Doherty
2026-08-20 11:03:02 -04:00
46 changed files with 5411 additions and 208 deletions
@@ -0,0 +1,111 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// Composition of the OPC UA data-plane authentication + authorization seams (#520), extracted from
/// <c>Program.cs</c> so the role-conditional branches are reachable from a test.
/// </summary>
/// <remarks>
/// A store decorator registered in one branch of a role conditional is exactly the
/// "built but never wired" shape, and <c>Program.cs</c>'s top-level statements cannot be driven by a
/// test — so the wiring moved here rather than the guard being written against a comment. The
/// branches are unchanged from the ones that lived inline; see <c>ApiKeyStoreRegistrationTests</c>.
/// </remarks>
public static class DataPlaneAuthRegistration
{
/// <summary>
/// Registers the shared clock, the node-scoped ACL gate and the role-appropriate API-key store +
/// user authenticator.
/// </summary>
/// <param name="services">The service collection.</param>
/// <param name="hasAdmin">Whether this node carries the <c>admin</c> role (and therefore a ConfigDb).</param>
/// <param name="hasDriver">Whether this node carries the <c>driver</c> role.</param>
/// <returns>The same collection, for chaining.</returns>
public static IServiceCollection AddOtOpcUaDataPlaneAuth(
this IServiceCollection services, bool hasAdmin, bool hasDriver)
{
ArgumentNullException.ThrowIfNull(services);
// ONE clock for the whole authorization surface. The authorizer, the key-store bound and the
// revalidation timer all read time, and DataPlaneAuthorizer already documents what a second,
// diverging clock costs: every request denied, with a verdict indistinguishable from "no grant
// matched". TryAdd so a test (or a future host) can substitute one before this runs.
services.TryAddSingleton(TimeProvider.System);
// #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because
// OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves
// constructors by whether every parameter is registered, not by whether the parameter type is nullable.
// That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations below
// both carry a comment about; an absent registration is a node that will not start.
services.AddSingleton<IPermissionEvaluator>(sp =>
new TriePermissionEvaluator(sp.GetRequiredService<PermissionTrieCache>()));
services.AddSingleton(sp => new DataPlaneAuthorizer(
sp.GetRequiredService<IDataPlaneAuthoritySource>(),
sp.GetRequiredService<IPermissionEvaluator>(),
sp.GetRequiredService<IOptions<AclOptions>>().Value.Enabled,
sp.GetRequiredService<ILoggerFactory>().CreateLogger<DataPlaneAuthorizer>(),
sp.GetRequiredService<TimeProvider>()));
if (hasAdmin)
{
// A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect
// on the very next activation rather than on the next deployment. Preferred wherever it is
// available, including on a fused admin+driver node.
// The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds
// the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which
// ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration),
// NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services
// rule is what keeps the redundancy-singleton registration in Program.cs from recursing into
// ActorSystem construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter.
//
// arch-review 03/S18 — and it is WRAPPED. The lookup runs inline in the SDK's synchronous
// impersonation callback, under a SessionManager-wide lock, with CancellationToken.None: an
// unanswered ConfigDb query therefore stalls every session activation in the server. The
// decorator adds the deadline that bounds that, plus a short TTL that turns a burst of
// activations into one round trip. It cannot admit anything the inner store did not.
services.AddSingleton<IApiKeyStore>(sp => new BoundedApiKeyStore(
new ConfigDbApiKeyStore(
sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
ConfigDbApiKeyStore.ResolveClusterId(
sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
sp.GetRequiredService<TimeProvider>()));
services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
// Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
services.AddSingleton<IApiKeyAdminService>(sp => new ApiKeyAdminService(
sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
sp.GetRequiredService<IOptions<ApiKeyOptions>>().Value.Pepper));
}
else if (hasDriver)
{
// Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact.
// Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName
// token — safe, but it meant no machine client could authenticate against a site node at all.
// Deliberately NOT wrapped by the S18 bound: the artifact store is two dictionary probes on an
// in-memory snapshot, so it can neither hang nor be worth caching.
services.AddSingleton<IApiKeyStore, ArtifactApiKeyStore>();
services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
}
else
{
// Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent —
// OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so
// an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start.
services.AddSingleton<IOpcUaUserAuthenticator>(NullOpcUaUserAuthenticator.Instance);
}
return services;
}
}
@@ -0,0 +1,125 @@
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
using ZB.MOM.WW.OtOpcUa.Host.Grpc;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// Maps the mesh gRPC ingress endpoints (artifact serve, central ack, telemetry stream, node
/// command) and the LocalDb sync endpoint.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why AllowAnonymous (archreview 03/S14).</b> On an admin-role node
/// <c>AddOtOpcUaAuth</c> installs a global <c>FallbackPolicy = RequireAuthenticatedUser</c>
/// under the cookie scheme, which the authorization middleware applies to EVERY endpoint
/// without explicit authorization metadata — gRPC included. These endpoints authenticate
/// with a per-cluster PSK bearer inside a gRPC interceptor, which runs INSIDE the endpoint
/// invocation, i.e. after that middleware. Without this opt-out a correctly-keyed node call
/// is challenged by the cookie handler before the interceptor's verdict matters: artifact
/// fetches fail, deploy acks are dropped, deployments seal TimedOut. Anonymous is safe here
/// because all five interceptors fail CLOSED — an unset key rejects every call and the
/// presented token is compared with FixedTimeEquals.
/// </para>
/// <para>
/// <b>Do not "tidy" this into Program.cs.</b> A standing census guard
/// (<c>MeshGrpcAnonymousMetadataTests</c>) drives THIS method, so a map that moves back
/// inline leaves the guard asserting a shape production no longer has — which is exactly
/// how the AdminUI cookie <c>FallbackPolicy</c> silently blocked every mesh gRPC call on a
/// login-enabled admin node for seven phases of live gates.
/// </para>
/// </remarks>
public static class MeshGrpcEndpoints
{
/// <summary>
/// Maps the mesh gRPC ingress endpoints and the LocalDb sync endpoint, gated on the same
/// role + dedicated-port checks <c>Program.cs</c> used inline before this extraction.
/// </summary>
/// <param name="app">The built application to map endpoints on.</param>
/// <param name="hasAdmin">Whether this node carries the <c>admin</c> role.</param>
/// <param name="hasDriver">Whether this node carries the <c>driver</c> role.</param>
/// <param name="syncListenPort">
/// The dedicated h2c port bound for the LocalDb passive sync endpoint, or <c>0</c> when it is
/// not bound.
/// </param>
/// <param name="configServeGrpcPort">
/// The dedicated h2c port bound for the ConfigServe artifact + central command endpoints, or
/// <c>0</c> when it is not bound.
/// </param>
/// <param name="telemetryListenPort">
/// The dedicated h2c port bound for the telemetry stream + node command endpoints, or
/// <c>0</c> when it is not bound.
/// </param>
public static void MapOtOpcUaMeshGrpcServices(
this WebApplication app,
bool hasAdmin,
bool hasDriver,
int syncListenPort,
int configServeGrpcPort,
int telemetryListenPort)
{
ArgumentNullException.ThrowIfNull(app);
// Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a
// default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be
// safe even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey
// configured — but not mapping it at all is the stronger default.
//
// AllowAnonymous (03/S14) via an empty-prefix MapGroup: MapZbLocalDbSync returns
// IEndpointRouteBuilder, NOT an IEndpointConventionBuilder, so .AllowAnonymous() cannot be
// chained onto it the way the other four maps below are. An empty-prefix group carries the
// AllowAnonymous convention to whatever it maps while leaving the gRPC route pattern
// byte-identical to the ungrouped map (verified by
// LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical).
if (hasDriver && syncListenPort > 0)
{
app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync();
}
// Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on
// admin-role nodes (which hold the central SQL connection) and only when a dedicated h2c
// port is bound. The ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes
// with no ConfigServe:ApiKey.
if (hasAdmin && configServeGrpcPort > 0)
{
// AllowAnonymous (03/S14): this is every FetchAndCache node's artifact fetch. Without the
// opt-out the AdminUI cookie FallbackPolicy 401s/redirects it before
// ConfigServeAuthInterceptor's verdict matters — a fleet-wide deploy-plane outage on any
// login-enabled admin node.
app.MapGrpcService<DeploymentArtifactGrpcService>().AllowAnonymous();
// Central command ingress (mesh gRPC command plane): the node→central deployment ack. It
// rides the SAME listener rather than claiming a port of its own — a second cleartext h2c
// port would buy nothing but another firewall rule, and both endpoints are admin-role and
// dialled by the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping
// it under Dps costs nothing (nothing dials it) whereas gating it would mean flipping the
// mode on a running fleet requires a central restart before any node can ack — the same
// reasoning that registers CentralCommunicationActor in both modes.
// CentralCommandAuthInterceptor (added at AddGrpc in Program.cs) is the ONLY inbound auth
// and fail-closes per cluster. AllowAnonymous (03/S14): this is every node's deploy ack,
// from every cluster — same reasoning as the artifact fetch above.
app.MapGrpcService<CentralCommandGrpcService>().AllowAnonymous();
}
// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
//
// The node-side command endpoint (mesh gRPC command plane, Phase 1) rides the SAME
// driver-role h2c listener rather than claiming a fourth port: central already dials this
// node here, both directions are per-cluster-keyed, and a separate port would be one more
// thing to open on every site firewall for no isolation gain. Mapping it is what makes
// MeshCommandAuthInterceptor live — it is registered in Program.cs and is a pass-through
// until a call matches this service's method prefix. It fail-closes with no
// MeshTransport:GrpcPsk, so an operator who maps the service without setting a key gets a
// closed endpoint, not an open one.
if (hasDriver && telemetryListenPort > 0)
{
// AllowAnonymous (03/S14) on both: on a fused admin+driver node (the docker-dev central
// pair, and any production fused central) these sit behind the SAME cookie FallbackPolicy
// as the ConfigServe endpoints above and would be 401/redirected identically without the
// opt-out — central's own telemetry dialer and node-command dispatch to its own pair.
app.MapGrpcService<TelemetryStreamGrpcService>().AllowAnonymous();
app.MapGrpcService<NodeCommandGrpcService>().AllowAnonymous();
}
}
}
@@ -0,0 +1,99 @@
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
/// <summary>
/// Runs the node manager's monitored-item revalidation pass on a low-frequency tick, so a key that
/// expires by the CLOCK reaches a live subscription without waiting for a deployment (arch-review
/// 03/S19).
/// </summary>
/// <remarks>
/// <para>
/// Every request-shaped operation re-checks the calling key's usability per call and so honours
/// <c>ExpiresUtc</c> the moment it passes. A subscription is authorized once and then streams, and
/// streaming is how machine consumers actually read this server — so before this the only thing
/// that could stand a subscription down was a deployment.
/// </para>
/// <para>
/// ⚠️ <b>The tick must never throw.</b> An unhandled exception on a timer callback thread takes
/// the process down. Every tick is wrapped; a faulting pass is logged and the timer keeps running,
/// because the next tick is the recovery.
/// </para>
/// <para>
/// ⚠️ <b>Each tick gets a fresh memo, and that is the pass's own constraint, not this class's.</b>
/// <c>RevalidateMonitoredItems</c> allocates its verdict dictionary per call — see the remarks on
/// <c>OtOpcUaNodeManager.OnAuthorityChanged</c>. Nothing here may hoist it across ticks, or an
/// expiry landing between two ticks would be served the earlier tick's allow.
/// </para>
/// <para>
/// It adds no decision logic: the verdict comes from the authorizer's own
/// <see cref="TimeProvider"/>, and the stand-down (queue <c>BadUserAccessDenied</c>, then detach)
/// and its restore are the pass's, unchanged. The pass early-returns unless ACL enforcement is on,
/// so this is inert on the default configuration.
/// </para>
/// </remarks>
public sealed class AclRevalidationTimer : IDisposable
{
private readonly Action _revalidate;
private readonly ILogger _log;
private readonly ITimer? _timer;
private volatile bool _disposed;
/// <summary>Initializes a new instance of the <see cref="AclRevalidationTimer"/> class.</summary>
/// <param name="revalidate">The revalidation pass to run on each tick.</param>
/// <param name="interval">
/// How often to run it. Non-positive arms no timer at all — the deploy-triggered-only behaviour
/// that predates 03/S19.
/// </param>
/// <param name="time">The shared clock; its <c>CreateTimer</c> is what arms the tick.</param>
/// <param name="log">The logger.</param>
public AclRevalidationTimer(Action revalidate, TimeSpan interval, TimeProvider time, ILogger log)
{
ArgumentNullException.ThrowIfNull(revalidate);
ArgumentNullException.ThrowIfNull(time);
ArgumentNullException.ThrowIfNull(log);
_revalidate = revalidate;
_log = log;
if (interval <= TimeSpan.Zero)
{
// Say so once. A disabled guard that logs nothing is indistinguishable from a guard that is
// running and finding nothing, which is precisely the confusion 03/S19 is about.
_log.LogInformation(
"AclRevalidationTimer: disabled — {Section}:RevalidationIntervalSeconds is not positive, "
+ "so live monitored items are re-authorized only when a deployment lands",
AclOptions.SectionName);
return;
}
_timer = time.CreateTimer(_ => Tick(), state: null, interval, interval);
_log.LogInformation(
"AclRevalidationTimer: live monitored items will be re-authorized every {Interval}", interval);
}
/// <summary>Runs one revalidation pass, absorbing anything it throws.</summary>
/// <remarks>Exposed so a test can drive the callback body deterministically rather than by waiting.</remarks>
public void Tick()
{
if (_disposed) return;
try
{
_revalidate();
}
catch (Exception ex)
{
_log.LogError(ex, "AclRevalidationTimer: the monitored-item revalidation pass threw; "
+ "the timer keeps running and the next tick is the recovery");
}
}
/// <inheritdoc />
public void Dispose()
{
_disposed = true;
_timer?.Dispose();
}
}
@@ -18,9 +18,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
/// <b>This runs inline on session activation.</b> The SDK raises <c>ImpersonateUser</c>
/// synchronously while holding a SessionManager-wide event lock, so a slow authenticator does not
/// stall one request — it serializes every session activation server-wide. That is why
/// verification is a single indexed lookup plus one HMAC, and why there is no directory call,
/// no retry, and no network I/O on this path at all. It is the main practical advantage of keys
/// over the LDAP bind this replaces.
/// verification is a single indexed lookup plus one HMAC, with no directory call and no retry —
/// the main practical advantage of keys over the LDAP bind this replaces.
/// </para>
/// <para>
/// ⚠️ <b>"No network I/O" is TRUE ONLY ON A DRIVER-ONLY NODE</b> (arch-review 03/S18). The claim
/// describes <see cref="ArtifactApiKeyStore"/>, which is two dictionary probes on an in-memory
/// snapshot. On an <b>admin or fused</b> node <see cref="IApiKeyStore"/> is
/// <see cref="ConfigDbApiKeyStore"/>, and a UserName activation costs <b>two</b> EF round trips
/// (the key row, then the principal's <c>NodeAcl</c> rows) — inside this same lock. What bounds
/// that is <see cref="BoundedApiKeyStore"/>, which wraps it in production; the deadline it applies
/// surfaces here as an ordinary throw and is denied by the catch-all below, like any other store
/// failure.
/// </para>
/// <para>
/// <b>Fail closed, and say little.</b> Every rejection returns the same opaque message. The
@@ -0,0 +1,240 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
/// <summary>
/// Bounds what an <see cref="IApiKeyStore"/> can cost the OPC UA impersonation callback (arch-review
/// 03/S18). Wraps <see cref="ConfigDbApiKeyStore"/> on a node that has a ConfigDb; the in-memory
/// <see cref="ArtifactApiKeyStore"/> needs neither half and is deliberately left unwrapped.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why a deadline is the fix and the cache is not.</b> The SDK raises impersonation
/// synchronously under a <c>SessionManager</c>-wide lock, so a store that does not answer does not
/// stall one activation — it stalls every activation in the server. A cache lowers how OFTEN the
/// store is asked; only a deadline bounds how LONG one ask may hold that lock.
/// </para>
/// <para>
/// ⚠️ <b>Every bounded outcome DENIES.</b> A deadline exceeded, a faulted inner store and a
/// cancelled call all surface as an exception into
/// <see cref="ApiKeyOpcUaUserAuthenticator"/>'s catch-all, which returns the same opaque denial as
/// a bad credential. Nothing here can admit a session the inner store did not authorize — the
/// decorator can only return what the inner store returned, or throw.
/// </para>
/// <para>
/// ⚠️ <b>Faults and timeouts are NEVER cached.</b> Caching a deny would be fail-closed and is
/// still wrong: it would extend a ConfigDb outage past the outage, denying valid keys for a
/// further TTL after the database came back. Only a COMPLETED lookup — a hit or an authoritative
/// miss — is stored.
/// </para>
/// </remarks>
public sealed class BoundedApiKeyStore : IApiKeyStore
{
/// <summary>
/// Entries either cache may hold before it is cleared wholesale.
/// </summary>
/// <remarks>
/// Wholesale rather than per-entry eviction, for the reason <c>SessionAclDecisionCache</c> already
/// documents: picking WHICH entry to drop needs knowledge the cache does not have, and a
/// repopulate is cheap and cannot be wrong. The ceiling exists so a key-id guessing flood is a
/// bounded memory event — negative results are cached deliberately, so that a flood of unknown ids
/// does not become a SQL flood.
/// </remarks>
public const int MaxCacheEntries = 1024;
private readonly IApiKeyStore _inner;
private readonly IOptions<ApiKeyOptions> _options;
private readonly ILogger<BoundedApiKeyStore> _log;
private readonly TimeProvider _time;
private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
new(StringComparer.Ordinal);
/// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
/// <param name="inner">The store being bounded.</param>
/// <param name="options">The <c>Security:ApiKeys</c> options carrying the deadline and TTL.</param>
/// <param name="log">The logger.</param>
/// <param name="time">
/// The shared clock. The same instance the authorizer reads — see
/// <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
/// detail.
/// </param>
public BoundedApiKeyStore(
IApiKeyStore inner,
IOptions<ApiKeyOptions> options,
ILogger<BoundedApiKeyStore> log,
TimeProvider time)
{
ArgumentNullException.ThrowIfNull(inner);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(log);
ArgumentNullException.ThrowIfNull(time);
_inner = inner;
_options = options;
_log = log;
_time = time;
}
/// <summary>The store being bounded.</summary>
/// <remarks>
/// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
/// of a role conditional is the "built but never wired" shape, and a guard that cannot see what is
/// inside the wrapper cannot tell a correctly-wired node from one wrapping the wrong store.
/// </remarks>
public IApiKeyStore Inner => _inner;
/// <summary>The clock this bound's deadline and TTL are measured on.</summary>
/// <remarks>Same reason as <see cref="Inner"/>: the shared-clock invariant is only pinnable by reference.</remarks>
public TimeProvider Clock => _time;
/// <inheritdoc />
/// <remarks>
/// Keyed by <paramref name="apiKeyId"/> alone. <paramref name="utcNow"/> is deliberately not part
/// of the key — it changes on every call, so including it would defeat the cache entirely — but a
/// cached hit IS re-checked against it before being served, so the TTL can never resurrect a key
/// that has expired by the clock in the meantime. The TTL therefore bounds how stale an
/// <c>Enabled</c> revocation may be, and nothing else.
/// </remarks>
public async Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
{
var ttl = Ttl;
if (ttl > TimeSpan.Zero
&& TryServe(_keys, apiKeyId, ttl, out var cached)
&& !IsExpired(cached, utcNow))
{
// Metered HERE rather than inside TryServe: an entry that is inside the TTL but whose key has
// since expired is not a hit — it falls through to a store read — so counting it in TryServe
// would report cache hits that never happened.
MeterCacheHit();
return cached;
}
var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
.ConfigureAwait(false);
if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
return fresh;
}
/// <inheritdoc />
/// <remarks>Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.</remarks>
public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
{
var ttl = Ttl;
if (ttl > TimeSpan.Zero && TryServe(_grants, principal, ttl, out var cached))
{
MeterCacheHit();
return cached;
}
var fresh = await BoundAsync(token => _inner.FindGrantsAsync(principal, token), ct)
.ConfigureAwait(false);
if (ttl > TimeSpan.Zero) Store(_grants, principal, fresh);
return fresh;
}
/// <summary>The configured TTL; non-positive means the cache is off.</summary>
private TimeSpan Ttl => TimeSpan.FromSeconds(_options.Value.StoreCacheSeconds);
/// <summary>Whether a cached lookup has since expired by the clock.</summary>
/// <param name="cached">The cached lookup, possibly null (an authoritative miss).</param>
/// <param name="utcNow">The caller's current time.</param>
/// <returns><see langword="true"/> when the cached key must not be served.</returns>
private static bool IsExpired(ApiKeyLookup? cached, DateTime utcNow) =>
cached?.Key.ExpiresUtc is { } expires && expires <= utcNow;
/// <summary>Counts one lookup the cache actually answered.</summary>
private static void MeterCacheHit() =>
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "cache_hit"));
/// <summary>Serves an entry if one is present and still inside the TTL.</summary>
/// <typeparam name="T">The cached value type.</typeparam>
/// <param name="cache">The cache to read.</param>
/// <param name="key">The cache key.</param>
/// <param name="ttl">How long an entry stays servable.</param>
/// <param name="value">The cached value, when this returns true.</param>
/// <returns><see langword="true"/> on a hit.</returns>
private bool TryServe<T>(
ConcurrentDictionary<string, Entry<T>> cache, string key, TimeSpan ttl, out T value)
{
if (cache.TryGetValue(key, out var entry) && _time.GetUtcNow() - entry.StampedUtc < ttl)
{
value = entry.Value;
return true;
}
value = default!;
return false;
}
/// <summary>Stores a COMPLETED lookup, clearing wholesale first if the ceiling has been reached.</summary>
/// <typeparam name="T">The cached value type.</typeparam>
/// <param name="cache">The cache to write.</param>
/// <param name="key">The cache key.</param>
/// <param name="value">The value to cache.</param>
private void Store<T>(ConcurrentDictionary<string, Entry<T>> cache, string key, T value)
{
if (cache.Count >= MaxCacheEntries) cache.Clear();
cache[key] = new Entry<T>(_time.GetUtcNow(), value);
}
/// <summary>One cached lookup and the instant it was taken.</summary>
/// <typeparam name="T">The cached value type.</typeparam>
/// <param name="StampedUtc">When the inner store answered.</param>
/// <param name="Value">What it answered.</param>
private readonly record struct Entry<T>(DateTimeOffset StampedUtc, T Value);
/// <summary>Runs one inner call under the configured deadline.</summary>
/// <typeparam name="T">The inner call's result type.</typeparam>
/// <param name="call">The inner call, taking the deadline-linked token.</param>
/// <param name="ct">The caller's token — in production always <c>None</c>, see remarks.</param>
/// <returns>Whatever the inner store returned.</returns>
/// <remarks>
/// <para>
/// The deadline is created HERE rather than received from the caller because there is no caller
/// to receive one from: <c>OpcUaApplicationHost</c> block-bridges the SDK's synchronous
/// impersonation callback with <see cref="CancellationToken.None"/>, so the <c>ct</c> the store
/// sees in production is always uncancellable.
/// </para>
/// <para>
/// ⚠️ <b>It re-throws rather than returning <see langword="null"/>.</b> <c>null</c> means
/// "no such usable key", which the authenticator logs as <i>denied — no enabled, unexpired key
/// with id X</i>. A timeout is not that, and an operator reading that line during a ConfigDb
/// outage would go hunting for a key problem that does not exist. The distinct Warning below is
/// the honest signal; the denial itself is unchanged either way.
/// </para>
/// </remarks>
private async Task<T> BoundAsync<T>(Func<CancellationToken, Task<T>> call, CancellationToken ct)
{
var timeoutMs = _options.Value.StoreTimeoutMs;
if (timeoutMs <= 0) return await call(ct).ConfigureAwait(false);
// The deadline runs on the INJECTED clock (the TimeProvider ctor overload), not on a plain
// CancelAfter, so it is the same TimeProvider the authorizer and the revalidation timer read.
// Linked with the caller's token so a genuine caller cancellation is still honoured — and so the
// catch below can tell the two apart.
using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(timeoutMs), _time);
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct, deadline.Token);
try
{
return await call(cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(
1, new KeyValuePair<string, object?>("outcome", "timeout"));
_log.LogWarning(
"ApiKeyStore: the key store did not answer within {TimeoutMs} ms — denying this activation. "
+ "Every OPC UA session activation on this server is serialized behind this call; "
+ "check ConfigDb health.",
timeoutMs);
throw;
}
}
}
@@ -14,8 +14,19 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
/// <b>A driver-only node has no ConfigDb since per-cluster mesh Phase 4</b>, so this store cannot be
/// registered there; that node uses <see cref="ArtifactApiKeyStore"/> instead (#520 stage 4).
/// <para>
/// Reading per request is what makes revocation immediate here, which is why this store is
/// preferred wherever a ConfigDb exists — including on a fused admin+driver node.
/// Reading per activation is what makes revocation prompt here, which is why this store is
/// preferred wherever a ConfigDb exists — including on a fused admin+driver node. ⚠️ It reaches a
/// <b>new session activation</b> only: an already-open session re-checks its key against the
/// in-memory artifact snapshot on every operation, not against this store, so clearing
/// <c>Enabled</c> here does not stop an open session until the next deployment. See
/// <c>docs/security.md</c>.
/// </para>
/// <para>
/// ⚠️ <b>In production this store is WRAPPED by <see cref="BoundedApiKeyStore"/></b> (arch-review
/// 03/S18), because these queries run inline in the SDK's impersonation callback under a
/// SessionManager-wide lock. A call may therefore be cancelled by the wrapper's deadline even
/// though the caller passed <see cref="CancellationToken.None"/> — the resulting
/// <see cref="OperationCanceledException"/> is a denial, not a bug.
/// </para>
/// <para>
/// ⚠️ <b>The ConfigDb holds the WHOLE fleet's ACL rows, not just this node's cluster's.</b> That is
@@ -38,6 +38,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
private readonly IDataPlaneAuthoritySource _dataPlaneAuthority;
private readonly DataPlaneAuthorizer? _dataPlaneAuthorizer;
private readonly ServerHistorianOptions _serverHistorianOptions;
private readonly AclOptions _aclOptions;
private readonly TimeProvider _time;
private readonly Func<ActorSystem> _actorSystemAccessor;
private readonly ActorRegistry _actorRegistry;
private readonly ILoggerFactory _loggerFactory;
@@ -45,6 +47,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
private OpcUaApplicationHost? _appHost;
private OtOpcUaSdkServer? _server;
private AclRevalidationTimer? _revalidationTimer;
/// <summary>
/// Initializes a new instance of the OtOpcUaServerHostedService class.
@@ -77,6 +80,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
/// <param name="dataPlaneAuthorizer">#520 stage 5 — the node-scoped ACL gate. Nullable, and null means
/// "not wired" rather than "deny": an unregistered gate must not lock a node out of writes to protect a
/// feature that is off by default. Once wired AND enabled, every unresolvable case inside it denies.</param>
/// <param name="aclOptions">#520 / arch-review 03/S19 — the <c>Security:Acl</c> options, read for
/// <see cref="AclOptions.RevalidationIntervalSeconds"/>. Required, not optional: the built-in DI
/// container does not fall back to a default for an unregistered service, the same trap the
/// <c>IOpcUaUserAuthenticator</c> and <c>IDataPlaneAuthoritySource</c> parameters carry comments
/// about.</param>
/// <param name="time">The shared clock — the one instance the authorizer and the API-key store bound
/// also read. Its <c>CreateTimer</c> arms the revalidation tick.</param>
/// <param name="loggerFactory">The logger factory for creating loggers.</param>
public OtOpcUaServerHostedService(
IOptions<OpcUaApplicationHostOptions> options,
@@ -89,9 +99,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
Func<ActorSystem> actorSystemAccessor,
ActorRegistry actorRegistry,
DataPlaneAuthorizer? dataPlaneAuthorizer,
IOptions<AclOptions> aclOptions,
TimeProvider time,
ILoggerFactory loggerFactory)
{
ArgumentNullException.ThrowIfNull(aclOptions);
ArgumentNullException.ThrowIfNull(time);
_dataPlaneAuthorizer = dataPlaneAuthorizer;
_aclOptions = aclOptions.Value;
_time = time;
_options = options.Value;
_deferredSink = deferredSink;
_deferredServiceLevel = deferredServiceLevel;
@@ -269,9 +286,41 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
}
// arch-review 03/S19 — LAST, after the node manager is known non-null and after the ACL gate is
// attached, so the first tick cannot land on a half-wired server.
StartRevalidationTimer();
_logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
}
/// <summary>
/// Arms the clock-driven monitored-item revalidation tick (arch-review 03/S19).
/// </summary>
/// <remarks>
/// Created unconditionally: the pass it drives early-returns unless the ACL gate is wired AND
/// <c>Security:Acl:Enabled</c> is on, so it stays inert on the default configuration, and
/// <c>RevalidationIntervalSeconds = 0</c> arms no timer at all.
/// <para>
/// The node manager is resolved <b>per tick</b> rather than captured: <see cref="_server"/> is
/// replaced across a restart, and a captured reference would go on revalidating a dead
/// manager's item map — the same detach hazard the authority-source setter documents.
/// </para>
/// <para>Separate from <see cref="StartAsync"/> so the timer's lifecycle is testable without booting the SDK.</para>
/// </remarks>
internal void StartRevalidationTimer()
{
// Dispose before overwriting. StartAsync runs again when a host restarts the service around the
// same instance, and an overwritten field would leave the previous timer armed with nothing left
// to dispose it — the same dispose-before-overwrite rule the driver reconnect paths carry.
_revalidationTimer?.Dispose();
_revalidationTimer = new AclRevalidationTimer(
() => _server?.NodeManager?.RevalidateMonitoredItems(),
TimeSpan.FromSeconds(_aclOptions.RevalidationIntervalSeconds),
_time,
_loggerFactory.CreateLogger<AclRevalidationTimer>());
}
/// <summary>
/// Stops the OPC UA server asynchronously.
/// </summary>
@@ -279,6 +328,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
/// <returns>A task that represents the asynchronous operation.</returns>
public Task StopAsync(CancellationToken cancellationToken)
{
// FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
// authority and gate have already been cleared is the shape this whole teardown block exists to
// prevent — a stopping node sheds the trigger before it sheds what the trigger drives.
_revalidationTimer?.Dispose();
_revalidationTimer = null;
// Revert to Null adapters so any in-flight writes from a poison-pilled actor don't hit a
// half-disposed NodeManager.
_deferredSink.SetSink(null);
@@ -304,6 +359,11 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
// Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
// not leave a tick armed against a server that is about to go away.
_revalidationTimer?.Dispose();
_revalidationTimer = null;
if (_appHost is not null) await _appHost.DisposeAsync();
}
}
+10 -91
View File
@@ -398,55 +398,10 @@ if (hasDriver)
builder.Services.AddSingleton<PermissionTrieCache>();
}
// #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because
// OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves
// constructors by whether every parameter is registered, not by whether the parameter type is nullable.
// That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations above
// both carry a comment about; an absent registration is a node that will not start.
builder.Services.AddSingleton<IPermissionEvaluator>(sp =>
new TriePermissionEvaluator(sp.GetRequiredService<PermissionTrieCache>()));
builder.Services.AddSingleton(sp => new DataPlaneAuthorizer(
sp.GetRequiredService<IDataPlaneAuthoritySource>(),
sp.GetRequiredService<IPermissionEvaluator>(),
sp.GetRequiredService<IOptions<AclOptions>>().Value.Enabled,
sp.GetRequiredService<ILoggerFactory>().CreateLogger<DataPlaneAuthorizer>()));
if (hasAdmin)
{
// A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect
// on the very next request rather than on the next deployment. Preferred wherever it is available,
// including on a fused admin+driver node.
// The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds
// the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which
// ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration),
// NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services
// rule is what keeps the redundancy-singleton registration below from recursing into ActorSystem
// construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter (legacy behaviour).
builder.Services.AddSingleton<IApiKeyStore>(sp => new ConfigDbApiKeyStore(
sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
ConfigDbApiKeyStore.ResolveClusterId(sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)));
builder.Services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
// Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
builder.Services.AddSingleton<IApiKeyAdminService>(sp => new ApiKeyAdminService(
sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
sp.GetRequiredService<IOptions<ApiKeyOptions>>().Value.Pepper));
}
else if (hasDriver)
{
// Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact.
// Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName
// token — safe, but it meant no machine client could authenticate against a site node at all.
builder.Services.AddSingleton<IApiKeyStore, ArtifactApiKeyStore>();
builder.Services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
}
else
{
// Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent —
// OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so
// an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start.
builder.Services.AddSingleton<IOpcUaUserAuthenticator>(NullOpcUaUserAuthenticator.Instance);
}
// #520 + arch-review 03/S18 — the node-scoped ACL gate, the shared clock and the role-appropriate
// API-key store + user authenticator, all in one extension so the role-conditional branches are
// reachable from a test (Program.cs's top-level statements are not). See DataPlaneAuthRegistration.
builder.Services.AddOtOpcUaDataPlaneAuth(hasAdmin, hasDriver);
// Bind + validate the OPC UA host options the same way (fail-fast at start via ValidateOnStart)
// and let OtOpcUaServerHostedService consume the validated IOptions instance rather than
@@ -740,48 +695,12 @@ if (hasAdmin)
app.MapOtOpcUaDeployApi(app.Configuration);
}
// Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a
// default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be safe
// even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey configured — but
// not mapping it at all is the stronger default.
if (hasDriver && syncListenPort > 0)
{
app.MapZbLocalDbSync();
}
// Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on admin-role
// nodes (which hold the central SQL connection) and only when a dedicated h2c port is bound. The
// ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes with no ConfigServe:ApiKey.
if (hasAdmin && configServeGrpcPort > 0)
{
app.MapGrpcService<DeploymentArtifactGrpcService>();
// Central command ingress (mesh gRPC command plane): the node→central deployment ack. It
// rides the SAME listener rather than claiming a port of its own — a second cleartext h2c port
// would buy nothing but another firewall rule, and both endpoints are admin-role and dialled by
// the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping it under Dps
// costs nothing (nothing dials it) whereas gating it would mean flipping the mode on a running
// fleet requires a central restart before any node can ack — the same reasoning that registers
// CentralCommunicationActor in both modes. CentralCommandAuthInterceptor (added at AddGrpc
// above) is the ONLY inbound auth and fail-closes per cluster.
app.MapGrpcService<CentralCommandGrpcService>();
}
// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
//
// The node-side command endpoint (mesh gRPC command plane, Phase 1) rides the SAME driver-role
// h2c listener rather than claiming a fourth port: central already dials this node here, both
// directions are per-cluster-keyed, and a separate port would be one more thing to open on every
// site firewall for no isolation gain. Mapping it is what makes MeshCommandAuthInterceptor live —
// it is registered above and is a pass-through until a call matches this service's method prefix.
// It fail-closes with no MeshTransport:GrpcPsk, so an operator who maps the service without setting
// a key gets a closed endpoint, not an open one.
if (hasDriver && telemetryListenPort > 0)
{
app.MapGrpcService<TelemetryStreamGrpcService>();
app.MapGrpcService<NodeCommandGrpcService>();
}
// Mesh gRPC ingress: the LocalDb passive sync endpoint, the ConfigServe artifact + central
// command endpoints, and the telemetry-stream + node-command endpoints. Extracted to
// MeshGrpcEndpoints so a standing census test drives the SAME production mapping (see that
// file's remarks — archreview 03/S14).
app.MapOtOpcUaMeshGrpcServices(
hasAdmin, hasDriver, syncListenPort, configServeGrpcPort, telemetryListenPort);
app.MapOtOpcUaHealth();
app.MapOtOpcUaMetrics();
@@ -13,6 +13,13 @@
<NoWarn>$(NoWarn);NU1608</NoWarn>
</PropertyGroup>
<ItemGroup>
<!-- OtOpcUaServerHostedService.StartRevalidationTimer is internal: it is a lifecycle seam the
tests drive so the ACL revalidation timer (arch-review 03/S19) can be asserted without
booting the real OPC UA SDK, not a surface for callers. -->
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Host.Tests"/>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Akka.Hosting"/>
<PackageReference Include="Serilog.AspNetCore"/>