Merge lane C: control plane, host, security (#733)
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Fail-fast startup validator for <see cref="ApiKeyOptions"/> (R03:2.3). The pepper used to be
|
||||
/// checked only when the first key was verified — <c>ApiKeySecret.Verify</c> throws on an empty one —
|
||||
/// so a site node deployed without <c>Security__ApiKeys__Pepper</c> booted green, passed health,
|
||||
/// applied deployments and denied every machine client with the same opaque "Invalid API key" until
|
||||
/// someone read the log; a central node with a wrong pepper minted keys that could verify nowhere.
|
||||
/// Every other security section already fails host start on a bad value; this one now does too.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Registered for <c>hasDriver || hasAdmin</c>: a driver node needs the pepper to VERIFY keys, an admin
|
||||
/// node to MINT them. The minimum length is a floor against a placeholder (<c>"x"</c>) passing as
|
||||
/// configured, not a strength claim — the pepper is a server-side secret, not a password.
|
||||
/// </remarks>
|
||||
public sealed class ApiKeyOptionsValidator : OptionsValidatorBase<ApiKeyOptions>
|
||||
{
|
||||
/// <summary>The shortest pepper accepted (characters, after trimming).</summary>
|
||||
public const int MinPepperLength = 16;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, ApiKeyOptions options)
|
||||
{
|
||||
var pepper = options.Pepper?.Trim() ?? string.Empty;
|
||||
builder.RequireThat(pepper.Length > 0,
|
||||
$"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " +
|
||||
"supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " +
|
||||
"every API key (driver) or mint keys that can never verify (admin).");
|
||||
builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength,
|
||||
$"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters.");
|
||||
builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username),
|
||||
$"{ApiKeyOptions.SectionName}:Username must not be blank.");
|
||||
builder.RequireThat(options.StoreTimeoutMs >= 0,
|
||||
$"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline).");
|
||||
builder.RequireThat(options.StoreCacheSeconds >= 0,
|
||||
$"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache).");
|
||||
}
|
||||
}
|
||||
@@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration
|
||||
sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
|
||||
sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
|
||||
sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
|
||||
sp.GetRequiredService<TimeProvider>()));
|
||||
sp.GetRequiredService<TimeProvider>(),
|
||||
// R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as
|
||||
// ApiKeyOptions, Host-owned knobs.
|
||||
ApiKeyStoreGuardOptions.Bind(sp.GetService<IConfiguration>())));
|
||||
services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
|
||||
|
||||
// Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
|
||||
|
||||
@@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Health;
|
||||
|
||||
@@ -28,11 +29,40 @@ public static class HealthEndpoints
|
||||
/// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to
|
||||
/// <c>true</c> so admin/fused callers (including the test harnesses) keep the probe unchanged.
|
||||
/// </param>
|
||||
/// <param name="hasDriver">
|
||||
/// Whether this node carries the <c>driver</c> role and therefore serves an OPC UA endpoint. When
|
||||
/// <c>true</c> the <see cref="OpcUaEndpointHealthCheck"/> joins the ready tier, and the active tier on a
|
||||
/// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading
|
||||
/// the <see cref="IOpcUaEndpointState"/> the host registers alongside <c>OtOpcUaServerHostedService</c>.
|
||||
/// Defaults to <c>false</c> so admin-only callers and the existing harnesses register nothing new.
|
||||
/// </param>
|
||||
/// <returns>The same service collection, for chaining.</returns>
|
||||
public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true)
|
||||
public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false)
|
||||
{
|
||||
var checks = services.AddHealthChecks();
|
||||
|
||||
// A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve.
|
||||
// The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6):
|
||||
// - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may
|
||||
// route to it;
|
||||
// - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState
|
||||
// (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would
|
||||
// take the operator's only console out of rotation precisely when they need it to diagnose the
|
||||
// failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands
|
||||
// the peer takes Primary and the active tier follows it on its own.
|
||||
// Factory-registered so the state singleton is resolved per probe.
|
||||
if (hasDriver)
|
||||
{
|
||||
var tags = hasAdmin
|
||||
? new[] { ZbHealthTags.Ready }
|
||||
: new[] { ZbHealthTags.Ready, ZbHealthTags.Active };
|
||||
checks.Add(new HealthCheckRegistration(
|
||||
OpcUaEndpointHealthCheck.Name,
|
||||
sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService<IOpcUaEndpointState>()),
|
||||
failureStatus: null,
|
||||
tags: tags));
|
||||
}
|
||||
|
||||
// configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config
|
||||
// from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so
|
||||
// the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is
|
||||
/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy
|
||||
/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets
|
||||
/// connection-refused.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Registered on driver-role nodes only (<see cref="HealthEndpoints.AddOtOpcUaHealth"/>); an
|
||||
/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused
|
||||
/// admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier
|
||||
/// is what Traefik routes the AdminUI on.
|
||||
/// </remarks>
|
||||
public sealed class OpcUaEndpointHealthCheck : IHealthCheck
|
||||
{
|
||||
/// <summary>The registration name, as it appears in the health JSON.</summary>
|
||||
public const string Name = "opcua-endpoint";
|
||||
|
||||
private readonly IOpcUaEndpointState _state;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OpcUaEndpointHealthCheck"/> class.</summary>
|
||||
/// <param name="state">The endpoint lifecycle fact the hosted service maintains.</param>
|
||||
public OpcUaEndpointHealthCheck(IOpcUaEndpointState state)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
_state = state;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var data = new Dictionary<string, object> { ["status"] = _state.Status.ToString() };
|
||||
var result = _state.Status switch
|
||||
{
|
||||
OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data),
|
||||
OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data),
|
||||
OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data),
|
||||
_ => HealthCheckResult.Unhealthy(
|
||||
"OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data),
|
||||
};
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// The pre-authentication guards <see cref="BoundedApiKeyStore"/> puts in front of the ConfigDb key
|
||||
/// lookup (R03:3.3). Bound from the same <c>Security:ApiKeys</c> section as <c>ApiKeyOptions</c> but
|
||||
/// owned by the Host, where the store lives.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why.</b> The negative cache was keyed by key id, so a flood of DISTINCT well-formed
|
||||
/// <c>otua_<random>_x</c> credentials missed it every time and cost one indexed SQL query per
|
||||
/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close
|
||||
/// that: a <b>known-id set</b> (every <c>ApiKeyId</c> the ConfigDb holds, refreshed every
|
||||
/// <c>StoreCacheSeconds</c>) rejects an unknown id in memory, and a <b>miss budget</b> (token
|
||||
/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the
|
||||
/// set is unavailable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback
|
||||
/// (the <c>IOpcUaUserAuthenticator</c> seam) does not carry the client's address. Per-endpoint
|
||||
/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ApiKeyStoreGuardOptions
|
||||
{
|
||||
/// <summary>The configuration section the knobs live in (shared with <c>ApiKeyOptions</c>).</summary>
|
||||
public const string SectionName = "Security:ApiKeys";
|
||||
|
||||
/// <summary>
|
||||
/// Whether the known-id set is consulted. <see langword="false"/> restores a store read per unknown
|
||||
/// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact
|
||||
/// store is never wrapped).
|
||||
/// </summary>
|
||||
public bool KnownIdSet { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// How many store MISSES (lookups that reach the inner store and find no key) may burst before
|
||||
/// further lookups for unknown ids are refused without a store read. <c>0</c> disables the budget.
|
||||
/// </summary>
|
||||
public int MissBurst { get; set; } = 100;
|
||||
|
||||
/// <summary>How many miss tokens are restored per second, up to <see cref="MissBurst"/>.</summary>
|
||||
public double MissRefillPerSecond { get; set; } = 20;
|
||||
|
||||
/// <summary>Binds the section, with defaults for anything absent.</summary>
|
||||
/// <param name="configuration">The app configuration; null yields the defaults.</param>
|
||||
/// <returns>The bound options.</returns>
|
||||
public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration)
|
||||
{
|
||||
var options = new ApiKeyStoreGuardOptions();
|
||||
configuration?.GetSection(SectionName).Bind(options);
|
||||
return options;
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,33 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
|
||||
new(StringComparer.Ordinal);
|
||||
private readonly ApiKeyStoreGuardOptions _guard;
|
||||
|
||||
// R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds
|
||||
// (single-flight; a refresh that fails or times out keeps the previous set). Null until the first
|
||||
// successful load, or when the inner store cannot list ids — then the per-id path runs as before.
|
||||
// The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile
|
||||
// reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read
|
||||
// a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed.
|
||||
private volatile KnownIdSnapshot? _knownIds;
|
||||
private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1);
|
||||
|
||||
// Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is
|
||||
// shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked
|
||||
// one store's capability into every other consumer of the same options).
|
||||
private volatile bool _listingUnsupported;
|
||||
|
||||
/// <summary>The known-id set and the instant it was loaded, published as one unit.</summary>
|
||||
/// <param name="Ids">The ids the inner store holds.</param>
|
||||
/// <param name="StampedUtc">When they were read.</param>
|
||||
private sealed record KnownIdSnapshot(IReadOnlySet<string> Ids, DateTimeOffset StampedUtc);
|
||||
|
||||
// R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards
|
||||
// the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the
|
||||
// configured rate instead of one SQL round trip per attempt under the SessionManager lock.
|
||||
private readonly object _bucketGate = new();
|
||||
private double _missTokens;
|
||||
private DateTimeOffset _missRefilledUtc;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
|
||||
/// <param name="inner">The store being bounded.</param>
|
||||
@@ -66,11 +93,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
/// <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
|
||||
/// detail.
|
||||
/// </param>
|
||||
/// <param name="guard">The pre-authentication guards (R03:3.3); defaults when omitted.</param>
|
||||
public BoundedApiKeyStore(
|
||||
IApiKeyStore inner,
|
||||
IOptions<ApiKeyOptions> options,
|
||||
ILogger<BoundedApiKeyStore> log,
|
||||
TimeProvider time)
|
||||
TimeProvider time,
|
||||
ApiKeyStoreGuardOptions? guard = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
@@ -81,8 +110,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
_options = options;
|
||||
_log = log;
|
||||
_time = time;
|
||||
_guard = guard ?? new ApiKeyStoreGuardOptions();
|
||||
_missTokens = Math.Max(0, _guard.MissBurst);
|
||||
_missRefilledUtc = time.GetUtcNow();
|
||||
}
|
||||
|
||||
/// <summary>The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards.</summary>
|
||||
public IReadOnlySet<string>? KnownKeyIds => _knownIds?.Ids;
|
||||
|
||||
/// <summary>The store being bounded.</summary>
|
||||
/// <remarks>
|
||||
/// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
|
||||
@@ -115,12 +150,103 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
|
||||
return cached;
|
||||
}
|
||||
|
||||
// R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked.
|
||||
var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false);
|
||||
if (known is not null && !known.Contains(apiKeyId))
|
||||
{
|
||||
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "unknown_id"));
|
||||
return null;
|
||||
}
|
||||
|
||||
// R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known
|
||||
// to exist (a known id is a real key being verified, never a guess).
|
||||
if (known is null && !TryTakeMissToken())
|
||||
{
|
||||
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "miss_budget"));
|
||||
throw new InvalidOperationException(
|
||||
"ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / "
|
||||
+ "MissRefillPerSecond); refusing this lookup without a store read.");
|
||||
}
|
||||
|
||||
var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
|
||||
.ConfigureAwait(false);
|
||||
if (fresh is not null && known is null) RefundMissToken();
|
||||
if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
|
||||
return fresh;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The known-id set, refreshed when older than <paramref name="ttl"/>. Single-flight: concurrent
|
||||
/// callers during a refresh use the previous set (or, before the first load, the per-id path). A
|
||||
/// refresh that fails or exceeds the deadline logs and keeps the previous set.
|
||||
/// </summary>
|
||||
/// <param name="ttl">The refresh interval (the cache TTL); non-positive disables the set.</param>
|
||||
/// <param name="ct">The caller's token.</param>
|
||||
/// <returns>The set, or null.</returns>
|
||||
private async Task<IReadOnlySet<string>?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct)
|
||||
{
|
||||
if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null;
|
||||
|
||||
var now = _time.GetUtcNow();
|
||||
var current = _knownIds;
|
||||
if (current is not null && now - current.StampedUtc < ttl) return current.Ids;
|
||||
|
||||
if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids;
|
||||
try
|
||||
{
|
||||
// Re-check under the gate: another caller may have refreshed while this one waited.
|
||||
if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids;
|
||||
var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false);
|
||||
if (ids is null)
|
||||
{
|
||||
// Unsupported by the inner store: stop asking — for this decorator only.
|
||||
_listingUnsupported = true;
|
||||
return null;
|
||||
}
|
||||
_knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow());
|
||||
return ids;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning(ex,
|
||||
"ApiKeyStore: refreshing the known key-id set failed; {Mode}",
|
||||
current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force");
|
||||
return current?.Ids;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_knownIdsRefresh.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled.</summary>
|
||||
/// <returns><see langword="false"/> when the budget is exhausted.</returns>
|
||||
private bool TryTakeMissToken()
|
||||
{
|
||||
var burst = _guard.MissBurst;
|
||||
if (burst <= 0) return true;
|
||||
lock (_bucketGate)
|
||||
{
|
||||
var now = _time.GetUtcNow();
|
||||
var elapsed = (now - _missRefilledUtc).TotalSeconds;
|
||||
if (elapsed > 0)
|
||||
{
|
||||
_missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond));
|
||||
_missRefilledUtc = now;
|
||||
}
|
||||
if (_missTokens < 1) return false;
|
||||
_missTokens -= 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A lookup that reached the store and HIT was not a miss: its token goes back.</summary>
|
||||
private void RefundMissToken()
|
||||
{
|
||||
if (_guard.MissBurst <= 0) return;
|
||||
lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1);
|
||||
}
|
||||
|
||||
// Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
|
||||
|
||||
@@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore
|
||||
return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
|
||||
{
|
||||
await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false);
|
||||
// One projection over the whole (small) table — the unique index on ApiKeyId makes this an index
|
||||
// scan — so an unknown id can be refused in memory instead of costing a query per attempt.
|
||||
var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false);
|
||||
return ids.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>Loads a principal's grants, confined to this node's cluster.</summary>
|
||||
/// <remarks>
|
||||
/// <b>The full grant set, not just the <c>Cluster</c>-scoped rows</b>. The old
|
||||
|
||||
@@ -35,6 +35,16 @@ public interface IApiKeyStore
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The matching ACL rows; empty when none.</returns>
|
||||
Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// Every <c>ApiKeyId</c> the store holds (enabled or not), for the pre-authentication known-id set
|
||||
/// (R03:3.3); <see langword="null"/> when the store cannot cheaply list them, in which case callers
|
||||
/// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry
|
||||
/// and the secret are still decided by <see cref="FindAsync"/>.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The id set, or null when unsupported.</returns>
|
||||
Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult<IReadOnlySet<string>?>(null);
|
||||
}
|
||||
|
||||
/// <summary>An API key resolved for authentication, with the ACL grants naming its principal.</summary>
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
/// <summary>Lifecycle of this node's OPC UA endpoint as <see cref="OtOpcUaServerHostedService"/> drives it.</summary>
|
||||
public enum OpcUaEndpointStatus
|
||||
{
|
||||
/// <summary>The hosted service has not run <c>StartAsync</c> yet.</summary>
|
||||
NotStarted,
|
||||
|
||||
/// <summary>The SDK is listening and the address-space / ServiceLevel sinks are bound.</summary>
|
||||
Started,
|
||||
|
||||
/// <summary>The SDK failed to start (port in use, PKI unwritable, bad <c>OpcUa:*</c> override, …).</summary>
|
||||
Failed,
|
||||
|
||||
/// <summary>The hosted service stopped the SDK; the port is released.</summary>
|
||||
Stopped,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed
|
||||
/// start was one Error line: <c>/health/ready</c> stayed 200, the node joined its mesh, could be
|
||||
/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got
|
||||
/// connection-refused from the node the fleet called healthy.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Consumed by <c>OpcUaEndpointHealthCheck</c> (Ready + Active tiers). The redundancy election and the
|
||||
/// ServiceLevel calculator do not read it yet — they live outside the Host (<c>Cluster</c> /
|
||||
/// <c>Runtime</c>); feeding them is the Wave 3 follow-up the lane report names. Until then a fused
|
||||
/// node whose SDK failed is visible (503) but still Primary-eligible.
|
||||
/// </remarks>
|
||||
public interface IOpcUaEndpointState
|
||||
{
|
||||
/// <summary>The endpoint's current lifecycle status.</summary>
|
||||
OpcUaEndpointStatus Status { get; }
|
||||
|
||||
/// <summary>Why the start failed, when <see cref="Status"/> is <see cref="OpcUaEndpointStatus.Failed"/>.</summary>
|
||||
string? FailureReason { get; }
|
||||
}
|
||||
|
||||
/// <summary>Mutable <see cref="IOpcUaEndpointState"/>; written by the hosted service, read by the health check.</summary>
|
||||
public sealed class OpcUaEndpointState : IOpcUaEndpointState
|
||||
{
|
||||
private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted;
|
||||
private volatile string? _failureReason;
|
||||
|
||||
/// <inheritdoc />
|
||||
public OpcUaEndpointStatus Status => _status;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string? FailureReason => _failureReason;
|
||||
|
||||
/// <summary>Records a successful SDK start.</summary>
|
||||
public void MarkStarted()
|
||||
{
|
||||
_failureReason = null;
|
||||
_status = OpcUaEndpointStatus.Started;
|
||||
}
|
||||
|
||||
/// <summary>Records a failed SDK start.</summary>
|
||||
/// <param name="reason">The failure, as the health probe should report it.</param>
|
||||
public void MarkFailed(string reason)
|
||||
{
|
||||
_failureReason = reason;
|
||||
_status = OpcUaEndpointStatus.Failed;
|
||||
}
|
||||
|
||||
/// <summary>Records that the hosted service stopped the SDK.</summary>
|
||||
public void MarkStopped()
|
||||
{
|
||||
_failureReason = null;
|
||||
_status = OpcUaEndpointStatus.Stopped;
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
@@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
|
||||
private readonly HistorianUpdaterHolder _historianUpdater;
|
||||
private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
|
||||
private readonly OpcUaEndpointState _endpointState;
|
||||
private readonly bool _rethrowStartFailure;
|
||||
|
||||
private OpcUaApplicationHost? _appHost;
|
||||
private OtOpcUaSdkServer? _server;
|
||||
@@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
/// <param name="historyUpdateRoleView">The HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
|
||||
/// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
|
||||
/// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
|
||||
/// <param name="endpointState">The endpoint lifecycle fact this service maintains for the
|
||||
/// <c>opcua-endpoint</c> health probe (R03:2.1). Optional: a private instance is kept when none is registered,
|
||||
/// so a harness that never probes health is unchanged.</param>
|
||||
/// <param name="roles">The node's resolved roles. A node with the driver role and NO admin role exists only
|
||||
/// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and
|
||||
/// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through
|
||||
/// the health probe instead. Null (harnesses) behaves as fused.</param>
|
||||
public OtOpcUaServerHostedService(
|
||||
IOptions<OpcUaApplicationHostOptions> options,
|
||||
DeferredAddressSpaceSink deferredSink,
|
||||
@@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
ILoggerFactory loggerFactory,
|
||||
IDriverPrimaryRoleSource? primaryRoleSource = null,
|
||||
HistorianUpdaterHolder? historianUpdater = null,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null)
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null,
|
||||
OpcUaEndpointState? endpointState = null,
|
||||
EffectiveRoles? roles = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(aclOptions);
|
||||
ArgumentNullException.ThrowIfNull(time);
|
||||
@@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_primaryRoleSource = primaryRoleSource;
|
||||
_historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
|
||||
_historyUpdateRoleView = historyUpdateRoleView;
|
||||
_endpointState = endpointState ?? new OpcUaEndpointState();
|
||||
_rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false };
|
||||
_logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
|
||||
}
|
||||
|
||||
@@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy
|
||||
// (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits
|
||||
// so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct
|
||||
// the OpcUa:* config without a process bounce — but it is visibly not ready.
|
||||
_endpointState.MarkFailed(ex.Message);
|
||||
await ReleaseFailedStartAsync().ConfigureAwait(false);
|
||||
if (_rethrowStartFailure)
|
||||
{
|
||||
_logger.LogCritical(ex,
|
||||
"OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " +
|
||||
"node justifies staying up; failing host start so the supervisor restarts the process");
|
||||
throw;
|
||||
}
|
||||
_logger.LogError(ex,
|
||||
"OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op");
|
||||
// Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot.
|
||||
// Operators see the failure via the logs + can correct config without a process bounce
|
||||
// of the whole binary.
|
||||
"OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " +
|
||||
"OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null");
|
||||
_endpointState.MarkFailed("SDK reported started but the node manager is null");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
|
||||
_endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message);
|
||||
try
|
||||
{
|
||||
await StopAsync(cancellationToken).ConfigureAwait(false);
|
||||
DetachSinks();
|
||||
}
|
||||
catch (Exception cleanupError)
|
||||
{
|
||||
_logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
|
||||
}
|
||||
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
|
||||
async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
|
||||
() => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
|
||||
if (released)
|
||||
{
|
||||
_appHost = null;
|
||||
_server = null;
|
||||
}
|
||||
await ReleaseEndpointAsync().ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// attached, so the first tick cannot land on a half-wired server.
|
||||
StartRevalidationTimer();
|
||||
|
||||
_endpointState.MarkStarted();
|
||||
_logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases an SDK whose <c>StartAsync</c> threw. The application instance may hold a half-opened
|
||||
/// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the
|
||||
/// transport is up); disposing it is the only way to be sure. The state stays Failed.
|
||||
/// </summary>
|
||||
private async Task ReleaseFailedStartAsync()
|
||||
{
|
||||
var appHost = _appHost;
|
||||
_appHost = null;
|
||||
_server = null;
|
||||
if (appHost is null) return;
|
||||
try
|
||||
{
|
||||
await appHost.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the SDK and disposes the application host, releasing endpoint ownership only when both
|
||||
/// succeeded (<see cref="FailedServiceLevelBindingCleanup.StopAndDisposeAsync"/>): on a failure the
|
||||
/// fields are retained so <see cref="DisposeAsync"/> retries at host disposal.
|
||||
/// </summary>
|
||||
private async Task ReleaseEndpointAsync()
|
||||
{
|
||||
var server = _server;
|
||||
var appHost = _appHost;
|
||||
if (server is null && appHost is null) return;
|
||||
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
|
||||
async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); },
|
||||
() => appHost?.DisposeAsync() ?? ValueTask.CompletedTask,
|
||||
_logger).ConfigureAwait(false);
|
||||
if (released)
|
||||
{
|
||||
_appHost = null;
|
||||
_server = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
|
||||
/// (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
|
||||
@@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the OPC UA server asynchronously.
|
||||
/// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown,
|
||||
/// the other hosted services' StopAsync), serving the last-written values with an emptied authority:
|
||||
/// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was
|
||||
/// up but inert instead of the connection-refused that triggers their failover. <see cref="DisposeAsync"/>
|
||||
/// stays idempotent with this: ownership is nulled once the stop succeeded.
|
||||
/// </remarks>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
DetachSinks();
|
||||
await ReleaseEndpointAsync().ConfigureAwait(false);
|
||||
if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor
|
||||
/// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running
|
||||
/// until <see cref="ReleaseEndpointAsync"/>.
|
||||
/// </summary>
|
||||
private void DetachSinks()
|
||||
{
|
||||
// FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
|
||||
// authority and gate have already been cleared is the shape this whole teardown block exists to
|
||||
@@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// request rather than let the seams fall back to their role checks — a stopping node should shed
|
||||
// enforcement in the same order it acquired it.
|
||||
_server?.SetDataPlaneAuthorizer(null);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
// Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
|
||||
// not leave a tick armed against a server that is about to go away.
|
||||
// not leave a tick armed against a server that is about to go away. A StopAsync that released the
|
||||
// endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left
|
||||
// them set, and this is the retry.
|
||||
_revalidationTimer?.Dispose();
|
||||
_revalidationTimer = null;
|
||||
|
||||
if (_appHost is not null) await _appHost.DisposeAsync();
|
||||
var appHost = _appHost;
|
||||
_appHost = null;
|
||||
_server = null;
|
||||
if (appHost is not null) await appHost.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua");
|
||||
// central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it
|
||||
// unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string.
|
||||
// A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected.
|
||||
// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it
|
||||
// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a
|
||||
// startup error.
|
||||
builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
|
||||
// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this
|
||||
// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3):
|
||||
// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is
|
||||
// present. A node with neither role (dev-only) has no key path and keeps the plain bind.
|
||||
if (hasDriver || hasAdmin)
|
||||
builder.Services.AddValidatedOptions<ApiKeyOptions, ApiKeyOptionsValidator>(
|
||||
builder.Configuration, ApiKeyOptions.SectionName);
|
||||
else
|
||||
builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
|
||||
// Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config
|
||||
// change rather than a redeploy, and what it swaps rather than adds.
|
||||
builder.Services.Configure<AclOptions>(builder.Configuration.GetSection(AclOptions.SectionName));
|
||||
@@ -452,6 +457,11 @@ if (hasDriver)
|
||||
// mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
|
||||
// DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
|
||||
builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
|
||||
// The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint
|
||||
// health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the
|
||||
// interface, so the writer and the probe cannot disagree.
|
||||
builder.Services.AddSingleton<OpcUaEndpointState>();
|
||||
builder.Services.AddSingleton<IOpcUaEndpointState>(sp => sp.GetRequiredService<OpcUaEndpointState>());
|
||||
builder.Services.AddHostedService<OtOpcUaServerHostedService>();
|
||||
}
|
||||
|
||||
@@ -554,8 +564,9 @@ if (hasAdmin)
|
||||
builder.Services.AddOtOpcUaSecrets(builder.Configuration);
|
||||
|
||||
// hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
|
||||
// probing it would throw when the readiness endpoint resolves the factory.
|
||||
builder.Services.AddOtOpcUaHealth(hasAdmin);
|
||||
// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the
|
||||
// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green.
|
||||
builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver);
|
||||
builder.Services.AddOtOpcUaObservability(builder.Configuration);
|
||||
|
||||
// gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,
|
||||
|
||||
Reference in New Issue
Block a user