Merge lane C: control plane, host, security (#733)

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 14:09:08 -04:00
61 changed files with 6603 additions and 453 deletions
@@ -0,0 +1,41 @@
using ZB.MOM.WW.Configuration;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
/// <summary>
/// Fail-fast startup validator for <see cref="ApiKeyOptions"/> (R03:2.3). The pepper used to be
/// checked only when the first key was verified — <c>ApiKeySecret.Verify</c> throws on an empty one —
/// so a site node deployed without <c>Security__ApiKeys__Pepper</c> booted green, passed health,
/// applied deployments and denied every machine client with the same opaque "Invalid API key" until
/// someone read the log; a central node with a wrong pepper minted keys that could verify nowhere.
/// Every other security section already fails host start on a bad value; this one now does too.
/// </summary>
/// <remarks>
/// Registered for <c>hasDriver || hasAdmin</c>: a driver node needs the pepper to VERIFY keys, an admin
/// node to MINT them. The minimum length is a floor against a placeholder (<c>"x"</c>) passing as
/// configured, not a strength claim — the pepper is a server-side secret, not a password.
/// </remarks>
public sealed class ApiKeyOptionsValidator : OptionsValidatorBase<ApiKeyOptions>
{
/// <summary>The shortest pepper accepted (characters, after trimming).</summary>
public const int MinPepperLength = 16;
/// <inheritdoc />
protected override void Validate(ValidationBuilder builder, ApiKeyOptions options)
{
var pepper = options.Pepper?.Trim() ?? string.Empty;
builder.RequireThat(pepper.Length > 0,
$"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " +
"supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " +
"every API key (driver) or mint keys that can never verify (admin).");
builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength,
$"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters.");
builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username),
$"{ApiKeyOptions.SectionName}:Username must not be blank.");
builder.RequireThat(options.StoreTimeoutMs >= 0,
$"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline).");
builder.RequireThat(options.StoreCacheSeconds >= 0,
$"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache).");
}
}
@@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration
sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
sp.GetRequiredService<TimeProvider>()));
sp.GetRequiredService<TimeProvider>(),
// R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as
// ApiKeyOptions, Host-owned knobs.
ApiKeyStoreGuardOptions.Bind(sp.GetService<IConfiguration>())));
services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
// Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
@@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
namespace ZB.MOM.WW.OtOpcUa.Host.Health;
@@ -28,11 +29,40 @@ public static class HealthEndpoints
/// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to
/// <c>true</c> so admin/fused callers (including the test harnesses) keep the probe unchanged.
/// </param>
/// <param name="hasDriver">
/// Whether this node carries the <c>driver</c> role and therefore serves an OPC UA endpoint. When
/// <c>true</c> the <see cref="OpcUaEndpointHealthCheck"/> joins the ready tier, and the active tier on a
/// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading
/// the <see cref="IOpcUaEndpointState"/> the host registers alongside <c>OtOpcUaServerHostedService</c>.
/// Defaults to <c>false</c> so admin-only callers and the existing harnesses register nothing new.
/// </param>
/// <returns>The same service collection, for chaining.</returns>
public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true)
public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false)
{
var checks = services.AddHealthChecks();
// A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve.
// The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6):
// - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may
// route to it;
// - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState
// (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would
// take the operator's only console out of rotation precisely when they need it to diagnose the
// failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands
// the peer takes Primary and the active tier follows it on its own.
// Factory-registered so the state singleton is resolved per probe.
if (hasDriver)
{
var tags = hasAdmin
? new[] { ZbHealthTags.Ready }
: new[] { ZbHealthTags.Ready, ZbHealthTags.Active };
checks.Add(new HealthCheckRegistration(
OpcUaEndpointHealthCheck.Name,
sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService<IOpcUaEndpointState>()),
failureStatus: null,
tags: tags));
}
// configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config
// from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so
// the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged.
@@ -0,0 +1,47 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
namespace ZB.MOM.WW.OtOpcUa.Host.Health;
/// <summary>
/// Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is
/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy
/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets
/// connection-refused.
/// </summary>
/// <remarks>
/// Registered on driver-role nodes only (<see cref="HealthEndpoints.AddOtOpcUaHealth"/>); an
/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused
/// admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier
/// is what Traefik routes the AdminUI on.
/// </remarks>
public sealed class OpcUaEndpointHealthCheck : IHealthCheck
{
/// <summary>The registration name, as it appears in the health JSON.</summary>
public const string Name = "opcua-endpoint";
private readonly IOpcUaEndpointState _state;
/// <summary>Initializes a new instance of the <see cref="OpcUaEndpointHealthCheck"/> class.</summary>
/// <param name="state">The endpoint lifecycle fact the hosted service maintains.</param>
public OpcUaEndpointHealthCheck(IOpcUaEndpointState state)
{
ArgumentNullException.ThrowIfNull(state);
_state = state;
}
/// <inheritdoc />
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
var data = new Dictionary<string, object> { ["status"] = _state.Status.ToString() };
var result = _state.Status switch
{
OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data),
OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data),
OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data),
_ => HealthCheckResult.Unhealthy(
"OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data),
};
return Task.FromResult(result);
}
}
@@ -0,0 +1,56 @@
using Microsoft.Extensions.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
/// <summary>
/// The pre-authentication guards <see cref="BoundedApiKeyStore"/> puts in front of the ConfigDb key
/// lookup (R03:3.3). Bound from the same <c>Security:ApiKeys</c> section as <c>ApiKeyOptions</c> but
/// owned by the Host, where the store lives.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why.</b> The negative cache was keyed by key id, so a flood of DISTINCT well-formed
/// <c>otua_&lt;random&gt;_x</c> credentials missed it every time and cost one indexed SQL query per
/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close
/// that: a <b>known-id set</b> (every <c>ApiKeyId</c> the ConfigDb holds, refreshed every
/// <c>StoreCacheSeconds</c>) rejects an unknown id in memory, and a <b>miss budget</b> (token
/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the
/// set is unavailable.
/// </para>
/// <para>
/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback
/// (the <c>IOpcUaUserAuthenticator</c> seam) does not carry the client's address. Per-endpoint
/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane.
/// </para>
/// </remarks>
public sealed class ApiKeyStoreGuardOptions
{
/// <summary>The configuration section the knobs live in (shared with <c>ApiKeyOptions</c>).</summary>
public const string SectionName = "Security:ApiKeys";
/// <summary>
/// Whether the known-id set is consulted. <see langword="false"/> restores a store read per unknown
/// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact
/// store is never wrapped).
/// </summary>
public bool KnownIdSet { get; set; } = true;
/// <summary>
/// How many store MISSES (lookups that reach the inner store and find no key) may burst before
/// further lookups for unknown ids are refused without a store read. <c>0</c> disables the budget.
/// </summary>
public int MissBurst { get; set; } = 100;
/// <summary>How many miss tokens are restored per second, up to <see cref="MissBurst"/>.</summary>
public double MissRefillPerSecond { get; set; } = 20;
/// <summary>Binds the section, with defaults for anything absent.</summary>
/// <param name="configuration">The app configuration; null yields the defaults.</param>
/// <returns>The bound options.</returns>
public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration)
{
var options = new ApiKeyStoreGuardOptions();
configuration?.GetSection(SectionName).Bind(options);
return options;
}
}
@@ -56,6 +56,33 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
new(StringComparer.Ordinal);
private readonly ApiKeyStoreGuardOptions _guard;
// R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds
// (single-flight; a refresh that fails or times out keeps the previous set). Null until the first
// successful load, or when the inner store cannot list ids — then the per-id path runs as before.
// The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile
// reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read
// a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed.
private volatile KnownIdSnapshot? _knownIds;
private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1);
// Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is
// shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked
// one store's capability into every other consumer of the same options).
private volatile bool _listingUnsupported;
/// <summary>The known-id set and the instant it was loaded, published as one unit.</summary>
/// <param name="Ids">The ids the inner store holds.</param>
/// <param name="StampedUtc">When they were read.</param>
private sealed record KnownIdSnapshot(IReadOnlySet<string> Ids, DateTimeOffset StampedUtc);
// R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards
// the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the
// configured rate instead of one SQL round trip per attempt under the SessionManager lock.
private readonly object _bucketGate = new();
private double _missTokens;
private DateTimeOffset _missRefilledUtc;
/// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
/// <param name="inner">The store being bounded.</param>
@@ -66,11 +93,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
/// <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
/// detail.
/// </param>
/// <param name="guard">The pre-authentication guards (R03:3.3); defaults when omitted.</param>
public BoundedApiKeyStore(
IApiKeyStore inner,
IOptions<ApiKeyOptions> options,
ILogger<BoundedApiKeyStore> log,
TimeProvider time)
TimeProvider time,
ApiKeyStoreGuardOptions? guard = null)
{
ArgumentNullException.ThrowIfNull(inner);
ArgumentNullException.ThrowIfNull(options);
@@ -81,8 +110,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
_options = options;
_log = log;
_time = time;
_guard = guard ?? new ApiKeyStoreGuardOptions();
_missTokens = Math.Max(0, _guard.MissBurst);
_missRefilledUtc = time.GetUtcNow();
}
/// <summary>The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards.</summary>
public IReadOnlySet<string>? KnownKeyIds => _knownIds?.Ids;
/// <summary>The store being bounded.</summary>
/// <remarks>
/// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
@@ -115,12 +150,103 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
return cached;
}
// R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked.
var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false);
if (known is not null && !known.Contains(apiKeyId))
{
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "unknown_id"));
return null;
}
// R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known
// to exist (a known id is a real key being verified, never a guess).
if (known is null && !TryTakeMissToken())
{
OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "miss_budget"));
throw new InvalidOperationException(
"ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / "
+ "MissRefillPerSecond); refusing this lookup without a store read.");
}
var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
.ConfigureAwait(false);
if (fresh is not null && known is null) RefundMissToken();
if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
return fresh;
}
/// <summary>
/// The known-id set, refreshed when older than <paramref name="ttl"/>. Single-flight: concurrent
/// callers during a refresh use the previous set (or, before the first load, the per-id path). A
/// refresh that fails or exceeds the deadline logs and keeps the previous set.
/// </summary>
/// <param name="ttl">The refresh interval (the cache TTL); non-positive disables the set.</param>
/// <param name="ct">The caller's token.</param>
/// <returns>The set, or null.</returns>
private async Task<IReadOnlySet<string>?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct)
{
if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null;
var now = _time.GetUtcNow();
var current = _knownIds;
if (current is not null && now - current.StampedUtc < ttl) return current.Ids;
if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids;
try
{
// Re-check under the gate: another caller may have refreshed while this one waited.
if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids;
var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false);
if (ids is null)
{
// Unsupported by the inner store: stop asking — for this decorator only.
_listingUnsupported = true;
return null;
}
_knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow());
return ids;
}
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
{
_log.LogWarning(ex,
"ApiKeyStore: refreshing the known key-id set failed; {Mode}",
current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force");
return current?.Ids;
}
finally
{
_knownIdsRefresh.Release();
}
}
/// <summary>Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled.</summary>
/// <returns><see langword="false"/> when the budget is exhausted.</returns>
private bool TryTakeMissToken()
{
var burst = _guard.MissBurst;
if (burst <= 0) return true;
lock (_bucketGate)
{
var now = _time.GetUtcNow();
var elapsed = (now - _missRefilledUtc).TotalSeconds;
if (elapsed > 0)
{
_missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond));
_missRefilledUtc = now;
}
if (_missTokens < 1) return false;
_missTokens -= 1;
return true;
}
}
/// <summary>A lookup that reached the store and HIT was not a miss: its token goes back.</summary>
private void RefundMissToken()
{
if (_guard.MissBurst <= 0) return;
lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1);
}
// Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.
/// <inheritdoc />
public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
@@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore
return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
{
await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false);
// One projection over the whole (small) table — the unique index on ApiKeyId makes this an index
// scan — so an unknown id can be refused in memory instead of costing a query per attempt.
var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false);
return ids.ToHashSet(StringComparer.Ordinal);
}
/// <summary>Loads a principal's grants, confined to this node's cluster.</summary>
/// <remarks>
/// <b>The full grant set, not just the <c>Cluster</c>-scoped rows</b>. The old
@@ -35,6 +35,16 @@ public interface IApiKeyStore
/// <param name="ct">The cancellation token.</param>
/// <returns>The matching ACL rows; empty when none.</returns>
Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct);
/// <summary>
/// Every <c>ApiKeyId</c> the store holds (enabled or not), for the pre-authentication known-id set
/// (R03:3.3); <see langword="null"/> when the store cannot cheaply list them, in which case callers
/// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry
/// and the secret are still decided by <see cref="FindAsync"/>.
/// </summary>
/// <param name="ct">The cancellation token.</param>
/// <returns>The id set, or null when unsupported.</returns>
Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult<IReadOnlySet<string>?>(null);
}
/// <summary>An API key resolved for authentication, with the ACL grants naming its principal.</summary>
@@ -0,0 +1,73 @@
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
/// <summary>Lifecycle of this node's OPC UA endpoint as <see cref="OtOpcUaServerHostedService"/> drives it.</summary>
public enum OpcUaEndpointStatus
{
/// <summary>The hosted service has not run <c>StartAsync</c> yet.</summary>
NotStarted,
/// <summary>The SDK is listening and the address-space / ServiceLevel sinks are bound.</summary>
Started,
/// <summary>The SDK failed to start (port in use, PKI unwritable, bad <c>OpcUa:*</c> override, …).</summary>
Failed,
/// <summary>The hosted service stopped the SDK; the port is released.</summary>
Stopped,
}
/// <summary>
/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed
/// start was one Error line: <c>/health/ready</c> stayed 200, the node joined its mesh, could be
/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got
/// connection-refused from the node the fleet called healthy.
/// </summary>
/// <remarks>
/// Consumed by <c>OpcUaEndpointHealthCheck</c> (Ready + Active tiers). The redundancy election and the
/// ServiceLevel calculator do not read it yet — they live outside the Host (<c>Cluster</c> /
/// <c>Runtime</c>); feeding them is the Wave 3 follow-up the lane report names. Until then a fused
/// node whose SDK failed is visible (503) but still Primary-eligible.
/// </remarks>
public interface IOpcUaEndpointState
{
/// <summary>The endpoint's current lifecycle status.</summary>
OpcUaEndpointStatus Status { get; }
/// <summary>Why the start failed, when <see cref="Status"/> is <see cref="OpcUaEndpointStatus.Failed"/>.</summary>
string? FailureReason { get; }
}
/// <summary>Mutable <see cref="IOpcUaEndpointState"/>; written by the hosted service, read by the health check.</summary>
public sealed class OpcUaEndpointState : IOpcUaEndpointState
{
private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted;
private volatile string? _failureReason;
/// <inheritdoc />
public OpcUaEndpointStatus Status => _status;
/// <inheritdoc />
public string? FailureReason => _failureReason;
/// <summary>Records a successful SDK start.</summary>
public void MarkStarted()
{
_failureReason = null;
_status = OpcUaEndpointStatus.Started;
}
/// <summary>Records a failed SDK start.</summary>
/// <param name="reason">The failure, as the health probe should report it.</param>
public void MarkFailed(string reason)
{
_failureReason = reason;
_status = OpcUaEndpointStatus.Failed;
}
/// <summary>Records that the hosted service stopped the SDK.</summary>
public void MarkStopped()
{
_failureReason = null;
_status = OpcUaEndpointStatus.Stopped;
}
}
@@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
@@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
private readonly HistorianUpdaterHolder _historianUpdater;
private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
private readonly OpcUaEndpointState _endpointState;
private readonly bool _rethrowStartFailure;
private OpcUaApplicationHost? _appHost;
private OtOpcUaSdkServer? _server;
@@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
/// <param name="historyUpdateRoleView">The HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
/// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
/// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
/// <param name="endpointState">The endpoint lifecycle fact this service maintains for the
/// <c>opcua-endpoint</c> health probe (R03:2.1). Optional: a private instance is kept when none is registered,
/// so a harness that never probes health is unchanged.</param>
/// <param name="roles">The node's resolved roles. A node with the driver role and NO admin role exists only
/// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and
/// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through
/// the health probe instead. Null (harnesses) behaves as fused.</param>
public OtOpcUaServerHostedService(
IOptions<OpcUaApplicationHostOptions> options,
DeferredAddressSpaceSink deferredSink,
@@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
ILoggerFactory loggerFactory,
IDriverPrimaryRoleSource? primaryRoleSource = null,
HistorianUpdaterHolder? historianUpdater = null,
IHistoryUpdateRoleView? historyUpdateRoleView = null)
IHistoryUpdateRoleView? historyUpdateRoleView = null,
OpcUaEndpointState? endpointState = null,
EffectiveRoles? roles = null)
{
ArgumentNullException.ThrowIfNull(aclOptions);
ArgumentNullException.ThrowIfNull(time);
@@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
_primaryRoleSource = primaryRoleSource;
_historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
_historyUpdateRoleView = historyUpdateRoleView;
_endpointState = endpointState ?? new OpcUaEndpointState();
_rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false };
_logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
}
@@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
}
catch (Exception ex)
{
// R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy
// (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits
// so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct
// the OpcUa:* config without a process bounce — but it is visibly not ready.
_endpointState.MarkFailed(ex.Message);
await ReleaseFailedStartAsync().ConfigureAwait(false);
if (_rethrowStartFailure)
{
_logger.LogCritical(ex,
"OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " +
"node justifies staying up; failing host start so the supervisor restarts the process");
throw;
}
_logger.LogError(ex,
"OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op");
// Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot.
// Operators see the failure via the logs + can correct config without a process bounce
// of the whole binary.
"OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " +
"OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section");
return;
}
@@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
{
_logger.LogWarning(
"OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null");
_endpointState.MarkFailed("SDK reported started but the node manager is null");
return;
}
@@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
{
_logger.LogError(ex,
"OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
_endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message);
try
{
await StopAsync(cancellationToken).ConfigureAwait(false);
DetachSinks();
}
catch (Exception cleanupError)
{
_logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
}
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
() => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
if (released)
{
_appHost = null;
_server = null;
}
await ReleaseEndpointAsync().ConfigureAwait(false);
return;
}
@@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
// attached, so the first tick cannot land on a half-wired server.
StartRevalidationTimer();
_endpointState.MarkStarted();
_logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
}
/// <summary>
/// Releases an SDK whose <c>StartAsync</c> threw. The application instance may hold a half-opened
/// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the
/// transport is up); disposing it is the only way to be sure. The state stays Failed.
/// </summary>
private async Task ReleaseFailedStartAsync()
{
var appHost = _appHost;
_appHost = null;
_server = null;
if (appHost is null) return;
try
{
await appHost.DisposeAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw");
}
}
/// <summary>
/// Stops the SDK and disposes the application host, releasing endpoint ownership only when both
/// succeeded (<see cref="FailedServiceLevelBindingCleanup.StopAndDisposeAsync"/>): on a failure the
/// fields are retained so <see cref="DisposeAsync"/> retries at host disposal.
/// </summary>
private async Task ReleaseEndpointAsync()
{
var server = _server;
var appHost = _appHost;
if (server is null && appHost is null) return;
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); },
() => appHost?.DisposeAsync() ?? ValueTask.CompletedTask,
_logger).ConfigureAwait(false);
if (released)
{
_appHost = null;
_server = null;
}
}
/// <summary>
/// The <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
/// (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
@@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
}
/// <summary>
/// Stops the OPC UA server asynchronously.
/// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes.
/// </summary>
/// <remarks>
/// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown,
/// the other hosted services' StopAsync), serving the last-written values with an emptied authority:
/// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was
/// up but inert instead of the connection-refused that triggers their failover. <see cref="DisposeAsync"/>
/// stays idempotent with this: ownership is nulled once the stop succeeded.
/// </remarks>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task StopAsync(CancellationToken cancellationToken)
public async Task StopAsync(CancellationToken cancellationToken)
{
DetachSinks();
await ReleaseEndpointAsync().ConfigureAwait(false);
if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped();
}
/// <summary>
/// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor
/// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running
/// until <see cref="ReleaseEndpointAsync"/>.
/// </summary>
private void DetachSinks()
{
// FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
// authority and gate have already been cleared is the shape this whole teardown block exists to
@@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
// request rather than let the seams fall back to their role checks — a stopping node should shed
// enforcement in the same order it acquired it.
_server?.SetDataPlaneAuthorizer(null);
return Task.CompletedTask;
}
/// <summary>
@@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
public async ValueTask DisposeAsync()
{
// Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
// not leave a tick armed against a server that is about to go away.
// not leave a tick armed against a server that is about to go away. A StopAsync that released the
// endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left
// them set, and this is the retry.
_revalidationTimer?.Dispose();
_revalidationTimer = null;
if (_appHost is not null) await _appHost.DisposeAsync();
var appHost = _appHost;
_appHost = null;
_server = null;
if (appHost is not null) await appHost.DisposeAsync();
}
}
+17 -6
View File
@@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua");
// central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it
// unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string.
// A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected.
// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it
// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a
// startup error.
builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this
// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3):
// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is
// present. A node with neither role (dev-only) has no key path and keeps the plain bind.
if (hasDriver || hasAdmin)
builder.Services.AddValidatedOptions<ApiKeyOptions, ApiKeyOptionsValidator>(
builder.Configuration, ApiKeyOptions.SectionName);
else
builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
// Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config
// change rather than a redeploy, and what it swaps rather than adds.
builder.Services.Configure<AclOptions>(builder.Configuration.GetSection(AclOptions.SectionName));
@@ -452,6 +457,11 @@ if (hasDriver)
// mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
// DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
// The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint
// health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the
// interface, so the writer and the probe cannot disagree.
builder.Services.AddSingleton<OpcUaEndpointState>();
builder.Services.AddSingleton<IOpcUaEndpointState>(sp => sp.GetRequiredService<OpcUaEndpointState>());
builder.Services.AddHostedService<OtOpcUaServerHostedService>();
}
@@ -554,8 +564,9 @@ if (hasAdmin)
builder.Services.AddOtOpcUaSecrets(builder.Configuration);
// hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
// probing it would throw when the readiness endpoint resolves the factory.
builder.Services.AddOtOpcUaHealth(hasAdmin);
// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the
// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green.
builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver);
builder.Services.AddOtOpcUaObservability(builder.Configuration);
// gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,