Merge branch 'master' into fix/r3-02-stale-good-family

# Conflicts:
#	src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
#	tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs

@@ -0,0 +1,111 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+/// <summary>
+///     Composition of the OPC UA data-plane authentication + authorization seams (#520), extracted from
+///     <c>Program.cs</c> so the role-conditional branches are reachable from a test.
+/// </summary>
+/// <remarks>
+///     A store decorator registered in one branch of a role conditional is exactly the
+///     "built but never wired" shape, and <c>Program.cs</c>'s top-level statements cannot be driven by a
+///     test — so the wiring moved here rather than the guard being written against a comment. The
+///     branches are unchanged from the ones that lived inline; see <c>ApiKeyStoreRegistrationTests</c>.
+/// </remarks>
+public static class DataPlaneAuthRegistration
+{
+    /// <summary>
+    ///     Registers the shared clock, the node-scoped ACL gate and the role-appropriate API-key store +
+    ///     user authenticator.
+    /// </summary>
+    /// <param name="services">The service collection.</param>
+    /// <param name="hasAdmin">Whether this node carries the <c>admin</c> role (and therefore a ConfigDb).</param>
+    /// <param name="hasDriver">Whether this node carries the <c>driver</c> role.</param>
+    /// <returns>The same collection, for chaining.</returns>
+    public static IServiceCollection AddOtOpcUaDataPlaneAuth(
+        this IServiceCollection services, bool hasAdmin, bool hasDriver)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+
+        // ONE clock for the whole authorization surface. The authorizer, the key-store bound and the
+        // revalidation timer all read time, and DataPlaneAuthorizer already documents what a second,
+        // diverging clock costs: every request denied, with a verdict indistinguishable from "no grant
+        // matched". TryAdd so a test (or a future host) can substitute one before this runs.
+        services.TryAddSingleton(TimeProvider.System);
+
+        // #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because
+        // OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves
+        // constructors by whether every parameter is registered, not by whether the parameter type is nullable.
+        // That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations below
+        // both carry a comment about; an absent registration is a node that will not start.
+        services.AddSingleton<IPermissionEvaluator>(sp =>
+            new TriePermissionEvaluator(sp.GetRequiredService<PermissionTrieCache>()));
+        services.AddSingleton(sp => new DataPlaneAuthorizer(
+            sp.GetRequiredService<IDataPlaneAuthoritySource>(),
+            sp.GetRequiredService<IPermissionEvaluator>(),
+            sp.GetRequiredService<IOptions<AclOptions>>().Value.Enabled,
+            sp.GetRequiredService<ILoggerFactory>().CreateLogger<DataPlaneAuthorizer>(),
+            sp.GetRequiredService<TimeProvider>()));
+
+        if (hasAdmin)
+        {
+            // A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect
+            // on the very next activation rather than on the next deployment. Preferred wherever it is
+            // available, including on a fused admin+driver node.
+            // The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds
+            // the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which
+            // ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration),
+            // NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services
+            // rule is what keeps the redundancy-singleton registration in Program.cs from recursing into
+            // ActorSystem construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter.
+            //
+            // arch-review 03/S18 — and it is WRAPPED. The lookup runs inline in the SDK's synchronous
+            // impersonation callback, under a SessionManager-wide lock, with CancellationToken.None: an
+            // unanswered ConfigDb query therefore stalls every session activation in the server. The
+            // decorator adds the deadline that bounds that, plus a short TTL that turns a burst of
+            // activations into one round trip. It cannot admit anything the inner store did not.
+            services.AddSingleton<IApiKeyStore>(sp => new BoundedApiKeyStore(
+                new ConfigDbApiKeyStore(
+                    sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
+                    ConfigDbApiKeyStore.ResolveClusterId(
+                        sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
+                sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
+                sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
+                sp.GetRequiredService<TimeProvider>()));
+            services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
+
+            // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
+            services.AddSingleton<IApiKeyAdminService>(sp => new ApiKeyAdminService(
+                sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
+                sp.GetRequiredService<IOptions<ApiKeyOptions>>().Value.Pepper));
+        }
+        else if (hasDriver)
+        {
+            // Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact.
+            // Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName
+            // token — safe, but it meant no machine client could authenticate against a site node at all.
+            // Deliberately NOT wrapped by the S18 bound: the artifact store is two dictionary probes on an
+            // in-memory snapshot, so it can neither hang nor be worth caching.
+            services.AddSingleton<IApiKeyStore, ArtifactApiKeyStore>();
+            services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
+        }
+        else
+        {
+            // Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent —
+            // OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so
+            // an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start.
+            services.AddSingleton<IOpcUaUserAuthenticator>(NullOpcUaUserAuthenticator.Instance);
+        }
+
+        return services;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/MeshGrpcEndpoints.cs

@@ -0,0 +1,125 @@
+using ZB.MOM.WW.LocalDb.Replication;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
+using ZB.MOM.WW.OtOpcUa.Host.Grpc;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+/// <summary>
+///     Maps the mesh gRPC ingress endpoints (artifact serve, central ack, telemetry stream, node
+///     command) and the LocalDb sync endpoint.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Why AllowAnonymous (archreview 03/S14).</b> On an admin-role node
+///         <c>AddOtOpcUaAuth</c> installs a global <c>FallbackPolicy = RequireAuthenticatedUser</c>
+///         under the cookie scheme, which the authorization middleware applies to EVERY endpoint
+///         without explicit authorization metadata — gRPC included. These endpoints authenticate
+///         with a per-cluster PSK bearer inside a gRPC interceptor, which runs INSIDE the endpoint
+///         invocation, i.e. after that middleware. Without this opt-out a correctly-keyed node call
+///         is challenged by the cookie handler before the interceptor's verdict matters: artifact
+///         fetches fail, deploy acks are dropped, deployments seal TimedOut. Anonymous is safe here
+///         because all five interceptors fail CLOSED — an unset key rejects every call and the
+///         presented token is compared with FixedTimeEquals.
+///     </para>
+///     <para>
+///         <b>Do not "tidy" this into Program.cs.</b> A standing census guard
+///         (<c>MeshGrpcAnonymousMetadataTests</c>) drives THIS method, so a map that moves back
+///         inline leaves the guard asserting a shape production no longer has — which is exactly
+///         how the AdminUI cookie <c>FallbackPolicy</c> silently blocked every mesh gRPC call on a
+///         login-enabled admin node for seven phases of live gates.
+///     </para>
+/// </remarks>
+public static class MeshGrpcEndpoints
+{
+    /// <summary>
+    ///     Maps the mesh gRPC ingress endpoints and the LocalDb sync endpoint, gated on the same
+    ///     role + dedicated-port checks <c>Program.cs</c> used inline before this extraction.
+    /// </summary>
+    /// <param name="app">The built application to map endpoints on.</param>
+    /// <param name="hasAdmin">Whether this node carries the <c>admin</c> role.</param>
+    /// <param name="hasDriver">Whether this node carries the <c>driver</c> role.</param>
+    /// <param name="syncListenPort">
+    ///     The dedicated h2c port bound for the LocalDb passive sync endpoint, or <c>0</c> when it is
+    ///     not bound.
+    /// </param>
+    /// <param name="configServeGrpcPort">
+    ///     The dedicated h2c port bound for the ConfigServe artifact + central command endpoints, or
+    ///     <c>0</c> when it is not bound.
+    /// </param>
+    /// <param name="telemetryListenPort">
+    ///     The dedicated h2c port bound for the telemetry stream + node command endpoints, or
+    ///     <c>0</c> when it is not bound.
+    /// </param>
+    public static void MapOtOpcUaMeshGrpcServices(
+        this WebApplication app,
+        bool hasAdmin,
+        bool hasDriver,
+        int syncListenPort,
+        int configServeGrpcPort,
+        int telemetryListenPort)
+    {
+        ArgumentNullException.ThrowIfNull(app);
+
+        // Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a
+        // default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be
+        // safe even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey
+        // configured — but not mapping it at all is the stronger default.
+        //
+        // AllowAnonymous (03/S14) via an empty-prefix MapGroup: MapZbLocalDbSync returns
+        // IEndpointRouteBuilder, NOT an IEndpointConventionBuilder, so .AllowAnonymous() cannot be
+        // chained onto it the way the other four maps below are. An empty-prefix group carries the
+        // AllowAnonymous convention to whatever it maps while leaving the gRPC route pattern
+        // byte-identical to the ungrouped map (verified by
+        // LocalDbSync_AllowAnonymousGroup_LeavesTheRoutePatternByteIdentical).
+        if (hasDriver && syncListenPort > 0)
+        {
+            app.MapGroup(string.Empty).AllowAnonymous().MapZbLocalDbSync();
+        }
+
+        // Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on
+        // admin-role nodes (which hold the central SQL connection) and only when a dedicated h2c
+        // port is bound. The ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes
+        // with no ConfigServe:ApiKey.
+        if (hasAdmin && configServeGrpcPort > 0)
+        {
+            // AllowAnonymous (03/S14): this is every FetchAndCache node's artifact fetch. Without the
+            // opt-out the AdminUI cookie FallbackPolicy 401s/redirects it before
+            // ConfigServeAuthInterceptor's verdict matters — a fleet-wide deploy-plane outage on any
+            // login-enabled admin node.
+            app.MapGrpcService<DeploymentArtifactGrpcService>().AllowAnonymous();
+
+            // Central command ingress (mesh gRPC command plane): the node→central deployment ack. It
+            // rides the SAME listener rather than claiming a port of its own — a second cleartext h2c
+            // port would buy nothing but another firewall rule, and both endpoints are admin-role and
+            // dialled by the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping
+            // it under Dps costs nothing (nothing dials it) whereas gating it would mean flipping the
+            // mode on a running fleet requires a central restart before any node can ack — the same
+            // reasoning that registers CentralCommunicationActor in both modes.
+            // CentralCommandAuthInterceptor (added at AddGrpc in Program.cs) is the ONLY inbound auth
+            // and fail-closes per cluster. AllowAnonymous (03/S14): this is every node's deploy ack,
+            // from every cluster — same reasoning as the artifact fetch above.
+            app.MapGrpcService<CentralCommandGrpcService>().AllowAnonymous();
+        }
+
+        // Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
+        // TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
+        //
+        // The node-side command endpoint (mesh gRPC command plane, Phase 1) rides the SAME
+        // driver-role h2c listener rather than claiming a fourth port: central already dials this
+        // node here, both directions are per-cluster-keyed, and a separate port would be one more
+        // thing to open on every site firewall for no isolation gain. Mapping it is what makes
+        // MeshCommandAuthInterceptor live — it is registered in Program.cs and is a pass-through
+        // until a call matches this service's method prefix. It fail-closes with no
+        // MeshTransport:GrpcPsk, so an operator who maps the service without setting a key gets a
+        // closed endpoint, not an open one.
+        if (hasDriver && telemetryListenPort > 0)
+        {
+            // AllowAnonymous (03/S14) on both: on a fused admin+driver node (the docker-dev central
+            // pair, and any production fused central) these sit behind the SAME cookie FallbackPolicy
+            // as the ConfigServe endpoints above and would be 401/redirected identically without the
+            // opt-out — central's own telemetry dialer and node-command dispatch to its own pair.
+            app.MapGrpcService<TelemetryStreamGrpcService>().AllowAnonymous();
+            app.MapGrpcService<NodeCommandGrpcService>().AllowAnonymous();
+        }
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/AclRevalidationTimer.cs

@@ -0,0 +1,99 @@
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>
+///     Runs the node manager's monitored-item revalidation pass on a low-frequency tick, so a key that
+///     expires by the CLOCK reaches a live subscription without waiting for a deployment (arch-review
+///     03/S19).
+/// </summary>
+/// <remarks>
+///     <para>
+///         Every request-shaped operation re-checks the calling key's usability per call and so honours
+///         <c>ExpiresUtc</c> the moment it passes. A subscription is authorized once and then streams, and
+///         streaming is how machine consumers actually read this server — so before this the only thing
+///         that could stand a subscription down was a deployment.
+///     </para>
+///     <para>
+///         ⚠️ <b>The tick must never throw.</b> An unhandled exception on a timer callback thread takes
+///         the process down. Every tick is wrapped; a faulting pass is logged and the timer keeps running,
+///         because the next tick is the recovery.
+///     </para>
+///     <para>
+///         ⚠️ <b>Each tick gets a fresh memo, and that is the pass's own constraint, not this class's.</b>
+///         <c>RevalidateMonitoredItems</c> allocates its verdict dictionary per call — see the remarks on
+///         <c>OtOpcUaNodeManager.OnAuthorityChanged</c>. Nothing here may hoist it across ticks, or an
+///         expiry landing between two ticks would be served the earlier tick's allow.
+///     </para>
+///     <para>
+///         It adds no decision logic: the verdict comes from the authorizer's own
+///         <see cref="TimeProvider"/>, and the stand-down (queue <c>BadUserAccessDenied</c>, then detach)
+///         and its restore are the pass's, unchanged. The pass early-returns unless ACL enforcement is on,
+///         so this is inert on the default configuration.
+///     </para>
+/// </remarks>
+public sealed class AclRevalidationTimer : IDisposable
+{
+    private readonly Action _revalidate;
+    private readonly ILogger _log;
+    private readonly ITimer? _timer;
+    private volatile bool _disposed;
+
+    /// <summary>Initializes a new instance of the <see cref="AclRevalidationTimer"/> class.</summary>
+    /// <param name="revalidate">The revalidation pass to run on each tick.</param>
+    /// <param name="interval">
+    ///     How often to run it. Non-positive arms no timer at all — the deploy-triggered-only behaviour
+    ///     that predates 03/S19.
+    /// </param>
+    /// <param name="time">The shared clock; its <c>CreateTimer</c> is what arms the tick.</param>
+    /// <param name="log">The logger.</param>
+    public AclRevalidationTimer(Action revalidate, TimeSpan interval, TimeProvider time, ILogger log)
+    {
+        ArgumentNullException.ThrowIfNull(revalidate);
+        ArgumentNullException.ThrowIfNull(time);
+        ArgumentNullException.ThrowIfNull(log);
+
+        _revalidate = revalidate;
+        _log = log;
+
+        if (interval <= TimeSpan.Zero)
+        {
+            // Say so once. A disabled guard that logs nothing is indistinguishable from a guard that is
+            // running and finding nothing, which is precisely the confusion 03/S19 is about.
+            _log.LogInformation(
+                "AclRevalidationTimer: disabled — {Section}:RevalidationIntervalSeconds is not positive, "
+                + "so live monitored items are re-authorized only when a deployment lands",
+                AclOptions.SectionName);
+            return;
+        }
+
+        _timer = time.CreateTimer(_ => Tick(), state: null, interval, interval);
+        _log.LogInformation(
+            "AclRevalidationTimer: live monitored items will be re-authorized every {Interval}", interval);
+    }
+
+    /// <summary>Runs one revalidation pass, absorbing anything it throws.</summary>
+    /// <remarks>Exposed so a test can drive the callback body deterministically rather than by waiting.</remarks>
+    public void Tick()
+    {
+        if (_disposed) return;
+
+        try
+        {
+            _revalidate();
+        }
+        catch (Exception ex)
+        {
+            _log.LogError(ex, "AclRevalidationTimer: the monitored-item revalidation pass threw; "
+                              + "the timer keeps running and the next tick is the recovery");
+        }
+    }
+
+    /// <inheritdoc />
+    public void Dispose()
+    {
+        _disposed = true;
+        _timer?.Dispose();
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyOpcUaUserAuthenticator.cs

@@ -18,9 +18,18 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
 ///         <b>This runs inline on session activation.</b> The SDK raises <c>ImpersonateUser</c>
 ///         synchronously while holding a SessionManager-wide event lock, so a slow authenticator does not
 ///         stall one request — it serializes every session activation server-wide. That is why
-///         verification is a single indexed lookup plus one HMAC, and why there is no directory call,
-///         no retry, and no network I/O on this path at all. It is the main practical advantage of keys
-///         over the LDAP bind this replaces.
+///         verification is a single indexed lookup plus one HMAC, with no directory call and no retry —
+///         the main practical advantage of keys over the LDAP bind this replaces.
+///     </para>
+///     <para>
+///         ⚠️ <b>"No network I/O" is TRUE ONLY ON A DRIVER-ONLY NODE</b> (arch-review 03/S18). The claim
+///         describes <see cref="ArtifactApiKeyStore"/>, which is two dictionary probes on an in-memory
+///         snapshot. On an <b>admin or fused</b> node <see cref="IApiKeyStore"/> is
+///         <see cref="ConfigDbApiKeyStore"/>, and a UserName activation costs <b>two</b> EF round trips
+///         (the key row, then the principal's <c>NodeAcl</c> rows) — inside this same lock. What bounds
+///         that is <see cref="BoundedApiKeyStore"/>, which wraps it in production; the deadline it applies
+///         surfaces here as an ordinary throw and is denied by the catch-all below, like any other store
+///         failure.
 ///     </para>
 ///     <para>
 ///         <b>Fail closed, and say little.</b> Every rejection returns the same opaque message. The

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs

@@ -0,0 +1,240 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Commons.Observability;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>
+///     Bounds what an <see cref="IApiKeyStore"/> can cost the OPC UA impersonation callback (arch-review
+///     03/S18). Wraps <see cref="ConfigDbApiKeyStore"/> on a node that has a ConfigDb; the in-memory
+///     <see cref="ArtifactApiKeyStore"/> needs neither half and is deliberately left unwrapped.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Why a deadline is the fix and the cache is not.</b> The SDK raises impersonation
+///         synchronously under a <c>SessionManager</c>-wide lock, so a store that does not answer does not
+///         stall one activation — it stalls every activation in the server. A cache lowers how OFTEN the
+///         store is asked; only a deadline bounds how LONG one ask may hold that lock.
+///     </para>
+///     <para>
+///         ⚠️ <b>Every bounded outcome DENIES.</b> A deadline exceeded, a faulted inner store and a
+///         cancelled call all surface as an exception into
+///         <see cref="ApiKeyOpcUaUserAuthenticator"/>'s catch-all, which returns the same opaque denial as
+///         a bad credential. Nothing here can admit a session the inner store did not authorize — the
+///         decorator can only return what the inner store returned, or throw.
+///     </para>
+///     <para>
+///         ⚠️ <b>Faults and timeouts are NEVER cached.</b> Caching a deny would be fail-closed and is
+///         still wrong: it would extend a ConfigDb outage past the outage, denying valid keys for a
+///         further TTL after the database came back. Only a COMPLETED lookup — a hit or an authoritative
+///         miss — is stored.
+///     </para>
+/// </remarks>
+public sealed class BoundedApiKeyStore : IApiKeyStore
+{
+    /// <summary>
+    ///     Entries either cache may hold before it is cleared wholesale.
+    /// </summary>
+    /// <remarks>
+    ///     Wholesale rather than per-entry eviction, for the reason <c>SessionAclDecisionCache</c> already
+    ///     documents: picking WHICH entry to drop needs knowledge the cache does not have, and a
+    ///     repopulate is cheap and cannot be wrong. The ceiling exists so a key-id guessing flood is a
+    ///     bounded memory event — negative results are cached deliberately, so that a flood of unknown ids
+    ///     does not become a SQL flood.
+    /// </remarks>
+    public const int MaxCacheEntries = 1024;
+
+    private readonly IApiKeyStore _inner;
+    private readonly IOptions<ApiKeyOptions> _options;
+    private readonly ILogger<BoundedApiKeyStore> _log;
+    private readonly TimeProvider _time;
+    private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
+    private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
+        new(StringComparer.Ordinal);
+
+    /// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
+    /// <param name="inner">The store being bounded.</param>
+    /// <param name="options">The <c>Security:ApiKeys</c> options carrying the deadline and TTL.</param>
+    /// <param name="log">The logger.</param>
+    /// <param name="time">
+    ///     The shared clock. The same instance the authorizer reads — see
+    ///     <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
+    ///     detail.
+    /// </param>
+    public BoundedApiKeyStore(
+        IApiKeyStore inner,
+        IOptions<ApiKeyOptions> options,
+        ILogger<BoundedApiKeyStore> log,
+        TimeProvider time)
+    {
+        ArgumentNullException.ThrowIfNull(inner);
+        ArgumentNullException.ThrowIfNull(options);
+        ArgumentNullException.ThrowIfNull(log);
+        ArgumentNullException.ThrowIfNull(time);
+
+        _inner = inner;
+        _options = options;
+        _log = log;
+        _time = time;
+    }
+
+    /// <summary>The store being bounded.</summary>
+    /// <remarks>
+    ///     Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
+    ///     of a role conditional is the "built but never wired" shape, and a guard that cannot see what is
+    ///     inside the wrapper cannot tell a correctly-wired node from one wrapping the wrong store.
+    /// </remarks>
+    public IApiKeyStore Inner => _inner;
+
+    /// <summary>The clock this bound's deadline and TTL are measured on.</summary>
+    /// <remarks>Same reason as <see cref="Inner"/>: the shared-clock invariant is only pinnable by reference.</remarks>
+    public TimeProvider Clock => _time;
+
+    /// <inheritdoc />
+    /// <remarks>
+    ///     Keyed by <paramref name="apiKeyId"/> alone. <paramref name="utcNow"/> is deliberately not part
+    ///     of the key — it changes on every call, so including it would defeat the cache entirely — but a
+    ///     cached hit IS re-checked against it before being served, so the TTL can never resurrect a key
+    ///     that has expired by the clock in the meantime. The TTL therefore bounds how stale an
+    ///     <c>Enabled</c> revocation may be, and nothing else.
+    /// </remarks>
+    public async Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
+    {
+        var ttl = Ttl;
+        if (ttl > TimeSpan.Zero
+            && TryServe(_keys, apiKeyId, ttl, out var cached)
+            && !IsExpired(cached, utcNow))
+        {
+            // Metered HERE rather than inside TryServe: an entry that is inside the TTL but whose key has
+            // since expired is not a hit — it falls through to a store read — so counting it in TryServe
+            // would report cache hits that never happened.
+            MeterCacheHit();
+            return cached;
+        }
+
+        var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
+            .ConfigureAwait(false);
+        if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
+        return fresh;
+    }
+
+    /// <inheritdoc />
+    /// <remarks>Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.</remarks>
+    public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
+    {
+        var ttl = Ttl;
+        if (ttl > TimeSpan.Zero && TryServe(_grants, principal, ttl, out var cached))
+        {
+            MeterCacheHit();
+            return cached;
+        }
+
+        var fresh = await BoundAsync(token => _inner.FindGrantsAsync(principal, token), ct)
+            .ConfigureAwait(false);
+        if (ttl > TimeSpan.Zero) Store(_grants, principal, fresh);
+        return fresh;
+    }
+
+    /// <summary>The configured TTL; non-positive means the cache is off.</summary>
+    private TimeSpan Ttl => TimeSpan.FromSeconds(_options.Value.StoreCacheSeconds);
+
+    /// <summary>Whether a cached lookup has since expired by the clock.</summary>
+    /// <param name="cached">The cached lookup, possibly null (an authoritative miss).</param>
+    /// <param name="utcNow">The caller's current time.</param>
+    /// <returns><see langword="true"/> when the cached key must not be served.</returns>
+    private static bool IsExpired(ApiKeyLookup? cached, DateTime utcNow) =>
+        cached?.Key.ExpiresUtc is { } expires && expires <= utcNow;
+
+    /// <summary>Counts one lookup the cache actually answered.</summary>
+    private static void MeterCacheHit() =>
+        OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "cache_hit"));
+
+    /// <summary>Serves an entry if one is present and still inside the TTL.</summary>
+    /// <typeparam name="T">The cached value type.</typeparam>
+    /// <param name="cache">The cache to read.</param>
+    /// <param name="key">The cache key.</param>
+    /// <param name="ttl">How long an entry stays servable.</param>
+    /// <param name="value">The cached value, when this returns true.</param>
+    /// <returns><see langword="true"/> on a hit.</returns>
+    private bool TryServe<T>(
+        ConcurrentDictionary<string, Entry<T>> cache, string key, TimeSpan ttl, out T value)
+    {
+        if (cache.TryGetValue(key, out var entry) && _time.GetUtcNow() - entry.StampedUtc < ttl)
+        {
+            value = entry.Value;
+            return true;
+        }
+
+        value = default!;
+        return false;
+    }
+
+    /// <summary>Stores a COMPLETED lookup, clearing wholesale first if the ceiling has been reached.</summary>
+    /// <typeparam name="T">The cached value type.</typeparam>
+    /// <param name="cache">The cache to write.</param>
+    /// <param name="key">The cache key.</param>
+    /// <param name="value">The value to cache.</param>
+    private void Store<T>(ConcurrentDictionary<string, Entry<T>> cache, string key, T value)
+    {
+        if (cache.Count >= MaxCacheEntries) cache.Clear();
+        cache[key] = new Entry<T>(_time.GetUtcNow(), value);
+    }
+
+    /// <summary>One cached lookup and the instant it was taken.</summary>
+    /// <typeparam name="T">The cached value type.</typeparam>
+    /// <param name="StampedUtc">When the inner store answered.</param>
+    /// <param name="Value">What it answered.</param>
+    private readonly record struct Entry<T>(DateTimeOffset StampedUtc, T Value);
+
+    /// <summary>Runs one inner call under the configured deadline.</summary>
+    /// <typeparam name="T">The inner call's result type.</typeparam>
+    /// <param name="call">The inner call, taking the deadline-linked token.</param>
+    /// <param name="ct">The caller's token — in production always <c>None</c>, see remarks.</param>
+    /// <returns>Whatever the inner store returned.</returns>
+    /// <remarks>
+    ///     <para>
+    ///         The deadline is created HERE rather than received from the caller because there is no caller
+    ///         to receive one from: <c>OpcUaApplicationHost</c> block-bridges the SDK's synchronous
+    ///         impersonation callback with <see cref="CancellationToken.None"/>, so the <c>ct</c> the store
+    ///         sees in production is always uncancellable.
+    ///     </para>
+    ///     <para>
+    ///         ⚠️ <b>It re-throws rather than returning <see langword="null"/>.</b> <c>null</c> means
+    ///         "no such usable key", which the authenticator logs as <i>denied — no enabled, unexpired key
+    ///         with id X</i>. A timeout is not that, and an operator reading that line during a ConfigDb
+    ///         outage would go hunting for a key problem that does not exist. The distinct Warning below is
+    ///         the honest signal; the denial itself is unchanged either way.
+    ///     </para>
+    /// </remarks>
+    private async Task<T> BoundAsync<T>(Func<CancellationToken, Task<T>> call, CancellationToken ct)
+    {
+        var timeoutMs = _options.Value.StoreTimeoutMs;
+        if (timeoutMs <= 0) return await call(ct).ConfigureAwait(false);
+
+        // The deadline runs on the INJECTED clock (the TimeProvider ctor overload), not on a plain
+        // CancelAfter, so it is the same TimeProvider the authorizer and the revalidation timer read.
+        // Linked with the caller's token so a genuine caller cancellation is still honoured — and so the
+        // catch below can tell the two apart.
+        using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(timeoutMs), _time);
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct, deadline.Token);
+
+        try
+        {
+            return await call(cts.Token).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+        {
+            OtOpcUaTelemetry.ApiKeyStoreBounded.Add(
+                1, new KeyValuePair<string, object?>("outcome", "timeout"));
+            _log.LogWarning(
+                "ApiKeyStore: the key store did not answer within {TimeoutMs} ms — denying this activation. "
+                + "Every OPC UA session activation on this server is serialized behind this call; "
+                + "check ConfigDb health.",
+                timeoutMs);
+            throw;
+        }
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs

@@ -14,8 +14,19 @@ namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
 ///     <b>A driver-only node has no ConfigDb since per-cluster mesh Phase 4</b>, so this store cannot be
 ///     registered there; that node uses <see cref="ArtifactApiKeyStore"/> instead (#520 stage 4).
 ///     <para>
-///         Reading per request is what makes revocation immediate here, which is why this store is
-///         preferred wherever a ConfigDb exists — including on a fused admin+driver node.
+///         Reading per activation is what makes revocation prompt here, which is why this store is
+///         preferred wherever a ConfigDb exists — including on a fused admin+driver node. ⚠️ It reaches a
+///         <b>new session activation</b> only: an already-open session re-checks its key against the
+///         in-memory artifact snapshot on every operation, not against this store, so clearing
+///         <c>Enabled</c> here does not stop an open session until the next deployment. See
+///         <c>docs/security.md</c>.
+///     </para>
+///     <para>
+///         ⚠️ <b>In production this store is WRAPPED by <see cref="BoundedApiKeyStore"/></b> (arch-review
+///         03/S18), because these queries run inline in the SDK's impersonation callback under a
+///         SessionManager-wide lock. A call may therefore be cancelled by the wrapper's deadline even
+///         though the caller passed <see cref="CancellationToken.None"/> — the resulting
+///         <see cref="OperationCanceledException"/> is a denial, not a bug.
 ///     </para>
 ///     <para>
 ///         ⚠️ <b>The ConfigDb holds the WHOLE fleet's ACL rows, not just this node's cluster's.</b> That is

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs

@@ -38,6 +38,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     private readonly IDataPlaneAuthoritySource _dataPlaneAuthority;
     private readonly DataPlaneAuthorizer? _dataPlaneAuthorizer;
     private readonly ServerHistorianOptions _serverHistorianOptions;
+    private readonly AclOptions _aclOptions;
+    private readonly TimeProvider _time;
     private readonly Func<ActorSystem> _actorSystemAccessor;
     private readonly ActorRegistry _actorRegistry;
     private readonly ILoggerFactory _loggerFactory;
@@ -45,6 +47,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
 
     private OpcUaApplicationHost? _appHost;
     private OtOpcUaSdkServer? _server;
+    private AclRevalidationTimer? _revalidationTimer;
 
     /// <summary>
     /// Initializes a new instance of the OtOpcUaServerHostedService class.
@@ -77,6 +80,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     /// <param name="dataPlaneAuthorizer">#520 stage 5 — the node-scoped ACL gate. Nullable, and null means
     /// "not wired" rather than "deny": an unregistered gate must not lock a node out of writes to protect a
     /// feature that is off by default. Once wired AND enabled, every unresolvable case inside it denies.</param>
+    /// <param name="aclOptions">#520 / arch-review 03/S19 — the <c>Security:Acl</c> options, read for
+    /// <see cref="AclOptions.RevalidationIntervalSeconds"/>. Required, not optional: the built-in DI
+    /// container does not fall back to a default for an unregistered service, the same trap the
+    /// <c>IOpcUaUserAuthenticator</c> and <c>IDataPlaneAuthoritySource</c> parameters carry comments
+    /// about.</param>
+    /// <param name="time">The shared clock — the one instance the authorizer and the API-key store bound
+    /// also read. Its <c>CreateTimer</c> arms the revalidation tick.</param>
     /// <param name="loggerFactory">The logger factory for creating loggers.</param>
     public OtOpcUaServerHostedService(
         IOptions<OpcUaApplicationHostOptions> options,
@@ -89,9 +99,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         Func<ActorSystem> actorSystemAccessor,
         ActorRegistry actorRegistry,
         DataPlaneAuthorizer? dataPlaneAuthorizer,
+        IOptions<AclOptions> aclOptions,
+        TimeProvider time,
         ILoggerFactory loggerFactory)
     {
+        ArgumentNullException.ThrowIfNull(aclOptions);
+        ArgumentNullException.ThrowIfNull(time);
+
         _dataPlaneAuthorizer = dataPlaneAuthorizer;
+        _aclOptions = aclOptions.Value;
+        _time = time;
         _options = options.Value;
         _deferredSink = deferredSink;
         _deferredServiceLevel = deferredServiceLevel;
@@ -269,9 +286,41 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
                 _loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
         }
 
+        // arch-review 03/S19 — LAST, after the node manager is known non-null and after the ACL gate is
+        // attached, so the first tick cannot land on a half-wired server.
+        StartRevalidationTimer();
+
         _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
     }
 
+    /// <summary>
+    ///     Arms the clock-driven monitored-item revalidation tick (arch-review 03/S19).
+    /// </summary>
+    /// <remarks>
+    ///     Created unconditionally: the pass it drives early-returns unless the ACL gate is wired AND
+    ///     <c>Security:Acl:Enabled</c> is on, so it stays inert on the default configuration, and
+    ///     <c>RevalidationIntervalSeconds = 0</c> arms no timer at all.
+    ///     <para>
+    ///         The node manager is resolved <b>per tick</b> rather than captured: <see cref="_server"/> is
+    ///         replaced across a restart, and a captured reference would go on revalidating a dead
+    ///         manager's item map — the same detach hazard the authority-source setter documents.
+    ///     </para>
+    ///     <para>Separate from <see cref="StartAsync"/> so the timer's lifecycle is testable without booting the SDK.</para>
+    /// </remarks>
+    internal void StartRevalidationTimer()
+    {
+        // Dispose before overwriting. StartAsync runs again when a host restarts the service around the
+        // same instance, and an overwritten field would leave the previous timer armed with nothing left
+        // to dispose it — the same dispose-before-overwrite rule the driver reconnect paths carry.
+        _revalidationTimer?.Dispose();
+
+        _revalidationTimer = new AclRevalidationTimer(
+            () => _server?.NodeManager?.RevalidateMonitoredItems(),
+            TimeSpan.FromSeconds(_aclOptions.RevalidationIntervalSeconds),
+            _time,
+            _loggerFactory.CreateLogger<AclRevalidationTimer>());
+    }
+
     /// <summary>
     /// Stops the OPC UA server asynchronously.
     /// </summary>
@@ -279,6 +328,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     /// <returns>A task that represents the asynchronous operation.</returns>
     public Task StopAsync(CancellationToken cancellationToken)
     {
+        // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
+        // authority and gate have already been cleared is the shape this whole teardown block exists to
+        // prevent — a stopping node sheds the trigger before it sheds what the trigger drives.
+        _revalidationTimer?.Dispose();
+        _revalidationTimer = null;
+
         // Revert to Null adapters so any in-flight writes from a poison-pilled actor don't hit a
         // half-disposed NodeManager.
         _deferredSink.SetSink(null);
@@ -304,6 +359,11 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     /// <returns>A task that represents the asynchronous operation.</returns>
     public async ValueTask DisposeAsync()
     {
+        // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
+        // not leave a tick armed against a server that is about to go away.
+        _revalidationTimer?.Dispose();
+        _revalidationTimer = null;
+
         if (_appHost is not null) await _appHost.DisposeAsync();
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -398,55 +398,10 @@ if (hasDriver)
         builder.Services.AddSingleton<PermissionTrieCache>();
     }
 
-    // #520 stage 5 — the node-scoped ACL gate. Registered on EVERY node, not just driver ones, because
-    // OtOpcUaServerHostedService takes it as a constructor parameter and the built-in container resolves
-    // constructors by whether every parameter is registered, not by whether the parameter type is nullable.
-    // That is the same trap the IOpcUaUserAuthenticator and IDataPlaneAuthoritySource registrations above
-    // both carry a comment about; an absent registration is a node that will not start.
-    builder.Services.AddSingleton<IPermissionEvaluator>(sp =>
-        new TriePermissionEvaluator(sp.GetRequiredService<PermissionTrieCache>()));
-    builder.Services.AddSingleton(sp => new DataPlaneAuthorizer(
-        sp.GetRequiredService<IDataPlaneAuthoritySource>(),
-        sp.GetRequiredService<IPermissionEvaluator>(),
-        sp.GetRequiredService<IOptions<AclOptions>>().Value.Enabled,
-        sp.GetRequiredService<ILoggerFactory>().CreateLogger<DataPlaneAuthorizer>()));
-
-    if (hasAdmin)
-    {
-        // A node WITH a ConfigDb reads keys straight from it, which is what makes revocation take effect
-        // on the very next request rather than on the next deployment. Preferred wherever it is available,
-        // including on a fused admin+driver node.
-        // The ClusterId confines the grants this node honours to its OWN cluster — the ConfigDb holds
-        // the whole fleet's ACL rows, unlike the artifact a driver-only node reads, which
-        // ParseDataPlaneAuthority already scoped. Taken from AkkaClusterOptions (pure configuration),
-        // NOT from IClusterRoleInfo: that depends on the ActorSystem, and the same options-not-services
-        // rule is what keeps the redundancy-singleton registration below from recursing into ActorSystem
-        // construction. Null on a node with no cluster-{ClusterId} role ⇒ no filter (legacy behaviour).
-        builder.Services.AddSingleton<IApiKeyStore>(sp => new ConfigDbApiKeyStore(
-            sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
-            ConfigDbApiKeyStore.ResolveClusterId(sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)));
-        builder.Services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
-
-        // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.
-        builder.Services.AddSingleton<IApiKeyAdminService>(sp => new ApiKeyAdminService(
-            sp.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>(),
-            sp.GetRequiredService<IOptions<ApiKeyOptions>>().Value.Pepper));
-    }
-    else if (hasDriver)
-    {
-        // Phase 4 left a driver-only node with no ConfigDb, so its keys ride the deployment artifact.
-        // Before stage 4 this node fell through to NullOpcUaUserAuthenticator and denied EVERY UserName
-        // token — safe, but it meant no machine client could authenticate against a site node at all.
-        builder.Services.AddSingleton<IApiKeyStore, ArtifactApiKeyStore>();
-        builder.Services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
-    }
-    else
-    {
-        // Neither role: nothing serves a data plane here. MUST be registered explicitly, not left absent —
-        // OtOpcUaServerHostedService takes IOpcUaUserAuthenticator as a REQUIRED constructor parameter, so
-        // an unregistered seam is not a quiet fallback to deny-all; it fails DI and the node does not start.
-        builder.Services.AddSingleton<IOpcUaUserAuthenticator>(NullOpcUaUserAuthenticator.Instance);
-    }
+    // #520 + arch-review 03/S18 — the node-scoped ACL gate, the shared clock and the role-appropriate
+    // API-key store + user authenticator, all in one extension so the role-conditional branches are
+    // reachable from a test (Program.cs's top-level statements are not). See DataPlaneAuthRegistration.
+    builder.Services.AddOtOpcUaDataPlaneAuth(hasAdmin, hasDriver);
 
     // Bind + validate the OPC UA host options the same way (fail-fast at start via ValidateOnStart)
     // and let OtOpcUaServerHostedService consume the validated IOptions instance rather than
@@ -740,48 +695,12 @@ if (hasAdmin)
     app.MapOtOpcUaDeployApi(app.Configuration);
 }
 
-// Passive LocalDb sync endpoint. Gated on the same port check that bound the listener, so a
-// default-OFF or admin-only graph carries no sync surface whatsoever. Mapping it would be safe
-// even unauthenticated — LocalDbSyncAuthInterceptor fail-closes with no ApiKey configured — but
-// not mapping it at all is the stronger default.
-if (hasDriver && syncListenPort > 0)
-{
-    app.MapZbLocalDbSync();
-}
-
-// Central config-serve artifact endpoint (per-cluster mesh Phase 3). Mapped only on admin-role
-// nodes (which hold the central SQL connection) and only when a dedicated h2c port is bound. The
-// ConfigServeAuthInterceptor is the ONLY inbound auth and fail-closes with no ConfigServe:ApiKey.
-if (hasAdmin && configServeGrpcPort > 0)
-{
-    app.MapGrpcService<DeploymentArtifactGrpcService>();
-
-    // Central command ingress (mesh gRPC command plane): the node→central deployment ack. It
-    // rides the SAME listener rather than claiming a port of its own — a second cleartext h2c port
-    // would buy nothing but another firewall rule, and both endpoints are admin-role and dialled by
-    // the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping it under Dps
-    // costs nothing (nothing dials it) whereas gating it would mean flipping the mode on a running
-    // fleet requires a central restart before any node can ack — the same reasoning that registers
-    // CentralCommunicationActor in both modes. CentralCommandAuthInterceptor (added at AddGrpc
-    // above) is the ONLY inbound auth and fail-closes per cluster.
-    app.MapGrpcService<CentralCommandGrpcService>();
-}
-
-// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
-// TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey).
-//
-// The node-side command endpoint (mesh gRPC command plane, Phase 1) rides the SAME driver-role
-// h2c listener rather than claiming a fourth port: central already dials this node here, both
-// directions are per-cluster-keyed, and a separate port would be one more thing to open on every
-// site firewall for no isolation gain. Mapping it is what makes MeshCommandAuthInterceptor live —
-// it is registered above and is a pass-through until a call matches this service's method prefix.
-// It fail-closes with no MeshTransport:GrpcPsk, so an operator who maps the service without setting
-// a key gets a closed endpoint, not an open one.
-if (hasDriver && telemetryListenPort > 0)
-{
-    app.MapGrpcService<TelemetryStreamGrpcService>();
-    app.MapGrpcService<NodeCommandGrpcService>();
-}
+// Mesh gRPC ingress: the LocalDb passive sync endpoint, the ConfigServe artifact + central
+// command endpoints, and the telemetry-stream + node-command endpoints. Extracted to
+// MeshGrpcEndpoints so a standing census test drives the SAME production mapping (see that
+// file's remarks — archreview 03/S14).
+app.MapOtOpcUaMeshGrpcServices(
+    hasAdmin, hasDriver, syncListenPort, configServeGrpcPort, telemetryListenPort);
 
 app.MapOtOpcUaHealth();
 app.MapOtOpcUaMetrics();

src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj

@@ -13,6 +13,13 @@
         <NoWarn>$(NoWarn);NU1608</NoWarn>
     </PropertyGroup>
 
+    <ItemGroup>
+        <!-- OtOpcUaServerHostedService.StartRevalidationTimer is internal: it is a lifecycle seam the
+             tests drive so the ACL revalidation timer (arch-review 03/S19) can be asserted without
+             booting the real OPC UA SDK, not a surface for callers. -->
+        <InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Host.Tests"/>
+    </ItemGroup>
+
     <ItemGroup>
         <PackageReference Include="Akka.Hosting"/>
         <PackageReference Include="Serilog.AspNetCore"/>

src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs

@@ -317,12 +317,21 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
     ///     Re-authorizes live subscriptions when a deployment installs a new authorization generation.
     /// </summary>
     /// <remarks>
-    ///     ⚠️ <b>This is the only trigger, and the pass's memo depends on that.</b>
-    ///     <see cref="IsSubscriptionStillPermitted"/> memoizes key revocation and expiry for the span of one
-    ///     pass — safe only while revalidation is deploy-triggered, because a pass is then a momentary
-    ///     event whose verdicts cannot go stale within it. A future periodic trigger must give each tick a
-    ///     fresh memo (it already does — the dictionary is pass-local) and must not be tempted to reuse one
-    ///     across ticks, or an expiry landing between ticks would be served the previous tick's allow.
+    ///     <para>
+    ///         <b>There are TWO triggers</b> (arch-review 03/S19): this one, the deployment axis; and the
+    ///         host's low-frequency <c>AclRevalidationTimer</c>, the clock axis — a key's
+    ///         <c>ExpiresUtc</c> passes with the clock, not with a deployment, so a deploy-only trigger
+    ///         left an expired key streaming to a live subscription indefinitely on a fleet that had been
+    ///         deployed once and left running.
+    ///     </para>
+    ///     <para>
+    ///         ⚠️ <b>The second trigger is safe only because the memo is pass-local.</b>
+    ///         <see cref="IsSubscriptionStillPermitted"/> memoizes key revocation and expiry for the span
+    ///         of one pass, which is sound because a pass is a momentary event whose verdicts cannot go
+    ///         stale within it — its dictionary is constructed per call. Hoisting it across ticks, to save
+    ///         the walk, would serve an expiry that landed between two ticks the earlier tick's allow, and
+    ///         re-open exactly the gap the timer closes.
+    ///     </para>
     /// </remarks>
     /// <param name="snapshot">The newly published snapshot; unused — the current state is re-read per item.</param>
     private void OnAuthorityChanged(DataPlaneAuthoritySnapshot snapshot) => RevalidateMonitoredItems();
@@ -3414,9 +3423,10 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
     ///     </para>
     ///     <para>
     ///         This memo does cover the key expiry check that the authorizer's own cache excludes, but only
-    ///         for the duration of one pass — and a subscription is re-checked at a deployment, never on a
-    ///         timer, so an expiry that lands inside a single pass was already going to be caught by the
-    ///         next deployment rather than by this one.
+    ///         for the duration of one pass. Since arch-review 03/S19 a subscription is re-checked on the
+    ///         host's low-frequency timer as well as at a deployment, so an expiry that lands inside a
+    ///         single pass is caught by the next TICK — one interval later, not one deployment later. What
+    ///         makes that hold is that the memo is constructed per call and never reused across passes.
     ///     </para>
     /// </remarks>
     private bool IsSubscriptionStillPermitted(

src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/DataPlaneAuthorizer.cs

@@ -108,6 +108,15 @@ public sealed class DataPlaneAuthorizer
     /// </summary>
     public bool Enabled { get; }
 
+    /// <summary>The clock this gate evaluates key expiry against.</summary>
+    /// <remarks>
+    ///     Exposed so the composition root's wiring can be asserted by reference. The remarks on
+    ///     <c>UserAuthorizationState</c> record what a second, diverging clock costs: every request denied,
+    ///     with a verdict indistinguishable from "no grant matched". One registered singleton threaded into
+    ///     every consumer is the fix, and a test that cannot see which instance landed here cannot pin it.
+    /// </remarks>
+    public TimeProvider Clock => _time;
+
     /// <summary>
     ///     Memoized grant walks for the request-rate seams (#520 stage 6). See
     ///     <see cref="SessionAclDecisionCache"/> for what it is and is not allowed to cache.

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -82,6 +82,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
     /// </remarks>
     public static readonly TimeSpan DefaultApplyWedgeTimeout = TimeSpan.FromMinutes(5);
 
+    /// <summary>Minimum interval between two supervision reprovisions of the SAME driver child (archreview
+    /// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the
+    /// reprovision hook makes every restart do ShutdownAsync + InitializeAsync + SubscribeAsync — so an
+    /// un-limited hook turns a driver that faults on every value into a reconnect storm against its device.
+    /// Matched to <see cref="DriverInstanceActor.DefaultReconnectInterval"/>: the reprovision drives the same
+    /// connect work, so it should not run more often than an ordinary reconnect attempt.</summary>
+    public static readonly TimeSpan DefaultReprovisionBackoff = TimeSpan.FromSeconds(10);
+
     /// <summary>
     ///     How long after a driver reconnects the 03/S15 recovery audit waits before reporting refs still
     ///     latched <see cref="OpcUaStatus.BadNoCommunication"/>. Sized so an ordinary poll driver has had
@@ -352,9 +360,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
     /// the config (#516). Bounded by the number of driver children with a delta in flight.</summary>
     private readonly Dictionary<CorrelationId, DriverInstanceSpec> _pendingDelta = new();
 
-    /// <summary>The composition from the most-recent apply (set at the END of
-    /// <see cref="PushDesiredSubscriptions"/>). Null until the first apply.</summary>
-    private AddressSpaceComposition? _lastComposition;
+    /// <summary>One driver child's desired subscription payload — exactly the
+    /// <see cref="DriverInstanceActor.SetDesiredSubscriptions"/> the apply's bulk push sent it.</summary>
+    /// <param name="ValueRefs">The non-alarm raw tags' RawPaths this driver should keep subscribed.</param>
+    /// <param name="AlarmRefs">The alarm-bearing raw tags' RawPaths that un-gate this driver's
+    /// <see cref="IAlarmSource"/> feed.</param>
+    private readonly record struct DesiredSubscriptionSet(
+        IReadOnlyList<string> ValueRefs,
+        IReadOnlyList<string> AlarmRefs);
+
+    /// <summary>Per-driver desired subscription sets from the most-recent apply. THE authority: the bulk
+    /// push, <see cref="SpawnChild"/> and the supervision-reprovision handler all send from here, so a
+    /// child obtained by ANY route carries the same set. Cleared and repopulated whole on every apply,
+    /// alongside the routing maps, so renames/removals can never leave a stale entry. Empty until the
+    /// first apply — a child spawned before any apply is handed nothing, which is what it should have.
+    /// <para>It is only ever rebuilt by a SUCCESSFUL apply, so after a failed one it deliberately lingers on
+    /// the last-good composition — matching the address space, drivers and subscriptions that also keep
+    /// serving through a failed apply (#485/#486). A respawn or reprovision in that window therefore hands
+    /// the child the last-known-good set, not an empty one.</para></summary>
+    private readonly Dictionary<string, DesiredSubscriptionSet> _desiredByDriver = new(StringComparer.Ordinal);
 
     /// <summary>
     ///     Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
@@ -430,6 +454,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
     /// </summary>
     private readonly Dictionary<string, HashSet<string>> _commsLatched = new(StringComparer.Ordinal);
 
+    /// <summary>Per-driver rate limit for the supervision-reprovision hook (see
+    /// <see cref="DefaultReprovisionBackoff"/>).</summary>
+    private readonly TimeSpan _reprovisionBackoff;
+
+    /// <summary>When each driver was last reprovisioned after a supervision restart, so a crash loop is
+    /// paced rather than amplified. Bounded by the number of driver children; an entry for a driver that is
+    /// later removed simply stops being consulted (the handler resolves <c>_children</c> first).</summary>
+    private readonly Dictionary<string, DateTime> _lastReprovisionUtc = new(StringComparer.Ordinal);
+
     /// <summary>
     ///     Host of this node's LocalDb replication partner — the only node that holds a copy of this
     ///     node's alarm queue, and therefore the only node it may ever stand down in favour of.
@@ -581,6 +614,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         IHistorizationRoleView? historizationRoleView = null,
         Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
         TimeSpan? applyWedgeTimeout = null,
+        TimeSpan? reprovisionBackoff = null,
         TimeSpan? commsRecoveryAuditDelay = null) =>
         // WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
         // expression tree. Six IActorRef? parameters and several interface-typed ones mean a
@@ -593,7 +627,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
             loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
             deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
             alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
-            sharesHistorizationScopeWithLocal, applyWedgeTimeout, commsRecoveryAuditDelay));
+            sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay));
 
     /// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
     /// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -683,9 +717,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         IHistorizationRoleView? historizationRoleView = null,
         Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
         TimeSpan? applyWedgeTimeout = null,
+        TimeSpan? reprovisionBackoff = null,
         TimeSpan? commsRecoveryAuditDelay = null)
     {
         _applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
+        _reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
         _commsRecoveryAuditDelay = commsRecoveryAuditDelay ?? DefaultCommsRecoveryAuditDelay;
         _historizationRoleView = historizationRoleView;
         _sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal;
@@ -1121,6 +1157,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
         Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
         Receive<RestartDriver>(HandleRestartDriver);
+        Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
+        Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
         Receive<ReconnectDriver>(HandleReconnectDriver);
         Receive<RouteNodeWrite>(HandleRouteNodeWrite);
         Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1177,6 +1215,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
         Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
         Receive<RestartDriver>(HandleRestartDriver);
+        Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
+        Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
         Receive<ReconnectDriver>(HandleReconnectDriver);
         Receive<RouteNodeWrite>(HandleRouteNodeWrite);
         Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1943,6 +1983,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         Receive<GetDiagnostics>(HandleGetDiagnostics);
         Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale());
         Receive<RestartDriver>(HandleRestartDriver);
+        Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
+        Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
         Receive<ReconnectDriver>(HandleReconnectDriver);
         Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
         // The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
@@ -2885,23 +2927,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
             }
         }
 
-        // One authored-only push (value refs + alarm refs from the maps built above), shared by the bulk loop AND
-        // the dropped-driver fallback so the two CANNOT drift: the fallback's correctness depends on sending the
-        // SAME payload the bulk loop would have, so it's a shared helper (structural), not a comment-maintained
-        // invariant. An EMPTY set is valid — the child's Connected handler routes it to Unsubscribe (dropping a
-        // stale handle) rather than a spurious subscribe. Returns the value-ref count for the bulk-loop log total.
-        int SendAuthoredOnly(IActorRef actor, string driverId)
+        // Cache the per-driver payload as THE authority, in the same clear-and-repopulate pass as the routing
+        // maps above so a rename/removal can never leave a stale entry behind. Keyed over the UNION of the two
+        // ref maps AND the running children: a child with no authored tags must get an explicit EMPTY set (which
+        // it routes to Unsubscribe), not be absent from the authority — absence and empty must not differ here.
+        _desiredByDriver.Clear();
+        foreach (var driverId in refsByDriver.Keys
+                     .Concat(alarmRefsByDriver.Keys)
+                     .Concat(_children.Keys)
+                     .Distinct(StringComparer.Ordinal))
         {
-            var refs = refsByDriver.TryGetValue(driverId, out var r) ? r : Array.Empty<string>();
-            var alarmRefs = alarmRefsByDriver.TryGetValue(driverId, out var ar) ? ar : Array.Empty<string>();
-            actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions(refs, SubscriptionPublishingInterval, alarmRefs));
-            return refs.Count;
+            _desiredByDriver[driverId] = new DesiredSubscriptionSet(
+                refsByDriver.TryGetValue(driverId, out var r) ? r : Array.Empty<string>(),
+                alarmRefsByDriver.TryGetValue(driverId, out var ar) ? ar : Array.Empty<string>());
         }
 
         var total = 0;
         foreach (var (driverId, entry) in _children)
         {
-            total += SendAuthoredOnly(entry.Actor, driverId);
+            total += SendDesiredSubscriptions(entry.Actor, driverId);
         }
 
         if (total > 0)
@@ -2936,11 +2980,34 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
             _log.Info("DriverHost {Node}: applied {Count} Equipment ScriptedAlarm(s) to the ScriptedAlarm host",
                 _localNode, composition.EquipmentScriptedAlarms.Count);
         }
+    }
 
-        // Cache the applied composition LAST. Set here (not in HandleArtifactForApply) so both the fresh-apply and
-        // bootstrap-restore paths — which both route through this method — leave a current composition.
-        _lastComposition = composition;
-
+    /// <summary>
+    ///     Send <paramref name="driverId"/>'s desired subscription payload to <paramref name="actor"/>.
+    /// </summary>
+    /// <remarks>
+    ///     The ONE place a <see cref="DriverInstanceActor.SetDesiredSubscriptions"/> is built, so the apply's
+    ///     bulk push, a respawn (<see cref="SpawnChild"/>) and a supervision reprovision
+    ///     (<c>HandleReprovisionRequested</c>) cannot drift. That is structural rather than
+    ///     comment-maintained on purpose: a second sender that assembled its own payload is exactly how
+    ///     03/S16 happened — the restart path got a child without the set that makes a child useful.
+    ///     <para>
+    ///         An EMPTY set is valid and MEANINGFUL — the child routes it to <c>Unsubscribe</c> (dropping a
+    ///         stale handle) rather than to a spurious subscribe — so a driver missing from
+    ///         <see cref="_desiredByDriver"/> is sent an empty set, never skipped.
+    ///     </para>
+    /// </remarks>
+    /// <param name="actor">The driver child to hand the set to.</param>
+    /// <param name="driverId">The driver instance whose set to send.</param>
+    /// <returns>The value-ref count, for the bulk loop's log total.</returns>
+    private int SendDesiredSubscriptions(IActorRef actor, string driverId)
+    {
+        var set = _desiredByDriver.TryGetValue(driverId, out var s)
+            ? s
+            : new DesiredSubscriptionSet(Array.Empty<string>(), Array.Empty<string>());
+        actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions(
+            set.ValueRefs, SubscriptionPublishingInterval, set.AlarmRefs));
+        return set.ValueRefs.Count;
     }
 
     private void SpawnChild(DriverInstanceSpec spec)
@@ -3014,6 +3081,29 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
 
         _children[spec.DriverInstanceId] = new ChildEntry(child, spec, stub);
 
+        // EVERY spawn hands the child its desired set — deploy reconcile, operator Restart (03/S16), and the
+        // supervision reprovision. Sending it HERE rather than at each call site is the point: a child that
+        // exists without its subscription set is a dead driver, so the two facts are established together.
+        // Before this, the only sender was the apply's bulk push, so an operator's Restart produced a child
+        // that connected and then read nothing until the next deployment — the #488 periodic reconcile cannot
+        // help, because it early-returns on an empty desired set.
+        //
+        // ORDERING, and its one accepted cost: ApplyDriverSpecs (which spawns) runs BEFORE
+        // ApplyDesiredSubscriptions (which rebuilds _desiredByDriver), so on a deploy that RESPAWNS a child
+        // (#516 — the merged DriverConfig folds in the raw tags, so a tag-set edit changes the spec) the new
+        // child is handed the PREVIOUS apply's set, and nothing at all on the first deploy. That transient is
+        // superseded a moment later by the bulk push, on the same sender→receiver pair, so Akka's per-pair
+        // ordering guarantees the child CONVERGES on the new set — but it is NOT guaranteed to be inert on
+        // the way there: the child runs concurrently with the rest of this handler, so a driver whose
+        // InitializeAsync completes quickly can reach Connected and subscribe the stale set before the bulk
+        // push arrives, then re-subscribe. The cost is one extra subscribe/unsubscribe round-trip on a
+        // respawning deploy (HandleSubscribeAsync drops the prior subscription before establishing the new
+        // one, so it converges); a real device driver, whose initialise opens a socket, rarely wins that race.
+        // Deliberately not "fixed" by withholding the send: a child that never gets a set is the 03/S16
+        // defect, and the operator-Restart and supervision paths have no bulk push coming at all.
+        // Sent after InitializeRequested above, which keeps init-before-subscribe ordering.
+        SendDesiredSubscriptions(child, spec.DriverInstanceId);
+
         // v3 (WP7): if the (real, non-stub) driver consumes other tags' live values, spawn a mux-adapter
         // child that registers its dependency refs on the per-node dependency mux and forwards changes
         // into the driver. Requires a wired mux (dev/None deployments have none — then calc inputs simply
@@ -3164,6 +3254,87 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
         SpawnChild(entry.Spec);
     }
 
+    /// <summary>
+    ///     Re-provision a driver child that Akka's supervision just RESTARTED in place (archreview 03/S6,
+    ///     03/S16) — the third and least visible of the three incarnation routes.
+    /// </summary>
+    /// <remarks>
+    ///     A restart recreates the actor object around the SAME driver instance (the Props closure captured
+    ///     it) without going through <see cref="SpawnChild"/>, so neither the <c>InitializeRequested</c> nor
+    ///     the desired-subscription send that a spawn performs ever happens — and the child's
+    ///     <see cref="IActorRef"/> is unchanged, so <c>_children</c> needs no maintenance and the host has no
+    ///     other notification that a new incarnation exists. Its <c>PreRestart</c> ran <c>PostStop</c>, which
+    ///     called <c>ShutdownAsync</c> on the driver, so re-initialising is not optional: without it the
+    ///     child sits in <c>Connecting</c> forever around a shut-down driver.
+    ///     <para>
+    ///         The dependency-consumer mux adapter is a SEPARATE child of this host and is unaffected by the
+    ///         instance actor restarting, so it is deliberately not touched here.
+    ///     </para>
+    /// </remarks>
+    /// <param name="msg">The restarted child's self-announcement.</param>
+    private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg) =>
+        ReprovisionChild(msg, Sender);
+
+    /// <summary>A reprovision held back by <see cref="_reprovisionBackoff"/>, carrying the incarnation it
+    /// was requested by so the identity check still applies when it is re-delivered by the timer.</summary>
+    /// <param name="Request">The original request.</param>
+    /// <param name="Child">The incarnation that asked; re-validated against <c>_children</c> on delivery,
+    /// so a driver restarted or removed in the meantime discards it.</param>
+    private sealed record ReprovisionDeferred(
+        DriverInstanceActor.ReprovisionRequested Request, IActorRef Child);
+
+    private static string ReprovisionTimerKey(string driverInstanceId) => "reprovision-" + driverInstanceId;
+
+    private void ReprovisionChild(DriverInstanceActor.ReprovisionRequested msg, IActorRef child)
+    {
+        // Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision.
+        if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return;
+
+        // Route by INCARNATION, not just by id. This message is delivered asynchronously, so one can still
+        // be in flight from an OLD incarnation after HandleRestartDriver stopped that child and spawned a
+        // replacement under the same DriverInstanceId. Delivered to the new child — which is in Connecting,
+        // where InitializeRequested IS handled — it would start a second, concurrent InitializeAsync,
+        // racing the dispose-before-overwrite that #527's teardown exists to serialise
+        // (ModbusDriver._transport, MqttDriver._connection each leak a transport on that race).
+        if (!Equals(child, entry.Actor))
+        {
+            _log.Debug(
+                "DriverHost {Node}: dropping a reprovision for {Id} from a superseded incarnation ({Stale})",
+                _localNode, msg.DriverInstanceId, child);
+            return;
+        }
+
+        // Rate-limit per driver (review F3). DEFERRED, never dropped: the last request must still be
+        // honoured, or a child whose final crash fell inside the window is left un-provisioned forever —
+        // exactly the 03/S16 failure this hook exists to close. One timer key per driver, so a burst
+        // collapses to a single pending reprovision rather than a queue of them.
+        var now = DateTime.UtcNow;
+        if (_lastReprovisionUtc.TryGetValue(msg.DriverInstanceId, out var last) &&
+            now - last < _reprovisionBackoff)
+        {
+            var delay = _reprovisionBackoff - (now - last);
+            _log.Warning(
+                "DriverHost {Node}: driver child {Id} was restarted by supervision again after {Reason} " +
+                "within {Backoff} — deferring the re-provision by {Delay} (crash loop?)",
+                _localNode, msg.DriverInstanceId, msg.Reason, _reprovisionBackoff, delay);
+            Timers.StartSingleTimer(
+                ReprovisionTimerKey(msg.DriverInstanceId), new ReprovisionDeferred(msg, child), delay);
+            return;
+        }
+
+        _lastReprovisionUtc[msg.DriverInstanceId] = now;
+
+        // A crash that Akka silently restarted is otherwise invisible to operators; name it.
+        _log.Warning(
+            "DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning",
+            _localNode, msg.DriverInstanceId, msg.Reason);
+
+        // A stub child has no real driver to initialise (its Stubbed behaviour no-ops InitializeRequested
+        // anyway), but it still stores a desired set, so only the initialise is conditional.
+        if (!entry.Stubbed) entry.Actor.Tell(new DriverInstanceActor.InitializeRequested(entry.Spec.DriverConfig));
+        SendDesiredSubscriptions(entry.Actor, msg.DriverInstanceId);
+    }
+
     private void HandleReconnectDriver(ReconnectDriver msg)
     {
         // DPS broadcast — only act if this node hosts the requested instance.

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs

@@ -42,6 +42,25 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
     /// restored → Good) — independently of alarm transitions, since a comms-lost driver emits no alarm
     /// events. Fire-and-forget, mirroring <see cref="DeltaApplied"/>.</summary>
     public sealed record ConnectivityChanged(string DriverInstanceId, bool Connected);
+
+    /// <summary>
+    ///     Sent to the parent from <see cref="PostRestart"/> so the host re-provisions THIS incarnation
+    ///     (archreview 03/S6, 03/S16).
+    /// </summary>
+    /// <remarks>
+    ///     A supervision restart recreates the actor object in place: the desired subscription sets are gone
+    ///     (they are instance fields), and <c>PreRestart</c> has already run <see cref="PostStop"/> — which
+    ///     called <see cref="IDriver.ShutdownAsync"/> on the driver object the Props closure re-uses. Nothing
+    ///     else re-sends <see cref="InitializeRequested"/>: its only sender is
+    ///     <c>DriverHostActor.SpawnChild</c>, and a restart does NOT respawn (the child's
+    ///     <see cref="IActorRef"/> is unchanged, so the host is not otherwise notified that a new incarnation
+    ///     exists at all). Without this the restarted child sits in <c>Connecting</c> forever around a
+    ///     shut-down driver, with nothing subscribed — silently, since Akka logs the fault but the recovery
+    ///     is invisible.
+    /// </remarks>
+    /// <param name="DriverInstanceId">The driver instance that was restarted.</param>
+    /// <param name="Reason">The type name of the exception that tripped supervision, for the host's log.</param>
+    public sealed record ReprovisionRequested(string DriverInstanceId, string Reason);
     public sealed record ApplyDelta(string DriverConfigJson, CorrelationId Correlation);
     public sealed record ApplyResult(bool Success, string? Reason, CorrelationId Correlation);
     /// <summary>
@@ -252,6 +271,15 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
     /// idempotent across repeated <see cref="SetDesiredSubscriptions"/> pushes.</summary>
     private IAlarmSubscriptionHandle? _alarmSubscriptionHandle;
 
+    /// <summary>The alarm references the live <see cref="_alarmSubscriptionHandle"/> was actually established
+    /// with, so a CHANGED desired set can be told apart from a repeat of the same one.
+    /// <para>Without this the subscription latched: both <see cref="SubscribeDesiredAlarms"/> and
+    /// <see cref="HandleSubscribeAlarmsAsync"/> gate on "a handle already exists", which is the right guard
+    /// for an idempotent re-push and the WRONG one for a deploy that widens or narrows the alarm set —
+    /// drivers that filter on <c>sourceNodeIds</c> (OPC UA Client, AB CIP, FOCAS) would then never deliver a
+    /// newly-added alarm for the life of the connection. Empty whenever no handle is held.</para></summary>
+    private IReadOnlyList<string> _subscribedAlarmRefs = Array.Empty<string>();
+
     /// <summary>
     /// Gets or sets the timer scheduler for scheduling reconnection attempts.
     /// </summary>
@@ -375,6 +403,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
         Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
     }
 
+    /// <inheritdoc />
+    /// <remarks>
+    ///     Ask the host to re-provision this incarnation — see <see cref="ReprovisionRequested"/> for why a
+    ///     supervision restart otherwise leaves a dead child. <c>base.PostRestart</c> runs first because it
+    ///     runs <see cref="PreStart"/>, which re-attaches the rediscovery / host-status sources and restarts
+    ///     the health timer; the reprovision is the part <see cref="PreStart"/> cannot do, since only the
+    ///     host knows this driver's config and desired set.
+    /// </remarks>
+    protected override void PostRestart(Exception reason)
+    {
+        base.PostRestart(reason);
+        Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
+    }
+
     private void Stubbed()
     {
         // Stubbed drivers accept the standard message contracts but return deterministic
@@ -488,10 +530,14 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
             // Native-alarm analogue: un-gate the IAlarmSource feed when alarm tags are (now) present. The
             // common live path — a deploy delivers SetDesiredSubscriptions while the driver is already
             // Connected — flows through HERE, so the alarm subscribe must happen on this message, not only
-            // on Connected entry. Unlike the value path above, there is deliberately no unsubscribe-on-empty:
-            // a removed alarm tag's condition node is torn down on the address-space rebuild and
-            // DriverHostActor.ForwardNativeAlarm drops any transition whose ConditionId no longer maps, so a
-            // lingering session-less alarm subscription can never surface a removed alarm.
+            // on Connected entry. A CHANGED set re-subscribes (release-then-subscribe inside
+            // HandleSubscribeAlarmsAsync): unlike the value path, "a subscription already exists" is NOT
+            // sufficient, because several drivers filter their feed on the subscribed sourceNodeIds, so a
+            // widened set that never re-subscribed would silently never deliver the new alarm.
+            // Still deliberately no unsubscribe-on-EMPTY: a removed alarm tag's condition node is torn down
+            // on the address-space rebuild and DriverHostActor.ForwardNativeAlarm drops any transition whose
+            // ConditionId no longer maps, so a lingering session-less alarm subscription can never surface a
+            // removed alarm.
             SubscribeDesiredAlarms();
         });
         ReceiveAsync<SubscribeAlarms>(HandleSubscribeAlarmsAsync);
@@ -561,6 +607,19 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
     ///     state on sustained failure: the nine poll drivers via Degraded, MQTT via its live connection
     ///     state, Galaxy via its reconnect supervisor.
     ///     </para>
+    ///     <para>
+    ///     <b>The grace is per CONNECTION (03/S17).</b> The silence baseline is
+    ///     <c>max(LastSuccessfulRead, _connectedSinceUtc)</c>, not <c>LastSuccessfulRead ?? _connectedSinceUtc</c>.
+    ///     Because <c>LastSuccessfulRead</c> is <em>preserved</em> across Degraded/Faulted writes (see above),
+    ///     a device that accepts a connection but serves no data reconnects carrying a stamp that is already
+    ///     older than the threshold, so a <c>??</c> baseline re-trips on the FIRST health tick of every new
+    ///     connection — a reconnect cycle at health-tick cadence rather than the designed threshold. Taking
+    ///     the later of the two gives each connection its own full window, which is exactly what re-stamping
+    ///     <c>_connectedSinceUtc</c> on every Connected entry was for. Note this <b>paces</b> the cycle, it
+    ///     does not end it: a permanently half-broken device still reconnects once per threshold, and a
+    ///     driver that stops reading <i>within</i> a stable connection still trips on schedule (its read
+    ///     stamp is the later term, exactly as before).
+    ///     </para>
     /// </remarks>
     /// <param name="health">The health observed on this tick, or null when <see cref="IDriver.GetHealth"/>
     /// threw — a failed observation is not evidence of comms loss, so it never advances the watchdog.</param>
@@ -570,8 +629,15 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
         if (health is not { } observed) return;
         if (observed.State == DriverState.Healthy) return;
 
-        // A driver that has never reported a successful read is measured from the current Connected entry.
-        var lastEvidence = observed.LastSuccessfulRead ?? _connectedSinceUtc;
+        // Measure silence from the LATER of the last successful read and THIS connect — the grace is per
+        // connection. `??` was wrong: drivers PRESERVE LastSuccessfulRead across Degraded/Faulted writes (that
+        // preservation is the #527 design), so after a trip + reconnect to a device that accepts the connection
+        // but still serves no data, the stamp is already older than the threshold and the FIRST tick of the new
+        // connection re-trips — churn at health-tick cadence instead of the designed grace. `??` only ever helped
+        // a driver that had never read at all.
+        var lastEvidence = observed.LastSuccessfulRead is { } read && read > _connectedSinceUtc
+            ? read
+            : _connectedSinceUtc;
         var silence = DateTime.UtcNow - lastEvidence;
         if (silence < _commsLossTimeout) return;
 
@@ -1114,6 +1180,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
         // (GalaxyDriver.SubscribeAlarmsAsync clears the set before adding), so handles no longer accumulate
         // across reconnects. The gate (Count > 0) and the one-shot fan-out are unchanged.
         _alarmSubscriptionHandle = null;
+        _subscribedAlarmRefs = Array.Empty<string>();
     }
 
     /// <summary>Establish the native-alarm subscription that un-gates an <see cref="IAlarmSource"/> driver's
@@ -1123,10 +1190,17 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
     /// desired, and no subscription is yet established.</summary>
     private void SubscribeDesiredAlarms()
     {
-        if (_driver is IAlarmSource && _desiredAlarmRefs.Count > 0 && _alarmSubscriptionHandle is null)
+        if (_driver is IAlarmSource && _desiredAlarmRefs.Count > 0 && AlarmSubscriptionIsStale())
             Self.Tell(new SubscribeAlarms());
     }
 
+    /// <summary>True when the desired alarm refs are not the ones the live handle was established with —
+    /// either because there is no handle yet, or because a deploy changed the set. The second case is the
+    /// one a bare "handle is null" check misses; see <see cref="_subscribedAlarmRefs"/>.</summary>
+    private bool AlarmSubscriptionIsStale() =>
+        _alarmSubscriptionHandle is null ||
+        !_subscribedAlarmRefs.SequenceEqual(_desiredAlarmRefs, StringComparer.Ordinal);
+
     /// <summary>Calls the driver's <see cref="IAlarmSource.SubscribeAlarmsAsync"/> (bounded) to register the
     /// alarm subscription that un-gates its native feed, caching the returned handle. Re-checks the guard
     /// (the desired set may have cleared, or another SubscribeAlarms may have already established a handle,
@@ -1134,10 +1208,36 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
     /// stays gated until then.</summary>
     private async Task HandleSubscribeAlarmsAsync(SubscribeAlarms _)
     {
-        if (_driver is not IAlarmSource src || _desiredAlarmRefs.Count == 0 || _alarmSubscriptionHandle is not null)
+        if (_driver is not IAlarmSource src || _desiredAlarmRefs.Count == 0 || !AlarmSubscriptionIsStale())
             return;
         var refs = _desiredAlarmRefs;
         using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+
+        // A handle held against a DIFFERENT set must be released before re-subscribing, or a
+        // sourceNodeIds-filtering driver accumulates two overlapping subscriptions. Best-effort and
+        // deliberately non-fatal: the re-subscribe below is what actually matters, and a driver whose
+        // unsubscribe throws must not be left on the stale set because of it. Cleared FIRST so a throw
+        // cannot leave a handle that the staleness check would keep treating as live.
+        if (_alarmSubscriptionHandle is { } stale)
+        {
+            _alarmSubscriptionHandle = null;
+            _subscribedAlarmRefs = Array.Empty<string>();
+            try
+            {
+                await _invoker.ExecuteAsync(
+                    DriverCapability.AlarmSubscribe,
+                    _driverInstanceId,
+                    async ct => await src.UnsubscribeAlarmsAsync(stale, ct),
+                    cts.Token);
+            }
+            catch (Exception ex)
+            {
+                _log.Warning(ex,
+                    "DriverInstance {Id}: releasing the previous native-alarm subscription ({Diag}) threw; re-subscribing anyway",
+                    _driverInstanceId, stale.DiagnosticId);
+            }
+        }
+
         try
         {
             _alarmSubscriptionHandle = await _invoker.ExecuteAsync(
@@ -1145,6 +1245,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
                 _driverInstanceId,
                 async ct => await src.SubscribeAlarmsAsync(refs, ct),
                 cts.Token);
+            _subscribedAlarmRefs = refs;
             _log.Info("DriverInstance {Id}: native-alarm subscription established for {Count} alarm ref(s) ({Diag})",
                 _driverInstanceId, refs.Count, _alarmSubscriptionHandle.DiagnosticId);
         }