Merge lane C: control plane, host, security (#733)

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

Directory.Packages.props

@@ -35,6 +35,12 @@
     <PackageVersion Include="Grpc.Tools" Version="2.76.0" />
     <PackageVersion Include="libplctag" Version="1.5.2" />
     <PackageVersion Include="MessagePack" Version="2.5.301" />
+    <!-- First-party bearer validation for /auth/token's JWTs (R03:2.4, Lane C fix round 1). Same 10.0.x servicing
+         line as the other Microsoft.AspNetCore.* pins. It depends on Microsoft.IdentityModel.Protocols(.OpenIdConnect)
+         8.0.1; those two are pinned to 8.11.0 below (and referenced directly by the Security project, the JwtBearer
+         entry point, so the pin applies without transitive pinning) to keep the whole IdentityModel family on one
+         version with Microsoft.IdentityModel.Tokens / System.IdentityModel.Tokens.Jwt. -->
+    <PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.7" />
     <PackageVersion Include="Microsoft.AspNetCore.Authorization" Version="10.0.7" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.7" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection.EntityFrameworkCore" Version="10.0.7" />
@@ -82,6 +88,8 @@
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.7" />
     <PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.7" />
     <PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.7" />
+    <PackageVersion Include="Microsoft.IdentityModel.Protocols" Version="8.11.0" />
+    <PackageVersion Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="8.11.0" />
     <PackageVersion Include="Microsoft.IdentityModel.Tokens" Version="8.11.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
     <PackageVersion Include="Microsoft.Playwright" Version="1.51.0" />

docs/security.md

@@ -6,11 +6,14 @@
 > auth + role-grant pages live in `OtOpcUa.AdminUI`), and Admin auth policies
 > register from `OtOpcUa.Host/Program.cs` via `AddOtOpcUaAuth`
 > (`src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs`) rather
-> than in a separate Admin process. The Admin UI uses a **single Cookie
+> than in a separate Admin process. The Admin UI session is a **Cookie**
-> authentication scheme** — there is no `AddJwtBearer` pipeline. The
+> scheme; the cookie stores the `ClaimsPrincipal` directly. JWT is **opt-in by
-> `Security:Jwt` section configures `JwtTokenService`, which mints a JWT at the
+> `Security:Jwt:SigningKey`** (validated at host start, ≥32 bytes): with a key,
-> `/auth/token` endpoint for **external** consumers (OPC UA clients / automation
+> `/auth/token` mints an HS256 token for a **cookie session only** (a bearer
-> scripts); the cookie itself stores the `ClaimsPrincipal` directly. DataProtection
+> cannot renew itself), and a first-party `AddJwtBearer` scheme (`Bearer`,
+> `ValidAlgorithms = HS256`, parameters from `JwtTokenService.BuildValidationParameters`)
+> accepts it on `/api/*` and every `RequireAuthorization` endpoint through the
+> `OtOpcUa.CookieOrBearer` default policy scheme. Without a key neither exists. DataProtection
 > keys persist to the shared Config DB (`PersistKeysToDbContext<OtOpcUaConfigDbContext>`)
 > so cookies survive failover between admin-role nodes.
 >
@@ -245,6 +248,25 @@ with two `Security:ApiKeys` knobs:
   a completed lookup (a hit or an authoritative miss) is stored, and a cached hit whose `ExpiresUtc` has
   since passed is re-read rather than served, so the TTL can never resurrect an expired key.
 
+#### Pre-authentication guards (`KnownIdSet` / `MissBurst` / `MissRefillPerSecond`)
+
+The negative cache above is keyed by **key id**, so a flood of *distinct*, well-formed `otua_<random>_x`
+credentials missed it every time and cost one indexed SQL query per attempt — under the SDK's
+`SessionManager`-wide lock, with no credential required (review 03 §3.3). `BoundedApiKeyStore` now puts two
+guards in front of the store, bound from the same `Security:ApiKeys` section
+(`ApiKeyStoreGuardOptions`, Host-owned):
+
+| Key | Default | Meaning |
+|---|---|---|
+| `KnownIdSet` | `true` | Keep an in-memory set of every `ApiKeyId` the ConfigDb holds, refreshed every `StoreCacheSeconds` (single-flight; a failed refresh keeps the previous set). An id outside it is refused in memory — `otopcua.opcua.apikey_store.bounded{outcome=unknown_id}` — and the store is never asked. A key minted less than `StoreCacheSeconds` ago is refused until the next refresh (the same window `Enabled` revocation already has). `StoreCacheSeconds = 0` disables the set with the cache. |
+| `MissBurst` | `100` | Token bucket on lookups that **reach** the store and **miss**, for when the set is unavailable (not loaded yet, or the store cannot list ids). Exhausted ⇒ unknown ids are refused without a store read (`{outcome=miss_budget}`, a Warning-free deny through the authenticator's catch-all). A hit refunds its token, so legitimate clients are never charged. `0` disables the budget. |
+| `MissRefillPerSecond` | `20` | Refill rate of the miss bucket. |
+
+The budget is **process-wide, not per remote endpoint**: the OPC UA SDK's impersonation callback (the
+`IOpcUaUserAuthenticator` seam) does not carry the client's address, so per-endpoint accounting needs that
+seam widened in the OpcUaServer project (a Wave 3 follow-up). Neither guard applies to a driver-only node:
+its `ArtifactApiKeyStore` is two in-memory dictionary probes and is never wrapped.
+
 ### LDAP bind flow (`OtOpcUaLdapAuthService`)
 
 LDAP is configured under the `Security:Ldap` section (bound to `LdapOptions`, `src/Server/ZB.MOM.WW.OtOpcUa.Security/Ldap/LdapOptions.cs`, `SectionName = "Security:Ldap"`). The app authenticator is `OtOpcUaLdapAuthService` — a thin wrapper around the shared `ZB.MOM.WW.Auth.Ldap` directory client that adds two app-only concerns the shared library deliberately does not model: the `Enabled` master switch and `DevStubMode`. Since #520 it serves the **Admin UI cookie login (`/auth/login`) only** — `LdapOpcUaUserAuthenticator` is deleted and the OPC UA UserName path authenticates API keys instead (see [API-key identity](#api-key-identity-opc-ua-data-plane)).
@@ -1116,6 +1138,31 @@ See `archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md` (R3-01) for the fu
 record, and `docs/Configuration.md` §`MeshTransport` / §`ConfigSource`-`ConfigServe` and `docs/Telemetry.md`
 for the affected listener ports.
 
+### The known-cluster gate is fail-OPEN until central's first ClusterNode load (review 03 §4.2)
+
+`CentralCommandAuthInterceptor` verifies an inbound deploy ack against the PSK of the cluster named in
+`x-otopcua-cluster`. `MeshPskProvider` consults a **known-cluster gate** first — the set of cluster ids
+central read from `ClusterNode` on its last contact refresh — and refuses anything outside it in O(1),
+touching neither the config map nor the secret store. That gate is **inactive until the first successful
+load** (`CentralCommunicationActor` publishes it from `ContactsLoaded`; an empty publication also
+deactivates it, deliberately, so "the DB told me nothing" never becomes a fleet-wide authentication
+outage). During a central boot with the ConfigDb down, therefore:
+
+- an unauthenticated caller can name *any* `x-otopcua-cluster` value and make central perform one bounded
+  (3 s) secret-store read plus one Warning line per request — the cost/oracle the gate exists to remove;
+- **keys are still required**: nothing is admitted without the matching PSK, so this is a cost exposure,
+  not an authentication bypass;
+- the window is bounded by the bootstrap retry (15 × 2 s) and the `ContactRefreshSeconds` (60 s) periodic
+  refresh, and closes on the first load that succeeds. The actor logs the condition on every failed load
+  (*"The known-cluster gate was NOT refreshed either … it is INACTIVE"*).
+
+Since review 03 §3.4 the per-cluster keys are **refreshed asynchronously** from the same handler
+(`IMeshPskProvider.RefreshAsync`: concurrent bounded reads, atomic snapshot swap, a failed read keeps the
+previous key) instead of being invalidated on every refresh, so the first ack per cluster per minute no
+longer pays a synchronous store read on a gRPC thread. Accepted as documented, bounded behaviour; the
+alternative — refusing every cluster until SQL answers — would turn a ConfigDb outage into a deploy-ack
+outage on top of it.
+
 ---
 
 ## OTOPCUA0001 Analyzer — Compile-Time Guard

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs

@@ -26,7 +26,11 @@ public sealed class Deployment
     public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow;
 
     /// <summary>Gets or sets the serialized artifact blob containing the configuration.</summary>
-    public byte[] ArtifactBlob { get; init; } = Array.Empty<byte>();
+    /// <remarks>
+    ///     Settable (not init-only) so the retention sweep can EMPTY it on an old terminal row. An empty blob
+    ///     is "no artifact" to every reader (#485), so the audit row survives with its bytes retired.
+    /// </remarks>
+    public byte[] ArtifactBlob { get; set; } = Array.Empty<byte>();
 
     /// <summary>Gets or sets the row version for optimistic concurrency control.</summary>
     public byte[] RowVersion { get; set; } = Array.Empty<byte>();

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs

@@ -0,0 +1,36 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations
+{
+    /// <summary>
+    ///     Enables <c>ALLOW_SNAPSHOT_ISOLATION</c> on the config database so the deploy path can read every
+    ///     table inside one <c>SNAPSHOT</c> transaction (<c>DraftSnapshotFactory.FromConfigDbAsync</c>):
+    ///     the validator and the artifact composer then see one consistent state instead of two reads
+    ///     an AdminUI save can land between. No schema change — the model snapshot is untouched.
+    /// </summary>
+    /// <remarks>
+    ///     <c>ALTER DATABASE</c> cannot run inside the user transaction the migrator wraps each migration in,
+    ///     hence <c>suppressTransaction: true</c>. Turning the option ON does not need exclusive access (unlike
+    ///     <c>READ_COMMITTED_SNAPSHOT</c>); it waits for in-flight transactions and then takes effect. Down
+    ///     leaves it ON: a database that has run with it is not harmed by keeping it, and switching it OFF
+    ///     would fail any deploy running at that instant.
+    /// </remarks>
+    public partial class EnableSnapshotIsolation : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.Sql(
+                "ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON;",
+                suppressTransaction: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            // Deliberately a no-op; see the class remarks.
+        }
+    }
+}

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs

@@ -66,6 +66,23 @@ public sealed class DraftSnapshot
     /// silently dropped when the permission trie is built, so the grant has to be caught here instead.</summary>
     public IReadOnlyList<NodeAcl> NodeAcls { get; init; } = [];
 
+    /// <summary>
+    ///     Cluster rows. Not read by the row-level rules; carried so the deploy path validates the cluster
+    ///     topology (<see cref="DraftValidator.ValidateClusterTopology"/>) and composes the artifact from
+    ///     the SAME read as every other table — a second read between validate and compose is exactly how
+    ///     an edit ships unvalidated.
+    /// </summary>
+    public IReadOnlyList<ServerCluster> ServerClusters { get; init; } = [];
+
+    /// <summary>Cluster-node rows; see <see cref="ServerClusters"/> for why they ride the snapshot.</summary>
+    public IReadOnlyList<ClusterNode> ClusterNodes { get; init; } = [];
+
+    /// <summary>
+    ///     Data-plane API keys (peppered HMAC only, never a secret). Not validated; carried so the artifact
+    ///     composer ships the keys a driver-only node authenticates against from the one consistent read.
+    /// </summary>
+    public IReadOnlyList<ApiKey> ApiKeys { get; init; } = [];
+
     /// <summary>Prior Equipment rows (any generation, same cluster) for stability checks.</summary>
     public IReadOnlyList<Equipment> PriorEquipment { get; init; } = [];
 

src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs

@@ -1,3 +1,4 @@
+using System.Data;
 using Microsoft.EntityFrameworkCore;
 
 namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation;
@@ -21,6 +22,16 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation;
 ///         / <see cref="DraftSnapshot.Site"/> are left null so the path-length rule uses its
 ///         conservative upper bound.
 ///     </para>
+///     <para>
+///         <b>One consistent read.</b> On a relational provider the tables are read inside a
+///         <see cref="IsolationLevel.Snapshot"/> transaction, so a <c>/uns</c> or <c>/raw</c> save that
+///         lands mid-read cannot leave half the tables on the new state and half on the old. The deploy path
+///         validates AND composes its artifact from this one snapshot, which is what makes "the artifact the
+///         validator never saw" unrepresentable. SQL Server requires <c>ALLOW_SNAPSHOT_ISOLATION ON</c>
+///         (migration <c>EnableSnapshotIsolation</c>); without it the first query fails with error 3952 and
+///         the deploy is rejected rather than silently read unisolated. A non-relational provider (the
+///         InMemory test provider) and a context already inside a transaction read without one.
+///     </para>
 /// </remarks>
 public static class DraftSnapshotFactory
 {
@@ -29,6 +40,29 @@ public static class DraftSnapshotFactory
     /// <param name="ct">Cancellation token.</param>
     /// <returns>A snapshot populated from the live DB, ready for <see cref="DraftValidator.Validate"/>.</returns>
     public static async Task<DraftSnapshot> FromConfigDbAsync(OtOpcUaConfigDbContext db, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(db);
+
+        var ownsTransaction = db.Database.IsRelational() && db.Database.CurrentTransaction is null;
+        var tx = ownsTransaction
+            ? await db.Database.BeginTransactionAsync(IsolationLevel.Snapshot, ct).ConfigureAwait(false)
+            : null;
+        await using (tx)
+        {
+            var snapshot = await ReadAsync(db, ct).ConfigureAwait(false);
+            // Read-only: committing releases the snapshot's version-store reservation; nothing was written.
+            if (tx is not null) await tx.CommitAsync(ct).ConfigureAwait(false);
+            return snapshot;
+        }
+    }
+
+    // Every composed table is read ORDERED BY ITS NATURAL KEY, server-side. That order IS the deployment
+    // artifact's order (ConfigComposer.Compose serialises the snapshot as given): it is exactly the order the
+    // pre-R03:3.2 composer's own queries produced — the database collation's, case-insensitive on SQL Server —
+    // so the revision hash of an unchanged configuration is byte-identical across the refactor (fix round 1;
+    // pinned by ConfigComposerSqlServerHashStabilityTests). Re-sorting in memory with any other comparer would
+    // re-hash every mixed-case configuration and make drift detection report the whole fleet drifted.
+    private static async Task<DraftSnapshot> ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct)
         => new DraftSnapshot
         {
             // GenerationId=0 and null Enterprise/Site are intentional conservative fallbacks at the
@@ -36,23 +70,26 @@ public static class DraftSnapshotFactory
             // EquipmentUuid checks run in separate validator passes — no rule here reads these fields.
             GenerationId = 0,            // generation model dropped; placeholder (no rule reads it)
             ClusterId = string.Empty,    // global snapshot; rules compare entity ClusterId fields, not this
+            ServerClusters     = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct),
+            ClusterNodes       = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct),
             // v3: Namespace entity retired — no namespace list to materialize. The Raw tree
             // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length,
             // effective-leaf uniqueness).
-            RawFolders         = await db.RawFolders.AsNoTracking().ToListAsync(ct),
+            RawFolders         = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct),
-            DriverInstances    = await db.DriverInstances.AsNoTracking().ToListAsync(ct),
+            DriverInstances    = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct),
-            Devices            = await db.Devices.AsNoTracking().ToListAsync(ct),
+            Devices            = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct),
-            TagGroups          = await db.TagGroups.AsNoTracking().ToListAsync(ct),
+            TagGroups          = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct),
-            UnsAreas           = await db.UnsAreas.AsNoTracking().ToListAsync(ct),
+            UnsAreas           = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct),
-            UnsLines           = await db.UnsLines.AsNoTracking().ToListAsync(ct),
+            UnsLines           = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct),
-            Equipment          = await db.Equipment.AsNoTracking().ToListAsync(ct),
+            Equipment          = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct),
-            Tags               = await db.Tags.AsNoTracking().ToListAsync(ct),
+            Tags               = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct),
-            UnsTagReferences   = await db.UnsTagReferences.AsNoTracking().ToListAsync(ct),
+            UnsTagReferences   = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct),
-            VirtualTags        = await db.VirtualTags.AsNoTracking().ToListAsync(ct),
+            VirtualTags        = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct),
-            ScriptedAlarms     = await db.ScriptedAlarms.AsNoTracking().ToListAsync(ct),
+            ScriptedAlarms     = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct),
-            Scripts            = await db.Scripts.AsNoTracking().ToListAsync(ct),
+            Scripts            = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct),
-            PollGroups         = await db.PollGroups.AsNoTracking().ToListAsync(ct),
+            PollGroups         = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct),
-            NodeAcls           = await db.NodeAcls.AsNoTracking().ToListAsync(ct),
+            NodeAcls           = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct),
+            ApiKeys            = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct),
             PriorEquipment     = [],   // intentional: no prior-generation table to diff against at this level
             ActiveReservations = await db.ExternalIdReservations
                 .AsNoTracking()

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs

@@ -28,6 +28,12 @@ public sealed class AdminOperationsActor : ReceiveActor
     private readonly IReadOnlyDictionary<string, IDriverProbe> _probesByType;
     private readonly TagConfigValidationMode _tagConfigValidationMode;
 
+    /// <summary>
+    ///     The coordinator's apply deadline. A <c>Dispatching</c> row with no expected-ack rows that is older
+    ///     than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7).
+    /// </summary>
+    private readonly TimeSpan _applyDeadline;
+
     /// <summary>
     ///     Where central→node commands are handed off (per-cluster mesh), or
     ///     <see langword="null"/> to publish on the mediator directly.
@@ -36,6 +42,9 @@ public sealed class AdminOperationsActor : ReceiveActor
 
     private readonly ILoggingAdapter _log = Context.GetLogger();
 
+    /// <summary>The StartDeployment whose compose is running off-actor, or null. See <see cref="HandleStartDeployment"/>.</summary>
+    private StartDeployment? _composeInFlight;
+
     /// <summary>Creates actor props for the admin operations actor.</summary>
     /// <param name="dbFactory">Factory for creating config database contexts.</param>
     /// <param name="coordinator">Reference to the deployment coordinator actor.</param>
@@ -45,15 +54,17 @@ public sealed class AdminOperationsActor : ReceiveActor
     ///     Central comm actor commands are handed to, or <see langword="null"/> to publish on the
     ///     DistributedPubSub mediator directly (the behaviour before the per-cluster mesh).
     /// </param>
+    /// <param name="applyDeadline">The coordinator's apply deadline; defaults to <see cref="DeployTimings.ApplyDeadline"/> (the coordinator's default source).</param>
     /// <returns>Props configured to create an AdminOperationsActor.</returns>
     public static Props Props(
         IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
         IActorRef coordinator,
         IEnumerable<IDriverProbe> probes,
         TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn,
-        IActorRef? meshRouter = null) =>
+        IActorRef? meshRouter = null,
+        TimeSpan? applyDeadline = null) =>
         Akka.Actor.Props.Create(() =>
-            new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter));
+            new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline));
 
     /// <summary>Initializes a new instance of the AdminOperationsActor.</summary>
     /// <param name="dbFactory">Factory for creating config database contexts.</param>
@@ -61,27 +72,48 @@ public sealed class AdminOperationsActor : ReceiveActor
     /// <param name="probes">Driver probes registered in DI; keyed by DriverType (case-insensitive).</param>
     /// <param name="tagConfigValidationMode">Deploy-gate TagConfig strictness mode (Warn default | Error opt-in).</param>
     /// <param name="meshRouter">Central comm actor, or <see langword="null"/> for the mediator.</param>
+    /// <param name="applyDeadline">The coordinator's apply deadline; defaults to <see cref="DeployTimings.ApplyDeadline"/> (the coordinator's default source).</param>
     public AdminOperationsActor(
         IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
         IActorRef coordinator,
         IEnumerable<IDriverProbe> probes,
         TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn,
-        IActorRef? meshRouter = null)
+        IActorRef? meshRouter = null,
+        TimeSpan? applyDeadline = null)
     {
+        // The same source the coordinator's default reads (ConfigPublishCoordinator.DefaultApplyDeadline IS
+        // DeployTimings.ApplyDeadline), so the lost-dispatch closure and the coordinator's own deadline agree.
+        _applyDeadline = applyDeadline ?? DeployTimings.ApplyDeadline;
         _dbFactory = dbFactory;
         _coordinator = coordinator;
         _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase);
         _tagConfigValidationMode = tagConfigValidationMode;
         _meshRouter = meshRouter;
 
-        ReceiveAsync<StartDeployment>(HandleStartDeploymentAsync);
+        // Synchronous receives: the compose and the probe run off-actor and come back as messages, so
-        ReceiveAsync<TestDriverConnect>(HandleTestDriverConnectAsync);
+        // the mailbox is never suspended for the length of a DB snapshot or a PLC connect timeout.
+        Receive<StartDeployment>(HandleStartDeployment);
+        Receive<DeployComposed>(HandleDeployComposed);
+        Receive<DeployComposeFailed>(HandleDeployComposeFailed);
+        Receive<TestDriverConnect>(HandleTestDriverConnect);
         ReceiveAsync<RestartDriver>(HandleRestartDriverAsync);
         ReceiveAsync<ReconnectDriver>(HandleReconnectDriverAsync);
         Receive<AcknowledgeAlarmCommand>(HandleAcknowledgeAlarm);
         Receive<ShelveAlarmCommand>(HandleShelveAlarm);
     }
 
+    /// <summary>
+    ///     The reply text for an unexpected server-side failure (R03:4.1): a fixed sentence plus the request's
+    ///     correlation id, never <c>ex.Message</c> — a SqlException or UriFormatException message carries
+    ///     connection-string fragments and host names, and the reply reaches any ConfigEditor browser session.
+    ///     The detail is in the central log under the same correlation id.
+    /// </summary>
+    /// <param name="operation">The admin operation that failed.</param>
+    /// <param name="correlationId">The request's correlation token, as logged alongside the exception.</param>
+    /// <returns>The operator-facing message.</returns>
+    internal static string ServerFault(string operation, object correlationId) =>
+        $"{operation} failed on the server; see the central node log for correlation {correlationId}.";
+
     /// <summary>
     ///     Hands a central→node command to the mesh transport router, or publishes it directly when
     ///     no router is wired.
@@ -129,8 +161,8 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId}", msg.AlarmId);
+            _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId} (correlation {CorrelationId})", msg.AlarmId, msg.CorrelationId);
-            replyTo.Tell(new AcknowledgeAlarmResult(false, ex.Message, msg.CorrelationId));
+            replyTo.Tell(new AcknowledgeAlarmResult(false, ServerFault("Acknowledge", msg.CorrelationId), msg.CorrelationId));
         }
     }
 
@@ -176,215 +208,364 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId}", msg.Kind, msg.AlarmId);
+            _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId} (correlation {CorrelationId})", msg.Kind, msg.AlarmId, msg.CorrelationId);
-            replyTo.Tell(new ShelveAlarmResult(false, ex.Message, msg.CorrelationId));
+            replyTo.Tell(new ShelveAlarmResult(false, ServerFault("Shelve", msg.CorrelationId), msg.CorrelationId));
         }
     }
 
-    private async Task HandleStartDeploymentAsync(StartDeployment msg)
+    /// <summary>
+    ///     <c>StartDeployment</c>. The only serialised part is the actor-local <see cref="_composeInFlight"/>
+    ///     flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the
+    ///     thread pool and come back as a <see cref="DeployComposed"/> / <see cref="DeployComposeFailed"/>
+    ///     message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the
+    ///     in-flight query off the actor thread too) and alarm acks keep flowing while it runs.
+    /// </summary>
+    /// <remarks>
+    ///     <see cref="_composeInFlight"/> is what keeps the check correct once it is off-actor: it is set
+    ///     BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a
+    ///     second request is refused in memory for the whole check-plus-compose and two artifacts can never
+    ///     race for one coordinator.
+    /// </remarks>
+    private void HandleStartDeployment(StartDeployment msg)
     {
         var replyTo = Sender;
+        if (_composeInFlight is { } inFlightRequest)
+        {
+            replyTo.Tell(new StartDeploymentResult(
+                StartDeploymentOutcome.AnotherDeploymentInFlight,
+                DeploymentId: null,
+                RevisionHash: null,
+                Message: $"A deployment requested by {inFlightRequest.CreatedBy} is still being composed.",
+                msg.CorrelationId));
+            return;
+        }
+
+        _composeInFlight = msg;
+        // Captured here: Context/Sender are not available on the thread-pool thread.
+        var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown";
+        var self = Self;
+        Task.Run(async () =>
+            {
+                var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false);
+                return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false);
+            })
+            .PipeTo(self,
+                success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch),
+                failure: ex => new DeployComposeFailed(msg, replyTo, ex));
+    }
+
+    /// <summary>
+    ///     The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight —
+    ///     keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first.
+    /// </summary>
+    /// <param name="msg">The request.</param>
+    /// <returns>The refusal, or null to proceed.</returns>
+    /// <remarks>
+    ///     <para>
+    ///         <b>Lost dispatch (fix round 1 / I7).</b> A <c>Dispatching</c> row with ZERO
+    ///         <c>NodeDeploymentState</c> rows was never taken by the coordinator (its <c>HandleDispatch</c>
+    ///         seeds the expected set and flips the status in one <c>SaveChanges</c>). The coordinator's PreStart
+    ///         recovers such a row — but only one that already existed when it started. A row committed AFTER
+    ///         PreStart whose <c>Tell</c> never arrived (this actor died between the compose's commit and
+    ///         <see cref="HandleDeployComposed"/>) has no owner at all, and refused every later deploy forever.
+    ///         Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is
+    ///         closed <c>TimedOut</c> with a reason and the new deploy proceeds. A late <c>DispatchDeployment</c>
+    ///         for it is ignored by the coordinator's terminal-state guard.
+    ///     </para>
+    ///     <para>
+    ///         One indexed projection (<c>IX_Deployment_Status</c>) plus, only for a Dispatching row past the
+    ///         deadline, a count and a tracked update.
+    ///     </para>
+    /// </remarks>
+    private async Task<ComposeOutcome?> CheckInFlightAsync(StartDeployment msg)
+    {
+        await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false);
+        var inflight = await db.Deployments
+            .AsNoTracking()
+            .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
+            .OrderByDescending(d => d.CreatedAtUtc)
+            .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc })
+            .ToListAsync()
+            .ConfigureAwait(false);
+
+        foreach (var row in inflight)
+        {
+            if (row.Status == DeploymentStatus.Dispatching
+                && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline
+                && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false)
+                && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false))
+            {
+                continue;
+            }
+
+            return new ComposeOutcome(
+                new StartDeploymentResult(
+                    StartDeploymentOutcome.AnotherDeploymentInFlight,
+                    DeploymentId: new DeploymentId(row.DeploymentId),
+                    RevisionHash: null,
+                    Message: $"Deployment {row.DeploymentId:N} is still in flight.",
+                    msg.CorrelationId),
+                Dispatch: null);
+        }
+        return null;
+    }
+
+    /// <summary>Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion).</summary>
+    /// <param name="db">The context.</param>
+    /// <param name="deploymentId">The row.</param>
+    /// <returns><see langword="true"/> when the row is now closed.</returns>
+    private async Task<bool> CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId)
+    {
+        var stamp = await db.Deployments
+            .AsNoTracking()
+            .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching)
+            .Select(d => new { d.RowVersion })
+            .FirstOrDefaultAsync()
+            .ConfigureAwait(false);
+        if (stamp is null) return true;   // no longer Dispatching — not in flight as far as this check goes
+
+        // Stub update: the row's ArtifactBlob is never read just to change its status.
+        var stub = new Deployment
+        {
+            DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty,
+            RowVersion = stamp.RowVersion,
+            Status = DeploymentStatus.TimedOut,
+            FailureReason =
+                "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " +
+                "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.",
+        };
+        var entry = db.Deployments.Attach(stub);
+        entry.Property(d => d.Status).IsModified = true;
+        entry.Property(d => d.FailureReason).IsModified = true;
         try
         {
-            await using var db = await _dbFactory.CreateDbContextAsync();
+            await db.SaveChangesAsync().ConfigureAwait(false);
+        }
+        catch (DbUpdateConcurrencyException)
+        {
+            // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight.
+            return false;
+        }
+        _log.Warning(
+            "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " +
+            "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline);
+        return true;
+    }
 
-            // Refuse if any deployment is already in flight — keeps the coordinator's state
+    private void HandleDeployComposed(DeployComposed msg)
-            // unambiguous. The UI is expected to wait for the in-flight one to seal or fail.
+    {
-            var inflight = await db.Deployments
+        _composeInFlight = null;
-                .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
+        // The row is committed (the compose task completed), so the coordinator can be told now; it reads
-                .Select(d => d.DeploymentId)
+        // the row back by id when it seeds the expected-ack set.
-                .FirstOrDefaultAsync();
+        if (msg.Dispatch is { } dispatch) _coordinator.Tell(dispatch);
-            if (inflight != Guid.Empty)
+        msg.ReplyTo.Tell(msg.Result);
-            {
+    }
-                replyTo.Tell(new StartDeploymentResult(
-                    StartDeploymentOutcome.AnotherDeploymentInFlight,
-                    DeploymentId: new DeploymentId(inflight),
-                    RevisionHash: null,
-                    Message: $"Deployment {inflight:N} is still in flight.",
-                    msg.CorrelationId));
-                return;
-            }
 
-            // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject
+    private void HandleDeployComposeFailed(DeployComposeFailed msg)
-            // on ANY validation error. The config is now canonical — the company overlay loader emits
+    {
-            // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId
+        _composeInFlight = null;
-            // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions,
+        _log.Error(msg.Error, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.Request.CreatedBy, msg.Request.CorrelationId);
-            // …) gates the deploy. A green build in this repo does not prove the config is valid; this
+        msg.ReplyTo.Tell(new StartDeploymentResult(
-            // is the last guard before a bad address space (or a non-derived EquipmentId) ships.
+            StartDeploymentOutcome.Rejected,
-            // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can
+            DeploymentId: null,
-            // (rarely) be a false positive if an external-id reservation release has not yet been
+            RevisionHash: null,
-            // committed/visible when the snapshot is read — operators seeing a spurious one should
+            Message: ServerFault("StartDeployment", msg.Request.CorrelationId),
-            // check ExternalIdReservation state before re-submitting.
+            msg.Request.CorrelationId));
-            var draft = await DraftSnapshotFactory.FromConfigDbAsync(db);
+    }
-            var errors = DraftValidator.Validate(draft).ToList();
 
-            // Cluster-topology guard. The SQL
+    /// <summary>Outcome of the off-actor compose: the reply to send, and the dispatch to issue when Accepted.</summary>
-            // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode)
+    private sealed record ComposeOutcome(StartDeploymentResult Result, DispatchDeployment? Dispatch);
-            // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an
-            // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at
-            // Hot/Warm and the constraint stays green, which would boot the runtime into an
-            // InvalidTopology band. ValidateClusterTopology catches that drift, but it isn't carried on
-            // the generation-versioned DraftSnapshot (the cluster/node rows aren't versioned), so it must
-            // be run separately here against the live rows. Read-only (AsNoTracking); errors fold into the
-            // same reject summary alongside the snapshot rules so a deploy failing either check is
-            // rejected with both sets of messages. ClusterId-ordered for a deterministic summary.
-            var clusters = await db.ServerClusters
-                .AsNoTracking()
-                .OrderBy(c => c.ClusterId)
-                .ToListAsync();
-            var nodesByCluster = (await db.ClusterNodes
-                    .AsNoTracking()
-                    .ToListAsync())
-                .GroupBy(n => n.ClusterId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal);
-            foreach (var cluster in clusters)
-            {
-                var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns)
-                    ? (IReadOnlyList<ClusterNode>)ns
-                    : [];
-                errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes));
-            }
 
-            // Deploy-time TagConfig strictness inspection over equipment tags. The
+    private sealed record DeployComposed(
-            // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would
+        StartDeployment Request, IActorRef ReplyTo, StartDeploymentResult Result, DispatchDeployment? Dispatch);
-            // REJECT — a present-but-invalid enum value, an absent or malformed address field, a
-            // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads
-            // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum;
-            // it now rejects the tag, so these warnings describe rejections.) In Warn mode
-            // (default) these are non-blocking (logged + appended to the accepted result Message); in Error
-            // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed.
-            // Running servers are untouched either way — the gate only sees re-deploys.
-            // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId /
-            // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type.
-            var driverTypeById = draft.DriverInstances
-                .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal);
-            var driverByDevice = draft.Devices
-                .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal);
-            var deviceConfigById = draft.Devices
-                .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
-                .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal);
-            var tagConfigWarnings = new List<(string TagId, string Text)>();
-            foreach (var t in draft.Tags)
-            {
-                if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue;
-                if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue;
-                foreach (var w in EquipmentTagConfigInspector.Inspect(
-                    driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId)))
-                    tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}"));
-            }
-            if (tagConfigWarnings.Count > 0)
-            {
-                if (_tagConfigValidationMode == TagConfigValidationMode.Error)
-                    errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId)));
-                else
-                    foreach (var x in tagConfigWarnings)
-                        _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text);
-            }
 
-            if (errors.Count > 0)
+    private sealed record DeployComposeFailed(StartDeployment Request, IActorRef ReplyTo, Exception Error);
-            {
+
-                var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}"));
+    /// <summary>
-                _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}",
+    ///     The deploy compose, off the actor thread. Touches no actor state: everything it needs is in
-                    errors.Count, summary);
+    ///     <paramref name="msg"/>, <paramref name="sourceNode"/> and the readonly fields captured at
-                replyTo.Tell(new StartDeploymentResult(
+    ///     construction (<see cref="_dbFactory"/>, <see cref="_tagConfigValidationMode"/>, <see cref="_log"/> —
+    ///     Akka's logging adapter is thread-safe).
+    /// </summary>
+    private async Task<ComposeOutcome> ComposeDeploymentAsync(StartDeployment msg, string sourceNode)
+    {
+        await using var db = await _dbFactory.CreateDbContextAsync();
+
+        // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject
+        // on ANY validation error. The config is now canonical — the company overlay loader emits
+        // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId
+        // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions,
+        // …) gates the deploy. A green build in this repo does not prove the config is valid; this
+        // is the last guard before a bad address space (or a non-derived EquipmentId) ships.
+        // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can
+        // (rarely) be a false positive if an external-id reservation release has not yet been
+        // committed/visible when the snapshot is read — operators seeing a spurious one should
+        // check ExternalIdReservation state before re-submitting.
+        var draft = await DraftSnapshotFactory.FromConfigDbAsync(db);
+        var errors = DraftValidator.Validate(draft).ToList();
+
+        // Cluster-topology guard. The SQL
+        // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode)
+        // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an
+        // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at
+        // Hot/Warm and the constraint stays green, which would boot the runtime into an
+        // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of
+        // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows.
+        // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing
+        // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary.
+        // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT
+        // transaction), not a second query that could observe a later edit.
+        var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList();
+        var nodesByCluster = draft.ClusterNodes
+            .GroupBy(n => n.ClusterId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal);
+        foreach (var cluster in clusters)
+        {
+            var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns)
+                ? (IReadOnlyList<ClusterNode>)ns
+                : [];
+            errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes));
+        }
+
+        // Deploy-time TagConfig strictness inspection over equipment tags. The
+        // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would
+        // REJECT — a present-but-invalid enum value, an absent or malformed address field, a
+        // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads
+        // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum;
+        // it now rejects the tag, so these warnings describe rejections.) In Warn mode
+        // (default) these are non-blocking (logged + appended to the accepted result Message); in Error
+        // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed.
+        // Running servers are untouched either way — the gate only sees re-deploys.
+        // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId /
+        // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type.
+        var driverTypeById = draft.DriverInstances
+            .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal);
+        var driverByDevice = draft.Devices
+            .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal);
+        var deviceConfigById = draft.Devices
+            .GroupBy(d => d.DeviceId, StringComparer.Ordinal)
+            .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal);
+        var tagConfigWarnings = new List<(string TagId, string Text)>();
+        foreach (var t in draft.Tags)
+        {
+            if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue;
+            if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue;
+            foreach (var w in EquipmentTagConfigInspector.Inspect(
+                driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId)))
+                tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}"));
+        }
+        if (tagConfigWarnings.Count > 0)
+        {
+            if (_tagConfigValidationMode == TagConfigValidationMode.Error)
+                errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId)));
+            else
+                foreach (var x in tagConfigWarnings)
+                    _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text);
+        }
+
+        if (errors.Count > 0)
+        {
+            var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}"));
+            _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}",
+                errors.Count, summary);
+            return new ComposeOutcome(
+                new StartDeploymentResult(
                     StartDeploymentOutcome.Rejected,
                     DeploymentId: null,
                     RevisionHash: null,
                     Message: summary,
-                    msg.CorrelationId));
+                    msg.CorrelationId),
-                return;
+                Dispatch: null);
+        }
+
+        // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough)
+        // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc
+        // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct
+        // sources are counted (the compile cache keys on source, so duplicates collapse to one unit).
+        // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard
+        // reject; this advisory rides in the Accepted Message so the UI can show it.
+        // advisory is declared outside the inner try so the Accepted reply below can still read it even
+        // when the guardrail throws. A failed estimate must NEVER block a valid deploy — the caller's
+        // failure branch is reserved for genuine seal/save failures.
+        string? advisory = null;
+        try
+        {
+            const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07)
+            // Projected from the snapshot already in hand — Script.SourceCode used to be read three times.
+            var compiled = draft.Scripts
+                .Select(s => s.SourceCode)
+                .Where(src => !PassthroughScript.TryMatch(src, out _))
+                .Distinct(StringComparer.Ordinal)
+                .Count();
+            if (compiled > 0)
+            {
+                var estMiB = compiled * PerScriptMiB;
+                _log.Warning(
+                    "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it",
+                    compiled, estMiB);
+                advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)";
             }
+        }
+        catch (Exception ex)
+        {
+            // Guardrail is advisory-only — a failed estimate must never block a valid deploy.
+            _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped");
+            advisory = null;
+        }
 
-            // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough)
+        // Composed from the validated snapshot, never from a second read: an edit that lands between
-            // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc
+        // validation and here is simply not in this artifact (R03:3.2).
-            // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct
+        var artifact = ConfigComposer.Compose(draft);
-            // sources are counted (the compile cache keys on source, so duplicates collapse to one unit).
+        var deploymentId = DeploymentId.NewId();
-            // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard
+        var revHash = RevisionHash.Parse(artifact.RevisionHash);
-            // reject; this advisory rides in the Accepted Message so the UI can show it.
-            // advisory is declared outside the inner try so the Accepted reply below can still read it even
-            // when the guardrail query throws (e.g. transient SQL). A failed estimate must NEVER block a
-            // valid deploy — the outer catch is reserved for genuine seal/save failures.
-            string? advisory = null;
-            try
-            {
-                const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07)
-                var scriptSources = await db.Scripts.AsNoTracking().Select(s => s.SourceCode).ToListAsync();
-                var compiled = scriptSources
-                    .Where(src => !PassthroughScript.TryMatch(src, out _))
-                    .Distinct(StringComparer.Ordinal)
-                    .Count();
-                if (compiled > 0)
-                {
-                    var estMiB = compiled * PerScriptMiB;
-                    _log.Warning(
-                        "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it",
-                        compiled, estMiB);
-                    advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)";
-                }
-            }
-            catch (Exception ex)
-            {
-                // Guardrail is advisory-only — a failed estimate must never block a valid deploy.
-                _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped");
-                advisory = null;
-            }
 
-            var artifact = await ConfigComposer.SnapshotAndFlattenAsync(db);
+        db.Deployments.Add(new Deployment
-            var deploymentId = DeploymentId.NewId();
+        {
-            var revHash = RevisionHash.Parse(artifact.RevisionHash);
+            DeploymentId = deploymentId.Value,
+            RevisionHash = artifact.RevisionHash,
+            Status = DeploymentStatus.Dispatching,
+            CreatedBy = msg.CreatedBy,
+            ArtifactBlob = artifact.Blob,
+        });
 
-            db.Deployments.Add(new Deployment
+        // Marker ConfigEdit row so the audit timeline shows the deployment snapshot.
-            {
+        db.ConfigEdits.Add(new ConfigEdit
-                DeploymentId = deploymentId.Value,
+        {
-                RevisionHash = artifact.RevisionHash,
+            EntityType = "Deployment",
-                Status = DeploymentStatus.Dispatching,
+            EntityId = deploymentId.Value,
-                CreatedBy = msg.CreatedBy,
+            FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}",
-                ArtifactBlob = artifact.Blob,
+            EditedBy = msg.CreatedBy,
-            });
+            SourceNode = sourceNode,
+        });
 
-            // Marker ConfigEdit row so the audit timeline shows the deployment snapshot.
+        await db.SaveChangesAsync();
-            db.ConfigEdits.Add(new ConfigEdit
-            {
-                EntityType = "Deployment",
-                EntityId = deploymentId.Value,
-                FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}",
-                EditedBy = msg.CreatedBy,
-                SourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown",
-            });
 
-            await db.SaveChangesAsync();
+        // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so
+        // the operator sees the non-blocking strictness findings on a successful deploy.
+        var messageParts = new List<string>();
+        if (advisory is not null) messageParts.Add(advisory);
+        messageParts.AddRange(tagConfigWarnings.Select(x => x.Text));
+        var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null;
 
-            _coordinator.Tell(new DispatchDeployment(deploymentId, revHash, msg.CorrelationId));
+        return new ComposeOutcome(
-
+            new StartDeploymentResult(
-            // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so
-            // the operator sees the non-blocking strictness findings on a successful deploy.
-            var messageParts = new List<string>();
-            if (advisory is not null) messageParts.Add(advisory);
-            messageParts.AddRange(tagConfigWarnings.Select(x => x.Text));
-            var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null;
-
-            replyTo.Tell(new StartDeploymentResult(
                 StartDeploymentOutcome.Accepted,
                 deploymentId,
                 revHash,
                 Message: acceptedMessage,
-                msg.CorrelationId));
+                msg.CorrelationId),
-        }
+            new DispatchDeployment(deploymentId, revHash, msg.CorrelationId));
-        catch (Exception ex)
-        {
-            _log.Error(ex, "StartDeployment failed for {CreatedBy}", msg.CreatedBy);
-            replyTo.Tell(new StartDeploymentResult(
-                StartDeploymentOutcome.Rejected,
-                DeploymentId: null,
-                RevisionHash: null,
-                Message: ex.Message,
-                msg.CorrelationId));
-        }
     }
 
-    private async Task HandleTestDriverConnectAsync(TestDriverConnect msg)
+    /// <summary>
+    ///     Test Connect runs on the thread pool and pipes its reply straight to the requester: a probe
+    ///     against a black-holed PLC (60 s clamp) must not suspend the singleton's mailbox (R03:3.1). Every
+    ///     path inside the task yields a <see cref="TestDriverConnectResult"/>, so the requester never
+    ///     sees a <c>Status.Failure</c>.
+    /// </summary>
+    private void HandleTestDriverConnect(TestDriverConnect msg)
     {
         var replyTo = Sender;
         if (!_probesByType.TryGetValue(msg.DriverType, out var probe))
@@ -397,6 +578,11 @@ public sealed class AdminOperationsActor : ReceiveActor
             return;
         }
 
+        Task.Run(() => ProbeAsync(probe, msg)).PipeTo(replyTo);
+    }
+
+    private async Task<TestDriverConnectResult> ProbeAsync(IDriverProbe probe, TestDriverConnect msg)
+    {
         var clampedSec = Math.Clamp(msg.TimeoutSeconds, 1, 60);
         var timeout = TimeSpan.FromSeconds(clampedSec);
         using var cts = new CancellationTokenSource(timeout);
@@ -406,28 +592,28 @@ public sealed class AdminOperationsActor : ReceiveActor
         {
             var result = await probe.ProbeAsync(msg.ConfigJson, timeout, cts.Token);
             sw.Stop();
-            replyTo.Tell(new TestDriverConnectResult(
+            return new TestDriverConnectResult(
                 result.Ok,
                 result.Message,
                 result.Ok ? sw.Elapsed.TotalMilliseconds : (double?)null,
-                msg.CorrelationId));
+                msg.CorrelationId);
         }
         catch (OperationCanceledException)
         {
-            replyTo.Tell(new TestDriverConnectResult(
+            return new TestDriverConnectResult(
                 false,
                 $"Probe timed out after {clampedSec}s.",
                 null,
-                msg.CorrelationId));
+                msg.CorrelationId);
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType);
+            _log.Error(ex, "Probe for {DriverType} threw (correlation {CorrelationId})", msg.DriverType, msg.CorrelationId);
-            replyTo.Tell(new TestDriverConnectResult(
+            return new TestDriverConnectResult(
                 false,
-                ex.Message,
+                ServerFault("Test Connect", msg.CorrelationId),
                 null,
-                msg.CorrelationId));
+                msg.CorrelationId);
         }
     }
 
@@ -457,8 +643,8 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId}", msg.DriverInstanceId);
+            _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId);
-            replyTo.Tell(new RestartDriverResult(false, ex.Message, msg.CorrelationId));
+            replyTo.Tell(new RestartDriverResult(false, ServerFault("RestartDriver", msg.CorrelationId), msg.CorrelationId));
         }
     }
 
@@ -487,8 +673,8 @@ public sealed class AdminOperationsActor : ReceiveActor
         }
         catch (Exception ex)
         {
-            _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId}", msg.DriverInstanceId);
+            _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId);
-            replyTo.Tell(new ReconnectDriverResult(false, ex.Message, msg.CorrelationId));
+            replyTo.Tell(new ReconnectDriverResult(false, ServerFault("ReconnectDriver", msg.CorrelationId), msg.CorrelationId));
         }
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs

@@ -1,17 +1,17 @@
 using System.IO.Compression;
 using System.Security.Cryptography;
 using System.Text.Json;
-using Microsoft.EntityFrameworkCore;
 using ZB.MOM.WW.OtOpcUa.Configuration;
-using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
 
 namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
 
 /// <summary>
 /// Pure snapshot composer: reads the current live-edit state from <see cref="OtOpcUaConfigDbContext"/>
 /// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash.
-/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots
+/// Determinism comes from reading every collection ordered by its natural key on the server
-/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the
+/// (<see cref="DraftSnapshotFactory"/>), so two snapshots over the same DB state always produce the same hash
+/// — and the same hash as the pre-R03:3.2 composer, which used those same queries; compression wraps the
 /// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see <see cref="Pack"/>.
 /// </summary>
 public static class ConfigComposer
@@ -25,45 +25,71 @@ public static class ConfigComposer
     };
 
     /// <summary>Reads the current configuration and returns a deterministic snapshot blob with revision hash.</summary>
+    /// <remarks>
+    ///     One consistent read (<see cref="DraftSnapshotFactory.FromConfigDbAsync"/>, a SNAPSHOT transaction on
+    ///     SQL Server) handed to <see cref="Compose"/>. The deploy path does not call this: it validates and
+    ///     composes from the one <see cref="DraftSnapshot"/> it already holds, so nothing can land between the
+    ///     validator's read and the composer's. This overload serves the drift probe and tests.
+    /// </remarks>
     /// <param name="db">The configuration database context.</param>
     /// <param name="ct">The cancellation token for the operation.</param>
     /// <returns>A deterministic <see cref="ConfigArtifact"/> containing the serialised snapshot bytes and its revision hash.</returns>
     public static async Task<ConfigArtifact> SnapshotAndFlattenAsync(
         OtOpcUaConfigDbContext db, CancellationToken ct = default)
+        => Compose(await DraftSnapshotFactory.FromConfigDbAsync(db, ct).ConfigureAwait(false));
+
+    /// <summary>
+    ///     Serialises an already-read <see cref="DraftSnapshot"/> into the deterministic artifact. Pure over
+    ///     its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now.
+    /// </summary>
+    /// <remarks>
+    ///     Serialises the collections IN THE ORDER GIVEN. Determinism comes from
+    ///     <see cref="DraftSnapshotFactory.FromConfigDbAsync"/>, which reads every table ordered by its natural key
+    ///     on the server; a snapshot built any other way must be ordered the same way or its hash will differ.
+    /// </remarks>
+    /// <param name="snapshot">The rows to compose — the SAME rows the validator was run over.</param>
+    /// <returns>The compressed artifact blob and its revision hash.</returns>
+    public static ConfigArtifact Compose(DraftSnapshot snapshot)
     {
+        ArgumentNullException.ThrowIfNull(snapshot);
+
         // v3 greenfield snapshot: the Raw tree (RawFolders → DriverInstances(+RawFolderId) → Devices(+DeviceConfig)
         // → TagGroups → Tags(DeviceId/TagGroupId, no EquipmentId/FolderPath/DriverInstanceId)) plus the UNS
         // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/
         // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit).
         // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256.
-        var snapshot = new
+        // NO re-sort here (fix round 1): the snapshot's collections arrive ordered by their natural key from
+        // DraftSnapshotFactory's server-side ORDER BY — the same order, and so the same bytes and RevisionHash,
+        // as the pre-R03:3.2 composer's own queries. An in-memory ordinal re-sort orders mixed-case keys
+        // differently from the SQL collation and silently re-hashed every unchanged configuration.
+        var ordered = new
         {
-            Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct),
+            Clusters = snapshot.ServerClusters,
-            Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct),
+            Nodes = snapshot.ClusterNodes,
-            RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct),
+            RawFolders = snapshot.RawFolders,
-            DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct),
+            DriverInstances = snapshot.DriverInstances,
-            Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct),
+            Devices = snapshot.Devices,
-            TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct),
+            TagGroups = snapshot.TagGroups,
-            Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct),
+            Equipment = snapshot.Equipment,
-            Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct),
+            Tags = snapshot.Tags,
-            UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct),
+            UnsTagReferences = snapshot.UnsTagReferences,
-            PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct),
+            PollGroups = snapshot.PollGroups,
-            UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct),
+            UnsAreas = snapshot.UnsAreas,
-            UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct),
+            UnsLines = snapshot.UnsLines,
-            NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct),
+            NodeAcls = snapshot.NodeAcls,
             // the data-plane credentials travel with the config, because a driver-only node
             // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every
             // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the
             // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob
             // still cannot be turned into a working credential. This is also why revoking a key reaches a
             // driver-only node only on the next deployment; see the ApiKey entity's remarks.
-            ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct),
+            ApiKeys = snapshot.ApiKeys,
-            Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct),
+            Scripts = snapshot.Scripts,
-            VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct),
+            VirtualTags = snapshot.VirtualTags,
-            ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct),
+            ScriptedAlarms = snapshot.ScriptedAlarms,
         };
 
-        return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions));
+        return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions));
     }
 
     /// <summary>

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs

@@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
 /// Dedup is two-layer: in-buffer (the <see cref="Dictionary{TKey, TValue}"/> below collapses
 /// duplicate EventIds before flush) and at the database via the filtered unique index
 /// <c>UX_ConfigAuditLog_EventId</c> (cross-restart safety — a retry of an already-flushed
-/// batch hits the constraint and we drop the duplicate insert without losing the rest of
+/// batch hits the constraint; <see cref="SaveBatch"/> drops exactly the already-persisted EventIds
-/// the batch).
+/// and re-saves the rest, so the duplicate is dropped without losing the rest of the batch).
 ///
 /// Implements the shared <see cref="IAuditWriter"/> seam: <see cref="WriteAsync"/> is a
 /// best-effort, never-throwing entry point that simply <c>Tell</c>s this actor and returns
@@ -92,23 +92,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
 
         try
         {
-            using var db = _dbFactory.CreateDbContext();
+            var dropped = SaveBatch(snapshot);
-            foreach (var evt in snapshot)
+            _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped);
-            {
-                db.ConfigAuditLogs.Add(new ConfigAuditLog
-                {
-                    Timestamp = evt.OccurredAtUtc.UtcDateTime,
-                    Principal = evt.Actor,
-                    EventType = $"{evt.Category}:{evt.Action}",
-                    NodeId = evt.SourceNode,
-                    DetailsJson = evt.DetailsJson,
-                    EventId = evt.EventId,
-                    CorrelationId = evt.CorrelationId,
-                    Outcome = evt.Outcome.ToString(),
-                });
-            }
-            db.SaveChanges();
-            _log.Debug("AuditWriter flushed {Count} events", snapshot.Count);
         }
         catch (Exception ex)
         {
@@ -116,6 +101,102 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
         }
     }
 
+    private static ConfigAuditLog ToRow(AuditEvent evt) => new()
+    {
+        Timestamp = evt.OccurredAtUtc.UtcDateTime,
+        Principal = evt.Actor,
+        EventType = $"{evt.Category}:{evt.Action}",
+        NodeId = evt.SourceNode,
+        DetailsJson = evt.DetailsJson,
+        EventId = evt.EventId,
+        CorrelationId = evt.CorrelationId,
+        Outcome = evt.Outcome.ToString(),
+    };
+
+    /// <summary>
+    ///     Persists one flush, dropping ONLY the rows the database will not take (R03:2.6, fix round 1 / I4).
+    /// </summary>
+    /// <param name="batch">The flush.</param>
+    /// <returns>How many rows were dropped.</returns>
+    /// <remarks>
+    ///     <para>
+    ///         <b>Why not detach <c>ex.Entries</c>.</b> SQL Server inserts the batch as one statement batch, so
+    ///         the <see cref="DbUpdateException"/> raised for ONE duplicate <c>EventId</c> names EVERY entry in
+    ///         the batch. Detaching what it names dropped the whole flush — the exact loss the fix was for
+    ///         (pinned against a real SQL Server by <c>On_sql_server_a_duplicate_event_id_costs_only_that_row</c>).
+    ///     </para>
+    ///     <para>
+    ///         So, on a rejection that names entries (a constraint violation; 2601/2627 for the unique index):
+    ///         (1) ask the database which of the batch's EventIds it already holds and drop exactly those —
+    ///         the at-least-once re-delivery the dedup contract expects; (2) re-save the rest as a batch; (3) if
+    ///         that is still rejected (some OTHER per-row constraint), fall back to row-by-row and drop only the
+    ///         rows that fail on their own. Every attempt uses a fresh context and fresh entities, so no state
+    ///         from a failed save leaks into the next one. A failure that names no entry (connection loss,
+    ///         deadlock) propagates — the whole-batch failure it always was. Bounded: at most 2 + N saves.
+    ///     </para>
+    /// </remarks>
+    private int SaveBatch(IReadOnlyList<AuditEvent> batch)
+    {
+        if (TrySave(batch, out var first)) return 0;
+        _log.Debug(first, "AuditWriter: a batch of {Count} was rejected; dropping only already-persisted EventIds", batch.Count);
+
+        HashSet<Guid> existing;
+        using (var db = _dbFactory.CreateDbContext())
+        {
+            var ids = batch.Select(e => (Guid?)e.EventId).ToList();
+            existing = db.ConfigAuditLogs
+                .AsNoTracking()
+                .Where(r => ids.Contains(r.EventId))
+                .Select(r => r.EventId!.Value)
+                .ToHashSet();
+        }
+        foreach (var id in existing)
+            _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", id);
+
+        var remaining = batch.Where(e => !existing.Contains(e.EventId)).ToList();
+        var dropped = batch.Count - remaining.Count;
+        if (remaining.Count == 0 || TrySave(remaining, out _)) return dropped;
+
+        // Still rejected with no duplicate left to blame: isolate the offending row(s).
+        foreach (var evt in remaining)
+        {
+            if (TrySave([evt], out var rowError)) continue;
+            dropped++;
+            if (IsUniqueViolation(rowError!))
+                _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", evt.EventId);
+            else
+                _log.Warning(rowError, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", evt.EventId);
+        }
+        return dropped;
+    }
+
+    /// <summary>Saves <paramref name="events"/> on a fresh context; a rejection that names entries is reported, anything else throws.</summary>
+    /// <param name="events">The rows to save.</param>
+    /// <param name="rejection">The row-level rejection, when this returns false.</param>
+    /// <returns><see langword="true"/> when saved.</returns>
+    private bool TrySave(IReadOnlyList<AuditEvent> events, out DbUpdateException? rejection)
+    {
+        using var db = _dbFactory.CreateDbContext();
+        foreach (var evt in events) db.ConfigAuditLogs.Add(ToRow(evt));
+        try
+        {
+            db.SaveChanges();
+            rejection = null;
+            return true;
+        }
+        catch (DbUpdateException ex) when (ex.Entries.Count > 0 || IsUniqueViolation(ex))
+        {
+            rejection = ex;
+            return false;
+        }
+    }
+
+    /// <summary>SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection.</summary>
+    /// <param name="ex">The update failure.</param>
+    /// <returns><see langword="true"/> for a duplicate-key rejection.</returns>
+    private static bool IsUniqueViolation(DbUpdateException ex) =>
+        ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 };
+
     /// <inheritdoc />
     protected override void PreRestart(Exception reason, object message)
     {

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs

@@ -201,6 +201,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
             LoadContactsFromDb();
         });
         Receive<ContactsLoaded>(HandleContactsLoaded);
+        Receive<PskRefreshed>(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount));
+        Receive<PskRefreshFailed>(msg => _log.Warning(msg.Error,
+            "Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh"));
         Receive<CommandDispatched>(HandleCommandDispatched);
 
         // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler
@@ -710,6 +713,20 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
 
         _targets = msg.Targets;
 
+        // R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port
+        // moved). Only the resolved dialer is asked; nothing is resolved just to prune it.
+        if (_dialer is { } dialer)
+        {
+            try
+            {
+                dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal));
+            }
+            catch (Exception ex)
+            {
+                _log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh");
+            }
+        }
+
         if (_targets.Count == 0)
         {
             _log.Warning(
@@ -780,7 +797,37 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
         }
 
         pskProvider.SetKnownClusters(msg.ClusterIds);
-        pskProvider.InvalidateAll();
+
+        // R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically,
+        // instead of dropping the cache and making the first ack per cluster re-read the store on a
+        // gRPC thread. A provider that does not pre-resolve (the interface default, and every fake)
+        // still drops its cache here, synchronously, exactly as before.
+        CancellationToken ct;
+        try
+        {
+            ct = _lifecycle.Token;
+        }
+        catch (ObjectDisposedException)
+        {
+            return; // Stopping.
+        }
+        var self = Self;
+        Task refresh;
+        try
+        {
+            refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct);
+        }
+        catch (Exception ex)
+        {
+            _log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh");
+            refresh = Task.CompletedTask;
+        }
+        if (!refresh.IsCompletedSuccessfully)
+        {
+            refresh.PipeTo(self,
+                success: () => new PskRefreshed(msg.ClusterIds.Count),
+                failure: ex => new PskRefreshFailed(ex));
+        }
 
         if (!_gatePublished)
         {
@@ -795,6 +842,14 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
     /// <summary>Internal tick asking for a contact-point refresh.</summary>
     public sealed record RefreshContacts;
 
+    /// <summary>The asynchronous PSK refresh (R03:3.4) completed.</summary>
+    /// <param name="ClusterCount">How many clusters were refreshed.</param>
+    public sealed record PskRefreshed(int ClusterCount);
+
+    /// <summary>The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service.</summary>
+    /// <param name="Error">The failure.</param>
+    public sealed record PskRefreshFailed(Exception Error);
+
     /// <summary>Result of a contact-point load, piped back from the DB read.</summary>
     /// <param name="Targets">
     ///     The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs

@@ -106,6 +106,27 @@ public interface IMeshPskProvider
 
     /// <summary>Drops every cached key.</summary>
     void InvalidateAll();
+
+    /// <summary>
+    ///     Re-reads the keys for <paramref name="clusterIds"/> from the store OFF the request path and
+    ///     swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep
+    ///     serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose
+    ///     secret the store reports ABSENT is revoked (no key from then on). Replaces
+    ///     "invalidate on every refresh", which made the first inbound ack per cluster per
+    ///     <c>ContactRefreshSeconds</c> block a gRPC thread on a synchronous, 3 s-bounded store read.
+    /// </summary>
+    /// <param name="clusterIds">The clusters central currently knows (the same set given to <see cref="SetKnownClusters"/>).</param>
+    /// <param name="ct">Cancels the refresh.</param>
+    /// <returns>Completes when the swap has happened.</returns>
+    /// <remarks>
+    ///     The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup
+    ///     re-read — so a fake or a non-caching provider needs nothing more.
+    /// </remarks>
+    Task RefreshAsync(IReadOnlyCollection<string> clusterIds, CancellationToken ct)
+    {
+        InvalidateAll();
+        return Task.CompletedTask;
+    }
 }
 
 /// <inheritdoc />
@@ -144,6 +165,16 @@ public sealed class MeshPskProvider : IMeshPskProvider
     private readonly ConcurrentDictionary<string, Lazy<string?>> _cache =
         new(StringComparer.OrdinalIgnoreCase);
 
+    /// <summary>
+    ///     Keys pre-resolved by <see cref="RefreshAsync"/>, consulted after the config map and before the
+    ///     lazy <see cref="_cache"/>. Swapped wholesale (volatile reference to an immutable dictionary),
+    ///     never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next
+    ///     one. A cluster missing from it — never refreshed, or whose read failed — falls through to the
+    ///     lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key".
+    /// </summary>
+    private volatile IReadOnlyDictionary<string, string> _refreshed =
+        new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
     /// <summary>
     ///     The known-cluster gate's membership set, or <see langword="null"/> when the gate is
     ///     inactive (never published, or published empty — see <see cref="SetKnownClusters"/>).
@@ -210,6 +241,13 @@ public sealed class MeshPskProvider : IMeshPskProvider
             return true;
         }
 
+        // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all.
+        if (_refreshed.TryGetValue(clusterId, out var refreshed))
+        {
+            psk = refreshed;
+            return true;
+        }
+
         var entry = _cache.GetOrAdd(
             clusterId,
             static (id, self) => new Lazy<string?>(
@@ -288,7 +326,124 @@ public sealed class MeshPskProvider : IMeshPskProvider
     }
 
     /// <inheritdoc />
-    public void InvalidateAll() => _cache.Clear();
+    public void InvalidateAll()
+    {
+        _cache.Clear();
+        _refreshed = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+    }
+
+    /// <summary>
+    ///     Ceiling on concurrent store reads during one <see cref="RefreshAsync"/>. A fleet of N clusters used to
+    ///     fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most
+    ///     this many at once now.
+    /// </summary>
+    public const int MaxConcurrentRefreshReads = 8;
+
+    /// <inheritdoc />
+    public async Task RefreshAsync(IReadOnlyCollection<string> clusterIds, CancellationToken ct)
+    {
+        ArgumentNullException.ThrowIfNull(clusterIds);
+
+        var previous = _refreshed;
+        var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
+        var results = new ConcurrentDictionary<string, StoreRead>(StringComparer.OrdinalIgnoreCase);
+
+        // Bounded fan-out; each read bounded like the synchronous path.
+        await Parallel.ForEachAsync(
+                ids,
+                new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct },
+                async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false))
+            .ConfigureAwait(false);
+
+        // Three outcomes per cluster (fix round 1 / I8):
+        //  - Found     → the new key;
+        //  - NotFound  → the store answered that the secret does not exist: a REVOCATION — the cluster gets no
+        //                key (and its lazily cached one is evicted below), so a deleted secret stops working
+        //                within one refresh instead of living on for the life of the process;
+        //  - Failed    → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key —
+        //                a transient fault must not open a window in which the cluster has no key at all.
+        var next = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        var answered = new List<string>();
+        foreach (var id in ids)
+        {
+            var read = results[id];
+            switch (read.Outcome)
+            {
+                case StoreReadOutcome.Found:
+                    next[id] = read.Value!;
+                    answered.Add(id);
+                    break;
+                case StoreReadOutcome.NotFound:
+                    answered.Add(id);
+                    break;
+                default:
+                    if (previous.TryGetValue(id, out var kept)) next[id] = kept;
+                    break;
+            }
+        }
+
+        // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new
+        // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too).
+        // Entries for clusters whose read failed are left alone.
+        _refreshed = next;
+        foreach (var id in answered) _cache.TryRemove(id, out _);
+    }
+
+    private enum StoreReadOutcome { Found, NotFound, Failed }
+
+    private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value);
+
+    /// <summary>
+    ///     The asynchronous twin of <see cref="ResolveFromStore"/>: same name derivation, same bound — awaited
+    ///     instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor —
+    ///     but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read".
+    /// </summary>
+    /// <param name="clusterId">The application <c>Cluster</c> id.</param>
+    /// <param name="ct">The caller's token, linked under the store-read bound.</param>
+    /// <returns>The outcome and, when found, the key.</returns>
+    private async Task<StoreRead> ResolveFromStoreAsync(string clusterId, CancellationToken ct)
+    {
+        SecretName name;
+        try
+        {
+            name = new SecretName(SecretNamePrefix + clusterId);
+        }
+        catch (ArgumentException ex)
+        {
+            _logger.LogWarning(
+                ex,
+                "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be "
+                + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or "
+                + "rename the cluster to [a-z0-9._/-].",
+                clusterId);
+            return new StoreRead(StoreReadOutcome.Failed, null);
+        }
+
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        cts.CancelAfter(StoreReadTimeout);
+        try
+        {
+            var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false);
+            if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value);
+
+            _logger.LogWarning(
+                "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is "
+                + "REVOKED — commands to that cluster and acks from it are refused until it is restored.",
+                name.Value,
+                clusterId);
+            return new StoreRead(StoreReadOutcome.NotFound, null);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(
+                ex,
+                "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the "
+                + "previously resolved key (if any) stays in service until the next refresh.",
+                clusterId,
+                name.Value);
+            return new StoreRead(StoreReadOutcome.Failed, null);
+        }
+    }
 
     /// <summary>
     ///     Reads <c>OT-MESH-PSK-{clusterId}</c> from the Secrets store, converting every failure mode

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs

@@ -34,6 +34,15 @@ public interface IMeshCommandDialer
     ///     <see cref="StatusCode.Unauthenticated"/> rather than dialled unauthenticated.
     /// </exception>
     Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct);
+
+    /// <summary>
+    ///     Releases any per-endpoint resources for endpoints NOT in <paramref name="nodeEndpoints"/>
+    ///     (R03:1.3). Central calls it on every contact refresh with the current dispatch set, so a
+    ///     renamed host or moved port does not leave the old channel and its idle HTTP/2 connection
+    ///     alive for the life of the process. Default no-op for dialers that cache nothing.
+    /// </summary>
+    /// <param name="nodeEndpoints">The endpoints still in the dispatch set.</param>
+    void RetainOnly(IReadOnlyCollection<string> nodeEndpoints) { }
 }
 
 /// <inheritdoc />
@@ -181,6 +190,30 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable
         await call.ResponseAsync.ConfigureAwait(false);
     }
 
+    /// <inheritdoc />
+    public void RetainOnly(IReadOnlyCollection<string> nodeEndpoints)
+    {
+        ArgumentNullException.ThrowIfNull(nodeEndpoints);
+        if (_disposed) return;
+
+        var keep = nodeEndpoints as IReadOnlySet<string> ?? nodeEndpoints.ToHashSet(StringComparer.Ordinal);
+        foreach (var endpoint in _channels.Keys)
+        {
+            if (keep.Contains(endpoint)) continue;
+            // A dial that already took this channel out of the cache before the removal fails once, as
+            // a named per-node RpcException the actor logs; the next dial to that endpoint (if it ever
+            // returns to the set) builds a fresh channel. Bounded to the dials in flight at the instant
+            // of a contact refresh, the same accepted race Dispose documents.
+            if (_channels.TryRemove(endpoint, out var channel))
+            {
+                channel.Dispose();
+                _log.LogInformation(
+                    "Mesh dialer released the cached channel for {Endpoint}: it is no longer in the dispatch set",
+                    endpoint);
+            }
+        }
+    }
+
     /// <summary>Marks the dialer disposed and disposes every cached gRPC channel.</summary>
     public void Dispose()
     {

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs

@@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
         DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self));
 
         using var db = _dbFactory.CreateDbContext();
+        // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never
+        // reads — and this query runs on every singleton start and hand-over.
         var inflight = db.Deployments
             .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
             .OrderByDescending(d => d.CreatedAtUtc)
+            .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc })
             .FirstOrDefault();
         if (inflight is null) return;
 
@@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             .AsNoTracking()
             .ToList();
 
+        // Resume the deadline timer using the remaining time. The deadline runs from when the
+        // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
+        // proxy — we don't track a separate "dispatched at" column).
+        var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc;
+        var remaining = _applyDeadline - elapsed;
+
+        if (nodeStates.Count == 0)
+        {
+            // The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by
+            // HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed:
+            // HandleDispatch threw before its SaveChanges (this actor restarted and the message was
+            // dropped), the process died between AdminOps' row write and the coordinator's Tell, or the
+            // Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to
+            // let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight
+            // forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply
+            // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that
+            // keeps throwing ends up on the other branch once the deadline passes); past it the row is
+            // closed as TimedOut with a reason that names the condition.
+            if (remaining > TimeSpan.Zero)
+            {
+                _log.Warning(
+                    "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " +
+                    "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
+                    "re-dispatching with {Remaining} of the apply deadline left",
+                    _current, remaining);
+                Self.Tell(new DispatchDeployment(
+                    _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
+            }
+            else
+            {
+                Self.Tell(new DeadlineElapsed(_current.Value));
+            }
+            return;
+        }
+
         _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer);
         foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying))
         {
@@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId));
         }
 
-        // Resume the deadline timer using the remaining time. The deadline runs from when the
-        // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
-        // proxy — we don't track a separate "dispatched at" column).
-        var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc;
-        var remaining = _applyDeadline - elapsed;
         if (remaining <= TimeSpan.Zero)
         {
             Self.Tell(new DeadlineElapsed(_current.Value));
@@ -201,6 +234,33 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
     private void HandleDispatch(DispatchDeployment msg)
     {
+        // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already
+        // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message
+        // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it.
+        //
+        // The RowVersion read HERE is the one the AwaitingApplyAcks flip below is conditioned on (re-review 1):
+        // reading a fresh one at flip time left a window in which AdminOps' lost-dispatch closure could commit
+        // TimedOut between the guard and the flip, and the flip would silently reopen the row. With the guard's
+        // RowVersion, any change since the guard read makes the flip a concurrency conflict instead.
+        byte[]? guardRowVersion;
+        using (var check = _dbFactory.CreateDbContext())
+        {
+            var guard = check.Deployments
+                .AsNoTracking()
+                .Where(d => d.DeploymentId == msg.DeploymentId.Value)
+                .Select(d => new { d.Status, d.RowVersion })
+                .FirstOrDefault();
+            if (guard?.Status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut)
+            {
+                _log.Warning(
+                    "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)",
+                    msg.DeploymentId, guard.Status);
+                if (_current == msg.DeploymentId) ResetForNext();
+                return;
+            }
+            guardRowVersion = guard?.RowVersion;
+        }
+
         _current = msg.DeploymentId;
         _acks.Clear();
         _degraded.Clear();
@@ -208,22 +268,51 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
         // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack.
         using (var db = _dbFactory.CreateDbContext())
         {
-            // Discovery reuses this context rather than opening a second one — it reads the same
+            // Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows
-            // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single
+            // already seeded for this deployment ARE the expected set — re-discovering would both duplicate
-            // context keeps the expected set and the seeded rows consistent by construction.
+            // the primary key and let a node toggled since the first attempt change who is expected.
-            _expectedAcks = DiscoverDriverNodes(db);
+            var seeded = db.NodeDeploymentStates
-
+                .AsNoTracking()
-            foreach (var node in _expectedAcks)
+                .Where(x => x.DeploymentId == msg.DeploymentId.Value)
+                .Select(x => x.NodeId)
+                .ToList();
+            if (seeded.Count > 0)
             {
-                db.NodeDeploymentStates.Add(new NodeDeploymentState
+                _expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer);
-                {
+            }
-                    NodeId = node.Value,
+            else
-                    DeploymentId = msg.DeploymentId.Value,
+            {
-                    Status = NodeDeploymentStatus.Applying,
+                // Discovery reuses this context rather than opening a second one — it reads the same
-                });
+                // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single
+                // context keeps the expected set and the seeded rows consistent by construction.
+                _expectedAcks = DiscoverDriverNodes(db);
+
+                foreach (var node in _expectedAcks)
+                {
+                    db.NodeDeploymentStates.Add(new NodeDeploymentState
+                    {
+                        NodeId = node.Value,
+                        DeploymentId = msg.DeploymentId.Value,
+                        Status = NodeDeploymentStatus.Applying,
+                    });
+                }
+            }
+            UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks, rowVersion: guardRowVersion);
+            try
+            {
+                db.SaveChanges();
+            }
+            catch (DbUpdateConcurrencyException)
+            {
+                // The row moved after the guard read (on SQL Server: AdminOps closed it as a lost dispatch). The
+                // expected-set rows and the flip were one SaveChanges, so nothing was written; nothing is
+                // published and the coordinator is free for the next deployment.
+                _log.Warning(
+                    "DispatchDeployment {Id} abandoned: the deployment changed state while it was being dispatched " +
+                    "(closed as a lost dispatch?) — not reopening it", msg.DeploymentId);
+                ResetForNext();
+                return;
             }
-            UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks);
-            db.SaveChanges();
         }
 
         // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing
@@ -323,11 +412,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
             // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on
             // each NodeDeploymentState row) and the seal logs at Warning.
             var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList();
-            var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value);
+            SetFailureReason(db, Truncate(
-            if (d is not null)
+                $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " +
-                d.FailureReason = Truncate(
+                "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."));
-                    $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " +
-                    "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.");
             db.SaveChanges();
             _log.Warning(
                 "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}",
@@ -360,7 +447,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
                 msg.DeploymentId, _current);
             return;
         }
-        if (_acks.Count == _expectedAcks.Count)
+        // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the
+        // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped.
+        if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count)
         {
             // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere.
             return;
@@ -368,6 +457,19 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
         using var db = _dbFactory.CreateDbContext();
         UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut);
+        if (_expectedAcks.Count == 0)
+        {
+            SetFailureReason(db, Truncate(
+                "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " +
+                "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."));
+            db.SaveChanges();
+            _log.Warning(
+                "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " +
+                "expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy",
+                _current.Value, _applyDeadline);
+            ResetForNext();
+            return;
+        }
         db.SaveChanges();
         // Name the silent nodes, not just the counts. Since the expected set became DB-derived this
         // is the failure an operator meets after deploying while a configured node is switched off,
@@ -394,13 +496,63 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
 
     public sealed record DeadlineElapsed(DeploymentId DeploymentId);
 
+    /// <summary>
+    ///     Stages a status change WITHOUT materialising the row: the row carries the whole <c>ArtifactBlob</c>
+    ///     (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub
+    ///     carrying only the key and the row's current RowVersion is attached and the changed columns marked
+    ///     modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged
+    ///     stub stays tracked so <see cref="SetFailureReason"/> can add to the same UPDATE.
+    /// </summary>
     private static void UpdateDeploymentStatus(
-        OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false)
+        OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null)
     {
-        var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value);
+        var entry = TrackStub(db, id, rowVersion);
-        if (d is null) return;
+        if (entry is null) return;
-        d.Status = status;
+        entry.Entity.Status = status;
-        if (sealNow) d.SealedAtUtc = DateTime.UtcNow;
+        entry.Property(x => x.Status).IsModified = true;
+        if (sealNow)
+        {
+            entry.Entity.SealedAtUtc = DateTime.UtcNow;
+            entry.Property(x => x.SealedAtUtc).IsModified = true;
+        }
+    }
+
+    /// <summary>Stages the current deployment's failure reason onto the stub <see cref="UpdateDeploymentStatus"/> staged.</summary>
+    private void SetFailureReason(OtOpcUaConfigDbContext db, string reason)
+    {
+        if (_current is null) return;
+        var entry = TrackStub(db, _current.Value);
+        if (entry is null) return;
+        entry.Entity.FailureReason = reason;
+        entry.Property(x => x.FailureReason).IsModified = true;
+    }
+
+    /// <summary>The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone.</summary>
+    /// <param name="db">The context.</param>
+    /// <param name="id">The deployment.</param>
+    /// <param name="rowVersion">
+    ///     The RowVersion the update must be conditioned on, when the caller already read one (the dispatch
+    ///     guard); <see langword="null"/> reads the current one.
+    /// </param>
+    private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry<Deployment>? TrackStub(
+        OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null)
+    {
+        var tracked = db.ChangeTracker.Entries<Deployment>().FirstOrDefault(e => e.Entity.DeploymentId == id.Value);
+        if (tracked is not null) return tracked;
+        if (rowVersion is null)
+        {
+            var stamp = db.Deployments
+                .AsNoTracking()
+                .Where(x => x.DeploymentId == id.Value)
+                .Select(x => new { x.RowVersion })
+                .FirstOrDefault();
+            if (stamp is null) return null;
+            rowVersion = stamp.RowVersion;
+        }
+        return db.Deployments.Attach(new Deployment
+        {
+            DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = rowVersion,
+        });
     }
 
     /// <summary>

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs

@@ -0,0 +1,190 @@
+using Akka.Actor;
+using Akka.Event;
+using Microsoft.EntityFrameworkCore;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
+
+/// <summary>
+///     Admin-role cluster singleton that periodically empties the <c>ArtifactBlob</c> of old, terminal
+///     deployments (R03:1.2). See <see cref="DeploymentRetentionOptions"/> for the policy and
+///     <see cref="SweepAsync"/> for the rule, which is pure over a context so it is unit-testable.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Empties, never deletes, and never nulls.</b> The column is <c>NOT NULL varbinary(max)</c>, and
+///         every reader (<c>DriverHostActor.LoadArtifactBlob</c>, <c>OpcUaPublishActor</c>,
+///         <c>DeploymentArtifactService</c>) already treats a zero-length blob as "no artifact" (#485: empty
+///         bytes are not an empty configuration — the apply fails and the node keeps last-known-good). An
+///         empty blob therefore needs no schema change and no reader change, where a NULL would need both.
+///     </para>
+///     <para>
+///         <b>Never touches an in-flight row</b> (Dispatching / AwaitingApplyAcks): a node may still be
+///         fetching it. The newest <see cref="DeploymentRetentionOptions.KeepSealed"/> terminal rows keep
+///         their blob whatever their age, so the "latest sealed" every boot path reads is always intact.
+///     </para>
+/// </remarks>
+public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers
+{
+    private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
+    private readonly DeploymentRetentionOptions _options;
+    private readonly ILoggingAdapter _log = Context.GetLogger();
+    private const string SweepTimerKey = "retention-sweep";
+
+    /// <summary>Gets or sets the timer scheduler.</summary>
+    public ITimerScheduler Timers { get; set; } = null!;
+
+    /// <summary>Creates the actor props.</summary>
+    /// <param name="dbFactory">The config DB factory.</param>
+    /// <param name="options">The retention policy.</param>
+    /// <returns>The props.</returns>
+    public static Props Props(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentRetentionOptions options) =>
+        Akka.Actor.Props.Create(() => new DeploymentRetentionActor(dbFactory, options));
+
+    /// <summary>Initializes a new instance of the <see cref="DeploymentRetentionActor"/> class.</summary>
+    /// <param name="dbFactory">The config DB factory.</param>
+    /// <param name="options">The retention policy.</param>
+    public DeploymentRetentionActor(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentRetentionOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(dbFactory);
+        ArgumentNullException.ThrowIfNull(options);
+        _dbFactory = dbFactory;
+        _options = options;
+
+        Receive<Sweep>(_ => RunSweep());
+        Receive<SweepCompleted>(msg =>
+        {
+            if (msg.Pruned > 0)
+                _log.Info("Deployment retention emptied the artifact blob on {Count} old deployment(s)", msg.Pruned);
+            else
+                _log.Debug("Deployment retention sweep found nothing to prune");
+            if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg);
+        });
+        Receive<SweepFailed>(msg =>
+        {
+            _log.Warning(msg.Error, "Deployment retention sweep failed; the next sweep retries");
+            if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg);
+        });
+    }
+
+    /// <inheritdoc />
+    protected override void PreStart()
+    {
+        if (!_options.Enabled)
+        {
+            _log.Info("Deployment retention is disabled (Deployments:Retention:Enabled=false); artifact blobs are kept forever");
+            return;
+        }
+        var interval = TimeSpan.FromMinutes(Math.Max(1, _options.IntervalMinutes));
+        var initial = TimeSpan.FromSeconds(Math.Max(0, _options.InitialDelaySeconds));
+        Timers.StartPeriodicTimer(SweepTimerKey, Sweep.Instance, initial, interval);
+    }
+
+    private void RunSweep()
+    {
+        var replyTo = Sender.IsNobody() ? null : Sender;
+        var self = Self;
+        var now = DateTime.UtcNow;
+        Task.Run(async () =>
+            {
+                await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false);
+                return await SweepAsync(db, _options, now).ConfigureAwait(false);
+            })
+            .PipeTo(self,
+                success: pruned => new SweepCompleted(pruned, replyTo),
+                failure: ex => new SweepFailed(ex, replyTo));
+    }
+
+    /// <summary>
+    ///     The retention rule. Empties the blob of every terminal deployment that is (a) not one of the
+    ///     newest <see cref="DeploymentRetentionOptions.KeepSealed"/> SEALED rows (at least one — the newest
+    ///     Sealed blob is never pruned), (b) not referenced by a
+    ///     <c>NodeDeploymentState</c> row started or applied within
+    ///     <see cref="DeploymentRetentionOptions.ReferencedDays"/>, and (c) still carrying bytes — at most
+    ///     <see cref="DeploymentRetentionOptions.MaxRowsPerSweep"/> per call.
+    /// </summary>
+    /// <param name="db">The config DB.</param>
+    /// <param name="options">The policy.</param>
+    /// <param name="utcNow">The sweep's clock.</param>
+    /// <param name="ct">Cancellation.</param>
+    /// <returns>How many rows were emptied.</returns>
+    public static async Task<int> SweepAsync(
+        OtOpcUaConfigDbContext db, DeploymentRetentionOptions options, DateTime utcNow, CancellationToken ct = default)
+    {
+        ArgumentNullException.ThrowIfNull(db);
+        ArgumentNullException.ThrowIfNull(options);
+
+        var keep = Math.Max(1, options.KeepSealed);
+        var referenceFloor = utcNow.AddDays(-Math.Max(0, options.ReferencedDays));
+        var batch = Math.Max(1, options.MaxRowsPerSweep);
+
+        // Projected: ids only. The candidate scan must not materialise blobs for rows it will keep.
+        var terminal = await db.Deployments
+            .AsNoTracking()
+            .Where(d => d.Status == DeploymentStatus.Sealed
+                        || d.Status == DeploymentStatus.PartiallyFailed
+                        || d.Status == DeploymentStatus.TimedOut)
+            .OrderByDescending(d => d.CreatedAtUtc)
+            .Select(d => new { d.DeploymentId, d.Status, HasBlob = d.ArtifactBlob.Length > 0 })
+            .ToListAsync(ct)
+            .ConfigureAwait(false);
+
+        // The keep window counts SEALED rows only (fix round 1 / I3). Counting every terminal row let a burst
+        // of failed deploys (TimedOut / PartiallyFailed) fill the window by itself and push the fleet's last
+        // good artifact — the newest Sealed row, which boot, drift detection and the publish fallback all
+        // read — out of it. `keep` is clamped to ≥ 1, so the newest Sealed row is never a candidate.
+        var keptSealed = terminal
+            .Where(d => d.Status == DeploymentStatus.Sealed)
+            .Take(keep)
+            .Select(d => d.DeploymentId)
+            .ToHashSet();
+        var aged = terminal
+            .Where(d => d.HasBlob && !keptSealed.Contains(d.DeploymentId))
+            .Select(d => d.DeploymentId)
+            .ToList();
+        if (aged.Count == 0) return 0;
+
+        var referenced = (await db.NodeDeploymentStates
+                .AsNoTracking()
+                .Where(s => aged.Contains(s.DeploymentId)
+                            && (s.StartedAtUtc >= referenceFloor
+                                || (s.AppliedAtUtc != null && s.AppliedAtUtc >= referenceFloor)))
+                .Select(s => s.DeploymentId)
+                .Distinct()
+                .ToListAsync(ct)
+                .ConfigureAwait(false))
+            .ToHashSet();
+
+        var targets = aged.Where(id => !referenced.Contains(id)).Take(batch).ToList();
+        if (targets.Count == 0) return 0;
+
+        // Tracked load of ONLY the rows being emptied: the update rides the row's real RowVersion on
+        // both providers, and the blob read is the one-time cost of retiring that row.
+        var rows = await db.Deployments
+            .Where(d => targets.Contains(d.DeploymentId))
+            .ToListAsync(ct)
+            .ConfigureAwait(false);
+        foreach (var row in rows) row.ArtifactBlob = [];
+        await db.SaveChangesAsync(ct).ConfigureAwait(false);
+        return rows.Count;
+    }
+
+    /// <summary>Asks for a sweep now; the sender (if any) receives the <see cref="SweepCompleted"/> / <see cref="SweepFailed"/>.</summary>
+    public sealed class Sweep
+    {
+        /// <summary>The singleton instance.</summary>
+        public static readonly Sweep Instance = new();
+        private Sweep() { }
+    }
+
+    /// <summary>A sweep finished.</summary>
+    /// <param name="Pruned">How many rows were emptied.</param>
+    /// <param name="ReplyTo">Who asked, when the sweep was requested rather than timer-driven.</param>
+    public sealed record SweepCompleted(int Pruned, IActorRef? ReplyTo);
+
+    /// <summary>A sweep threw.</summary>
+    /// <param name="Error">The failure.</param>
+    /// <param name="ReplyTo">Who asked, when the sweep was requested rather than timer-driven.</param>
+    public sealed record SweepFailed(Exception Error, IActorRef? ReplyTo);
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs

@@ -0,0 +1,55 @@
+using Microsoft.Extensions.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
+
+/// <summary>
+///     <c>Deployments:Retention</c> — how long a sealed deployment keeps its <c>ArtifactBlob</c> (R03:1.2).
+///     Every deploy stored the whole gzip'd fleet snapshot forever; nothing pruned it, so a year of routine
+///     edits was gigabytes of blobs. The retention sweep EMPTIES the blob on old rows and never deletes a
+///     row: the audit record (who deployed what revision, when, and how every node acked) survives.
+/// </summary>
+public sealed class DeploymentRetentionOptions
+{
+    /// <summary>The configuration section name.</summary>
+    public const string SectionName = "Deployments:Retention";
+
+    /// <summary>Master switch; <see langword="false"/> keeps every blob forever (the previous behaviour).</summary>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>
+    ///     How many of the most recent SEALED deployments keep their blob regardless of age (clamped to ≥ 1).
+    ///     Failed deploys (PartiallyFailed / TimedOut) do not count toward it, so a burst of them can never push
+    ///     the last good artifact out. A node booting from central SQL reads the LATEST sealed row's blob, so
+    ///     this must stay well above 1; 50 also keeps a comfortable re-deploy history.
+    /// </summary>
+    public int KeepSealed { get; set; } = 50;
+
+    /// <summary>
+    ///     A deployment any <c>NodeDeploymentState</c> row started or applied within this many days keeps its
+    ///     blob even when older than the <see cref="KeepSealed"/> window — a node that just applied it may
+    ///     still replay it on restart.
+    /// </summary>
+    public int ReferencedDays { get; set; } = 30;
+
+    /// <summary>How often the sweep runs.</summary>
+    public int IntervalMinutes { get; set; } = 60;
+
+    /// <summary>Delay before the first sweep after the singleton starts, so a boot is not a blob scan.</summary>
+    public int InitialDelaySeconds { get; set; } = 60;
+
+    /// <summary>
+    ///     Rows emptied per sweep. The sweep loads each candidate row (blob included) to update it under its
+    ///     RowVersion, so this bounds one sweep's memory and transaction size; a backlog drains over sweeps.
+    /// </summary>
+    public int MaxRowsPerSweep { get; set; } = 200;
+
+    /// <summary>Binds the section from configuration, with defaults for anything absent.</summary>
+    /// <param name="configuration">The app configuration; null yields the defaults.</param>
+    /// <returns>The bound options.</returns>
+    public static DeploymentRetentionOptions Bind(IConfiguration? configuration)
+    {
+        var options = new DeploymentRetentionOptions();
+        configuration?.GetSection(SectionName).Bind(options);
+        return options;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs

@@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
@@ -15,6 +16,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
 using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 
 namespace ZB.MOM.WW.OtOpcUa.ControlPlane;
@@ -30,6 +32,7 @@ public static class ServiceCollectionExtensions
     public const string AuditWriterSingletonName = "audit-writer";
     public const string RedundancyStateSingletonName = "redundancy-state";
     public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler";
+    public const string DeploymentRetentionSingletonName = "deployment-retention";
 
     /// <summary>
     /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — three with
@@ -109,7 +112,7 @@ public static class ServiceCollectionExtensions
                 // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks
                 // inside a props factory and races the spawn it is waiting for.
                 var meshRouter = registry.Get<CentralCommunicationActorKey>();
-                return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter);
+                return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter);
             },
             singletonOptions);
 
@@ -126,7 +129,10 @@ public static class ServiceCollectionExtensions
                     resolver.GetService<IConfiguration>()?["Deployment:TagConfigValidationMode"],
                     ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn;
                 var meshRouter = registry.Get<CentralCommunicationActorKey>();
-                return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter);
+                // The SAME deadline source as the coordinator above: AdminOps closes a lost dispatch only once it
+                // is past the coordinator's deadline, so the two must never drift.
+                return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter,
+                    applyDeadline: DeployTimings.ApplyDeadline);
             },
             singletonOptions);
 
@@ -181,6 +187,21 @@ public static class ServiceCollectionExtensions
             singletonOptions,
             createProxyToo: false);
 
+        // Deployment artifact retention (R03:1.2): empties ArtifactBlob on old terminal deployments so the
+        // Deployment table stops growing by one fleet snapshot per edit. Admin singleton for the same
+        // reason as the reconciler (one sweeper per fleet, and only central holds the ConfigDb);
+        // proxy-less because nothing addresses it in production — a test sends Sweep directly.
+        builder.WithSingleton<DeploymentRetentionKey>(
+            DeploymentRetentionSingletonName,
+            (system, registry, resolver) =>
+            {
+                var dbFactory = resolver.GetService<IDbContextFactory<OtOpcUaConfigDbContext>>();
+                var options = DeploymentRetentionOptions.Bind(resolver.GetService<IConfiguration>());
+                return DeploymentRetentionActor.Props(dbFactory, options);
+            },
+            singletonOptions,
+            createProxyToo: false);
+
         return builder;
     }
 
@@ -256,4 +277,5 @@ public sealed class AdminOperationsActorKey { }
 public sealed class AuditWriterActorKey { }
 public sealed class RedundancyStateActorKey { }
 public sealed class ClusterNodeAddressReconcilerKey { }
+public sealed class DeploymentRetentionKey { }
 public sealed class CentralCommunicationActorKey { }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs

@@ -0,0 +1,41 @@
+using ZB.MOM.WW.Configuration;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+/// <summary>
+///     Fail-fast startup validator for <see cref="ApiKeyOptions"/> (R03:2.3). The pepper used to be
+///     checked only when the first key was verified — <c>ApiKeySecret.Verify</c> throws on an empty one —
+///     so a site node deployed without <c>Security__ApiKeys__Pepper</c> booted green, passed health,
+///     applied deployments and denied every machine client with the same opaque "Invalid API key" until
+///     someone read the log; a central node with a wrong pepper minted keys that could verify nowhere.
+///     Every other security section already fails host start on a bad value; this one now does too.
+/// </summary>
+/// <remarks>
+///     Registered for <c>hasDriver || hasAdmin</c>: a driver node needs the pepper to VERIFY keys, an admin
+///     node to MINT them. The minimum length is a floor against a placeholder (<c>"x"</c>) passing as
+///     configured, not a strength claim — the pepper is a server-side secret, not a password.
+/// </remarks>
+public sealed class ApiKeyOptionsValidator : OptionsValidatorBase<ApiKeyOptions>
+{
+    /// <summary>The shortest pepper accepted (characters, after trimming).</summary>
+    public const int MinPepperLength = 16;
+
+    /// <inheritdoc />
+    protected override void Validate(ValidationBuilder builder, ApiKeyOptions options)
+    {
+        var pepper = options.Pepper?.Trim() ?? string.Empty;
+        builder.RequireThat(pepper.Length > 0,
+            $"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " +
+            "supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " +
+            "every API key (driver) or mint keys that can never verify (admin).");
+        builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength,
+            $"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters.");
+        builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username),
+            $"{ApiKeyOptions.SectionName}:Username must not be blank.");
+        builder.RequireThat(options.StoreTimeoutMs >= 0,
+            $"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline).");
+        builder.RequireThat(options.StoreCacheSeconds >= 0,
+            $"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache).");
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs

@@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration
                         sp.GetRequiredService<IOptions<AkkaClusterOptions>>().Value)),
                 sp.GetRequiredService<IOptions<ApiKeyOptions>>(),
                 sp.GetRequiredService<ILoggerFactory>().CreateLogger<BoundedApiKeyStore>(),
-                sp.GetRequiredService<TimeProvider>()));
+                sp.GetRequiredService<TimeProvider>(),
+                // R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as
+                // ApiKeyOptions, Host-owned knobs.
+                ApiKeyStoreGuardOptions.Bind(sp.GetService<IConfiguration>())));
             services.AddSingleton<IOpcUaUserAuthenticator, ApiKeyOpcUaUserAuthenticator>();
 
             // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb.

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs

@@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication;
 using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Configuration;
 using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
 
 namespace ZB.MOM.WW.OtOpcUa.Host.Health;
 
@@ -28,11 +29,40 @@ public static class HealthEndpoints
     /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to
     /// <c>true</c> so admin/fused callers (including the test harnesses) keep the probe unchanged.
     /// </param>
+    /// <param name="hasDriver">
+    /// Whether this node carries the <c>driver</c> role and therefore serves an OPC UA endpoint. When
+    /// <c>true</c> the <see cref="OpcUaEndpointHealthCheck"/> joins the ready tier, and the active tier on a
+    /// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading
+    /// the <see cref="IOpcUaEndpointState"/> the host registers alongside <c>OtOpcUaServerHostedService</c>.
+    /// Defaults to <c>false</c> so admin-only callers and the existing harnesses register nothing new.
+    /// </param>
     /// <returns>The same service collection, for chaining.</returns>
-    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true)
+    public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false)
     {
         var checks = services.AddHealthChecks();
 
+        // A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve.
+        // The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6):
+        //  - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may
+        //    route to it;
+        //  - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState
+        //    (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would
+        //    take the operator's only console out of rotation precisely when they need it to diagnose the
+        //    failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands
+        //    the peer takes Primary and the active tier follows it on its own.
+        // Factory-registered so the state singleton is resolved per probe.
+        if (hasDriver)
+        {
+            var tags = hasAdmin
+                ? new[] { ZbHealthTags.Ready }
+                : new[] { ZbHealthTags.Ready, ZbHealthTags.Active };
+            checks.Add(new HealthCheckRegistration(
+                OpcUaEndpointHealthCheck.Name,
+                sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService<IOpcUaEndpointState>()),
+                failureStatus: null,
+                tags: tags));
+        }
+
         // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config
         // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so
         // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged.

src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs

@@ -0,0 +1,47 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Health;
+
+/// <summary>
+///     Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is
+///     listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy
+///     one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets
+///     connection-refused.
+/// </summary>
+/// <remarks>
+///     Registered on driver-role nodes only (<see cref="HealthEndpoints.AddOtOpcUaHealth"/>); an
+///     admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused
+///     admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier
+///     is what Traefik routes the AdminUI on.
+/// </remarks>
+public sealed class OpcUaEndpointHealthCheck : IHealthCheck
+{
+    /// <summary>The registration name, as it appears in the health JSON.</summary>
+    public const string Name = "opcua-endpoint";
+
+    private readonly IOpcUaEndpointState _state;
+
+    /// <summary>Initializes a new instance of the <see cref="OpcUaEndpointHealthCheck"/> class.</summary>
+    /// <param name="state">The endpoint lifecycle fact the hosted service maintains.</param>
+    public OpcUaEndpointHealthCheck(IOpcUaEndpointState state)
+    {
+        ArgumentNullException.ThrowIfNull(state);
+        _state = state;
+    }
+
+    /// <inheritdoc />
+    public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+    {
+        var data = new Dictionary<string, object> { ["status"] = _state.Status.ToString() };
+        var result = _state.Status switch
+        {
+            OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data),
+            OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data),
+            OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data),
+            _ => HealthCheckResult.Unhealthy(
+                "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data),
+        };
+        return Task.FromResult(result);
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs

@@ -0,0 +1,56 @@
+using Microsoft.Extensions.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>
+///     The pre-authentication guards <see cref="BoundedApiKeyStore"/> puts in front of the ConfigDb key
+///     lookup (R03:3.3). Bound from the same <c>Security:ApiKeys</c> section as <c>ApiKeyOptions</c> but
+///     owned by the Host, where the store lives.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <b>Why.</b> The negative cache was keyed by key id, so a flood of DISTINCT well-formed
+///         <c>otua_&lt;random&gt;_x</c> credentials missed it every time and cost one indexed SQL query per
+///         attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close
+///         that: a <b>known-id set</b> (every <c>ApiKeyId</c> the ConfigDb holds, refreshed every
+///         <c>StoreCacheSeconds</c>) rejects an unknown id in memory, and a <b>miss budget</b> (token
+///         bucket) bounds how many lookups that reach the store and MISS are allowed per second when the
+///         set is unavailable.
+///     </para>
+///     <para>
+///         The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback
+///         (the <c>IOpcUaUserAuthenticator</c> seam) does not carry the client's address. Per-endpoint
+///         accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane.
+///     </para>
+/// </remarks>
+public sealed class ApiKeyStoreGuardOptions
+{
+    /// <summary>The configuration section the knobs live in (shared with <c>ApiKeyOptions</c>).</summary>
+    public const string SectionName = "Security:ApiKeys";
+
+    /// <summary>
+    ///     Whether the known-id set is consulted. <see langword="false"/> restores a store read per unknown
+    ///     id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact
+    ///     store is never wrapped).
+    /// </summary>
+    public bool KnownIdSet { get; set; } = true;
+
+    /// <summary>
+    ///     How many store MISSES (lookups that reach the inner store and find no key) may burst before
+    ///     further lookups for unknown ids are refused without a store read. <c>0</c> disables the budget.
+    /// </summary>
+    public int MissBurst { get; set; } = 100;
+
+    /// <summary>How many miss tokens are restored per second, up to <see cref="MissBurst"/>.</summary>
+    public double MissRefillPerSecond { get; set; } = 20;
+
+    /// <summary>Binds the section, with defaults for anything absent.</summary>
+    /// <param name="configuration">The app configuration; null yields the defaults.</param>
+    /// <returns>The bound options.</returns>
+    public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration)
+    {
+        var options = new ApiKeyStoreGuardOptions();
+        configuration?.GetSection(SectionName).Bind(options);
+        return options;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs

@@ -56,6 +56,33 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
     private readonly ConcurrentDictionary<string, Entry<ApiKeyLookup?>> _keys = new(StringComparer.Ordinal);
     private readonly ConcurrentDictionary<string, Entry<IReadOnlyList<NodeAcl>>> _grants =
         new(StringComparer.Ordinal);
+    private readonly ApiKeyStoreGuardOptions _guard;
+
+    // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds
+    // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first
+    // successful load, or when the inner store cannot list ids — then the per-id path runs as before.
+    // The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile
+    // reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read
+    // a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed.
+    private volatile KnownIdSnapshot? _knownIds;
+    private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1);
+
+    // Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is
+    // shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked
+    // one store's capability into every other consumer of the same options).
+    private volatile bool _listingUnsupported;
+
+    /// <summary>The known-id set and the instant it was loaded, published as one unit.</summary>
+    /// <param name="Ids">The ids the inner store holds.</param>
+    /// <param name="StampedUtc">When they were read.</param>
+    private sealed record KnownIdSnapshot(IReadOnlySet<string> Ids, DateTimeOffset StampedUtc);
+
+    // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards
+    // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the
+    // configured rate instead of one SQL round trip per attempt under the SessionManager lock.
+    private readonly object _bucketGate = new();
+    private double _missTokens;
+    private DateTimeOffset _missRefilledUtc;
 
     /// <summary>Initializes a new instance of the <see cref="BoundedApiKeyStore"/> class.</summary>
     /// <param name="inner">The store being bounded.</param>
@@ -66,11 +93,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
     ///     <c>DataPlaneAuthRegistration</c> for why a second clock here would be a hazard rather than a
     ///     detail.
     /// </param>
+    /// <param name="guard">The pre-authentication guards (R03:3.3); defaults when omitted.</param>
     public BoundedApiKeyStore(
         IApiKeyStore inner,
         IOptions<ApiKeyOptions> options,
         ILogger<BoundedApiKeyStore> log,
-        TimeProvider time)
+        TimeProvider time,
+        ApiKeyStoreGuardOptions? guard = null)
     {
         ArgumentNullException.ThrowIfNull(inner);
         ArgumentNullException.ThrowIfNull(options);
@@ -81,8 +110,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
         _options = options;
         _log = log;
         _time = time;
+        _guard = guard ?? new ApiKeyStoreGuardOptions();
+        _missTokens = Math.Max(0, _guard.MissBurst);
+        _missRefilledUtc = time.GetUtcNow();
     }
 
+    /// <summary>The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards.</summary>
+    public IReadOnlySet<string>? KnownKeyIds => _knownIds?.Ids;
+
     /// <summary>The store being bounded.</summary>
     /// <remarks>
     ///     Exposed so the composition root's wiring can be asserted. A decorator registered in one branch
@@ -115,12 +150,103 @@ public sealed class BoundedApiKeyStore : IApiKeyStore
             return cached;
         }
 
+        // R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked.
+        var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false);
+        if (known is not null && !known.Contains(apiKeyId))
+        {
+            OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "unknown_id"));
+            return null;
+        }
+
+        // R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known
+        // to exist (a known id is a real key being verified, never a guess).
+        if (known is null && !TryTakeMissToken())
+        {
+            OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair<string, object?>("outcome", "miss_budget"));
+            throw new InvalidOperationException(
+                "ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / "
+                + "MissRefillPerSecond); refusing this lookup without a store read.");
+        }
+
         var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct)
             .ConfigureAwait(false);
+        if (fresh is not null && known is null) RefundMissToken();
         if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh);
         return fresh;
     }
 
+    /// <summary>
+    ///     The known-id set, refreshed when older than <paramref name="ttl"/>. Single-flight: concurrent
+    ///     callers during a refresh use the previous set (or, before the first load, the per-id path). A
+    ///     refresh that fails or exceeds the deadline logs and keeps the previous set.
+    /// </summary>
+    /// <param name="ttl">The refresh interval (the cache TTL); non-positive disables the set.</param>
+    /// <param name="ct">The caller's token.</param>
+    /// <returns>The set, or null.</returns>
+    private async Task<IReadOnlySet<string>?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct)
+    {
+        if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null;
+
+        var now = _time.GetUtcNow();
+        var current = _knownIds;
+        if (current is not null && now - current.StampedUtc < ttl) return current.Ids;
+
+        if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids;
+        try
+        {
+            // Re-check under the gate: another caller may have refreshed while this one waited.
+            if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids;
+            var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false);
+            if (ids is null)
+            {
+                // Unsupported by the inner store: stop asking — for this decorator only.
+                _listingUnsupported = true;
+                return null;
+            }
+            _knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow());
+            return ids;
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
+        {
+            _log.LogWarning(ex,
+                "ApiKeyStore: refreshing the known key-id set failed; {Mode}",
+                current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force");
+            return current?.Ids;
+        }
+        finally
+        {
+            _knownIdsRefresh.Release();
+        }
+    }
+
+    /// <summary>Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled.</summary>
+    /// <returns><see langword="false"/> when the budget is exhausted.</returns>
+    private bool TryTakeMissToken()
+    {
+        var burst = _guard.MissBurst;
+        if (burst <= 0) return true;
+        lock (_bucketGate)
+        {
+            var now = _time.GetUtcNow();
+            var elapsed = (now - _missRefilledUtc).TotalSeconds;
+            if (elapsed > 0)
+            {
+                _missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond));
+                _missRefilledUtc = now;
+            }
+            if (_missTokens < 1) return false;
+            _missTokens -= 1;
+            return true;
+        }
+    }
+
+    /// <summary>A lookup that reached the store and HIT was not a miss: its token goes back.</summary>
+    private void RefundMissToken()
+    {
+        if (_guard.MissBurst <= 0) return;
+        lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1);
+    }
+
     // Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect.
     /// <inheritdoc />
     public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs

@@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore
         return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false);
     }
 
+    /// <inheritdoc />
+    public async Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
+    {
+        await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false);
+        // One projection over the whole (small) table — the unique index on ApiKeyId makes this an index
+        // scan — so an unknown id can be refused in memory instead of costing a query per attempt.
+        var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false);
+        return ids.ToHashSet(StringComparer.Ordinal);
+    }
+
     /// <summary>Loads a principal's grants, confined to this node's cluster.</summary>
     /// <remarks>
     ///     <b>The full grant set, not just the <c>Cluster</c>-scoped rows</b>. The old

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs

@@ -35,6 +35,16 @@ public interface IApiKeyStore
     /// <param name="ct">The cancellation token.</param>
     /// <returns>The matching ACL rows; empty when none.</returns>
     Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct);
+
+    /// <summary>
+    ///     Every <c>ApiKeyId</c> the store holds (enabled or not), for the pre-authentication known-id set
+    ///     (R03:3.3); <see langword="null"/> when the store cannot cheaply list them, in which case callers
+    ///     fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry
+    ///     and the secret are still decided by <see cref="FindAsync"/>.
+    /// </summary>
+    /// <param name="ct">The cancellation token.</param>
+    /// <returns>The id set, or null when unsupported.</returns>
+    Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult<IReadOnlySet<string>?>(null);
 }
 
 /// <summary>An API key resolved for authentication, with the ACL grants naming its principal.</summary>

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs

@@ -0,0 +1,73 @@
+namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+/// <summary>Lifecycle of this node's OPC UA endpoint as <see cref="OtOpcUaServerHostedService"/> drives it.</summary>
+public enum OpcUaEndpointStatus
+{
+    /// <summary>The hosted service has not run <c>StartAsync</c> yet.</summary>
+    NotStarted,
+
+    /// <summary>The SDK is listening and the address-space / ServiceLevel sinks are bound.</summary>
+    Started,
+
+    /// <summary>The SDK failed to start (port in use, PKI unwritable, bad <c>OpcUa:*</c> override, …).</summary>
+    Failed,
+
+    /// <summary>The hosted service stopped the SDK; the port is released.</summary>
+    Stopped,
+}
+
+/// <summary>
+///     The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed
+///     start was one Error line: <c>/health/ready</c> stayed 200, the node joined its mesh, could be
+///     elected driver Primary and acked deploys as Applied through the Null sink — while clients got
+///     connection-refused from the node the fleet called healthy.
+/// </summary>
+/// <remarks>
+///     Consumed by <c>OpcUaEndpointHealthCheck</c> (Ready + Active tiers). The redundancy election and the
+///     ServiceLevel calculator do not read it yet — they live outside the Host (<c>Cluster</c> /
+///     <c>Runtime</c>); feeding them is the Wave 3 follow-up the lane report names. Until then a fused
+///     node whose SDK failed is visible (503) but still Primary-eligible.
+/// </remarks>
+public interface IOpcUaEndpointState
+{
+    /// <summary>The endpoint's current lifecycle status.</summary>
+    OpcUaEndpointStatus Status { get; }
+
+    /// <summary>Why the start failed, when <see cref="Status"/> is <see cref="OpcUaEndpointStatus.Failed"/>.</summary>
+    string? FailureReason { get; }
+}
+
+/// <summary>Mutable <see cref="IOpcUaEndpointState"/>; written by the hosted service, read by the health check.</summary>
+public sealed class OpcUaEndpointState : IOpcUaEndpointState
+{
+    private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted;
+    private volatile string? _failureReason;
+
+    /// <inheritdoc />
+    public OpcUaEndpointStatus Status => _status;
+
+    /// <inheritdoc />
+    public string? FailureReason => _failureReason;
+
+    /// <summary>Records a successful SDK start.</summary>
+    public void MarkStarted()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Started;
+    }
+
+    /// <summary>Records a failed SDK start.</summary>
+    /// <param name="reason">The failure, as the health probe should report it.</param>
+    public void MarkFailed(string reason)
+    {
+        _failureReason = reason;
+        _status = OpcUaEndpointStatus.Failed;
+    }
+
+    /// <summary>Records that the hosted service stopped the SDK.</summary>
+    public void MarkStopped()
+    {
+        _failureReason = null;
+        _status = OpcUaEndpointStatus.Stopped;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Core.Authorization;
@@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
     private readonly HistorianUpdaterHolder _historianUpdater;
     private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
+    private readonly OpcUaEndpointState _endpointState;
+    private readonly bool _rethrowStartFailure;
 
     private OpcUaApplicationHost? _appHost;
     private OtOpcUaSdkServer? _server;
@@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     /// <param name="historyUpdateRoleView">The HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
     /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
     /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
+    /// <param name="endpointState">The endpoint lifecycle fact this service maintains for the
+    /// <c>opcua-endpoint</c> health probe (R03:2.1). Optional: a private instance is kept when none is registered,
+    /// so a harness that never probes health is unchanged.</param>
+    /// <param name="roles">The node's resolved roles. A node with the driver role and NO admin role exists only
+    /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and
+    /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through
+    /// the health probe instead. Null (harnesses) behaves as fused.</param>
     public OtOpcUaServerHostedService(
         IOptions<OpcUaApplicationHostOptions> options,
         DeferredAddressSpaceSink deferredSink,
@@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         ILoggerFactory loggerFactory,
         IDriverPrimaryRoleSource? primaryRoleSource = null,
         HistorianUpdaterHolder? historianUpdater = null,
-        IHistoryUpdateRoleView? historyUpdateRoleView = null)
+        IHistoryUpdateRoleView? historyUpdateRoleView = null,
+        OpcUaEndpointState? endpointState = null,
+        EffectiveRoles? roles = null)
     {
         ArgumentNullException.ThrowIfNull(aclOptions);
         ArgumentNullException.ThrowIfNull(time);
@@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         _primaryRoleSource = primaryRoleSource;
         _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
         _historyUpdateRoleView = historyUpdateRoleView;
+        _endpointState = endpointState ?? new OpcUaEndpointState();
+        _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false };
         _logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
     }
 
@@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         }
         catch (Exception ex)
         {
+            // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy
+            // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits
+            // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct
+            // the OpcUa:* config without a process bounce — but it is visibly not ready.
+            _endpointState.MarkFailed(ex.Message);
+            await ReleaseFailedStartAsync().ConfigureAwait(false);
+            if (_rethrowStartFailure)
+            {
+                _logger.LogCritical(ex,
+                    "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " +
+                    "node justifies staying up; failing host start so the supervisor restarts the process");
+                throw;
+            }
             _logger.LogError(ex,
-                "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op");
+                "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " +
-            // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot.
+                "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section");
-            // Operators see the failure via the logs + can correct config without a process bounce
-            // of the whole binary.
             return;
         }
 
@@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         {
             _logger.LogWarning(
                 "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null");
+            _endpointState.MarkFailed("SDK reported started but the node manager is null");
             return;
         }
 
@@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         {
             _logger.LogError(ex,
                 "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
+            _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message);
             try
             {
-                await StopAsync(cancellationToken).ConfigureAwait(false);
+                DetachSinks();
             }
             catch (Exception cleanupError)
             {
                 _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
             }
-            var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
+            await ReleaseEndpointAsync().ConfigureAwait(false);
-                async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
-                () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
-            if (released)
-            {
-                _appHost = null;
-                _server = null;
-            }
             return;
         }
 
@@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         // attached, so the first tick cannot land on a half-wired server.
         StartRevalidationTimer();
 
+        _endpointState.MarkStarted();
         _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
     }
 
+    /// <summary>
+    ///     Releases an SDK whose <c>StartAsync</c> threw. The application instance may hold a half-opened
+    ///     listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the
+    ///     transport is up); disposing it is the only way to be sure. The state stays Failed.
+    /// </summary>
+    private async Task ReleaseFailedStartAsync()
+    {
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is null) return;
+        try
+        {
+            await appHost.DisposeAsync().ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw");
+        }
+    }
+
+    /// <summary>
+    ///     Stops the SDK and disposes the application host, releasing endpoint ownership only when both
+    ///     succeeded (<see cref="FailedServiceLevelBindingCleanup.StopAndDisposeAsync"/>): on a failure the
+    ///     fields are retained so <see cref="DisposeAsync"/> retries at host disposal.
+    /// </summary>
+    private async Task ReleaseEndpointAsync()
+    {
+        var server = _server;
+        var appHost = _appHost;
+        if (server is null && appHost is null) return;
+        var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
+            async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); },
+            () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask,
+            _logger).ConfigureAwait(false);
+        if (released)
+        {
+            _appHost = null;
+            _server = null;
+        }
+    }
+
     /// <summary>
     ///     The <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
     ///     (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
@@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     }
 
     /// <summary>
-    /// Stops the OPC UA server asynchronously.
+    /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes.
     /// </summary>
+    /// <remarks>
+    ///     R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown,
+    ///     the other hosted services' StopAsync), serving the last-written values with an emptied authority:
+    ///     reads succeeded on stale data while writes got "unavailable", and clients saw a server that was
+    ///     up but inert instead of the connection-refused that triggers their failover. <see cref="DisposeAsync"/>
+    ///     stays idempotent with this: ownership is nulled once the stop succeeded.
+    /// </remarks>
     /// <param name="cancellationToken">Cancellation token.</param>
     /// <returns>A task that represents the asynchronous operation.</returns>
-    public Task StopAsync(CancellationToken cancellationToken)
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        DetachSinks();
+        await ReleaseEndpointAsync().ConfigureAwait(false);
+        if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped();
+    }
+
+    /// <summary>
+    ///     Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor
+    ///     or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running
+    ///     until <see cref="ReleaseEndpointAsync"/>.
+    /// </summary>
+    private void DetachSinks()
     {
         // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose
         // authority and gate have already been cleared is the shape this whole teardown block exists to
@@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
         // request rather than let the seams fall back to their role checks — a stopping node should shed
         // enforcement in the same order it acquired it.
         _server?.SetDataPlaneAuthorizer(null);
-        return Task.CompletedTask;
     }
 
     /// <summary>
@@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
     public async ValueTask DisposeAsync()
     {
         // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must
-        // not leave a tick armed against a server that is about to go away.
+        // not leave a tick armed against a server that is about to go away. A StopAsync that released the
+        // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left
+        // them set, and this is the retry.
         _revalidationTimer?.Dispose();
         _revalidationTimer = null;
 
-        if (_appHost is not null) await _appHost.DisposeAsync();
+        var appHost = _appHost;
+        _appHost = null;
+        _server = null;
+        if (appHost is not null) await appHost.DisposeAsync();
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs

@@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua");
 // central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it
 // unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string.
 // A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected.
-// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it
+// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this
-// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a
+// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3):
-// startup error.
+// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is
-builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
+// present. A node with neither role (dev-only) has no key path and keeps the plain bind.
+if (hasDriver || hasAdmin)
+    builder.Services.AddValidatedOptions<ApiKeyOptions, ApiKeyOptionsValidator>(
+        builder.Configuration, ApiKeyOptions.SectionName);
+else
+    builder.Services.Configure<ApiKeyOptions>(builder.Configuration.GetSection(ApiKeyOptions.SectionName));
 // Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config
 // change rather than a redeploy, and what it swaps rather than adds.
 builder.Services.Configure<AclOptions>(builder.Configuration.GetSection(AclOptions.SectionName));
@@ -452,6 +457,11 @@ if (hasDriver)
     // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the
     // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once.
     builder.Services.TryAddSingleton<Func<ActorSystem>>(sp => () => sp.GetRequiredService<ActorSystem>());
+    // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint
+    // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the
+    // interface, so the writer and the probe cannot disagree.
+    builder.Services.AddSingleton<OpcUaEndpointState>();
+    builder.Services.AddSingleton<IOpcUaEndpointState>(sp => sp.GetRequiredService<OpcUaEndpointState>());
     builder.Services.AddHostedService<OtOpcUaServerHostedService>();
 }
 
@@ -554,8 +564,9 @@ if (hasAdmin)
 builder.Services.AddOtOpcUaSecrets(builder.Configuration);
 
 // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so
-// probing it would throw when the readiness endpoint resolves the factory.
+// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the
-builder.Services.AddOtOpcUaHealth(hasAdmin);
+// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green.
+builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver);
 builder.Services.AddOtOpcUaObservability(builder.Configuration);
 
 // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role,

src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs

@@ -0,0 +1,95 @@
+using System.Security.Claims;
+using Microsoft.AspNetCore.Components.Authorization;
+using Microsoft.AspNetCore.Http;
+using ZB.MOM.WW.Auth.AspNetCore;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Audit;
+
+/// <summary>
+///     <see cref="IAuditActorAccessor"/> that reads the Blazor circuit's principal first (R03:2.5).
+/// </summary>
+/// <remarks>
+///     <para>
+///         <see cref="HttpAuditActorAccessor"/> reads <c>IHttpContextAccessor</c>, which is right for a
+///         minimal-API request but wrong inside a Blazor Server circuit: after the first render there is no
+///         current <c>HttpContext</c> on the circuit's threads, so every audit row written from an interactive
+///         page would have been stamped <c>"system"</c>. The circuit's identity lives in the scoped
+///         <see cref="AuthenticationStateProvider"/>, so that is consulted first and the HTTP context is the
+///         fallback for non-circuit scopes (where the provider is not registered at all).
+///     </para>
+///     <para>
+///         <c>CurrentActor</c> is synchronous by contract; the server-side provider's
+///         <see cref="AuthenticationStateProvider.GetAuthenticationStateAsync"/> is already completed in a
+///         circuit, and an incomplete task is treated as "no circuit identity" rather than blocked on.
+///     </para>
+/// </remarks>
+public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor
+{
+    private readonly IHttpContextAccessor _httpContextAccessor;
+    private readonly AuthenticationStateProvider? _authenticationState;
+
+    /// <summary>Initializes a new instance of the <see cref="AuthenticationStateAuditActorAccessor"/> class.</summary>
+    /// <param name="httpContextAccessor">The HTTP context accessor (minimal-API fallback).</param>
+    /// <param name="authenticationState">The circuit's authentication state, when resolved inside a Blazor circuit.</param>
+    public AuthenticationStateAuditActorAccessor(
+        IHttpContextAccessor httpContextAccessor,
+        AuthenticationStateProvider? authenticationState = null)
+    {
+        ArgumentNullException.ThrowIfNull(httpContextAccessor);
+        _httpContextAccessor = httpContextAccessor;
+        _authenticationState = authenticationState;
+    }
+
+    /// <inheritdoc />
+    public string? CurrentActor
+    {
+        get
+        {
+            if (_authenticationState is not null)
+            {
+                var actor = TryResolveCircuitActor(_authenticationState);
+                if (actor is not null) return actor;
+            }
+            return ResolveActor(_httpContextAccessor.HttpContext?.User);
+        }
+    }
+
+    /// <summary>
+    ///     The circuit's actor, or null when this scope has no circuit identity. The server-side provider
+    ///     is registered for EVERY scope on an AdminUI node, but only a circuit calls
+    ///     <c>SetAuthenticationState</c>; anywhere else (a minimal-API request, a hosted service's scope)
+    ///     <c>GetAuthenticationStateAsync</c> throws <see cref="InvalidOperationException"/> synchronously
+    ///     (fix round 1 / I5). That is "no circuit", never an audit failure — the caller falls back to
+    ///     the HTTP context.
+    /// </summary>
+    /// <param name="provider">The scoped provider.</param>
+    /// <returns>The actor, or null.</returns>
+    private static string? TryResolveCircuitActor(AuthenticationStateProvider provider)
+    {
+        Task<AuthenticationState> task;
+        try
+        {
+            task = provider.GetAuthenticationStateAsync();
+        }
+        catch (InvalidOperationException)
+        {
+            return null;
+        }
+        return task.IsCompletedSuccessfully ? ResolveActor(task.Result.User) : null;
+    }
+
+    /// <summary>The claim-resolution rule shared with <see cref="HttpAuditActorAccessor"/>.</summary>
+    /// <param name="user">The principal, or null.</param>
+    /// <returns>The actor string, or null when unauthenticated.</returns>
+    internal static string? ResolveActor(ClaimsPrincipal? user)
+    {
+        if (user?.Identity?.IsAuthenticated != true)
+            return null;
+
+        // Prefer the canonical login-name claim; fall back to the Name claim or
+        // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name).
+        return user.FindFirst(ZbClaimTypes.Username)?.Value
+               ?? user.FindFirst(ZbClaimTypes.Name)?.Value
+               ?? user.Identity.Name;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs

@@ -1,5 +1,4 @@
 using Microsoft.AspNetCore.Http;
-using ZB.MOM.WW.Auth.AspNetCore;
 
 namespace ZB.MOM.WW.OtOpcUa.Security.Audit;
 
@@ -17,10 +16,10 @@ namespace ZB.MOM.WW.OtOpcUa.Security.Audit;
 ///   <see cref="System.Security.Principal.IIdentity.Name"/>, in that order.</item>
 /// </list>
 /// <para>
-/// Registered as <b>scoped</b> in <see cref="ZB.MOM.WW.OtOpcUa.Security.ServiceCollectionExtensions.AddOtOpcUaAuth"/>
+/// No longer the registered implementation: <see cref="AuthenticationStateAuditActorAccessor"/> is, because
-/// so that it correctly follows the request scope used by Blazor Server interactive components
+/// a Blazor circuit has no <c>HttpContext</c> after its first render and this accessor would stamp every row
-/// and minimal-API endpoints. <c>IHttpContextAccessor</c> is registered by
+/// written from an interactive page <c>"system"</c>. Kept as the explicit HTTP-only variant for non-circuit
-/// <c>AddOtOpcUaAuth</c> via <c>services.AddHttpContextAccessor()</c>.
+/// hosts and tests.
 /// </para>
 /// </remarks>
 public sealed class HttpAuditActorAccessor : IAuditActorAccessor
@@ -35,19 +34,6 @@ public sealed class HttpAuditActorAccessor : IAuditActorAccessor
     }
 
     /// <inheritdoc />
-    public string? CurrentActor
+    public string? CurrentActor =>
-    {
+        AuthenticationStateAuditActorAccessor.ResolveActor(_httpContextAccessor.HttpContext?.User);
-        get
-        {
-            var user = _httpContextAccessor.HttpContext?.User;
-            if (user?.Identity?.IsAuthenticated != true)
-                return null;
-
-            // Prefer the canonical login-name claim; fall back to the Name claim or
-            // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name).
-            return user.FindFirst(ZbClaimTypes.Username)?.Value
-                   ?? user.FindFirst(ZbClaimTypes.Name)?.Value
-                   ?? user.Identity.Name;
-        }
-    }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs

@@ -0,0 +1,39 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Primitives;
+using ZB.MOM.WW.OtOpcUa.Security.Jwt;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Auth;
+
+/// <summary>The authentication scheme names OtOpcUa registers.</summary>
+public static class OtOpcUaAuthSchemes
+{
+    /// <summary>
+    ///     The bearer scheme that validates the HS256 tokens <c>/auth/token</c> mints: the first-party
+    ///     <c>AddJwtBearer</c> handler over <see cref="JwtTokenService.BuildValidationParameters"/>.
+    ///     Registered only when <see cref="JwtOptions.IsConfigured"/>.
+    /// </summary>
+    public const string Bearer = "Bearer";
+
+    /// <summary>
+    ///     The default scheme: a policy scheme that forwards to <see cref="Bearer"/> when the request
+    ///     carries an <c>Authorization: Bearer</c> header and to the cookie scheme otherwise, so one
+    ///     <c>RequireAuthorization()</c> / policy accepts a browser session AND a machine token.
+    /// </summary>
+    public const string CookieOrBearer = "OtOpcUa.CookieOrBearer";
+
+    /// <summary>Whether the request presents a bearer token (the selector's whole rule).</summary>
+    /// <param name="context">The request.</param>
+    /// <returns><see langword="true"/> when an <c>Authorization: Bearer …</c> header is present.</returns>
+    public static bool HasBearerToken(HttpContext context) =>
+        TryGetBearerToken(context.Request.Headers.Authorization, out _);
+
+    internal static bool TryGetBearerToken(StringValues authorization, out string token)
+    {
+        token = string.Empty;
+        var header = authorization.ToString();
+        const string prefix = "Bearer ";
+        if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false;
+        token = header[prefix.Length..].Trim();
+        return token.Length > 0;
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs

@@ -1,79 +0,0 @@
-using System.Net.Http.Json;
-using System.Security.Claims;
-using Microsoft.AspNetCore.Components.Authorization;
-using Microsoft.Extensions.Logging;
-
-namespace ZB.MOM.WW.OtOpcUa.Security.Blazor;
-
-/// <summary>
-/// Blazor Server <see cref="AuthenticationStateProvider"/> that snapshots the cookie-backed
-/// principal supplied at circuit boot and polls <c>/auth/ping</c> every 60 seconds to detect
-/// expiry. Mirrors ScadaLink's CentralUI implementation.
-/// </summary>
-public sealed class CookieAuthenticationStateProvider : AuthenticationStateProvider, IAsyncDisposable
-{
-    private static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(60);
-
-    private readonly HttpClient _http;
-    private readonly ILogger<CookieAuthenticationStateProvider> _logger;
-    private readonly CancellationTokenSource _cts = new();
-    private ClaimsPrincipal _current;
-    private Task? _pingLoop;
-
-    /// <summary>Initializes a new instance of the CookieAuthenticationStateProvider class.</summary>
-    /// <param name="initial">The initial claims principal from circuit boot.</param>
-    /// <param name="http">The HTTP client for authentication ping requests.</param>
-    /// <param name="logger">The logger for diagnostic messages.</param>
-    public CookieAuthenticationStateProvider(
-        ClaimsPrincipal initial,
-        HttpClient http,
-        ILogger<CookieAuthenticationStateProvider> logger)
-    {
-        _current = initial;
-        _http = http;
-        _logger = logger;
-    }
-
-    /// <inheritdoc />
-    public override Task<AuthenticationState> GetAuthenticationStateAsync()
-    {
-        _pingLoop ??= Task.Run(() => PingLoopAsync(_cts.Token));
-        return Task.FromResult(new AuthenticationState(_current));
-    }
-
-    private async Task PingLoopAsync(CancellationToken ct)
-    {
-        try
-        {
-            while (!ct.IsCancellationRequested)
-            {
-                await Task.Delay(PingInterval, ct).ConfigureAwait(false);
-                var resp = await _http.GetAsync("/auth/ping", ct).ConfigureAwait(false);
-                if (!resp.IsSuccessStatusCode && _current.Identity?.IsAuthenticated == true)
-                {
-                    _logger.LogInformation("/auth/ping returned {Code}; notifying circuit", (int)resp.StatusCode);
-                    _current = new ClaimsPrincipal(new ClaimsIdentity());
-                    NotifyAuthenticationStateChanged(
-                        Task.FromResult(new AuthenticationState(_current)));
-                }
-            }
-        }
-        catch (OperationCanceledException) { /* expected on shutdown */ }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Auth ping loop terminated unexpectedly");
-        }
-    }
-
-    /// <summary>Cancels the background ping loop, awaits its completion, and disposes the cancellation token source.</summary>
-    /// <returns>A task that represents the asynchronous operation.</returns>
-    public async ValueTask DisposeAsync()
-    {
-        _cts.Cancel();
-        if (_pingLoop is not null)
-        {
-            try { await _pingLoop.ConfigureAwait(false); } catch { /* swallow shutdown errors */ }
-        }
-        _cts.Dispose();
-    }
-}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs

@@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using ZB.MOM.WW.Auth.AspNetCore;
 using ZB.MOM.WW.Auth.Abstractions.Roles;
 using ZB.MOM.WW.OtOpcUa.Security.Jwt;
@@ -23,6 +24,17 @@ public static class AuthEndpoints
     /// <summary>Response for a token issue request.</summary>
     public sealed record TokenResponse(string Token);
 
+    /// <summary>
+    ///     The <c>/auth/token</c> policy: an authenticated principal from the COOKIE scheme. Naming the scheme
+    ///     makes the authorization middleware authenticate that scheme alone, so an <c>Authorization: Bearer</c>
+    ///     header is never consulted here (with login disabled the auto-login handler sits under the same
+    ///     scheme name, so the dev rig is unchanged).
+    /// </summary>
+    internal static readonly AuthorizationPolicy CookieSessionOnly =
+        new AuthorizationPolicyBuilder(CookieAuthenticationDefaults.AuthenticationScheme)
+            .RequireAuthenticatedUser()
+            .Build();
+
     /// <summary>Maps OtOpcUa authentication endpoints to the application route builder.</summary>
     /// <param name="app">The endpoint route builder.</param>
     /// <returns>The endpoint route builder for chaining.</returns>
@@ -36,7 +48,17 @@ public static class AuthEndpoints
         // AddOtOpcUaAuth fallback policy that would otherwise 401 the request.
         app.MapPost("/auth/login", (Delegate)LoginAsync).AllowAnonymous().DisableAntiforgery();
         app.MapGet("/auth/ping", (Delegate)Ping).AllowAnonymous();
-        app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization();
+        // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured
+        // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call.
+        if (app.ServiceProvider.GetRequiredService<IOptions<JwtOptions>>().Value.IsConfigured)
+            // COOKIE scheme only (fix round 1 / I2): the default scheme also accepts a bearer, and a bearer
+            // that could mint a fresh token would renew itself forever — outliving its own expiry and the
+            // user's LDAP membership. A token is minted only for an interactive (cookie) session.
+            app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(CookieSessionOnly);
+        else
+            app.ServiceProvider.GetService<ILoggerFactory>()?
+                .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints")
+                .LogInformation("Security:Jwt:SigningKey is not configured — /auth/token is not mapped and bearer tokens are not accepted");
         app.MapPost("/auth/logout", (Delegate)LogoutAsync).RequireAuthorization();
         return app;
     }
@@ -73,8 +95,13 @@ public static class AuthEndpoints
         {
             result = await ldap.AuthenticateAsync(username, password, ct);
         }
-        catch (Exception)
+        catch (Exception ex) when (ex is not OperationCanceledException)
         {
+            // R03:2.7 — a directory outage used to leave NOTHING in the server log from this path; the
+            // operator had to infer every 503 from the directory side.
+            http.RequestServices.GetService<ILoggerFactory>()?
+                .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints")
+                .LogWarning(ex, "LDAP authentication failed for {User}; answering 503", username);
             return Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
         }
 

src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs

@@ -15,4 +15,26 @@ public sealed class JwtOptions
 
     /// <summary>Default token expiry. Mirrors ScadaLink (15 min).</summary>
     public int ExpiryMinutes { get; set; } = 15;
+
+    /// <summary>
+    ///     Whether the JWT feature is on: a non-blank <see cref="SigningKey"/>. When off, <c>/auth/token</c>
+    ///     is not mapped and no bearer scheme is registered, so an admin node that never configured a key
+    ///     keeps booting exactly as before; when on, the key is validated at host start
+    ///     (<c>ValidateOnStart</c>) instead of throwing HTTP 500 on the first token request.
+    /// </summary>
+    public bool IsConfigured => !string.IsNullOrWhiteSpace(SigningKey);
+
+    /// <summary>The validation rule shared by the startup validator and the tests: why the options are invalid, or null.</summary>
+    /// <returns>A failure message, or <see langword="null"/> when valid.</returns>
+    public string? Validate()
+    {
+        if (!IsConfigured) return null;
+        var keyBytes = System.Text.Encoding.UTF8.GetByteCount(SigningKey);
+        if (keyBytes < MinSigningKeyBytes)
+            return $"{SectionName}:SigningKey must be at least {MinSigningKeyBytes} bytes (256 bits) UTF-8 for HMAC-SHA256; the configured key is {keyBytes} byte(s).";
+        if (string.IsNullOrWhiteSpace(Issuer)) return $"{SectionName}:Issuer must not be blank.";
+        if (string.IsNullOrWhiteSpace(Audience)) return $"{SectionName}:Audience must not be blank.";
+        if (ExpiryMinutes <= 0) return $"{SectionName}:ExpiryMinutes must be > 0.";
+        return null;
+    }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs

@@ -0,0 +1,19 @@
+using Microsoft.Extensions.Options;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Jwt;
+
+/// <summary>
+///     Startup validator for <see cref="JwtOptions"/> (R03:2.4): when a signing key is configured it must
+///     be usable, and the host must learn that at start (<c>ValidateOnStart</c>) rather than from an HTTP 500
+///     on the first <c>/auth/token</c>. An unconfigured section (no key) is valid — the feature is simply off.
+/// </summary>
+public sealed class JwtOptionsValidator : IValidateOptions<JwtOptions>
+{
+    /// <inheritdoc />
+    public ValidateOptionsResult Validate(string? name, JwtOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        var failure = options.Validate();
+        return failure is null ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failure);
+    }
+}

src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs

@@ -25,24 +25,12 @@ public sealed class JwtTokenService
     /// <summary>
     /// Role claim type used in the JWT payload.
     /// <para>
-    /// <b>Issued-only / no internal JwtBearer scheme:</b> OtOpcUa uses a single Cookie
+    /// The short "Role" key is what <see cref="Issue"/> mints and what external consumers of the token
-    /// authentication scheme; the JWT is minted by the <c>/auth/token</c> endpoint and
+    /// expect. The internal bearer scheme (<c>AddJwtBearer</c> in <c>AddOtOpcUaAuth</c>) validates with
-    /// consumed externally (e.g. by OPC-UA clients or automation scripts). There is no
+    /// <see cref="BuildValidationParameters"/>, which sets <c>RoleClaimType = RoleClaimType</c> and
-    /// <c>AddJwtBearer</c> pipeline in OtOpcUa — the cookie stores the
+    /// <c>NameClaimType = UsernameClaimType</c>, and turns <c>MapInboundClaims</c> off, so
-    /// <see cref="System.Security.Claims.ClaimsPrincipal"/> directly. Because no internal
+    /// <c>[Authorize(Roles=...)]</c> and <c>ClaimsPrincipal.IsInRole</c> resolve against this key. Never
-    /// bearer validation path exists, the short "Role" key is intentionally used here rather
+    /// construct <see cref="Microsoft.IdentityModel.Tokens.TokenValidationParameters"/> ad hoc.
-    /// than the long <see cref="ClaimTypes.Role"/> URI; external consumers receive exactly the
-    /// key they expect.
-    /// </para>
-    /// <para>
-    /// <b>If a JwtBearer scheme is ever added:</b> the
-    /// <see cref="Microsoft.IdentityModel.Tokens.TokenValidationParameters"/> passed to
-    /// <c>AddJwtBearer</c> MUST set <c>RoleClaimType = JwtTokenService.RoleClaimType</c> (and
-    /// <c>NameClaimType = JwtTokenService.UsernameClaimType</c>) so that
-    /// <c>[Authorize(Roles=...)]</c> and <c>ClaimsPrincipal.IsInRole</c> resolve correctly.
-    /// <see cref="BuildValidationParameters"/> is already wired to do this and MUST be used
-    /// rather than constructing <see cref="Microsoft.IdentityModel.Tokens.TokenValidationParameters"/>
-    /// ad hoc.
     /// </para>
     /// </summary>
     public const string RoleClaimType = "Role";
@@ -150,5 +138,9 @@ public sealed class JwtTokenService
         // if this is ever passed to AddJwtBearer. See RoleClaimType doc comment for rationale.
         RoleClaimType = RoleClaimType,
         NameClaimType = UsernameClaimType,
+        // Pinned to the one algorithm Issue signs with: an unsigned (alg:none) token or one signed with a
+        // different HMAC variant over the same key is refused on both the bearer and TryValidate paths.
+        ValidAlgorithms = [SecurityAlgorithms.HmacSha256],
+        RequireSignedTokens = true,
     };
 }

src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs

@@ -1,5 +1,6 @@
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.Cookies;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.DataProtection;
 using Microsoft.AspNetCore.Http;
@@ -19,9 +20,12 @@ using ZB.MOM.WW.OtOpcUa.Security.Ldap;
 namespace ZB.MOM.WW.OtOpcUa.Security;
 
 /// <summary>
-///     DI registration for OtOpcUa auth. Single Cookie scheme (the JWT lives inside the
+///     DI registration for OtOpcUa auth. The cookie scheme carries the browser session; when
-///     cookie as its credential payload); no JwtBearer parallel scheme. Matches ScadaBridge
+///     <c>Security:Jwt:SigningKey</c> is configured a <see cref="OtOpcUaAuthSchemes.Bearer"/> scheme
-///     structurally — see <c>docs/plans/2026-05-29-auth-alignment-design.md</c>.
+///     accepts the tokens <c>/auth/token</c> mints (R03:2.4), and the default scheme is a policy scheme
+///     that forwards to whichever the request presents — so <c>/api/*</c> and every
+///     <c>RequireAuthorization</c> endpoint take a browser session OR a machine token. Matches
+///     ScadaBridge structurally — see <c>docs/plans/2026-05-29-auth-alignment-design.md</c>.
 /// </summary>
 public static class ServiceCollectionExtensions
 {
@@ -36,7 +40,14 @@ public static class ServiceCollectionExtensions
     /// <returns>The same service collection, for chaining.</returns>
     public static IServiceCollection AddOtOpcUaAuth(this IServiceCollection services, IConfiguration configuration)
     {
-        services.AddOptions<JwtOptions>().Bind(configuration.GetSection(JwtOptions.SectionName));
+        // JWT is opt-in by key: with no SigningKey the feature is off (no /auth/token, no bearer scheme)
+        // and an admin node boots as before; with one, the key is validated at host START (R03:2.4) —
+        // the JwtTokenService constructor used to throw on the first /auth/token call (HTTP 500).
+        services.AddOptions<JwtOptions>()
+            .Bind(configuration.GetSection(JwtOptions.SectionName))
+            .ValidateOnStart();
+        services.AddSingleton<IValidateOptions<JwtOptions>, JwtOptionsValidator>();
+        var jwtConfigured = configuration.GetSection(JwtOptions.SectionName).Get<JwtOptions>()?.IsConfigured ?? false;
         services.AddOptions<OtOpcUaCookieOptions>().Bind(configuration.GetSection(OtOpcUaCookieOptions.SectionName));
         services.AddOptions<LdapOptions>().Bind(configuration.GetSection(LdapOptions.SectionName));
         services.AddOptions<AuthDisableLoginOptions>().Bind(configuration.GetSection(AuthDisableLoginOptions.SectionName));
@@ -53,7 +64,10 @@ public static class ServiceCollectionExtensions
         // as the Actor field when constructing a canonical ZB.MOM.WW.Audit.AuditEvent. Registered
         // Scoped so it correctly follows the request scope used by Blazor Server and minimal-API
         // endpoints.
-        services.TryAddScoped<IAuditActorAccessor, HttpAuditActorAccessor>();
+        // AuthenticationStateAuditActorAccessor reads the Blazor circuit's principal first (no HttpContext
+        // exists on a circuit's threads after the first render) and falls back to the HTTP context for
+        // minimal-API scopes, where AuthenticationStateProvider is not registered (R03:2.5).
+        services.TryAddScoped<IAuditActorAccessor, AuthenticationStateAuditActorAccessor>();
 
         // Singleton — OtOpcUaLdapAuthService is stateless (the shared-library directory client it
         // wraps opens/disposes an LdapConnection per call) and must be consumable by the Singleton
@@ -78,7 +92,39 @@ public static class ServiceCollectionExtensions
             .GetSection(AuthDisableLoginOptions.SectionName)
             .GetValue<bool>(nameof(AuthDisableLoginOptions.DisableLogin));
 
-        var authBuilder = services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme);
+        // The default scheme is the cookie-or-bearer selector; sign-in/out stay on the cookie scheme, which is
+        // the only one that can issue a session. With JWT off the selector always picks the cookie.
+        var authBuilder = services.AddAuthentication(o =>
+        {
+            o.DefaultScheme = OtOpcUaAuthSchemes.CookieOrBearer;
+            o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme;
+            o.DefaultSignOutScheme = CookieAuthenticationDefaults.AuthenticationScheme;
+        });
+        authBuilder.AddPolicyScheme(OtOpcUaAuthSchemes.CookieOrBearer, "Cookie or Bearer", o =>
+        {
+            o.ForwardDefaultSelector = ctx =>
+                jwtConfigured && OtOpcUaAuthSchemes.HasBearerToken(ctx)
+                    ? OtOpcUaAuthSchemes.Bearer
+                    : CookieAuthenticationDefaults.AuthenticationScheme;
+        });
+        if (jwtConfigured)
+        {
+            // First-party JwtBearer (R03:2.4, fix round 1 / I1) — the earlier hand-rolled handler is gone.
+            // Validation parameters come from the SAME JwtTokenService that mints, so issuer/audience/key/
+            // claim types can never drift; BuildValidationParameters pins ValidAlgorithms = HS256 (an
+            // alg:none or a different-HMAC token is refused). MapInboundClaims=false keeps the minted
+            // "Role"/"zb:username" claim keys verbatim, which RoleClaimType/NameClaimType name. A bad token
+            // is AuthenticateResult.Fail ⇒ 401 + WWW-Authenticate: Bearer, never a 500.
+            authBuilder.AddJwtBearer(OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { });
+            services.AddOptions<JwtBearerOptions>(OtOpcUaAuthSchemes.Bearer)
+                .Configure<JwtTokenService>((o, jwt) =>
+                {
+                    o.TokenValidationParameters = jwt.BuildValidationParameters();
+                    o.MapInboundClaims = false;
+                    o.SaveToken = false;
+                    o.IncludeErrorDetails = false;
+                });
+        }
         if (disableLogin)
         {
             // DEV/TEST ONLY: replace the cookie handler with an always-succeeding handler registered
@@ -142,8 +188,8 @@ public static class ServiceCollectionExtensions
 
         services.AddAuthorization(o =>
         {
-            o.FallbackPolicy = new AuthorizationPolicyBuilder(
+            // Named on the selector, not the cookie scheme, so the fallback accepts a bearer token too.
-                    CookieAuthenticationDefaults.AuthenticationScheme)
+            o.FallbackPolicy = new AuthorizationPolicyBuilder(OtOpcUaAuthSchemes.CookieOrBearer)
                 .RequireAuthenticatedUser()
                 .Build();
 

src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj

@@ -10,6 +10,10 @@
     </ItemGroup>
 
     <ItemGroup>
+        <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer"/>
+        <!-- Direct, to promote JwtBearer's 8.0.1 IdentityModel.Protocols dependencies to the central 8.11.0 pin. -->
+        <PackageReference Include="Microsoft.IdentityModel.Protocols"/>
+        <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect"/>
         <PackageReference Include="Microsoft.IdentityModel.Tokens"/>
         <PackageReference Include="System.IdentityModel.Tokens.Jwt"/>
         <PackageReference Include="ZB.MOM.WW.Auth.Abstractions"/>

tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs

@@ -0,0 +1,117 @@
+using Microsoft.Data.SqlClient;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
+
+namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests;
+
+/// <summary>
+///     R03:3.2 on the real provider: the migrated config database allows SNAPSHOT isolation (migration
+///     <c>EnableSnapshotIsolation</c>) and <see cref="DraftSnapshotFactory.FromConfigDbAsync"/> reads under
+///     it. The InMemory tests cannot see either — the provider has no transactions — so this is the one
+///     place the deploy path's "one consistent read" is proven against SQL Server.
+/// </summary>
+[Trait("Category", "SchemaCompliance")]
+[Collection(nameof(SchemaComplianceCollection))]
+public sealed class DraftSnapshotFactorySqlServerTests
+{
+    private readonly SchemaComplianceFixture _fixture;
+
+    /// <summary>Initializes a new instance of the <see cref="DraftSnapshotFactorySqlServerTests"/> class.</summary>
+    /// <param name="fixture">The migrated test database.</param>
+    public DraftSnapshotFactorySqlServerTests(SchemaComplianceFixture fixture) => _fixture = fixture;
+
+    /// <summary>The migration leaves the database with snapshot isolation allowed.</summary>
+    [Fact]
+    public void Migrated_database_allows_snapshot_isolation()
+    {
+        using var conn = _fixture.OpenConnection();
+        using var cmd = new SqlCommand(
+            "SELECT snapshot_isolation_state_desc FROM sys.databases WHERE name = DB_NAME();", conn);
+        cmd.ExecuteScalar().ShouldBe("ON");
+    }
+
+    /// <summary>
+    ///     A row committed by another connection while the snapshot read is in progress is NOT observed by
+    ///     the tables read after it — every table reflects the instant the transaction started. Positive
+    ///     control: a read started after the commit does see the row.
+    /// </summary>
+    [Fact]
+    public async Task FromConfigDb_reads_every_table_as_of_one_instant()
+    {
+        var clusterId = $"SNAP-{Guid.NewGuid():N}"[..24];
+        var scriptId = $"late-{Guid.NewGuid():N}";
+        await using (var seed = NewContext())
+        {
+            seed.ServerClusters.Add(new ServerCluster
+            {
+                ClusterId = clusterId, Name = "snapshot-read", Enterprise = "zb", Site = "test",
+                NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
+            });
+            await seed.SaveChangesAsync();
+        }
+
+        try
+        {
+            var options = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
+                .UseSqlServer(_fixture.ConnectionString)
+                .AddInterceptors(new LateEditInterceptor(_fixture.ConnectionString, clusterId, scriptId))
+                .Options;
+
+            await using (var db = new OtOpcUaConfigDbContext(options))
+            {
+                var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(db);
+                // The interceptor committed a Script while ServerClusters (the FIRST table read) was being
+                // materialised. Scripts is read later in the same transaction and must not see it.
+                snapshot.ServerClusters.ShouldContain(c => c.ClusterId == clusterId);
+                snapshot.Scripts.ShouldNotContain(s => s.ScriptId == scriptId);
+            }
+
+            await using (var verify = NewContext())
+            {
+                // Positive control: the edit really landed, and a read started afterwards sees it.
+                (await verify.Scripts.AnyAsync(s => s.ScriptId == scriptId)).ShouldBeTrue();
+                var later = await DraftSnapshotFactory.FromConfigDbAsync(verify);
+                later.Scripts.ShouldContain(s => s.ScriptId == scriptId);
+            }
+        }
+        finally
+        {
+            await using var cleanup = NewContext();
+            await cleanup.Scripts.Where(s => s.ScriptId == scriptId).ExecuteDeleteAsync();
+            await cleanup.ServerClusters.Where(c => c.ClusterId == clusterId).ExecuteDeleteAsync();
+        }
+    }
+
+    private OtOpcUaConfigDbContext NewContext() => new(
+        new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
+
+    /// <summary>
+    ///     Commits a Script on a SEPARATE connection the first time the snapshot read materialises the
+    ///     seeded ServerCluster row — i.e. mid-read, after the first table and before Scripts.
+    /// </summary>
+    private sealed class LateEditInterceptor(string connectionString, string clusterId, string scriptId)
+        : IMaterializationInterceptor
+    {
+        private int _fired;
+
+        public object InitializedInstance(MaterializationInterceptionData materializationData, object entity)
+        {
+            if (entity is ServerCluster c && c.ClusterId == clusterId && Interlocked.Exchange(ref _fired, 1) == 0)
+            {
+                using var other = new OtOpcUaConfigDbContext(
+                    new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseSqlServer(connectionString).Options);
+                other.Scripts.Add(new Script
+                {
+                    ScriptId = scriptId, Name = "late edit", SourceCode = "return 1;", SourceHash = "late-hash",
+                });
+                other.SaveChanges();
+            }
+            return entity;
+        }
+    }
+}

tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs

@@ -103,6 +103,36 @@ public sealed class DraftSnapshotFactoryTests : IDisposable
         snapshot.ActiveReservations[0].ReleasedAt.ShouldBeNull();
     }
 
+    /// <summary>
+    ///     R03:3.2 — the deploy path validates AND composes from ONE snapshot, so the tables the composer
+    ///     needs beyond the validator's set (clusters, nodes, API keys) must ride the same read.
+    /// </summary>
+    [Fact]
+    public async Task FromConfigDb_carries_clusters_nodes_and_api_keys_for_the_composer()
+    {
+        _db.ServerClusters.Add(new ServerCluster
+        {
+            ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
+            NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
+        });
+        _db.ClusterNodes.Add(new ClusterNode
+        {
+            NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1",
+            ApplicationUri = "urn:OtOpcUa:site-a-1", Enabled = true, CreatedBy = "test",
+        });
+        _db.ApiKeys.Add(new ApiKey
+        {
+            ApiKeyId = "k1", Name = "scada", SecretHash = "hash", Principal = "scadabridge", Enabled = true,
+        });
+        await _db.SaveChangesAsync();
+
+        var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(_db);
+
+        snapshot.ServerClusters.ShouldHaveSingleItem().ClusterId.ShouldBe("SITE-A");
+        snapshot.ClusterNodes.ShouldHaveSingleItem().NodeId.ShouldBe("site-a-1:4053");
+        snapshot.ApiKeys.ShouldHaveSingleItem().ApiKeyId.ShouldBe("k1");
+    }
+
     private void SeedEquipment(string equipmentId)
     {
         var uuid = Guid.NewGuid();

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs

@@ -0,0 +1,159 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     R03:3.1 — the admin singleton's mailbox used to be suspended for the whole of every
+///     <c>ReceiveAsync</c>: one Test Connect against a black-holed PLC (60 s clamp) or one large deploy
+///     compose stalled every other admin operation fleet-wide, alarm acks included. Both now run off the
+///     actor thread; only the "is one in flight" check stays serialised.
+/// </summary>
+public sealed class AdminOperationsActorMailboxTests : ControlPlaneActorTestBase
+{
+    private static readonly TimeSpan AckBudget = TimeSpan.FromSeconds(1);
+
+    /// <summary>An alarm ack completes in under a second while a 60 s Test Connect is still pending.</summary>
+    [Fact]
+    public void Alarm_ack_completes_while_a_60s_test_connect_is_pending()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        var coordinator = CreateTestProbe("coord");
+        var probe = new BlockingProbe();
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe }));
+        var connectClient = CreateTestProbe("connect-client");
+
+        connectClient.Send(actor, new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 60, Guid.NewGuid()));
+        probe.Started.Wait(PresenceBudget).ShouldBeTrue("the probe must be in flight before the ack is sent");
+
+        actor.Tell(new AcknowledgeAlarmCommand("alarm-1", "op", null, CorrelationId.NewId()));
+        var ack = ExpectMsg<AcknowledgeAlarmResult>(AckBudget);
+        ack.Ok.ShouldBeTrue();
+
+        // Positive control: the probe reply still lands, with the right sender, once the probe returns.
+        connectClient.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+        probe.Release(new DriverProbeResult(true, null, TimeSpan.FromMilliseconds(5)));
+        var result = connectClient.ExpectMsg<TestDriverConnectResult>(PresenceBudget);
+        result.Ok.ShouldBeTrue();
+        result.LatencyMs.ShouldNotBeNull();
+    }
+
+    /// <summary>A probe that times out still answers its own sender with the clamped timeout message.</summary>
+    [Fact]
+    public void Test_connect_timeout_is_reported_to_its_sender()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        var coordinator = CreateTestProbe("coord");
+        var probe = new BlockingProbe();   // never released: the CTS fires at the 1 s clamp floor
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe }));
+
+        actor.Tell(new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 1, Guid.NewGuid()));
+
+        var result = ExpectMsg<TestDriverConnectResult>(PresenceBudget);
+        result.Ok.ShouldBeFalse();
+        result.Message.ShouldNotBeNull();
+        result.Message.ShouldContain("timed out");
+    }
+
+    /// <summary>
+    ///     The deploy compose (snapshot + validate + gzip + row write) runs off the actor: an alarm ack
+    ///     completes while the compose is parked on a gated DB context, and the deploy still completes
+    ///     (Accepted + dispatched) once the gate opens.
+    /// </summary>
+    [Fact]
+    public void Alarm_ack_completes_while_a_deploy_compose_is_in_flight()
+    {
+        var gated = new GatedDbFactory(NewInMemoryDbFactory());
+        var coordinator = CreateTestProbe("coord");
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
+        var deployer = CreateTestProbe("deployer");
+
+        gated.Hold();
+        deployer.Send(actor, new StartDeployment("joe", CorrelationId.NewId()));
+        gated.Entered.Wait(PresenceBudget).ShouldBeTrue("the compose must be parked before the ack is sent");
+
+        actor.Tell(new AcknowledgeAlarmCommand("alarm-2", "op", null, CorrelationId.NewId()));
+        ExpectMsg<AcknowledgeAlarmResult>(AckBudget).Ok.ShouldBeTrue();
+
+        // Positive control: the deploy completes once released.
+        deployer.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+        gated.Release();
+        deployer.ExpectMsg<StartDeploymentResult>(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
+        coordinator.ExpectMsg<DispatchDeployment>(PresenceBudget);
+    }
+
+    /// <summary>
+    ///     Only the in-flight check is serialised — and it must still catch a second request that arrives
+    ///     while the first compose has not yet written its row (the DB alone would let both through).
+    /// </summary>
+    [Fact]
+    public void Second_StartDeployment_during_an_in_flight_compose_is_refused()
+    {
+        var gated = new GatedDbFactory(NewInMemoryDbFactory());
+        var coordinator = CreateTestProbe("coord");
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
+        var first = CreateTestProbe("first");
+        var second = CreateTestProbe("second");
+
+        gated.Hold();
+        first.Send(actor, new StartDeployment("joe", CorrelationId.NewId()));
+        gated.Entered.Wait(PresenceBudget).ShouldBeTrue();
+
+        second.Send(actor, new StartDeployment("kim", CorrelationId.NewId()));
+        second.ExpectMsg<StartDeploymentResult>(AckBudget).Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight);
+
+        gated.Release();
+        first.ExpectMsg<StartDeploymentResult>(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
+        coordinator.ExpectMsg<DispatchDeployment>(PresenceBudget);
+    }
+
+    private sealed class BlockingProbe : IDriverProbe
+    {
+        private readonly TaskCompletionSource<DriverProbeResult> _tcs =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        public ManualResetEventSlim Started { get; } = new();
+        public string DriverType => "Blocking";
+
+        public async Task<DriverProbeResult> ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct)
+        {
+            Started.Set();
+            return await _tcs.Task.WaitAsync(ct);
+        }
+
+        public void Release(DriverProbeResult result) => _tcs.TrySetResult(result);
+    }
+
+    /// <summary>
+    ///     Passes synchronous context creation through (the serialised in-flight check) and parks the
+    ///     asynchronous one (the off-actor compose) on a gate while <see cref="Hold"/> is in effect.
+    /// </summary>
+    private sealed class GatedDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        private volatile TaskCompletionSource? _gate;
+
+        public ManualResetEventSlim Entered { get; } = new();
+
+        public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        public void Release() => _gate?.TrySetResult();
+
+        public OtOpcUaConfigDbContext CreateDbContext() => inner.CreateDbContext();
+
+        public async Task<OtOpcUaConfigDbContext> CreateDbContextAsync(CancellationToken cancellationToken = default)
+        {
+            Entered.Set();
+            if (_gate is { } gate) await gate.Task.WaitAsync(cancellationToken);
+            return inner.CreateDbContext();
+        }
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs

@@ -0,0 +1,68 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     R03:4.1 — admin operations echoed <c>ex.Message</c> to the UI, so a SqlException's connection-string
+///     fragment reached any ConfigEditor browser session. The reply is now a fixed sentence plus the request's
+///     correlation id; the detail stays in the central log.
+/// </summary>
+public sealed class AdminOperationsActorServerFaultMessageTests : ControlPlaneActorTestBase
+{
+    private const string Secret = "Server=sql-prod-01;Password=hunter2";
+
+    [Fact]
+    public void StartDeployment_server_fault_reply_carries_the_correlation_id_and_not_the_exception_text()
+    {
+        var coordinator = CreateTestProbe("coord");
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(new ThrowingDbFactory(), coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
+        var correlationId = CorrelationId.NewId();
+
+        actor.Tell(new StartDeployment("joe", correlationId));
+
+        var reply = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        reply.Outcome.ShouldBe(StartDeploymentOutcome.Rejected);
+        reply.Message.ShouldNotBeNull();
+        reply.Message.ShouldNotContain("hunter2");
+        reply.Message.ShouldNotContain("sql-prod-01");
+        reply.Message.ShouldContain(correlationId.ToString());   // the operator can find the log line
+    }
+
+    [Fact]
+    public void TestDriverConnect_probe_throw_reply_carries_the_correlation_id_and_not_the_exception_text()
+    {
+        var coordinator = CreateTestProbe("coord");
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(NewInMemoryDbFactory(), coordinator.Ref, new IDriverProbe[] { new ThrowingProbe() }));
+        var correlationId = Guid.NewGuid();
+
+        actor.Tell(new TestDriverConnect("Throwing", "{}", 5, correlationId));
+
+        var reply = ExpectMsg<TestDriverConnectResult>(PresenceBudget);
+        reply.Ok.ShouldBeFalse();
+        reply.Message.ShouldNotBeNull();
+        reply.Message.ShouldNotContain("hunter2");
+        reply.Message.ShouldContain(correlationId.ToString());
+    }
+
+    private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        public OtOpcUaConfigDbContext CreateDbContext() =>
+            throw new InvalidOperationException("Cannot open database: " + Secret);
+    }
+
+    private sealed class ThrowingProbe : IDriverProbe
+    {
+        public string DriverType => "Throwing";
+        public Task<DriverProbeResult> ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) =>
+            throw new InvalidOperationException("probe blew up: " + Secret);
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs

@@ -0,0 +1,133 @@
+using System.IO.Compression;
+using System.Text;
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using Microsoft.EntityFrameworkCore.Storage;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     R03:3.2 — <c>StartDeployment</c> used to read the whole config DB twice: once for the validator,
+///     once for the composer. An edit landing between the two shipped an artifact the validator never saw.
+///     The deploy now validates and composes ONE snapshot, so an edit that lands mid-deploy is either
+///     validated (it was in the snapshot) or absent from the artifact (it was not) — never shipped unseen.
+/// </summary>
+public sealed class AdminOperationsActorSingleSnapshotTests : ControlPlaneActorTestBase
+{
+    private const string LateEquipmentId = "EQ-late-and-NOT-canonical";
+
+    /// <summary>
+    ///     An Equipment with a non-canonical id (the validator rejects those with <c>EquipmentIdNotDerived</c>)
+    ///     is committed by another context while the snapshot read is in progress, AFTER the Equipment table
+    ///     has been read. The validator cannot see it, so the deploy is Accepted — and the artifact must not
+    ///     carry it either. Positive control: the row really landed in the DB, and a deploy started after it
+    ///     is Rejected on exactly that rule.
+    /// </summary>
+    [Fact]
+    public void Edit_landing_between_validate_and_compose_cannot_ship_unvalidated()
+    {
+        // One shared root: an InMemory store is scoped to the internal service provider, and a context
+        // carrying an interceptor builds its own provider — without the shared root the intercepting
+        // context would read an EMPTY database and the whole test would be vacuous.
+        var dbName = Guid.NewGuid().ToString("N");
+        var root = new InMemoryDatabaseRoot();
+        var plain = new RootedDbFactory(dbName, root, interceptor: null);
+        using (var db = plain.CreateDbContext())
+        {
+            // A Tag to materialise — Tags is read AFTER Equipment by the snapshot factory, so an insert
+            // fired on the first Tag lands "between" the two tables of one read.
+            db.DriverInstances.Add(new DriverInstance
+            {
+                DriverInstanceId = "d", ClusterId = "", Name = "drv", DriverType = "Modbus", DriverConfig = "{}",
+            });
+            db.Devices.Add(new Device { DeviceId = "dev", DriverInstanceId = "d", Name = "dev", DeviceConfig = "{}" });
+            db.Tags.Add(new Tag
+            {
+                TagId = "tag-speed", DeviceId = "dev", Name = "speed", DataType = "Float",
+                AccessLevel = TagAccessLevel.Read, TagConfig = "{}",
+            });
+            db.SaveChanges();
+        }
+
+        var injecting = new RootedDbFactory(dbName, root, new LateEquipmentInterceptor(plain));
+        var coordinator = CreateTestProbe("coord");
+        var actor = Sys.ActorOf(AdminOperationsActor.Props(injecting, coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
+
+        actor.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+
+        var reply = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        reply.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, reply.Message);
+        coordinator.ExpectMsg<DispatchDeployment>(PresenceBudget);
+
+        using (var verify = plain.CreateDbContext())
+        {
+            // The injection happened...
+            verify.Equipment.Any(e => e.EquipmentId == LateEquipmentId).ShouldBeTrue();
+            // ...and the shipped artifact does not contain what the validator never saw.
+            var row = verify.Deployments.Single();
+            Inflate(row.ArtifactBlob).ShouldNotContain(LateEquipmentId);
+            // The coordinator is a probe here, so seal the row by hand to let the next deploy through.
+            row.Status = DeploymentStatus.Sealed;
+            verify.SaveChanges();
+        }
+
+        // Positive control: now that the row is visible to a fresh read, the gate rejects it.
+        actor.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var second = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        second.Outcome.ShouldBe(StartDeploymentOutcome.Rejected);
+        second.Message.ShouldNotBeNull();
+        second.Message.ShouldContain("EquipmentIdNotDerived");
+    }
+
+    private static string Inflate(byte[] blob)
+    {
+        using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress);
+        using var reader = new StreamReader(gz, Encoding.UTF8);
+        return reader.ReadToEnd();
+    }
+
+    /// <summary>On the first Tag materialised, commits a non-canonical Equipment through ANOTHER context.</summary>
+    private sealed class LateEquipmentInterceptor(IDbContextFactory<OtOpcUaConfigDbContext> other)
+        : IMaterializationInterceptor
+    {
+        private int _fired;
+
+        public object InitializedInstance(MaterializationInterceptionData materializationData, object entity)
+        {
+            if (entity is Tag && Interlocked.Exchange(ref _fired, 1) == 0)
+            {
+                using var db = other.CreateDbContext();
+                db.Equipment.Add(new Equipment
+                {
+                    EquipmentUuid = Guid.NewGuid(), EquipmentId = LateEquipmentId, Name = "late",
+                    UnsLineId = "line-a", MachineCode = "m",
+                });
+                db.SaveChanges();
+            }
+            return entity;
+        }
+    }
+
+    private sealed class RootedDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor? interceptor)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        public OtOpcUaConfigDbContext CreateDbContext()
+        {
+            var builder = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseInMemoryDatabase(dbName, root);
+            if (interceptor is not null) builder.AddInterceptors(interceptor);
+            return new OtOpcUaConfigDbContext(builder.Options);
+        }
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs

@@ -0,0 +1,191 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage;
+using Microsoft.EntityFrameworkCore.Update;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.Audit;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected
+///     row(s) and re-saves, so a duplicate <c>EventId</c> costs exactly that row.
+/// </summary>
+public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase
+{
+    [Fact]
+    public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted()
+    {
+        var dbName = Guid.NewGuid().ToString("N");
+        var root = new InMemoryDatabaseRoot();
+        var duplicate = Guid.NewGuid();
+        var rejecting = new RejectingDbFactory(dbName, root, new AlwaysRejectBatchContaining(duplicate));
+        var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting));
+
+        var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
+        ids.Insert(7, duplicate);
+        foreach (var id in ids) actor.Tell(NewEvent(id));
+
+        Watch(actor);
+        actor.Tell(PoisonPill.Instance);   // PostStop flushes
+        ExpectTerminated(actor);
+
+        using var db = rejecting.CreateDbContext();
+        var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
+        persisted.Count.ShouldBe(20, "every row but the rejected one lands");
+        persisted.ShouldNotContain(duplicate);
+        // Positive control: the rows on either side of the rejected one are present.
+        persisted.ShouldContain(ids[6]);
+        persisted.ShouldContain(ids[8]);
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I4 — a FAITHFUL SQL Server shape: the provider sends the batch as one statement batch,
+    ///     so the <see cref="DbUpdateException"/> for one duplicate names EVERY entry in it. Detaching
+    ///     <c>ex.Entries</c> therefore dropped the whole flush. The writer must drop only the rows whose
+    ///     EventId is already persisted and keep the rest.
+    /// </summary>
+    [Fact]
+    public void A_whole_batch_rejection_drops_only_the_rows_already_persisted()
+    {
+        var dbName = Guid.NewGuid().ToString("N");
+        var root = new InMemoryDatabaseRoot();
+        var factory = new RejectingDbFactory(dbName, root, new RejectWholeBatchOnDuplicate());
+        var duplicate = Guid.NewGuid();
+        using (var seed = factory.CreateDbContext())
+        {
+            seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate });
+            seed.SaveChanges();
+        }
+        var actor = Sys.ActorOf(AuditWriterActor.Props(factory));
+
+        var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
+        ids.Insert(7, duplicate);
+        foreach (var id in ids) actor.Tell(NewEvent(id));
+        Watch(actor);
+        actor.Tell(PoisonPill.Instance);
+        ExpectTerminated(actor);
+
+        using var db = factory.CreateDbContext();
+        var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
+        persisted.Count.ShouldBe(21, "the 20 new rows land beside the one that was already there");
+        persisted.Count(id => id == duplicate).ShouldBe(1);
+        db.ConfigAuditLogs.Single(r => r.EventId == duplicate).Principal.ShouldBe("earlier");
+    }
+
+    /// <summary>The same contract against the REAL provider and the real filtered unique index.</summary>
+    [Fact]
+    [Trait("Category", "SchemaCompliance")]
+    public void On_sql_server_a_duplicate_event_id_costs_only_that_row()
+    {
+        using var sql = new SqlServerTestDatabase();
+        var duplicate = Guid.NewGuid();
+        using (var seed = sql.NewContext())
+        {
+            seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate });
+            seed.SaveChanges();
+        }
+        var actor = Sys.ActorOf(AuditWriterActor.Props(sql.Factory()));
+
+        var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
+        ids.Insert(7, duplicate);
+        foreach (var id in ids) actor.Tell(NewEvent(id));
+        Watch(actor);
+        actor.Tell(PoisonPill.Instance);
+        ExpectTerminated(actor, PresenceBudget);
+
+        using var db = sql.NewContext();
+        var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
+        persisted.Count.ShouldBe(21);
+        persisted.Count(id => id == duplicate).ShouldBe(1);
+        foreach (var id in ids) persisted.ShouldContain(id);
+    }
+
+    /// <summary>A failure that names no entry is a whole-batch failure, exactly as before.</summary>
+    [Fact]
+    public void A_connection_level_failure_still_drops_the_batch_without_looping()
+    {
+        var dbName = Guid.NewGuid().ToString("N");
+        var root = new InMemoryDatabaseRoot();
+        var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries());
+        var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting));
+
+        for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid()));
+        Watch(actor);
+        actor.Tell(PoisonPill.Instance);
+        ExpectTerminated(actor);   // terminates ⇒ the retry loop did not spin
+
+        using var db = rejecting.CreateDbContext();
+        db.ConfigAuditLogs.Count().ShouldBe(0);
+    }
+
+    private static AuditEvent NewEvent(Guid eventId) => new()
+    {
+        EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe",
+        OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success,
+        DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(),
+    };
+
+    /// <summary>
+    ///     A row the database refuses on its own terms (not a duplicate — e.g. a per-row constraint), every
+    ///     time it is in the save. Faithful to SQL Server's batch shape: the exception names EVERY added
+    ///     entry, so only a row-by-row retry can isolate the offender.
+    /// </summary>
+    private sealed class AlwaysRejectBatchContaining(Guid eventId) : ISaveChangesInterceptor
+    {
+        public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
+        {
+            if (eventData.Context is null) return result;
+            var added = eventData.Context.ChangeTracker.Entries<ConfigAuditLog>()
+                .Where(e => e.State == EntityState.Added).ToList();
+            if (!added.Any(e => e.Entity.EventId == eventId)) return result;
+#pragma warning disable EF1001
+            var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList();
+#pragma warning restore EF1001
+            throw new DbUpdateException("The statement conflicted with a CHECK constraint (simulated, whole batch)", entries);
+        }
+    }
+
+    /// <summary>
+    ///     Models SQL Server's batched insert: when ANY added row's EventId is already stored, the whole save
+    ///     fails with an exception naming EVERY added entry (the InMemory provider enforces no unique index,
+    ///     so this interceptor is the constraint).
+    /// </summary>
+    private sealed class RejectWholeBatchOnDuplicate : ISaveChangesInterceptor
+    {
+        public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
+        {
+            if (eventData.Context is not OtOpcUaConfigDbContext ctx) return result;
+            var added = ctx.ChangeTracker.Entries<ConfigAuditLog>().Where(e => e.State == EntityState.Added).ToList();
+            var ids = added.Select(e => e.Entity.EventId).Where(id => id != null).ToList();
+            if (ids.Count == 0 || !ctx.ConfigAuditLogs.AsNoTracking().Any(r => ids.Contains(r.EventId))) return result;
+#pragma warning disable EF1001
+            var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList();
+#pragma warning restore EF1001
+            throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601, whole batch)", entries);
+        }
+    }
+
+    private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor
+    {
+        public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result) =>
+            throw new DbUpdateException("connection lost (simulated)");
+    }
+
+    private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        public OtOpcUaConfigDbContext CreateDbContext() => new(
+            new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
+                .UseInMemoryDatabase(dbName, root)
+                .AddInterceptors(interceptor)
+                .Options);
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs

@@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
     /// <summary>A spawned gRPC-mode actor plus the two seams the tests observe it through.</summary>
     private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk);
 
+    /// <summary>
+    ///     R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the
+    ///     channels of endpoints that left it.
+    /// </summary>
+    [Fact]
+    public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain()
+    {
+        var factory = NewInMemoryDbFactory();
+        SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false));
+        var rig = SpawnGrpcRig(factory);
+        AwaitContactsLoaded(rig);
+        // The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes.
+        rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
+        AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
+
+        using (var db = factory.CreateDbContext())
+        {
+            db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true;
+            db.SaveChanges();
+        }
+        rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts());
+
+        AwaitCondition(
+            () => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")),
+            TimeSpan.FromSeconds(5));
+        rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" });
+    }
+
     private GrpcRig SpawnGrpcRig(
         IDbContextFactory<OtOpcUaConfigDbContext> factory,
         RecordingDialer? dialer = null,
@@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
                 : Task.CompletedTask;
         }
 
+        /// <summary>Every endpoint set handed to <see cref="RetainOnly"/>, in order.</summary>
+        public IReadOnlyList<IReadOnlyCollection<string>> RetainCalls
+        {
+            get { lock (_gate) return _retainCalls.ToList(); }
+        }
+
+        private readonly List<IReadOnlyCollection<string>> _retainCalls = [];
+
+        public void RetainOnly(IReadOnlyCollection<string> nodeEndpoints)
+        {
+            lock (_gate) _retainCalls.Add(nodeEndpoints.ToList());
+        }
+
         /// <summary>One recorded dial.</summary>
         public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId);
     }

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs

@@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests
         Should.NotThrow(dialer.Dispose);
     }
 
+    /// <summary>
+    ///     R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is
+    ///     released on the next refresh instead of living, with its idle connection, for the life of central.
+    ///     Positive control: a dial to that endpoint afterwards rebuilds a channel.
+    /// </summary>
+    [Fact]
+    public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set()
+    {
+        var dialer = new MeshGrpcCommandDialer(
+            new FakePskProvider(("SITE-A", "psk-site-a")),
+            Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }),
+            NullLogger<MeshGrpcCommandDialer>.Instance);
+        foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" })
+        {
+            await Should.ThrowAsync<RpcException>(
+                () => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None));
+        }
+        dialer.CachedChannelCount.ShouldBe(2);
+
+        dialer.RetainOnly(new[] { "http://127.0.0.1:1" });
+
+        dialer.CachedChannelCount.ShouldBe(1);
+
+        await Should.ThrowAsync<RpcException>(
+            () => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None));
+        dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel");
+        Should.NotThrow(dialer.Dispose);
+    }
+
     [Fact]
     public void It_activates_from_the_container_despite_the_optional_client_factory_parameter()
     {

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs

@@ -0,0 +1,200 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
+using ZB.MOM.WW.Secrets.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
+
+/// <summary>
+///     R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the
+///     first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store
+///     read and was denied past the bound. <see cref="MeshPskProvider.RefreshAsync"/> pre-resolves the
+///     keys off the request path and swaps them in atomically.
+/// </summary>
+public sealed class MeshPskProviderRefreshTests
+{
+    private static MeshPskProvider Provider(ISecretResolver resolver) =>
+        new(Options.Create(new MeshTransportOptions()), resolver, NullLogger<MeshPskProvider>.Instance);
+
+    [Fact]
+    public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store()
+    {
+        var resolver = new GatedResolver { Value = "k-1" };
+        var provider = Provider(resolver);
+
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+        resolver.Calls.ShouldBe(1);
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+        psk.ShouldBe("k-1");
+        resolver.Calls.ShouldBe(1, "the request path did not consult the store");
+        provider.Matches("SITE-A", "k-1").ShouldBeTrue();
+    }
+
+    /// <summary>
+    ///     While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window);
+    ///     once it completes the rotated key is served. Positive control on both sides of the swap.
+    /// </summary>
+    [Fact]
+    public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then()
+    {
+        var resolver = new GatedResolver { Value = "old" };
+        var provider = Provider(resolver);
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+        provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue();
+        before.ShouldBe("old");
+
+        resolver.Value = "new";
+        resolver.Hold();
+        var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+        resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue();
+
+        // Mid-refresh: the old key, from memory — the store is NOT re-read on this path.
+        var callsMidRefresh = resolver.Calls;
+        provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue();
+        during.ShouldBe("old");
+        resolver.Calls.ShouldBe(callsMidRefresh);
+
+        resolver.Release();
+        await refresh;
+        provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue();
+        after.ShouldBe("new");
+    }
+
+    [Fact]
+    public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key()
+    {
+        var resolver = new GatedResolver { Value = "k-1" };
+        var provider = Provider(resolver);
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        resolver.Throw = new InvalidOperationException("store offline");
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+        psk.ShouldBe("k-1");
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure:
+    ///     the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown
+    ///     read keeps the previous key (the test above); a definitive not-found drops it.
+    /// </summary>
+    [Fact]
+    public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh()
+    {
+        var resolver = new GatedResolver { Value = "k-1" };
+        var provider = Provider(resolver);
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+        provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service");
+
+        resolver.Value = null;   // the secret was deleted
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
+        provider.Matches("SITE-A", "k-1").ShouldBeFalse();
+    }
+
+    /// <summary>The same revocation reaches a key that was first served from the LAZY cache, not a refresh.</summary>
+    [Fact]
+    public async Task A_deleted_secret_also_evicts_a_lazily_cached_key()
+    {
+        var resolver = new GatedResolver { Value = "k-lazy" };
+        var provider = Provider(resolver);
+        provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue();
+        lazy.ShouldBe("k-lazy");
+
+        resolver.Value = null;
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
+    }
+
+    /// <summary>Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once.</summary>
+    [Fact]
+    public async Task Refresh_store_reads_are_concurrency_bounded()
+    {
+        var resolver = new ConcurrencyTrackingResolver();
+        var provider = Provider(resolver);
+        var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList();
+
+        await provider.RefreshAsync(clusters, CancellationToken.None);
+
+        resolver.Calls.ShouldBe(40);
+        resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads);
+        resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded");
+        foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue();
+    }
+
+    private sealed class ConcurrencyTrackingResolver : ISecretResolver
+    {
+        private int _inFlight;
+        private int _max;
+        private int _calls;
+        public int MaxConcurrent => Volatile.Read(ref _max);
+        public int Calls => Volatile.Read(ref _calls);
+
+        public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
+        {
+            Interlocked.Increment(ref _calls);
+            var now = Interlocked.Increment(ref _inFlight);
+            int seen;
+            while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { }
+            await Task.Delay(20, ct);
+            Interlocked.Decrement(ref _inFlight);
+            return "k-" + name.Value;
+        }
+    }
+
+    /// <summary>A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour).</summary>
+    [Fact]
+    public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily()
+    {
+        var resolver = new GatedResolver { Value = "k-b" };
+        var provider = Provider(resolver);
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue();
+        psk.ShouldBe("k-b");
+        resolver.Calls.ShouldBe(2);
+    }
+
+    [Fact]
+    public async Task InvalidateAll_drops_refreshed_keys_too()
+    {
+        var resolver = new GatedResolver { Value = "k-1" };
+        var provider = Provider(resolver);
+        await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
+
+        provider.InvalidateAll();
+        resolver.Value = "k-2";
+        provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
+        psk.ShouldBe("k-2");
+    }
+
+    private sealed class GatedResolver : ISecretResolver
+    {
+        private int _calls;
+        private volatile TaskCompletionSource? _gate;
+
+        public string? Value { get; set; }
+        public Exception? Throw { get; set; }
+        public int Calls => Volatile.Read(ref _calls);
+        public ManualResetEventSlim Entered { get; } = new();
+
+        public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        public void Release() => _gate?.TrySetResult();
+
+        public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
+        {
+            Interlocked.Increment(ref _calls);
+            Entered.Set();
+            if (_gate is { } gate) await gate.Task.WaitAsync(ct);
+            if (Throw is not null) throw Throw;
+            return Value;
+        }
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs

@@ -0,0 +1,87 @@
+using System.IO.Compression;
+using System.Security.Cryptography;
+using System.Text.Json;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     Fix round 1 minor — revision-hash stability across the R03:3.2 refactor. The pre-change composer read
+///     every table with a SQL <c>ORDER BY</c> (the database's case-INSENSITIVE collation); the refactor re-sorted
+///     in memory with an ORDINAL comparer, which orders mixed-case keys differently ("B-1" before "a-2"). The
+///     same configuration would then hash differently after the upgrade: drift detection reports every node
+///     drifted, and a re-deploy of an unchanged config shows a "new" revision. This pins the composer's output
+///     BYTE-identical to a verbatim replica of the pre-change composer on the real provider.
+/// </summary>
+[Trait("Category", "SchemaCompliance")]
+public sealed class ConfigComposerSqlServerHashStabilityTests
+{
+    [Fact]
+    public async Task Composer_output_is_byte_identical_to_the_pre_change_composer_for_mixed_case_keys()
+    {
+        using var sql = new SqlServerTestDatabase();
+        await using (var seed = sql.NewContext())
+        {
+            // Collation order a-2, B-1, c-3; ordinal order B-1, a-2, c-3.
+            foreach (var id in new[] { "c-3", "B-1", "a-2" })
+                seed.Scripts.Add(new Script { ScriptId = id, Name = id, SourceCode = "return 1;", SourceHash = id });
+            await seed.SaveChangesAsync();
+        }
+
+        await using var legacyDb = sql.NewContext();
+        var legacy = await LegacySnapshotAndFlattenAsync(legacyDb);
+
+        await using var viaDb = sql.NewContext();
+        var current = await ConfigComposer.SnapshotAndFlattenAsync(viaDb);
+        current.Blob.ShouldBe(legacy.Blob);
+        current.RevisionHash.ShouldBe(legacy.RevisionHash);
+
+        // The deploy path's form (validate + compose from ONE snapshot) must agree too.
+        await using var deployDb = sql.NewContext();
+        var viaSnapshot = ConfigComposer.Compose(await DraftSnapshotFactory.FromConfigDbAsync(deployDb));
+        viaSnapshot.RevisionHash.ShouldBe(legacy.RevisionHash);
+    }
+
+    /// <summary>Verbatim replica of <c>ConfigComposer.SnapshotAndFlattenAsync</c> before R03:3.2 (ad2b5c40).</summary>
+    private static async Task<(byte[] Blob, string RevisionHash)> LegacySnapshotAndFlattenAsync(OtOpcUaConfigDbContext db)
+    {
+        var ct = CancellationToken.None;
+        var snapshot = new
+        {
+            Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct),
+            Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct),
+            RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct),
+            DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct),
+            Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct),
+            TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct),
+            Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct),
+            Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct),
+            UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct),
+            PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct),
+            UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct),
+            UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct),
+            NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct),
+            ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct),
+            Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct),
+            VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct),
+            ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct),
+        };
+        var json = JsonSerializer.SerializeToUtf8Bytes(snapshot, new JsonSerializerOptions
+        {
+            WriteIndented = false,
+            DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never,
+        });
+        using var output = new MemoryStream(json.Length / 8);
+        using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true))
+            gzip.Write(json, 0, json.Length);
+        var blob = output.ToArray();
+        return (blob, Convert.ToHexStringLower(SHA256.HashData(blob)));
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs

@@ -222,6 +222,47 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase
         withoutOverride.ResilienceConfig.ShouldBeNull("an unset override stays null ⇒ tier defaults at spawn");
     }
 
+    /// <summary>
+    ///     R03:3.2 — the deploy path composes from the <see cref="Configuration.Validation.DraftSnapshot"/> it
+    ///     validated rather than re-reading the DB. <see cref="ConfigComposer.Compose"/> over the factory's
+    ///     snapshot must therefore be byte-identical to the DB overload, or the drift probe (which still
+    ///     uses the DB overload) would report every deployment as drifted.
+    /// </summary>
+    [Fact]
+    public async Task Compose_over_the_factory_snapshot_is_byte_identical_to_the_db_overload()
+    {
+        var f = NewInMemoryDbFactory();
+        await using (var db = f.CreateDbContext())
+        {
+            db.ServerClusters.Add(NewCluster("cluster-b"));
+            db.ServerClusters.Add(NewCluster("cluster-a"));
+            db.ApiKeys.Add(new ApiKey
+            {
+                ApiKeyId = "k1", Name = "scada", SecretHash = "h", Principal = "scadabridge", CreatedUtc = FixedTimestamp,
+            });
+            db.Scripts.Add(new Script { ScriptId = "s-1", Name = "s", SourceCode = "return 1;", SourceHash = "x" });
+            await db.SaveChangesAsync();
+        }
+
+        var viaDb = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext());
+        var snapshot = await Configuration.Validation.DraftSnapshotFactory.FromConfigDbAsync(f.CreateDbContext());
+        var viaSnapshot = ConfigComposer.Compose(snapshot);
+
+        viaSnapshot.RevisionHash.ShouldBe(viaDb.RevisionHash);
+        viaSnapshot.Blob.ShouldBe(viaDb.Blob);
+        // The composed content is the snapshot's, not something re-read: the keys and clusters are in it.
+        var json = Inflate(viaSnapshot.Blob);
+        json.ShouldContain("\"ApiKeyId\":\"k1\"");
+        json.IndexOf("cluster-a", StringComparison.Ordinal).ShouldBeLessThan(json.IndexOf("cluster-b", StringComparison.Ordinal));
+    }
+
+    private static string Inflate(byte[] blob)
+    {
+        using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress);
+        using var reader = new StreamReader(gz, Encoding.UTF8);
+        return reader.ReadToEnd();
+    }
+
     private static readonly DateTime FixedTimestamp = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
 
     private static ServerCluster NewCluster(string id) => new()

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs

@@ -0,0 +1,440 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     Review 03 §2.2: a <c>Deployment</c> row left in <c>Dispatching</c> with NO <c>NodeDeploymentState</c>
+///     rows — <c>HandleDispatch</c> threw before its <c>SaveChanges</c>, or the process died between the
+///     AdminOps row write and the coordinator's <c>Tell</c> — used to be recovered with an EMPTY expected
+///     set, at which point the deadline hit the <c>0 == 0</c> "race" branch and returned without touching
+///     the row. Every later <c>StartDeployment</c> was then refused <c>AnotherDeploymentInFlight</c> forever.
+/// </summary>
+public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase
+{
+    private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64));
+    private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
+
+    /// <summary>
+    ///     A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it,
+    ///     the expected set is seeded from the enabled rows, and the node's ack seals it.
+    /// </summary>
+    [Fact]
+    public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        SeedNodes(dbFactory, "site-a-1:4053");
+
+        // No DispatchDeployment is ever sent — PreStart alone has to recover it.
+        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);
+
+        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
+            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>
+    ///     A lost dispatch that is already past the apply deadline is closed as <c>TimedOut</c> with a reason
+    ///     naming the condition — never left in <c>Dispatching</c> — and the next deployment proceeds
+    ///     (positive control: it is dispatched, acked and sealed through the same coordinator).
+    /// </summary>
+    [Fact]
+    public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
+        SeedNodes(dbFactory, "site-a-1:4053");
+
+        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value);
+            row.Status.ShouldBe(DeploymentStatus.TimedOut);
+            row.FailureReason.ShouldNotBeNull();
+            row.FailureReason.ShouldContain("lost");
+        }, duration: PresenceBudget);
+
+        // Positive control: the coordinator is not wedged — a fresh deployment runs end to end.
+        var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);
+
+        actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"),
+            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>
+    ///     An <c>AwaitingApplyAcks</c> row with no rows (the process died between the status flip and the
+    ///     empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh
+    ///     dispatch would — the recovery keys on the missing expected set, not on the status value.
+    /// </summary>
+    [Fact]
+    public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
+        Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+            db.NodeDeploymentStates.ShouldBeEmpty();
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>
+    ///     The brief's scenario end to end: <c>HandleDispatch</c> throws once (a transient fault seeding the
+    ///     expected set). The coordinator restarts, recovers the <c>Dispatching</c> row, re-dispatches it and
+    ///     the deployment completes; a second <c>StartDeployment</c> is then Accepted (positive control: a
+    ///     normal deploy seals).
+    /// </summary>
+    [Fact]
+    public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds()
+    {
+        var dbName = Guid.NewGuid().ToString("N");
+        var adminFactory = NewInMemoryDbFactory(dbName);
+        var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName));
+        SeedNodes(adminFactory, "site-a-1:4053");
+
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
+        var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty<IDriverProbe>()));
+
+        // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
+        // the actor would be stopped, not restarted). Identify is processed after creation, so a
+        // resolved selection proves PreStart completed.
+        await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);
+        faulting.FailNext();
+        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var first = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
+        first.DeploymentId.ShouldNotBeNull();
+
+        // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands.
+        AwaitAssert(() =>
+        {
+            faulting.Faulted.ShouldBeTrue();
+            using var db = adminFactory.CreateDbContext();
+            db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1);
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);
+
+        coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"),
+            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = adminFactory.CreateDbContext();
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget);
+
+        // Positive control: nothing is wedged — the next deploy is Accepted and seals too.
+        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var second = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
+        coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
+            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = adminFactory.CreateDbContext();
+            db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2);
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the
+    ///     coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between
+    ///     its commit and the Tell). Nothing would ever close it. The in-flight check now closes a
+    ///     Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next
+    ///     StartDeployment is Accepted (and seals — positive control on the pipeline).
+    /// </summary>
+    [Fact]
+    public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        SeedNodes(dbFactory, "site-a-1:4053");
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+        await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);   // PreStart saw nothing
+
+        var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
+        var admin = Sys.ActorOf(AdminOperationsActor.Props(
+            dbFactory, coordinator, Enumerable.Empty<IDriverProbe>(), applyDeadline: NoTimeout));
+
+        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var result = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message);
+
+        using (var db = dbFactory.CreateDbContext())
+        {
+            var row = db.Deployments.Single(d => d.DeploymentId == stale.Value);
+            row.Status.ShouldBe(DeploymentStatus.TimedOut);
+            row.FailureReason.ShouldNotBeNull();
+            row.FailureReason.ShouldContain("lost");
+        }
+
+        coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
+            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status
+                .ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>Control for the closure: the same row INSIDE the deadline is still in flight and still refuses.</summary>
+    [Fact]
+    public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment()
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        var coordinator = CreateTestProbe();
+        var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5));
+        var admin = Sys.ActorOf(AdminOperationsActor.Props(
+            dbFactory, coordinator.Ref, Enumerable.Empty<IDriverProbe>(), applyDeadline: NoTimeout));
+
+        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
+        var result = ExpectMsg<StartDeploymentResult>(PresenceBudget);
+        result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight);
+        result.DeploymentId.ShouldBe(pending);
+
+        using var db = dbFactory.CreateDbContext();
+        db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
+    }
+
+    /// <summary>
+    ///     Fix round 1 minor — a late or duplicated <c>DispatchDeployment</c> for a row that is already
+    ///     terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip
+    ///     it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right
+    ///     after it IS seeded (so the first message was processed, not merely not-yet-processed).
+    /// </summary>
+    [Theory]
+    [InlineData(DeploymentStatus.Sealed)]
+    [InlineData(DeploymentStatus.TimedOut)]
+    [InlineData(DeploymentStatus.PartiallyFailed)]
+    public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal)
+    {
+        var dbFactory = NewInMemoryDbFactory();
+        SeedNodes(dbFactory, "site-a-1:4053");
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
+
+        var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1));
+        coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId()));
+        var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId()));
+
+        AwaitAssert(() =>
+        {
+            using var db = dbFactory.CreateDbContext();
+            db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);
+
+        using var verify = dbFactory.CreateDbContext();
+        verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal);
+        verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0);
+    }
+
+    /// <summary>
+    ///     Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of
+    ///     materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the
+    ///     optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are
+    ///     untouched by the partial UPDATEs.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "SchemaCompliance")]
+    public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob()
+    {
+        using var sql = new SqlServerTestDatabase();
+        var factory = sql.Factory();
+        SeedNodes(factory, "site-a-1:4053");
+        var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray();
+        var id = DeploymentId.NewId();
+        using (var db = sql.NewContext())
+        {
+            db.Deployments.Add(new Deployment
+            {
+                DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching,
+                CreatedBy = "test", ArtifactBlob = blob,
+            });
+            db.SaveChanges();
+        }
+
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout));
+        AwaitAssert(() =>
+        {
+            using var db = sql.NewContext();
+            db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);   // PreStart re-drives the un-dispatched row
+        coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+
+        AwaitAssert(() =>
+        {
+            using var db = sql.NewContext();
+            var row = db.Deployments.AsNoTracking().Single();
+            row.Status.ShouldBe(DeploymentStatus.Sealed);
+            row.SealedAtUtc.ShouldNotBeNull();
+            row.ArtifactBlob.ShouldBe(blob);
+        }, duration: PresenceBudget);
+    }
+
+    /// <summary>
+    ///     Re-review 1 — the reopen race. AdminOps' lost-dispatch closure commits TimedOut AFTER the coordinator's
+    ///     terminal guard read the row as Dispatching but BEFORE the AwaitingApplyAcks flip. The flip is
+    ///     conditioned on the guard's RowVersion, so on SQL Server it becomes a concurrency conflict and the row
+    ///     stays TimedOut with no expected set. Positive control: the coordinator is not wedged — the next live
+    ///     dispatch is seeded and flipped.
+    /// </summary>
+    [Fact]
+    [Trait("Category", "SchemaCompliance")]
+    public async Task On_sql_server_a_closure_between_guard_and_flip_is_not_reopened()
+    {
+        using var sql = new SqlServerTestDatabase();
+        var inner = sql.Factory();
+        SeedNodes(inner, "site-a-1:4053");
+        var interposer = new InterposingDbFactory(inner);
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(interposer, NoTimeout));
+        await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);   // PreStart ran over an empty table
+
+        // The race proper: a new Dispatching row whose dispatch is interleaved with an AdminOps-style closure.
+        var victim = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        interposer.OnSecondCreate(() =>
+        {
+            using var db = sql.NewContext();
+            db.Database.ExecuteSqlRaw(
+                "UPDATE Deployment SET Status = {0}, FailureReason = N'closed as lost' WHERE DeploymentId = {1}",
+                (int)DeploymentStatus.TimedOut, victim.Value);
+        });
+        coordinator.Tell(new DispatchDeployment(victim, TestRevision, CorrelationId.NewId()));
+        AwaitAssert(() => interposer.Fired.ShouldBeTrue(), duration: PresenceBudget);
+
+        // Positive control + ordering: a live row dispatched after it IS flipped, so the victim's dispatch was processed.
+        var live = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = sql.NewContext();
+            db.Deployments.AsNoTracking().Single(d => d.DeploymentId == live.Value).Status
+                .ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+        }, duration: PresenceBudget);
+
+        using var verify = sql.NewContext();
+        verify.Deployments.AsNoTracking().Single(d => d.DeploymentId == victim.Value).Status
+            .ShouldBe(DeploymentStatus.TimedOut, "the closure must not be reopened");
+        verify.NodeDeploymentStates.Count(s => s.DeploymentId == victim.Value).ShouldBe(0);
+    }
+
+    /// <summary>Runs an action just before handing out the SECOND context after arming (HandleDispatch: guard, then flip).</summary>
+    private sealed class InterposingDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        private Action? _action;
+        private int _countdown;
+        public volatile bool Fired;
+
+        public void OnSecondCreate(Action action)
+        {
+            _action = action;
+            Volatile.Write(ref _countdown, 2);
+        }
+
+        public OtOpcUaConfigDbContext CreateDbContext()
+        {
+            if (Volatile.Read(ref _countdown) > 0 && Interlocked.Decrement(ref _countdown) == 0)
+            {
+                _action?.Invoke();
+                Fired = true;
+            }
+            return inner.CreateDbContext();
+        }
+    }
+
+    private static DeploymentId SeedDeployment(
+        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
+    {
+        var id = DeploymentId.NewId();
+        using var db = dbFactory.CreateDbContext();
+        db.Deployments.Add(new Deployment
+        {
+            DeploymentId = id.Value,
+            RevisionHash = TestRevision.Value,
+            Status = status,
+            CreatedBy = "test",
+            CreatedAtUtc = createdAtUtc,
+        });
+        db.SaveChanges();
+        return id;
+    }
+
+    private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
+    {
+        using var db = dbFactory.CreateDbContext();
+        db.ServerClusters.Add(new ServerCluster
+        {
+            ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
+            NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
+        });
+        foreach (var nodeId in nodeIds)
+        {
+            db.ClusterNodes.Add(new ClusterNode
+            {
+                NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
+                ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
+            });
+        }
+        db.SaveChanges();
+    }
+
+    /// <summary>Throws from exactly one <c>CreateDbContext</c> call after <see cref="FailNext"/>.</summary>
+    private sealed class OneShotFaultingDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        private int _armed;
+        public bool Faulted { get; private set; }
+
+        public void FailNext() => Interlocked.Exchange(ref _armed, 1);
+
+        public OtOpcUaConfigDbContext CreateDbContext()
+        {
+            if (Interlocked.Exchange(ref _armed, 0) == 1)
+            {
+                Faulted = true;
+                throw new InvalidOperationException("injected transient fault seeding the expected set");
+            }
+            return inner.CreateDbContext();
+        }
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs

@@ -0,0 +1,233 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the
+///     blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N,
+///     never one a node applied recently).
+/// </summary>
+public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase
+{
+    private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
+
+    [Fact]
+    public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact()
+    {
+        var f = NewInMemoryDbFactory();
+        var ids = Seed(f, sealedCount: 12, mixInTimedOut: false);   // ids[0] newest … ids[11] oldest
+        var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 };
+
+        int pruned;
+        await using (var db = f.CreateDbContext())
+            pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow);
+
+        pruned.ShouldBe(7);
+        await using var verify = f.CreateDbContext();
+        foreach (var id in ids.Take(5))
+            (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob");
+        foreach (var id in ids.Skip(5))
+            (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied");
+        // Rows survive: the audit record is intact.
+        (await verify.Deployments.CountAsync()).ShouldBe(12);
+
+        // Idempotent: a second sweep finds nothing.
+        await using var again = f.CreateDbContext();
+        (await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0);
+    }
+
+    [Fact]
+    public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row()
+    {
+        var f = NewInMemoryDbFactory();
+        var ids = Seed(f, sealedCount: 4);
+        var inFlight = Guid.NewGuid();
+        await using (var db = f.CreateDbContext())
+        {
+            db.ClusterNodes.Add(new ClusterNode
+            {
+                NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1",
+                ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test",
+            });
+            // ids[3] is the oldest; a node applied it yesterday, so it stays.
+            db.NodeDeploymentStates.Add(new NodeDeploymentState
+            {
+                NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied,
+                StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1),
+            });
+            // ids[2]: applied long ago — not protected.
+            db.NodeDeploymentStates.Add(new NodeDeploymentState
+            {
+                NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied,
+                StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90),
+            });
+            // An ancient in-flight row with a blob — never touched.
+            db.Deployments.Add(new Deployment
+            {
+                DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks,
+                CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 },
+            });
+            await db.SaveChangesAsync();
+        }
+        var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 };
+
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2);   // ids[1], ids[2]
+
+        await using var verify = f.CreateDbContext();
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest");
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty();
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago");
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday");
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight");
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I3 — the keep window counts SEALED rows, not "terminal" ones. A burst of 60 failed
+    ///     deploys used to fill a 50-row terminal window on its own and push the fleet's last good (Sealed)
+    ///     artifact out of it — the one every boot / drift / fallback path reads. The newest Sealed blob is
+    ///     never pruned; the unreferenced TimedOut rows are (positive control: the sweep did run).
+    /// </summary>
+    [Fact]
+    public async Task Sixty_timed_out_deploys_never_prune_the_newest_sealed_blob()
+    {
+        var f = NewInMemoryDbFactory();
+        var sealedId = Guid.NewGuid();
+        var timedOut = new List<Guid>();
+        await using (var db = f.CreateDbContext())
+        {
+            db.Deployments.Add(new Deployment
+            {
+                DeploymentId = sealedId, RevisionHash = new string('s', 64), Status = DeploymentStatus.Sealed,
+                CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-200), SealedAtUtc = SweepNow.AddDays(-200),
+                ArtifactBlob = new byte[] { 1, 2, 3 },
+            });
+            for (var i = 0; i < 60; i++)
+            {
+                var id = Guid.NewGuid();
+                timedOut.Add(id);
+                db.Deployments.Add(new Deployment
+                {
+                    DeploymentId = id, RevisionHash = new string('t', 64), Status = DeploymentStatus.TimedOut,
+                    CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), ArtifactBlob = new byte[] { 9 },
+                });
+            }
+            await db.SaveChangesAsync();
+        }
+        var options = new DeploymentRetentionOptions { KeepSealed = 50, ReferencedDays = 30, MaxRowsPerSweep = 200 };
+
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(60);
+
+        await using var verify = f.CreateDbContext();
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == sealedId)).ArtifactBlob
+            .ShouldNotBeEmpty("the newest Sealed artifact is never pruned");
+        (await verify.Deployments.CountAsync(d => timedOut.Contains(d.DeploymentId) && d.ArtifactBlob.Length == 0))
+            .ShouldBe(60);
+    }
+
+    /// <summary>Even with KeepSealed misconfigured to 0, the newest Sealed row keeps its blob.</summary>
+    [Fact]
+    public async Task The_newest_sealed_blob_survives_a_zero_keep_window()
+    {
+        var f = NewInMemoryDbFactory();
+        var ids = Seed(f, sealedCount: 3, mixInTimedOut: false);
+
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, new DeploymentRetentionOptions { KeepSealed = 0 }, SweepNow))
+                .ShouldBe(2);
+
+        await using var verify = f.CreateDbContext();
+        (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty();
+    }
+
+    [Fact]
+    public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls()
+    {
+        var f = NewInMemoryDbFactory();
+        Seed(f, sealedCount: 10);
+        var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 };
+
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3);
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3);
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2);
+        await using (var db = f.CreateDbContext())
+            (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0);
+    }
+
+    /// <summary>The actor runs the same rule off its thread and reports back.</summary>
+    [Fact]
+    public void Actor_sweeps_on_request_and_reports_the_count()
+    {
+        var f = NewInMemoryDbFactory();
+        Seed(f, sealedCount: 6);
+        var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions
+        {
+            KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60,
+        }));
+
+        actor.Tell(DeploymentRetentionActor.Sweep.Instance);
+
+        ExpectMsg<DeploymentRetentionActor.SweepCompleted>(PresenceBudget).Pruned.ShouldBe(2);
+        using var verify = f.CreateDbContext();
+        verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2);
+    }
+
+    [Fact]
+    public void Options_bind_from_configuration_with_defaults()
+    {
+        var defaults = DeploymentRetentionOptions.Bind(null);
+        defaults.Enabled.ShouldBeTrue();
+        defaults.KeepSealed.ShouldBe(50);
+        defaults.ReferencedDays.ShouldBe(30);
+        defaults.IntervalMinutes.ShouldBe(60);
+
+        var config = new ConfigurationBuilder()
+            .AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                ["Deployments:Retention:KeepSealed"] = "7",
+                ["Deployments:Retention:Enabled"] = "false",
+            })
+            .Build();
+        var bound = DeploymentRetentionOptions.Bind(config);
+        bound.KeepSealed.ShouldBe(7);
+        bound.Enabled.ShouldBeFalse();
+        bound.ReferencedDays.ShouldBe(30);
+    }
+
+    /// <summary>Seeds sealed deployments, newest first; each one hour older than the previous.</summary>
+    private static List<Guid> Seed(IDbContextFactory<OtOpcUaConfigDbContext> f, int sealedCount, bool mixInTimedOut = true)
+    {
+        var ids = new List<Guid>();
+        using var db = f.CreateDbContext();
+        for (var i = 0; i < sealedCount; i++)
+        {
+            var id = Guid.NewGuid();
+            ids.Add(id);
+            db.Deployments.Add(new Deployment
+            {
+                DeploymentId = id,
+                RevisionHash = new string('a', 64),
+                Status = mixInTimedOut && i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed,
+                CreatedBy = "test",
+                CreatedAtUtc = SweepNow.AddHours(-i),
+                SealedAtUtc = SweepNow.AddHours(-i),
+                ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i },
+            });
+        }
+        db.SaveChanges();
+        return ids;
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs

@@ -0,0 +1,77 @@
+using Microsoft.Data.SqlClient;
+using Microsoft.EntityFrameworkCore;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+/// <summary>
+///     A migrated, throw-away config database on the central SQL Server, for the few ControlPlane behaviours
+///     the InMemory provider cannot model (a SQL Server <c>DbUpdateException</c> names the WHOLE statement
+///     batch; collation-ordered reads). Same server / credentials / env overrides as
+///     <c>Configuration.Tests.SchemaComplianceFixture</c> (<c>OTOPCUA_CONFIG_TEST_SERVER</c>,
+///     <c>OTOPCUA_CONFIG_TEST_SA_PASSWORD</c>; default the shared docker host's dev SA), and the same database
+///     name shape, so that suite's 24 h orphan reaper also reclaims a database a killed run leaks here.
+/// </summary>
+public sealed class SqlServerTestDatabase : IDisposable
+{
+    private const string DefaultServer = "10.100.0.35,14330";
+    private const string DefaultSaPassword = "OtOpcUaDev_2026!";
+
+    /// <summary>Gets the connection string of the migrated database.</summary>
+    public string ConnectionString { get; }
+
+    private readonly string _databaseName;
+
+    /// <summary>Creates and migrates the database.</summary>
+    public SqlServerTestDatabase()
+    {
+        var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer;
+        var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword;
+        _databaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+        ConnectionString =
+            $"Server={server};Database={_databaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;";
+        using var ctx = NewContext();
+        ctx.Database.Migrate();
+    }
+
+    /// <summary>A new context on the database.</summary>
+    /// <returns>The context.</returns>
+    public OtOpcUaConfigDbContext NewContext() =>
+        new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseSqlServer(ConnectionString).Options);
+
+    /// <summary>A factory over <see cref="NewContext"/>.</summary>
+    /// <returns>The factory.</returns>
+    public IDbContextFactory<OtOpcUaConfigDbContext> Factory() => new ContextFactory(this);
+
+    /// <summary>Drops the database (best effort; a leak is reaped by the Configuration suite).</summary>
+    public void Dispose()
+    {
+        try
+        {
+            SqlConnection.ClearAllPools();
+            var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString;
+            using var conn = new SqlConnection(master);
+            conn.Open();
+            using var cmd = conn.CreateCommand();
+            cmd.CommandText = @"
+DECLARE @sql nvarchar(max);
+IF DB_ID(@name) IS NOT NULL
+BEGIN
+    SET @sql = N'ALTER DATABASE ' + QUOTENAME(@name) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; ' +
+               N'DROP DATABASE ' + QUOTENAME(@name) + N';';
+    EXEC sp_executesql @sql;
+END";
+            cmd.Parameters.Add(new SqlParameter("@name", _databaseName));
+            cmd.ExecuteNonQuery();
+        }
+        catch (Exception)
+        {
+            // Best effort — see the class remarks.
+        }
+    }
+
+    private sealed class ContextFactory(SqlServerTestDatabase owner) : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        public OtOpcUaConfigDbContext CreateDbContext() => owner.NewContext();
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs

@@ -0,0 +1,54 @@
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
+
+/// <summary>
+///     R03:2.3 — an unset or placeholder pepper must fail host start, not surface as "Invalid API key"
+///     on the first activation.
+/// </summary>
+public sealed class ApiKeyOptionsValidatorTests
+{
+    private static ValidateOptionsResult Validate(ApiKeyOptions options) =>
+        new ApiKeyOptionsValidator().Validate(Options.DefaultName, options);
+
+    [Theory]
+    [InlineData("")]
+    [InlineData("   ")]
+    public void Blank_pepper_fails(string pepper)
+    {
+        var result = Validate(new ApiKeyOptions { Pepper = pepper });
+        result.Failed.ShouldBeTrue();
+        result.FailureMessage.ShouldNotBeNull();
+        result.FailureMessage.ShouldContain("Pepper is required");
+    }
+
+    [Fact]
+    public void Short_pepper_fails()
+    {
+        var result = Validate(new ApiKeyOptions { Pepper = "x" });
+        result.Failed.ShouldBeTrue();
+        result.FailureMessage.ShouldNotBeNull();
+        result.FailureMessage.ShouldContain($"at least {ApiKeyOptionsValidator.MinPepperLength}");
+    }
+
+    [Fact]
+    public void Blank_username_fails()
+    {
+        var result = Validate(new ApiKeyOptions { Pepper = new string('p', 32), Username = " " });
+        result.Failed.ShouldBeTrue();
+        result.FailureMessage.ShouldNotBeNull();
+        result.FailureMessage.ShouldContain("Username");
+    }
+
+    /// <summary>Positive control: the docker-dev rig's pepper and the defaults pass unchanged.</summary>
+    [Fact]
+    public void Configured_pepper_passes()
+    {
+        Validate(new ApiKeyOptions { Pepper = "docker-dev-apikey-pepper-not-for-production" })
+            .Succeeded.ShouldBeTrue();
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs

@@ -49,6 +49,41 @@ public sealed class ActiveTierRegistrationTests
         ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active);
     }
 
+    /// <summary>
+    ///     R03:2.1 + fix round 1 / I6 (coordinator ruling) — the OPC UA endpoint probe's tiers by role:
+    ///     <list type="bullet">
+    ///         <item>driver-only: Ready + Active (it cannot serve and must not be routed to);</item>
+    ///         <item>fused admin+driver: Ready ONLY, until the ServiceLevel calculator consumes the fact
+    ///         (F-C1) — Traefik routes the AdminUI on <c>/health/active</c>, so a failed SDK on the active
+    ///         central node must not take the AdminUI out of rotation;</item>
+    ///         <item>admin-only: absent (it starts no endpoint to be degraded by).</item>
+    ///     </list>
+    /// </summary>
+    [Theory]
+    [InlineData(false, true, true, true)]
+    [InlineData(true, true, true, false)]
+    [InlineData(true, false, false, false)]
+    public void Opcua_endpoint_probe_tiers_follow_the_node_roles(bool hasAdmin, bool hasDriver, bool ready, bool active)
+    {
+        var registrations = new ServiceCollection()
+            .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: hasDriver)
+            .BuildServiceProvider()
+            .GetRequiredService<IOptions<HealthCheckServiceOptions>>()
+            .Value.Registrations
+            .Where(r => r.Name == OpcUaEndpointHealthCheck.Name)
+            .ToList();
+
+        if (!ready && !active)
+        {
+            registrations.ShouldBeEmpty();
+            return;
+        }
+
+        var registration = registrations.ShouldHaveSingleItem();
+        registration.Tags.Contains(ZbHealthTags.Ready).ShouldBe(ready);
+        registration.Tags.Contains(ZbHealthTags.Active).ShouldBe(active);
+    }
+
     [Fact]
     public void Active_tier_check_is_not_on_the_ready_tier()
     {

tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs

@@ -0,0 +1,203 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
+
+/// <summary>
+///     R03:3.3 — the negative cache was keyed by key id, so a flood of DISTINCT well-formed ids cost one
+///     SQL query per attempt under the SessionManager lock, no credential needed. Two guards:
+///     the known-id set (unknown ids refused in memory) and the miss budget (a token bucket on lookups
+///     that reach the store and miss, for when the set is unavailable).
+/// </summary>
+public sealed class BoundedApiKeyStoreKnownIdTests
+{
+    private static readonly DateTimeOffset T0 = new(2026, 10, 22, 12, 0, 0, TimeSpan.Zero);
+
+    [Fact]
+    public async Task An_unknown_id_is_refused_without_reaching_the_store_and_a_known_one_still_does()
+    {
+        var inner = new ListingStore("k-real");
+        var clock = new FakeClock(T0);
+        var sut = Bounded(inner, clock, cacheSeconds: 5);
+
+        (await sut.FindAsync("k-guess-1", T0.UtcDateTime, default)).ShouldBeNull();
+        (await sut.FindAsync("k-guess-2", T0.UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(0, "unknown ids never reach the store");
+        inner.ListCalls.ShouldBe(1, "one set load serves every unknown id inside the TTL");
+
+        // Positive control: the real id reaches the store and is found.
+        (await sut.FindAsync("k-real", T0.UtcDateTime, default)).ShouldNotBeNull();
+        inner.FindCalls.ShouldBe(1);
+    }
+
+    [Fact]
+    public async Task The_set_is_refreshed_after_the_TTL_so_a_newly_minted_key_becomes_visible()
+    {
+        var inner = new ListingStore("k-old");
+        var clock = new FakeClock(T0);
+        var sut = Bounded(inner, clock, cacheSeconds: 5);
+
+        (await sut.FindAsync("k-new", T0.UtcDateTime, default)).ShouldBeNull();
+        inner.Mint("k-new");
+        // Still inside the TTL: the stale set refuses it (the documented window).
+        clock.Advance(TimeSpan.FromSeconds(3));
+        (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(0);
+
+        clock.Advance(TimeSpan.FromSeconds(3));
+        (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
+        inner.ListCalls.ShouldBe(2);
+    }
+
+    [Fact]
+    public async Task A_failed_set_load_falls_back_to_the_per_id_lookup_and_keeps_an_older_set()
+    {
+        var inner = new ListingStore("k-real") { ListThrows = true };
+        var clock = new FakeClock(T0);
+        var sut = Bounded(inner, clock, cacheSeconds: 5);
+
+        // No set yet and the load fails: the per-id path runs (old behaviour), nothing is wedged.
+        (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(1);
+        sut.KnownKeyIds.ShouldBeNull();
+
+        inner.ListThrows = false;
+        clock.Advance(TimeSpan.FromSeconds(6));
+        (await sut.FindAsync("k-guess-2", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
+        sut.KnownKeyIds.ShouldNotBeNull();
+        inner.FindCalls.ShouldBe(1, "once the set loaded, the guess was refused in memory");
+
+        // A later failed refresh keeps the previous set in force.
+        inner.ListThrows = true;
+        clock.Advance(TimeSpan.FromSeconds(6));
+        (await sut.FindAsync("k-guess-3", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(1);
+    }
+
+    /// <summary>With no id set available, misses are rate-limited; hits are not charged; time refills.</summary>
+    [Fact]
+    public async Task The_miss_budget_bounds_store_misses_when_no_set_is_available()
+    {
+        var inner = new ListingStore("k-real") { SupportsListing = false };
+        var clock = new FakeClock(T0);
+        var sut = Bounded(inner, clock, cacheSeconds: 0, missBurst: 3, refillPerSecond: 1);
+
+        for (var i = 0; i < 3; i++)
+            (await sut.FindAsync($"k-guess-{i}", T0.UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(3);
+
+        // Budget exhausted: refused without a store read, as a throw the authenticator denies on.
+        await Should.ThrowAsync<InvalidOperationException>(() => sut.FindAsync("k-guess-9", T0.UtcDateTime, default));
+        inner.FindCalls.ShouldBe(3);
+
+        // Refill: one second restores one token.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        (await sut.FindAsync("k-guess-10", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(4);
+
+        // A hit is not a miss: it reaches the store even with the bucket empty? No — it needs a token to
+        // be attempted, but the token is refunded, so the next attempt still has it. Prove the refund.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        (await sut.FindAsync("k-real", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
+        (await sut.FindAsync("k-guess-11", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(6);
+    }
+
+    [Fact]
+    public async Task The_guards_are_off_when_the_cache_is_off_restoring_a_read_per_lookup()
+    {
+        var inner = new ListingStore("k-real");
+        var sut = Bounded(inner, new FakeClock(T0), cacheSeconds: 0, missBurst: 0);
+
+        (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
+        (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
+        inner.FindCalls.ShouldBe(2);
+        inner.ListCalls.ShouldBe(0);
+    }
+
+    /// <summary>
+    ///     Fix round 1 minor — an inner store that cannot list ids switches the known-id set off for THIS
+    ///     decorator only. It used to do so by writing <c>KnownIdSet = false</c> into the guard options it was
+    ///     handed, a shared options object, so one store's capability leaked into every other consumer.
+    ///     Positive control: a second decorator over a listing store on the same options instance still uses
+    ///     the set.
+    /// </summary>
+    [Fact]
+    public async Task An_unlisting_store_does_not_mutate_the_shared_guard_options()
+    {
+        var guard = new ApiKeyStoreGuardOptions { MissBurst = 100, MissRefillPerSecond = 20 };
+        var options = Options.Create(new ApiKeyOptions
+        {
+            Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = 5,
+        });
+        var clock = new FakeClock(T0);
+        var unlisting = new ListingStore("k-real") { SupportsListing = false };
+        var first = new BoundedApiKeyStore(unlisting, options, NullLogger<BoundedApiKeyStore>.Instance, clock, guard);
+
+        (await first.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
+        unlisting.FindCalls.ShouldBe(1, "no set: the per-id path ran");
+        guard.KnownIdSet.ShouldBeTrue("the shared options object is never written");
+
+        var listing = new ListingStore("k-real");
+        var second = new BoundedApiKeyStore(listing, options, NullLogger<BoundedApiKeyStore>.Instance, clock, guard);
+        (await second.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
+        listing.FindCalls.ShouldBe(0, "the second decorator still refuses unknown ids in memory");
+        listing.ListCalls.ShouldBe(1);
+    }
+
+    private static BoundedApiKeyStore Bounded(
+        IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new(
+        inner,
+        Options.Create(new ApiKeyOptions
+        {
+            Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = cacheSeconds,
+        }),
+        NullLogger<BoundedApiKeyStore>.Instance,
+        clock,
+        new ApiKeyStoreGuardOptions { MissBurst = missBurst, MissRefillPerSecond = refillPerSecond });
+
+    private sealed class ListingStore(params string[] ids) : IApiKeyStore
+    {
+        private readonly HashSet<string> _ids = ids.ToHashSet(StringComparer.Ordinal);
+        private int _findCalls;
+        private int _listCalls;
+
+        public bool SupportsListing { get; set; } = true;
+        public bool ListThrows { get; set; }
+        public int FindCalls => _findCalls;
+        public int ListCalls => _listCalls;
+
+        public void Mint(string id) => _ids.Add(id);
+
+        public Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
+        {
+            Interlocked.Increment(ref _findCalls);
+            return Task.FromResult(_ids.Contains(apiKeyId)
+                ? new ApiKeyLookup(new ApiKey { ApiKeyId = apiKeyId, Name = "k", SecretHash = "h", Principal = "p" }, [])
+                : null);
+        }
+
+        public Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct) =>
+            Task.FromResult<IReadOnlyList<NodeAcl>>([]);
+
+        public Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
+        {
+            if (!SupportsListing) return Task.FromResult<IReadOnlySet<string>?>(null);
+            Interlocked.Increment(ref _listCalls);
+            if (ListThrows) throw new InvalidOperationException("ConfigDb unreachable (simulated)");
+            return Task.FromResult<IReadOnlySet<string>?>(_ids.ToHashSet(StringComparer.Ordinal));
+        }
+    }
+
+    private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider
+    {
+        private DateTimeOffset _now = utcNow;
+        public override DateTimeOffset GetUtcNow() => _now;
+        public void Advance(TimeSpan by) => _now = _now.Add(by);
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs

@@ -363,7 +363,10 @@ public sealed class BoundedApiKeyStoreTests
     {
         var (_, inner) = SetUp();
         var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
-        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock);
+        // The miss budget (R03:3.3) is OFF here on purpose: this fact floods 1025 distinct unknown ids — the
+        // exact shape the budget refuses — to pin the cache CEILING, which is a separate rule.
+        var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock,
+            guard: new ApiKeyStoreGuardOptions { MissBurst = 0 });
 
         // Seed the real key, then flood past the ceiling with ids that do not exist.
         (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
@@ -477,7 +480,8 @@ public sealed class BoundedApiKeyStoreTests
         int timeoutMs,
         int cacheSeconds,
         TimeProvider? clock = null,
-        ILogger<BoundedApiKeyStore>? log = null) => new(
+        ILogger<BoundedApiKeyStore>? log = null,
+        ApiKeyStoreGuardOptions? guard = null) => new(
         inner,
         Options.Create(new ApiKeyOptions
         {
@@ -487,7 +491,8 @@ public sealed class BoundedApiKeyStoreTests
             StoreCacheSeconds = cacheSeconds,
         }),
         log ?? NullLogger<BoundedApiKeyStore>.Instance,
-        clock ?? TimeProvider.System);
+        clock ?? TimeProvider.System,
+        guard);
 
     private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new(
         store,

tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs

@@ -0,0 +1,213 @@
+using System.Net;
+using System.Net.Sockets;
+using Akka.Hosting;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Cluster;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using ZB.MOM.WW.OtOpcUa.Host.Health;
+using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer;
+using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
+
+/// <summary>
+///     R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined
+///     its mesh, and acked deploys through the Null sink while clients got connection-refused. The start
+///     outcome is now a fact (<see cref="IOpcUaEndpointState"/>) the ready/active tiers read, a driver-only
+///     node rethrows so the process exits, and <c>StopAsync</c> actually stops the SDK.
+/// </summary>
+public sealed class OtOpcUaServerHostedServiceStartFailureTests
+{
+    private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    ///     Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the
+    ///     real <c>/health/ready</c> endpoint answers 503 — while StartAsync itself returns (fused node).
+    ///     Positive control: the same endpoint answers 200 once the state is Started.
+    /// </summary>
+    [Fact]
+    public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503()
+    {
+        using var deadline = new CancellationTokenSource(Deadline);
+        using var squatter = SquatAllInterfaces(out var port);
+        var pki = NewPki();
+        var state = new OpcUaEndpointState();
+        var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null);
+
+        var service = NewService(port, pki, state, fused);
+        try
+        {
+            await service.StartAsync(deadline.Token);   // must NOT throw on a fused node
+
+            state.Status.ShouldBe(OpcUaEndpointStatus.Failed);
+            state.FailureReason.ShouldNotBeNullOrWhiteSpace();
+
+            var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext());
+            probe.Status.ShouldBe(HealthStatus.Unhealthy);
+            probe.Description.ShouldNotBeNull();
+            probe.Description.ShouldContain("failed to start");
+
+            await using var web = BuildHealthHost(state);
+            await web.StartAsync(deadline.Token);
+            using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) };
+
+            (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
+            (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
+
+            // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the
+            // 503 above measured the probe and not a broken health pipeline.
+            state.MarkStarted();
+            (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK);
+            await web.StopAsync(deadline.Token);
+        }
+        finally
+        {
+            await service.DisposeAsync();
+            squatter.Stop();
+            Cleanup(pki);
+        }
+    }
+
+    /// <summary>A driver-only node has nothing else to stay up for: the failed start propagates.</summary>
+    [Fact]
+    public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start()
+    {
+        using var deadline = new CancellationTokenSource(Deadline);
+        using var squatter = SquatAllInterfaces(out var port);
+        var pki = NewPki();
+        var state = new OpcUaEndpointState();
+        var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null);
+
+        var service = NewService(port, pki, state, driverOnly);
+        try
+        {
+            await Should.ThrowAsync<Exception>(() => service.StartAsync(deadline.Token));
+            state.Status.ShouldBe(OpcUaEndpointStatus.Failed);
+        }
+        finally
+        {
+            await service.DisposeAsync();
+            squatter.Stop();
+            Cleanup(pki);
+        }
+    }
+
+    /// <summary>
+    ///     A clean start is Started and listening; <c>StopAsync</c> closes the port (connection refused, not
+    ///     a stale-but-listening endpoint) and marks Stopped; <c>DisposeAsync</c> twice afterwards is a no-op.
+    /// </summary>
+    [Fact]
+    public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent()
+    {
+        using var deadline = new CancellationTokenSource(Deadline);
+        var port = FreePort();
+        var pki = NewPki();
+        var state = new OpcUaEndpointState();
+        var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null));
+        try
+        {
+            await service.StartAsync(deadline.Token);
+            state.Status.ShouldBe(OpcUaEndpointStatus.Started);
+            using (var client = new TcpClient())
+            {
+                await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token);
+                client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync");
+            }
+
+            await service.StopAsync(deadline.Token);
+
+            state.Status.ShouldBe(OpcUaEndpointStatus.Stopped);
+            using (var client = new TcpClient())
+            {
+                await Should.ThrowAsync<SocketException>(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token));
+            }
+
+            await service.DisposeAsync();
+            await service.DisposeAsync();   // idempotent
+        }
+        finally
+        {
+            await service.DisposeAsync();
+            Cleanup(pki);
+        }
+    }
+
+    private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) =>
+        new(
+            Options.Create(new OpcUaApplicationHostOptions
+            {
+                ApplicationName = "OtOpcUa.StartFailure",
+                ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}",
+                OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki,
+            }),
+            new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(),
+            NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance,
+            EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(),
+            () => throw new NotSupportedException("no actor system required"), new ActorRegistry(),
+            null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance,
+            endpointState: state, roles: roles);
+
+    /// <summary>The production health registration (driver role) over the supplied state, on a loopback Kestrel.</summary>
+    private static WebApplication BuildHealthHost(OpcUaEndpointState state)
+    {
+        var builder = WebApplication.CreateBuilder();
+        builder.WebHost.UseUrls("http://127.0.0.1:0");
+        builder.Logging.ClearProviders();
+        builder.Services.AddSingleton<IOpcUaEndpointState>(state);
+        builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true);
+        // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so
+        // the status measured is the endpoint's. (The registration shape itself is pinned separately.)
+        builder.Services.Configure<HealthCheckServiceOptions>(o =>
+        {
+            foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList())
+                o.Registrations.Remove(r);
+        });
+        var app = builder.Build();
+        app.MapOtOpcUaHealth();
+        return app;
+    }
+
+    /// <summary>
+    ///     Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the
+    ///     wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both
+    ///     sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of
+    ///     this test proved exactly that. The SDK also only fails when NEITHER family binds.
+    /// </summary>
+    private static TcpListener SquatAllInterfaces(out int port)
+    {
+        var squatter = new TcpListener(IPAddress.IPv6Any, 0);
+        squatter.Server.DualMode = true;
+        squatter.ExclusiveAddressUse = true;
+        squatter.Start();
+        port = ((IPEndPoint)squatter.LocalEndpoint).Port;
+        return squatter;
+    }
+
+    private static int FreePort()
+    {
+        using var probe = new TcpListener(IPAddress.Loopback, 0);
+        probe.Start();
+        var port = ((IPEndPoint)probe.LocalEndpoint).Port;
+        probe.Stop();
+        return port;
+    }
+
+    private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}");
+
+    private static void Cleanup(string pki)
+    {
+        if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true);
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs

@@ -0,0 +1,116 @@
+using System.Security.Claims;
+using Microsoft.AspNetCore.Components.Authorization;
+using Microsoft.AspNetCore.Http;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.Auth.AspNetCore;
+using ZB.MOM.WW.OtOpcUa.Security.Audit;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Tests.Audit;
+
+/// <summary>
+///     R03:2.5 — inside a Blazor circuit there is no <c>HttpContext</c> after the first render, so an
+///     HTTP-only accessor stamps every audit row from an interactive page <c>"system"</c>. The circuit's
+///     <see cref="AuthenticationStateProvider"/> is consulted first; the HTTP context is the fallback.
+/// </summary>
+public sealed class AuthenticationStateAuditActorAccessorTests
+{
+    [Fact]
+    public void Circuit_principal_wins_when_there_is_no_http_context()
+    {
+        var accessor = new AuthenticationStateAuditActorAccessor(
+            new HttpContextAccessorStub(null),
+            new FixedStateProvider(Authenticated(new Claim(ZbClaimTypes.Username, "alice"))));
+
+        accessor.CurrentActor.ShouldBe("alice");
+    }
+
+    [Fact]
+    public void Http_principal_is_the_fallback_outside_a_circuit()
+    {
+        var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "bob")) };
+        var accessor = new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), authenticationState: null);
+
+        accessor.CurrentActor.ShouldBe("bob");
+    }
+
+    [Fact]
+    public void Unauthenticated_circuit_falls_through_to_http_then_null()
+    {
+        var anonymousCircuit = new FixedStateProvider(new ClaimsPrincipal(new ClaimsIdentity()));
+
+        new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(null), anonymousCircuit)
+            .CurrentActor.ShouldBeNull();
+
+        var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Name, "carol")) };
+        new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), anonymousCircuit)
+            .CurrentActor.ShouldBe("carol");
+    }
+
+    /// <summary>An incomplete state task is "no circuit identity", never a block.</summary>
+    [Fact]
+    public void Pending_authentication_state_is_not_awaited()
+    {
+        var accessor = new AuthenticationStateAuditActorAccessor(
+            new HttpContextAccessorStub(null),
+            new PendingStateProvider());
+
+        accessor.CurrentActor.ShouldBeNull();
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I5 — the REAL server-side provider, resolved in a non-circuit scope (a minimal-API
+    ///     request, a background service), throws <see cref="InvalidOperationException"/> from
+    ///     <c>GetAuthenticationStateAsync</c> because nothing called <c>SetAuthenticationState</c>. That used
+    ///     to escape <c>CurrentActor</c>; it must fall back to the HTTP principal instead.
+    /// </summary>
+    [Fact]
+    public void Real_server_provider_outside_a_circuit_falls_back_to_the_http_principal()
+    {
+        var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "dave")) };
+        var accessor = new AuthenticationStateAuditActorAccessor(
+            new HttpContextAccessorStub(context),
+            new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider());
+
+        accessor.CurrentActor.ShouldBe("dave");
+
+        // No HTTP principal either ⇒ null, still no throw.
+        new AuthenticationStateAuditActorAccessor(
+                new HttpContextAccessorStub(null),
+                new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider())
+            .CurrentActor.ShouldBeNull();
+    }
+
+    /// <summary>Positive control on the real provider: once a circuit has set its state, that principal wins.</summary>
+    [Fact]
+    public void Real_server_provider_inside_a_circuit_serves_the_circuit_principal()
+    {
+        var provider = new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider();
+        provider.SetAuthenticationState(Task.FromResult(
+            new AuthenticationState(Authenticated(new Claim(ZbClaimTypes.Username, "erin")))));
+        var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "http-user")) };
+
+        new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), provider)
+            .CurrentActor.ShouldBe("erin");
+    }
+
+    private static ClaimsPrincipal Authenticated(params Claim[] claims) =>
+        new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role));
+
+    private sealed class HttpContextAccessorStub(HttpContext? context) : IHttpContextAccessor
+    {
+        public HttpContext? HttpContext { get; set; } = context;
+    }
+
+    private sealed class FixedStateProvider(ClaimsPrincipal user) : AuthenticationStateProvider
+    {
+        public override Task<AuthenticationState> GetAuthenticationStateAsync() =>
+            Task.FromResult(new AuthenticationState(user));
+    }
+
+    private sealed class PendingStateProvider : AuthenticationStateProvider
+    {
+        public override Task<AuthenticationState> GetAuthenticationStateAsync() =>
+            new TaskCompletionSource<AuthenticationState>().Task;
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs

@@ -11,6 +11,8 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.IdentityModel.Tokens;
 using Shouldly;
 using Xunit;
 using ZB.MOM.WW.Auth.AspNetCore;
@@ -37,6 +39,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
     private IHost _host = null!;
     private TestServer _server = null!;
     private readonly StubLdapGroupRoleMappingService _roleMappings = new();
+    private readonly CapturingLoggerProvider _logs = new();
 
     private static CancellationToken Ct => TestContext.Current.CancellationToken;
 
@@ -70,6 +73,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
                             ["Security:Ldap:GroupToRole:ReadOnly"] = "Viewer",
                         }).Build();
                     services.AddOtOpcUaAuth(configuration);
+                    services.AddLogging(l => l.AddProvider(_logs));
                     services.AddSingleton<ILdapAuthService, StubLdapAuthService>();
                     // The login handler now resolves the DB role-map service via DI to merge
                     // DB-backed grants on top of the appsettings baseline. Register the stub so
@@ -96,6 +100,10 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
                                 .ToArray();
                             return Results.Ok(claims);
                         }).RequireAuthorization();
+                        // A role-gated machine endpoint: proves a bearer principal's roles resolve through
+                        // IsInRole (the validation parameters pair RoleClaimType with the minted key).
+                        e.MapGet("/api/viewer-only", () => Results.Ok("viewer"))
+                            .RequireAuthorization(p => p.RequireRole("Viewer"));
                     });
                 });
             })
@@ -153,6 +161,158 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
             new AuthEndpoints.LoginRequest("ldap-down", "anything"), Ct);
 
         response.StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
+        // R03:2.7 — the outage is named in the server log, not inferred from the directory side.
+        _logs.Entries.Any(e =>
+                e.Level == LogLevel.Warning && e.Message.Contains("LDAP authentication failed") && e.Exception != null)
+            .ShouldBeTrue("a Warning naming the LDAP failure must be logged");
+    }
+
+    /// <summary>
+    ///     R03:2.4 — a token from <c>/auth/token</c> authenticates a request carrying it as a bearer:
+    ///     the fallback-protected root AND a role-gated endpoint (the bearer principal's "Role" claims
+    ///     resolve through <c>IsInRole</c>). No cookie is sent.
+    /// </summary>
+    [Fact]
+    public async Task Bearer_token_authenticates_api_requests_without_a_cookie()
+    {
+        var token = await LoginAndIssueTokenAsync();
+        var client = NewClient();
+
+        var root = new HttpRequestMessage(HttpMethod.Get, "/");
+        root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
+        (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
+
+        var gated = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only");
+        gated.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
+        var gatedResponse = await client.SendAsync(gated, Ct);
+        gatedResponse.StatusCode.ShouldBe(HttpStatusCode.OK);
+        (await gatedResponse.Content.ReadAsStringAsync(Ct)).ShouldContain("viewer");
+    }
+
+    /// <summary>A tampered or garbage bearer fails closed (401), it never falls through to anonymous.</summary>
+    [Theory]
+    [InlineData("not-a-jwt")]
+    [InlineData("tampered")]
+    public async Task Bad_bearer_token_is_rejected(string kind)
+    {
+        var token = kind == "tampered"
+            ? (await LoginAndIssueTokenAsync()) + "x"
+            : kind;
+        var client = NewClient();
+        var req = new HttpRequestMessage(HttpMethod.Get, "/");
+        req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
+        req.Headers.Add("X-Requested-With", "XMLHttpRequest");
+
+        var response = await client.SendAsync(req, Ct);
+
+        response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized);
+        response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer");
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I1 — the first-party JwtBearer handler pins HS256: an unsigned (<c>alg: none</c>)
+    ///     token carrying the right issuer/audience/roles, and a token signed with a different HMAC
+    ///     variant, both fail CLOSED with a 401 — never a 500, never a pass.
+    /// </summary>
+    [Theory]
+    [InlineData("none")]
+    [InlineData("HS384")]
+    public async Task Bearer_with_a_non_HS256_algorithm_is_rejected_with_401(string alg)
+    {
+        var token = alg == "none" ? UnsignedToken() : SignedToken(SecurityAlgorithms.HmacSha384);
+        var client = NewClient();
+        var req = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only");
+        req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
+
+        var response = await client.SendAsync(req, Ct);
+
+        response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized);
+        response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer");
+
+        // Positive control: the same claims signed HS256 with the configured key ARE accepted, so the
+        // 401 above measured the algorithm and not a broken token shape.
+        var good = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only");
+        good.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(
+            "Bearer", SignedToken(SecurityAlgorithms.HmacSha256));
+        (await client.SendAsync(good, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
+    }
+
+    /// <summary>
+    ///     Fix round 1 / I2 — <c>/auth/token</c> mints only for a COOKIE session. A bearer token used to
+    ///     be able to mint a fresh token for itself, so a stolen token could renew itself forever and
+    ///     outlive both its own expiry and the user's LDAP membership. Positive control: the cookie
+    ///     session that minted the bearer still gets a token.
+    /// </summary>
+    [Fact]
+    public async Task Auth_token_refuses_a_bearer_only_caller_and_serves_a_cookie_session()
+    {
+        var client = NewClient();
+        var loginResponse = await client.PostAsJsonAsync("/auth/login",
+            new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct);
+        loginResponse.EnsureSuccessStatusCode();
+
+        var cookieReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
+        AttachCookies(cookieReq, loginResponse);
+        var cookieResp = await client.SendAsync(cookieReq, Ct);
+        cookieResp.StatusCode.ShouldBe(HttpStatusCode.OK);
+        var bearer = (await cookieResp.Content.ReadFromJsonAsync<JsonElement>(Ct)).GetProperty("token").GetString()!;
+
+        // The bearer authenticates elsewhere (positive control on the token itself) …
+        var root = new HttpRequestMessage(HttpMethod.Get, "/");
+        root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer);
+        (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
+
+        // … but cannot renew itself.
+        var renew = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
+        renew.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer);
+        renew.Headers.Add("X-Requested-With", "XMLHttpRequest");
+        var renewResp = await client.SendAsync(renew, Ct);
+        renewResp.StatusCode.ShouldBeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden);
+        (await renewResp.Content.ReadAsStringAsync(Ct)).ShouldNotContain("token");
+    }
+
+    private const string TestSigningKey = "test-signing-key-with-at-least-32-bytes-of-utf8-content";
+
+    private static IEnumerable<System.Security.Claims.Claim> ViewerClaims() =>
+    [
+        new(JwtTokenService.UsernameClaimType, "mallory"),
+        new(JwtTokenService.RoleClaimType, "Viewer"),
+    ];
+
+    /// <summary>A token signed with <paramref name="algorithm"/> using the configured key bytes.</summary>
+    private static string SignedToken(string algorithm)
+    {
+        var keyBytes = System.Text.Encoding.UTF8.GetBytes(TestSigningKey);
+        // The SAME key bytes for every algorithm, so a rejection measures the algorithm pin alone.
+        var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken(
+            issuer: "otopcua-test", audience: "otopcua-test", claims: ViewerClaims(),
+            expires: DateTime.UtcNow.AddMinutes(5),
+            signingCredentials: new SigningCredentials(new SymmetricSecurityKey(keyBytes), algorithm));
+        return new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler().WriteToken(token);
+    }
+
+    /// <summary>An <c>alg: none</c> token with otherwise valid issuer/audience/lifetime/roles.</summary>
+    private static string UnsignedToken()
+    {
+        static string B64(string json) => Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(json);
+        var exp = DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeSeconds();
+        return B64("{\"alg\":\"none\",\"typ\":\"JWT\"}") + "." +
+               B64($"{{\"zb:username\":\"mallory\",\"Role\":\"Viewer\",\"iss\":\"otopcua-test\",\"aud\":\"otopcua-test\",\"exp\":{exp}}}") +
+               ".";
+    }
+
+    private async Task<string> LoginAndIssueTokenAsync()
+    {
+        var client = NewClient();
+        var loginResponse = await client.PostAsJsonAsync("/auth/login",
+            new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct);
+        loginResponse.EnsureSuccessStatusCode();
+        var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
+        AttachCookies(tokenReq, loginResponse);
+        var response = await client.SendAsync(tokenReq, Ct);
+        response.EnsureSuccessStatusCode();
+        var payload = await response.Content.ReadFromJsonAsync<JsonElement>(Ct);
+        return payload.GetProperty("token").GetString()!;
     }
 
     /// <summary>Tests that ping anonymous returns 401.</summary>

tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs

@@ -0,0 +1,25 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Tests;
+
+/// <summary>Collects every log entry written through the host's logger factory, for assertions on what was logged.</summary>
+internal sealed class CapturingLoggerProvider : ILoggerProvider
+{
+    public ConcurrentQueue<Entry> Entries { get; } = new();
+
+    public ILogger CreateLogger(string categoryName) => new Logger(categoryName, Entries);
+
+    public void Dispose() { }
+
+    internal sealed record Entry(string Category, LogLevel Level, string Message, Exception? Exception);
+
+    private sealed class Logger(string category, ConcurrentQueue<Entry> entries) : ILogger
+    {
+        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) =>
+            entries.Enqueue(new Entry(category, logLevel, formatter(state, exception), exception));
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs

@@ -0,0 +1,127 @@
+using System.Net;
+using System.Net.Http.Json;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Services;
+using ZB.MOM.WW.OtOpcUa.Security.Auth;
+using ZB.MOM.WW.OtOpcUa.Security.Endpoints;
+using ZB.MOM.WW.OtOpcUa.Security.Jwt;
+using ZB.MOM.WW.OtOpcUa.Security.Ldap;
+
+namespace ZB.MOM.WW.OtOpcUa.Security.Tests;
+
+/// <summary>
+///     R03:2.4 — JWT is opt-in by signing key. Unconfigured: <c>/auth/token</c> is not mapped (it used to
+///     answer HTTP 500 from the service constructor) and no bearer scheme exists. Configured badly: the
+///     host refuses to START rather than failing on the first token request.
+/// </summary>
+public sealed class JwtConfigurationGateTests
+{
+    private static CancellationToken Ct => TestContext.Current.CancellationToken;
+
+    [Fact]
+    public async Task Without_a_signing_key_auth_token_is_not_mapped_and_bearer_is_ignored()
+    {
+        using var host = await StartHostAsync(signingKey: null);
+        var server = host.GetTestServer();
+        var client = server.CreateClient();
+
+        // The login still works — the cookie path is untouched.
+        var login = await client.PostAsJsonAsync("/auth/login", new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct);
+        login.StatusCode.ShouldBe(HttpStatusCode.NoContent);
+
+        // /auth/token is simply absent (404), not a 500.
+        var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
+        foreach (var c in login.Headers.GetValues("Set-Cookie")) tokenReq.Headers.Add("Cookie", c.Split(';')[0]);
+        (await client.SendAsync(tokenReq, Ct)).StatusCode.ShouldBe(HttpStatusCode.NotFound);
+
+        // No bearer scheme is registered, so a bearer header is ignored and the cookie session still wins.
+        var schemes = host.Services.GetRequiredService<IAuthenticationSchemeProvider>();
+        (await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer)).ShouldBeNull();
+        var withBogusBearer = new HttpRequestMessage(HttpMethod.Get, "/");
+        foreach (var c in login.Headers.GetValues("Set-Cookie")) withBogusBearer.Headers.Add("Cookie", c.Split(';')[0]);
+        withBogusBearer.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "garbage");
+        (await client.SendAsync(withBogusBearer, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
+
+        await host.StopAsync(Ct);
+    }
+
+    /// <summary>Positive control for the gate: with a key, the bearer scheme IS registered.</summary>
+    [Fact]
+    public async Task With_a_signing_key_the_bearer_scheme_is_registered()
+    {
+        using var host = await StartHostAsync(signingKey: "test-signing-key-with-at-least-32-bytes-of-utf8-content");
+        var schemes = host.Services.GetRequiredService<IAuthenticationSchemeProvider>();
+        var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer);
+        bearer.ShouldNotBeNull();
+        bearer.HandlerType.ShouldBe(typeof(Microsoft.AspNetCore.Authentication.JwtBearer.JwtBearerHandler));
+        await host.StopAsync(Ct);
+    }
+
+    /// <summary>A configured-but-short key fails at host start (ValidateOnStart), not on the first /auth/token.</summary>
+    [Fact]
+    public async Task A_short_signing_key_fails_host_start()
+    {
+        var ex = await Should.ThrowAsync<OptionsValidationException>(async () =>
+        {
+            using var host = await StartHostAsync(signingKey: "too-short");
+        });
+        ex.Message.ShouldContain("SigningKey");
+    }
+
+    private static async Task<IHost> StartHostAsync(string? signingKey)
+    {
+        var dbName = $"jwt-gate-{Guid.NewGuid():N}";
+        var settings = new Dictionary<string, string?>();
+        if (signingKey is not null) settings["Security:Jwt:SigningKey"] = signingKey;
+        var configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build();
+
+        var host = new HostBuilder()
+            .ConfigureWebHost(web =>
+            {
+                web.UseTestServer();
+                web.ConfigureServices(services =>
+                {
+                    services.AddDbContextFactory<OtOpcUaConfigDbContext>(opt => opt.UseInMemoryDatabase(dbName));
+                    services.AddDbContext<OtOpcUaConfigDbContext>(opt => opt.UseInMemoryDatabase(dbName));
+                    services.AddRouting();
+                    services.AddOtOpcUaAuth(configuration);
+                    services.AddSingleton<ILdapAuthService, AlwaysAliceLdap>();
+                    services.AddSingleton<ILdapGroupRoleMappingService, NullLdapGroupRoleMappingService>();
+                });
+                web.Configure(app =>
+                {
+                    app.UseRouting();
+                    app.UseAuthentication();
+                    app.UseAuthorization();
+                    app.UseEndpoints(e =>
+                    {
+                        e.MapOtOpcUaAuth();
+                        e.MapGet("/", () => Results.Ok("authenticated")).RequireAuthorization();
+                    });
+                });
+            })
+            .Build();
+        await host.StartAsync(Ct);
+        return host;
+    }
+
+    private sealed class AlwaysAliceLdap : ILdapAuthService
+    {
+        public Task<LdapAuthResult> AuthenticateAsync(string username, string password, CancellationToken ct = default) =>
+            Task.FromResult(new LdapAuthResult(
+                Success: password == "valid-password", DisplayName: "Alice", Username: username,
+                Groups: [], Roles: [], Error: null));
+    }
+}