feat(secrets): enable Akka clustered secret replication on the docker-dev rig

Turns on Secrets:Replication:Enabled across all six docker-dev host nodes via
one x-secrets-env anchor: replication enabled, a 5s AnnounceInterval so
anti-entropy is observable during dev exercise, and the ONE shared dev-only
KEK every node requires (committed default per the rig's zero-operator-step
pattern, overridable via OTOPCUA_SECRETS_KEK; never reuse outside this rig).

Live-proven on the rig with Secrets 0.2.2: all six nodes start with NO
startup deadlock (the scadaproj#1 signature never appeared), the mesh forms,
and a secret seeded into site-a-1's local store alone converged to all five
peers within 60s — byte-identical ciphertext on all six nodes (same
sha256, kek_id sha256:f3f0c2056e7a), decrypt-verified on central-2 and
site-b-2. site-b-2 restart rejoined cleanly with the row intact; the fleet
also survived a full Docker-engine restart with replication enabled.

Found during verification, tracked separately: the AdminUI /admin/secrets
page renders but is non-interactive (Secrets.Ui RCL components carry no
@rendermode and the host applies render modes per-page), so Add secret does
nothing - pre-existing, unrelated to replication.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts

Directory.Packages.props

@@ -132,10 +132,10 @@
     <PackageVersion Include="ZB.MOM.WW.Auth.Abstractions" Version="0.1.1" />
     <PackageVersion Include="ZB.MOM.WW.Auth.Ldap" Version="0.1.1" />
     <PackageVersion Include="ZB.MOM.WW.Auth.AspNetCore" Version="0.1.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets" Version="0.2.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Abstractions" Version="0.2.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Ui" Version="0.2.1" />
-    <PackageVersion Include="ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" Version="0.2.1" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets" Version="0.2.2" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Abstractions" Version="0.2.2" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Ui" Version="0.2.2" />
+    <PackageVersion Include="ZB.MOM.WW.Secrets.Replicator.AkkaDotNet" Version="0.2.2" />
     <PackageVersion Include="ZB.MOM.WW.Audit" Version="0.1.0" />
     <PackageVersion Include="ZB.MOM.WW.Theme" Version="0.3.1" />
     <PackageVersion Include="ZB.MOM.WW.HistorianGateway.Client" Version="0.3.0" />

docker-dev/docker-compose.yml

@@ -47,6 +47,29 @@
 
 name: otopcua-dev
 
+# ── Shared secret-replication env (all six host nodes) ──────────────────────────
+# Akka peer-to-peer clustered secret replication rides the existing `otopcua` Akka
+# mesh (DistributedPubSub topic `zb-mom-ww-secrets`) across all six host nodes. Two
+# hard requirements are wired here, identically, on every node:
+#   1. Secrets__Replication__Enabled=true  → SecretsRegistration.AddOtOpcUaSecrets
+#      switches from plain AddZbSecrets to AddZbSecretsAkkaReplication + the eager
+#      SecretReplicationStarter hosted service, so every node joins anti-entropy.
+#   2. ZB_SECRETS_MASTER_KEY  → the ONE shared 32-byte base64 KEK. Every node MUST
+#      carry the SAME key or peers fail closed decrypting each other's rows
+#      (kek_id mismatch). appsettings.json binds Secrets:MasterKey:Source=Environment
+#      / EnvVarName=ZB_SECRETS_MASTER_KEY, so this env var IS the KEK on every node.
+# The KEK below is a committed DEV-ONLY default (matches the rig's zero-operator-step
+# philosophy, like the committed dev SQL password). Override for a fresh key with:
+#   OTOPCUA_SECRETS_KEK=$(openssl rand -base64 32) docker compose ... up -d
+# NEVER reuse this key outside this local dev rig.
+# AnnounceInterval is shortened to 5s (default 30s) so anti-entropy convergence is
+# observable quickly during dev exercise. Each node keeps its own local SQLite store
+# (Secrets:SqlitePath=otopcua-secrets.db, per-container); replication syncs them.
+x-secrets-env: &secrets-env
+  Secrets__Replication__Enabled: "true"
+  Secrets__Replication__AnnounceInterval: "00:00:05"
+  ZB_SECRETS_MASTER_KEY: "${OTOPCUA_SECRETS_KEK:-ZYGhIX0luS/XsevpCB2W18jYHMcqO6AjM9oXy+T6Zp4=}"
+
 services:
 
   sql:
@@ -145,6 +168,7 @@ services:
       sql: { condition: service_healthy }
       migrator: { condition: service_completed_successfully }
     environment:
+      <<: *secrets-env
       OTOPCUA_ROLES: "admin,driver"
       ASPNETCORE_URLS: "http://+:9000"
       ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;"
@@ -210,6 +234,7 @@ services:
       central-1: { condition: service_started }
       migrator: { condition: service_completed_successfully }
     environment:
+      <<: *secrets-env
       OTOPCUA_ROLES: "admin,driver"
       ASPNETCORE_URLS: "http://+:9000"
       ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;"
@@ -280,6 +305,7 @@ services:
       Cluster__PublicHostname: "site-a-1"
       Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
       Cluster__Roles__0: "driver"
+      <<: *secrets-env
       # Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/
       # mem_reservation are inherited from the *otopcua-host anchor.
       Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
@@ -304,6 +330,7 @@ services:
       Cluster__PublicHostname: "site-a-2"
       Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
       Cluster__Roles__0: "driver"
+      <<: *secrets-env
       Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
       Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning"
       GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}"
@@ -326,6 +353,7 @@ services:
       Cluster__PublicHostname: "site-b-1"
       Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
       Cluster__Roles__0: "driver"
+      <<: *secrets-env
       Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
       Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning"
       GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}"
@@ -346,6 +374,7 @@ services:
       Cluster__PublicHostname: "site-b-2"
       Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053"
       Cluster__Roles__0: "driver"
+      <<: *secrets-env
       Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning"
       Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning"
       GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}"

src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/SecretReplicationStarter.cs

@@ -24,15 +24,15 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
 ///         unconditional.
 ///     </para>
 ///     <para>
-///         <b>Known ineffective against ZB.MOM.WW.Secrets.Replicator.AkkaDotNet 0.2.0.</b> That
-///         version never binds its own <c>ISecretReplicator</c>:
-///         <c>AddZbSecretsAkkaReplication</c> calls <c>AddZbSecrets</c> first, which
-///         <c>TryAdd</c>s <c>NoOpSecretReplicator</c>, making the package's own subsequent
-///         <c>TryAddSingleton&lt;ISecretReplicator&gt;</c> a no-op. Resolving the store therefore
-///         builds a <c>ReplicatingSecretStore</c> around a no-op replicator and spawns no actor.
-///         This hook is correct and stays in place for when the library is fixed, but replication
-///         must be treated as <b>non-functional</b> until then — see the skipped test
-///         <c>SecretsReplicationRegistrationTests.The_startup_hook_actually_creates_the_replication_actor</c>.
+///         <b>History — this hook has met two library defects, both fixed.</b> Against 0.2.0 it was
+///         inert: the package never bound its own <c>ISecretReplicator</c> (TryAdd registration
+///         order), so this resolve built a <c>ReplicatingSecretStore</c> around a no-op sink and
+///         spawned no actor. Against 0.2.1 it was worse: the resolve <b>deadlocked the host at
+///         startup</b> — the package's DI graph had a circular singleton dependency, invisible to
+///         the container through factory lambdas (scadaproj#1). Fixed in 0.2.2 by deferring the
+///         cycle-closing invalidator edge;
+///         <c>SecretsReplicationRegistrationTests.The_startup_hook_actually_creates_the_replication_actor</c>
+///         exercises this hook against a built provider on a real single-node cluster.
 ///     </para>
 /// </remarks>
 /// <param name="services">Root provider used to resolve the (decorated) secret store exactly once.</param>

tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/SecretsReplicationRegistrationTests.cs

@@ -103,23 +103,13 @@ public sealed class SecretsReplicationRegistrationTests
     [Fact]
     public void Replication_enabled_decorates_the_store_so_writes_publish()
     {
-        // Asserted on the ServiceCollection rather than a built provider, deliberately. Resolving
-        // ISecretStore with replication on constructs ReplicatingSecretStore, which resolves
-        // ISecretReplicator, which eagerly spawns the replication actor, whose PreStart calls
-        // DistributedPubSub.Get(...) — unavailable until the node has joined a cluster. Against a
-        // plain ActorSystem that resolve hangs instead of failing.
-        //
-        // Akka.TestKit is the family convention for this (ScadaBridge's suites use it throughout)
-        // and would be the right tool, but it is NOT available here: Akka.TestKit.Xunit2 is
-        // xunit-v2-only, and this project is one of the 44 on xunit.v3. Adding it yields CS0433
-        // type conflicts. Directory.Packages.props documents the same constraint — AdminUI.Tests,
-        // ControlPlane.Tests and Runtime.Tests are deliberately held on xunit v2 precisely because
-        // no xunit.v3 TestKit ships as of Akka 1.5.62. Host wiring tests belong here, not in one of
-        // those three, so the descriptor is the right seam until Akka ships a v3 TestKit.
-        //
-        // Nothing is lost: the registration order IS the defect, and the library's own
-        // TwoNodeClusterReplicationTests already cover actor creation and convergence against a
-        // genuine 2-node cluster.
+        // Asserted on the ServiceCollection because the registration order IS the defect class this
+        // file guards (which descriptor won a TryAdd race), and the descriptor is where that is
+        // visible. An earlier revision of this comment ALSO claimed a provider-based resolve was
+        // impossible ("hangs against a plain ActorSystem — DistributedPubSub needs a joined
+        // cluster"). That hang was real but misattributed: it was a circular singleton dependency
+        // in the 0.2.1 package's own DI wiring (scadaproj#1), fixed in 0.2.2. The provider-based
+        // resolve is covered by The_startup_hook_actually_creates_the_replication_actor below.
         //
         // AddSingleton (not TryAdd) appends the decorator, and the LAST registration for a service
         // type is what the container resolves.
@@ -130,6 +120,66 @@ public sealed class SecretsReplicationRegistrationTests
             "with replication enabled the local store must be decorated so writes publish to peers");
     }
 
+    [Fact]
+    public async Task The_startup_hook_actually_creates_the_replication_actor()
+    {
+        // The test SecretReplicationStarter's docs have promised since the 0.2.x adoption, runnable
+        // now that the upstream deadlock is fixed: build the container the way the host does, start
+        // the hook, and prove the replication actor exists. On 0.2.0 this fails because the actor
+        // is never spawned (inert replicator); on 0.2.1 it deadlocks in the resolve (scadaproj#1).
+        //
+        // Akka.TestKit.Xunit2 is xunit-v2-only and this project is on xunit.v3 (CS0433), so this
+        // uses a plain self-joined single-node cluster — which is also all the actor needs: its
+        // constructor gets the DistributedPubSub mediator, no peers required.
+        ActorSystem system = ActorSystem.Create(
+            "otopcua-secrets-gate",
+            Akka.Configuration.ConfigurationFactory.ParseString("""
+                akka {
+                  loglevel = WARNING
+                  actor.provider = cluster
+                  remote.dot-netty.tcp {
+                    hostname = "127.0.0.1"
+                    public-hostname = "127.0.0.1"
+                    port = 0
+                  }
+                }
+                """));
+
+        try
+        {
+            var cluster = Akka.Cluster.Cluster.Get(system);
+            cluster.Join(cluster.SelfAddress);
+
+            using ServiceProvider sp = BuildProvider(replicationEnabled: true, system);
+
+            // Startup order mirrors the host: hosted services run, nothing else has resolved the
+            // store yet. The hook's resolve is the moment 0.2.1 hung forever, so it runs under a
+            // watchdog — a regression should fail the test, not the whole run.
+            var starter = sp.GetServices<IHostedService>().OfType<SecretReplicationStarter>().Single();
+
+            Task start = starter.StartAsync(TestContext.Current.CancellationToken);
+            Task first = await Task.WhenAny(
+                start, Task.Delay(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken));
+            first.ShouldBe(start,
+                "resolving ISecretStore from the startup hook did not complete — the scadaproj#1 "
+                + "DI deadlock has regressed");
+            await start;
+
+            // The store must be the replicating decorator, and the node's replication actor must
+            // genuinely exist under the configured name — not merely be registered.
+            sp.GetRequiredService<ISecretStore>().ShouldBeOfType<ReplicatingSecretStore>();
+
+            IActorRef actor = await system
+                .ActorSelection("/user/zb-secret-replication")
+                .ResolveOne(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
+            actor.Path.Name.ShouldBe("zb-secret-replication");
+        }
+        finally
+        {
+            await system.Terminate();
+        }
+    }
+
     [Fact]
     public void Replication_enabled_still_registers_the_undecorated_concrete_store()
     {