From c878fbbd03f9ff3b1afe27c50e0434f10e3d082f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 18 Jul 2026 16:00:14 -0400 Subject: [PATCH] feat(secrets): enable Akka clustered secret replication on the docker-dev rig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns on Secrets:Replication:Enabled across all six docker-dev host nodes via one x-secrets-env anchor: replication enabled, a 5s AnnounceInterval so anti-entropy is observable during dev exercise, and the ONE shared dev-only KEK every node requires (committed default per the rig's zero-operator-step pattern, overridable via OTOPCUA_SECRETS_KEK; never reuse outside this rig). Live-proven on the rig with Secrets 0.2.2: all six nodes start with NO startup deadlock (the scadaproj#1 signature never appeared), the mesh forms, and a secret seeded into site-a-1's local store alone converged to all five peers within 60s — byte-identical ciphertext on all six nodes (same sha256, kek_id sha256:f3f0c2056e7a), decrypt-verified on central-2 and site-b-2. site-b-2 restart rejoined cleanly with the row intact; the fleet also survived a full Docker-engine restart with replication enabled. Found during verification, tracked separately: the AdminUI /admin/secrets page renders but is non-interactive (Secrets.Ui RCL components carry no @rendermode and the host applies render modes per-page), so Add secret does nothing - pre-existing, unrelated to replication. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts --- docker-dev/docker-compose.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index d19fe9bd..af4cd50e 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -47,6 +47,29 @@ name: otopcua-dev +# ── Shared secret-replication env (all six host nodes) ────────────────────────── +# Akka peer-to-peer clustered secret replication rides the existing `otopcua` Akka +# mesh (DistributedPubSub topic `zb-mom-ww-secrets`) across all six host nodes. Two +# hard requirements are wired here, identically, on every node: +# 1. Secrets__Replication__Enabled=true → SecretsRegistration.AddOtOpcUaSecrets +# switches from plain AddZbSecrets to AddZbSecretsAkkaReplication + the eager +# SecretReplicationStarter hosted service, so every node joins anti-entropy. +# 2. ZB_SECRETS_MASTER_KEY → the ONE shared 32-byte base64 KEK. Every node MUST +# carry the SAME key or peers fail closed decrypting each other's rows +# (kek_id mismatch). appsettings.json binds Secrets:MasterKey:Source=Environment +# / EnvVarName=ZB_SECRETS_MASTER_KEY, so this env var IS the KEK on every node. +# The KEK below is a committed DEV-ONLY default (matches the rig's zero-operator-step +# philosophy, like the committed dev SQL password). Override for a fresh key with: +# OTOPCUA_SECRETS_KEK=$(openssl rand -base64 32) docker compose ... up -d +# NEVER reuse this key outside this local dev rig. +# AnnounceInterval is shortened to 5s (default 30s) so anti-entropy convergence is +# observable quickly during dev exercise. Each node keeps its own local SQLite store +# (Secrets:SqlitePath=otopcua-secrets.db, per-container); replication syncs them. +x-secrets-env: &secrets-env + Secrets__Replication__Enabled: "true" + Secrets__Replication__AnnounceInterval: "00:00:05" + ZB_SECRETS_MASTER_KEY: "${OTOPCUA_SECRETS_KEK:-ZYGhIX0luS/XsevpCB2W18jYHMcqO6AjM9oXy+T6Zp4=}" + services: sql: @@ -145,6 +168,7 @@ services: sql: { condition: service_healthy } migrator: { condition: service_completed_successfully } environment: + <<: *secrets-env OTOPCUA_ROLES: "admin,driver" ASPNETCORE_URLS: "http://+:9000" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" @@ -210,6 +234,7 @@ services: central-1: { condition: service_started } migrator: { condition: service_completed_successfully } environment: + <<: *secrets-env OTOPCUA_ROLES: "admin,driver" ASPNETCORE_URLS: "http://+:9000" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" @@ -280,6 +305,7 @@ services: Cluster__PublicHostname: "site-a-1" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + <<: *secrets-env # Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/ # mem_reservation are inherited from the *otopcua-host anchor. Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" @@ -304,6 +330,7 @@ services: Cluster__PublicHostname: "site-a-2" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + <<: *secrets-env Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" @@ -326,6 +353,7 @@ services: Cluster__PublicHostname: "site-b-1" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + <<: *secrets-env Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" @@ -346,6 +374,7 @@ services: Cluster__PublicHostname: "site-b-2" Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "driver" + <<: *secrets-env Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}"