# docker-dev/ — Mac-friendly THREE-mesh hub-and-spoke fleet for v2 development + manual UI exercise. # # Topology (per-cluster mesh Phase 6, 2026-07-24): THREE independent 2-node Akka # meshes, not one six-node mesh. Every node still runs the same ActorSystem name # `otopcua` and all six containers still sit on the ONE shared docker-dev network # — separation is purely by per-pair self-first Cluster__SeedNodes partitioning, # never by network isolation. This is deliberately faithful to production, where # the three meshes are separated by a WAN, not a firewall: nothing here stops # central-1 opening a bare TCP connection to site-a-1's Akka port, it just never # tries, because no seed list and no gossip ever names it. # # MAIN central-1 + central-2 (fused admin+driver, roles admin,driver,cluster-MAIN) # SITE-A site-a-1 + site-a-2 (driver-only, roles driver,cluster-SITE-A) # SITE-B site-b-1 + site-b-2 (driver-only, roles driver,cluster-SITE-B) # # Each pair seeds ONLY itself (self-first, partner-second) — no node lists a node # outside its own mesh as a seed. Logical/tenant separation (ServerCluster.ClusterId # rows MAIN / SITE-A / SITE-B in the one shared `OtOpcUa` ConfigDb) now COINCIDES # with mesh separation: a `cluster-` Akka role marks which mesh + which # ServerCluster row a node belongs to (RoleParser.ClusterRolePrefix = "cluster-"). # # Since central no longer shares a gossip ring with the site meshes, it reaches # them over the three explicit split-topology transports instead of gossip: # - Command plane: MeshTransport__Mode=ClusterClient (central dials each site # mesh's ClusterClientReceptionist; DPS is gossip-only and # would not cross the mesh boundary at all now) # - Live telemetry: Telemetry__Mode=Grpc on every driver node (all six) + # TelemetryDial__Mode=Grpc on the admin pair (central-1/2) # — the site node hosts the gRPC server, central dials in # - Deployed config: ConfigSource__Mode=FetchAndCache on the site nodes (already # true since Phase 3/4 — driver-only nodes have no ConfigDb # connection at all); central stays ConfigSource-less/Direct # and keeps serving via ConfigServe. # SplitTopologyTransportValidator (Cluster project) REFUSES to start any node # carrying a `cluster-*` role that is left on a DPS-shaped transport — these # three flips are therefore mandatory here, not optional dark switches. # # Stack: # sql SQL Server 2022 — hosts the one ConfigDb every node uses # cluster-seed one-shot mssql-tools job that INSERTs the ServerCluster + # ClusterNode rows scoping each tenant, then exits (idempotent) # # central-1, central-2 OTOPCUA_ROLES=admin,driver,cluster-MAIN — the ONLY UI + # deploy singleton, plus the MAIN cluster's OPC UA # publishers. Reachable at http://localhost:9200 (via # Traefik). Both are Akka seed nodes for the MAIN mesh # ONLY, each listing ITSELF first (2026-07-22) so either # can cold-start while the other is down. # site-a-1, site-a-2 OTOPCUA_ROLES=driver,cluster-SITE-A — driver-only # site-b-1, site-b-2 OTOPCUA_ROLES=driver,cluster-SITE-B — members of their # OWN 2-node mesh (self-first seeded within the pair, NOT # off central). They serve no UI, but they DO expose OPC UA # data-plane endpoints (4842-4845) and authenticate those # users against the same shared GLAuth (via the *ldap-env # anchor); central manages + deploys to them over the # ClusterClient/gRPC/FetchAndCache transports above, not # gossip. # # Auth is real LDAP against the shared GLAuth on the Linux Docker host # (10.100.0.35:3893, dc=zb,dc=local) — there is no LDAP container here. # EVERY host node carries the Security__Ldap__* block (shared *ldap-env anchor): # the driver-only site nodes serve OPC UA endpoints too, and LDAP options are # validated at boot on all of them (a node with no LDAP config crashes with # OptionsValidationException — transport None + AllowInsecure false). # Sign in `multi-role` / `password`. # # traefik PathPrefix(`/`) → central-1 / central-2 (the single UI route). # # OPC UA endpoints (host-side port → container 4840): # central-1 :4840 central-2 :4841 # site-a-1 :4842 site-a-2 :4843 # site-b-1 :4844 site-b-2 :4845 # # Headless deploy: POST http://localhost:9200/api/deployments with the # X-Api-Key header (Security__DeployApiKey = "docker-dev-deploy-key"). # # SQL persistence: the otopcua-mssql-data named volume keeps the ConfigDb schema # + seeded clusters across `docker compose up` cycles; without it a recreate # silently drops the OtOpcUa database. # # Per-cluster mesh Phase 5 (gRPC live-telemetry stream): every driver-role node (all six, # including the fused central pair) carries Telemetry__GrpcListenPort=4056 (dedicated h2c # listener, verified free against every other per-node port: Akka 4053, ConfigServe 4055 on # central, LocalDb sync 9001 on site-a, OPC UA 4840, HTTP 9000) + Telemetry__ApiKey # (committed dev key, matches the SQL password / secrets KEK / ConfigServe key exception). # central-1/central-2 also carry the matching TelemetryDial__ApiKey. As of Phase 6, Telemetry__Mode # is set to "Grpc" on all six nodes and TelemetryDial__Mode is set to "Grpc" on central-1/central-2 # ONLY — SplitTopologyTransportValidator requires it once a node carries a cluster-* role, so this # is no longer an optional live-gate step (see the Phase 5 comment history in git blame for the # prior dark-switch shape). The ClusterNode.GrpcPort column (seeded by # docker-dev/seed/seed-clusters.sql) points central's dial supervisor at each node's :4056. # # Usage: # docker compose -f docker-dev/docker-compose.yml up -d --build # open http://localhost:9200 # central Blazor admin UI # open http://localhost:8089 # Traefik dashboard (8080 is the sister scadalink stack) # # Tear-down: docker compose -f docker-dev/docker-compose.yml down -v name: otopcua-dev # ── Shared secret-replication env (all six host nodes) ────────────────────────── # Akka peer-to-peer clustered secret replication rides the existing `otopcua` Akka # mesh (DistributedPubSub topic `zb-mom-ww-secrets`) across all six host nodes. Two # hard requirements are wired here, identically, on every node: # 1. Secrets__Replication__Enabled=true → SecretsRegistration.AddOtOpcUaSecrets # switches from plain AddZbSecrets to AddZbSecretsAkkaReplication + the eager # SecretReplicationStarter hosted service, so every node joins anti-entropy. # 2. ZB_SECRETS_MASTER_KEY → the ONE shared 32-byte base64 KEK. Every node MUST # carry the SAME key or peers fail closed decrypting each other's rows # (kek_id mismatch). appsettings.json binds Secrets:MasterKey:Source=Environment # / EnvVarName=ZB_SECRETS_MASTER_KEY, so this env var IS the KEK on every node. # The KEK below is a committed DEV-ONLY default (matches the rig's zero-operator-step # philosophy, like the committed dev SQL password). Override for a fresh key with: # OTOPCUA_SECRETS_KEK=$(openssl rand -base64 32) docker compose ... up -d # NEVER reuse this key outside this local dev rig. # AnnounceInterval is shortened to 5s (default 30s) so anti-entropy convergence is # observable quickly during dev exercise. Each node keeps its own local SQLite store # (Secrets:SqlitePath=otopcua-secrets.db, per-container); replication syncs them. # # Per-cluster mesh Phase 6: this DPS topic now rides each pair's OWN 2-node mesh, not one # shared six-node mesh — so secret replication is PAIR-LOCAL. central-1/central-2 replicate # with each other; site-a-1/site-a-2 replicate with each other; site-b-1/site-b-2 replicate # with each other. There is no cross-cluster secret sync after the split (by design — the site # meshes don't share a gossip ring with central or with each other, consistent with Phase 4's # "driver-only nodes have no SQL, hence no SQL-backed cross-mesh hub" posture). No code change # was needed for this — Enabled=true + the same KEK on every node is still correct; it just now # converges within three independent 2-node groups instead of one six-node group. x-secrets-env: &secrets-env Secrets__Replication__Enabled: "true" Secrets__Replication__AnnounceInterval: "00:00:05" ZB_SECRETS_MASTER_KEY: "${OTOPCUA_SECRETS_KEK:-ZYGhIX0luS/XsevpCB2W18jYHMcqO6AjM9oXy+T6Zp4=}" # EVERY node that serves an OPC UA endpoint authenticates data-plane users against the shared GLAuth — # that includes the driver-only site nodes (ports 4842-4845), not just the admin/central pair. LDAP # options are validated at host start on ALL of them (Enabled=true by appsettings default, Transport=None, # AllowInsecure=false ⇒ OptionsValidationException), so a node that carries no LDAP config crashes at boot. # This anchor is merged into every host node's environment (via `<<: [*secrets-env, *ldap-env]`). Central # still also lists the block inline for readability; the explicit keys win over the merge, same values. x-ldap-env: &ldap-env Security__Ldap__Enabled: "true" Security__Ldap__DevStubMode: "false" Security__Ldap__Server: "10.100.0.35" Security__Ldap__Port: "3893" Security__Ldap__Transport: "None" Security__Ldap__AllowInsecure: "true" Security__Ldap__SearchBase: "dc=zb,dc=local" Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" Security__Ldap__ServiceAccountPassword: "serviceaccount123" Security__Ldap__GroupToRole__ReadOnly: "ReadOnly" Security__Ldap__GroupToRole__WriteOperate: "WriteOperate" Security__Ldap__GroupToRole__WriteTune: "WriteTune" Security__Ldap__GroupToRole__WriteConfigure: "WriteConfigure" Security__Ldap__GroupToRole__AlarmAck: "AlarmAck" # Readiness gate for a pair FOUNDER node (site-a-1 / site-b-1): passes once its Akka port 4053 is bound. # The partner (site-a-2 / site-b-2) waits on this via `depends_on: { condition: service_healthy }` so the # founder forms its 2-node mesh FIRST and the partner JOINS it — instead of both self-first seeds racing # and each forming its own single-node cluster (the split brain the Phase 6 live gate observed: 250/250, # both Primary). `service_started` is NOT enough — it fires the instant the container launches, long # before Akka binds. The image ships bash but no nc/curl, so the probe uses bash's /dev/tcp. start_period # gives the founder a head start to not just bind but FORM (a 1-node cluster forms within the same second # as the bind), so the partner's InitJoin lands on an existing cluster and is ack'd. x-akka-founder-healthcheck: &akka-founder-healthcheck test: ["CMD-SHELL", "bash -c ' UriFormatException "The URI is empty"). # To enable: OTOPCUA_HISTORIAN_ENABLED=true + OTOPCUA_HISTORIAN_ENDPOINT=https://:5222, and # export ServerHistorian__ApiKey=histgw__ at `docker compose up` (never commit the key). ServerHistorian__Enabled: "${OTOPCUA_HISTORIAN_ENABLED:-false}" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-}" ServerHistorian__ApiKey: "${ServerHistorian__ApiKey:-}" ServerHistorian__UseTls: "true" ServerHistorian__AllowUntrustedServerCertificate: "true" # Pin EF Core + ASP.NET Core to Warning so the per-poll Deployment SELECT / # "Executed DbCommand" Information|Debug lines stop flooding the Serilog # pipeline and starving the Akka cluster heartbeat thread. The host logs via # Serilog (AddZbSerilog → ReadFrom.Configuration); these env vars override # Serilog:MinimumLevel:Override:* (app/Akka levels are left untouched). Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # Consolidated pair-local config cache (ZB.MOM.WW.LocalDb). Every driver-role node # (central is admin,driver) gets one, on a per-node named volume so the cache survives # container recreates. The central pair leaves replication OFF (default) — only the # site-a pair replicates, as the enablement demo; site-b is the default-OFF pin. LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4840:4840" volumes: - otopcua-localdb-central-1:/app/data central-2: <<: *otopcua-host depends_on: sql: { condition: service_healthy } central-1: { condition: service_started } migrator: { condition: service_completed_successfully } environment: <<: [*secrets-env, *ldap-env] OTOPCUA_ROLES: "admin,driver,cluster-MAIN" ASPNETCORE_URLS: "http://+:9000" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "central-2" # Per-cluster mesh Phase 3 — serves the artifact to FetchAndCache site nodes (see central-1). # Always Direct; ConfigServe:ApiKey must match the site nodes' ConfigSource:ApiKey. ConfigServe__GrpcListenPort: "4055" ConfigServe__ApiKey: "configserve-docker-dev-key" ConfigSource__Mode: "Direct" # Per-cluster mesh Phase 5/6 (gRPC live-telemetry stream) — see central-1. central-2 carries # a cluster-* role + admin, so both Telemetry:Mode=Grpc and TelemetryDial:Mode=Grpc are # required (SplitTopologyTransportValidator); the dedicated :4056 listener binds regardless. Telemetry__GrpcListenPort: "4056" Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" TelemetryDial__Mode: "Grpc" TelemetryDial__ApiKey: "telemetry-docker-dev-key" # Both redundancy peers are seeds (#459 / ScadaBridge parity) — see central-1. SELF FIRST: # central-2 lists itself as seed-nodes[0], which is what lets it cold-start while central-1 # is down (it previously listed central-1 first and simply never came Up in that case). Per # per-cluster mesh Phase 6, central-2's seed list contains ONLY the MAIN pair — no site node. Cluster__SeedNodes__0: "akka.tcp://otopcua@central-2:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@central-1:4053" # Per-cluster mesh Phase 6: cluster-MAIN marks this node as a member of the MAIN mesh — see # central-1's Cluster__Roles comment. Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Cluster__Roles__2: "cluster-MAIN" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see central-1. # # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in # silence. Both central nodes are listed so a driver node's client can rotate to whichever one # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" Security__Jwt__Issuer: "otopcua-dev" Security__Jwt__Audience: "otopcua-dev" Security__Ldap__Enabled: "true" Security__Ldap__DevStubMode: "false" Security__Ldap__Server: "10.100.0.35" Security__Ldap__Port: "3893" Security__Ldap__Transport: "None" Security__Ldap__AllowInsecure: "true" Security__Ldap__SearchBase: "dc=zb,dc=local" Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" Security__Ldap__ServiceAccountPassword: "serviceaccount123" Security__DeployApiKey: "docker-dev-deploy-key" Security__Auth__DisableLogin: "true" # Data-plane group→role baseline (see central-1). Security__Ldap__GroupToRole__ReadOnly: "ReadOnly" Security__Ldap__GroupToRole__WriteOperate: "WriteOperate" Security__Ldap__GroupToRole__WriteTune: "WriteTune" Security__Ldap__GroupToRole__WriteConfigure: "WriteConfigure" Security__Ldap__GroupToRole__AlarmAck: "AlarmAck" # Server-side HistoryRead via the ZB.MOM.WW.HistorianGateway sidecar. DISABLED by default: the # docker-dev rig has no historian gateway to point at, and the historian-gateway cutover retired the # old Wonderware Host/Port/SharedSecret keys — the code now reads ServerHistorian__Endpoint and builds # a Uri from it, so Enabled=true with an empty Endpoint crashes the Host at startup # (HistorianGatewayClientAdapter.Create -> UriFormatException "The URI is empty"). # To enable: OTOPCUA_HISTORIAN_ENABLED=true + OTOPCUA_HISTORIAN_ENDPOINT=https://:5222, and # export ServerHistorian__ApiKey=histgw__ at `docker compose up` (never commit the key). ServerHistorian__Enabled: "${OTOPCUA_HISTORIAN_ENABLED:-false}" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-}" ServerHistorian__ApiKey: "${ServerHistorian__ApiKey:-}" ServerHistorian__UseTls: "true" ServerHistorian__AllowUntrustedServerCertificate: "true" # Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/ # mem_reservation are inherited from the *otopcua-host anchor. Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # Pair-local config cache; central pair leaves replication OFF (see central-1). LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4841:4840" volumes: - otopcua-localdb-central-2:/app/data # ── Site A cluster (2-node driver-only) ───────────────────────────────────── # Per-cluster mesh Phase 6: site-a-1 and site-a-2 form their OWN 2-node "SITE-A" Akka mesh, # separate from MAIN and SITE-B — self-first seeded WITHIN the pair, never off central. No UI, # no user auth; central manages + deploys to this mesh over ClusterClient/gRPC/FetchAndCache, # not gossip (central and site-a share no seed list, so they never form one ring). site-a-1: <<: *otopcua-host healthcheck: *akka-founder-healthcheck depends_on: sql: { condition: service_healthy } central-1: { condition: service_started } migrator: { condition: service_completed_successfully } environment: OTOPCUA_ROLES: "driver,cluster-SITE-A" # Per-cluster mesh Phase 4: driver-only site nodes hold NO ConfigDb connection string. Their # config comes from central via ConfigSource:Mode=FetchAndCache (gRPC fetch → LocalDb cache), their # scripted-alarm condition state lives in LocalDb, and central persists their deploy acks. Direct # mode is invalid here — ConfigSourceOptionsValidator fails start for a driver-only node set Direct. Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-a-1" # FetchAndCache is MANDATORY for driver-only nodes (validator-enforced): they fetch the deployed # artifact from central over gRPC (endpoints below) and read ONLY the LocalDb cache — no central # SQL config read (they have no connection string). Central stays Direct + keeps ConfigServe. # ConfigSource:ApiKey must equal central's ConfigServe:ApiKey (the interceptor is fail-closed). ConfigSource__Mode: "FetchAndCache" ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055" ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055" ConfigSource__ApiKey: "configserve-docker-dev-key" # Per-cluster mesh Phase 5/6 (gRPC live-telemetry stream) — see central-1/central-2. This # node carries cluster-SITE-A + driver, so SplitTopologyTransportValidator requires # Telemetry:Mode=Grpc (mandatory, not a dark switch, as of Phase 6). The dedicated :4056 # listener binds regardless (the node "always hosts" per docs/Telemetry.md). Port checked # free against this node's other ports: Akka 4053, OPC UA 4840, HTTP 9000, LocalDb sync 9001 # (below). ApiKey must equal central's TelemetryDial:ApiKey (fail-closed interceptor). Telemetry__GrpcListenPort: "4056" Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" # Per-cluster mesh Phase 6: site-a-1 seeds its OWN "SITE-A" mesh (itself first, site-a-2 # second) — it no longer seeds off central-1. Central is reached over ClusterClient/gRPC/ # FetchAndCache, never gossip; central and site-a share no seed list. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-1:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-a-2:4053" Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/central-2. Central dials this mesh's ClusterClientReceptionist via the contact # points below; there is no gossip path anymore. # # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in # silence. Both central nodes are listed so a driver node's client can rotate to whichever one # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: [*secrets-env, *ldap-env] # Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/ # mem_reservation are inherited from the *otopcua-host anchor. Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" # Resolved at runtime by GalaxyDriver.ResolveApiKey when a DriverInstance's # Gateway.ApiKeySecretRef = "env:GALAXY_MXGW_API_KEY". GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # Pair-local config cache + REPLICATION ON — the site-a pair is the enablement demo. # site-a-1 is the initiator: it binds the h2c sync listener on 9001 AND dials the peer. # Setting SyncListenPort makes Program.cs add a dedicated Http2-only Kestrel listener and # re-bind the primary HTTP port (an explicit Listen* otherwise discards ASPNETCORE_URLS). # Alarm store-and-forward buffer, Phase 2. Enabled on BOTH halves of the pair so the # buffer is a live, replicated table rather than an empty one — the live gate needs a real # queue to watch converge, and needs to see that only the Primary drains it. # # There is no HistorianGateway on this rig, so the endpoint below is deliberately # unresolvable: every drain attempt fails, which is exactly the historian-outage state the # buffer exists for. Note the endpoint must still be a syntactically valid absolute http(s) # URI or the host refuses to start — ServerHistorianOptionsValidator is consumer-gated, so # AlarmHistorian:Enabled=true makes it required even while ServerHistorian:Enabled=false. # # MaxAttempts is raised far above the production default of 10. With a permanently # unreachable gateway the default would dead-letter the whole queue about five minutes into # the outage, turning a buffering test into a dead-letter test. AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" # Required whenever the section is consumed: the gateway client validates its own options at # construction and throws on an empty key, so a keyless node crash-loops during Akka startup # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via # ServerHistorian__ApiKey from the environment. ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER # direction, so an http endpoint with the default UseTls=true crash-loops the host. ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__PeerAddress: "http://site-a-2:9001" # DEV-ONLY committed key (rig philosophy — like the SQL password + secrets KEK above). It # MUST be byte-identical on both nodes: the interceptor is fail-closed, so any mismatch # silently stops the pair converging. NEVER reuse this key outside this local dev rig. LocalDb__Replication__ApiKey: "dev-site-a-localdb-sync-key" # Row-count batching against gRPC's 4 MB message cap; artifact chunk rows are ≈171 KB, so # 16 × 171 KB ≈ 2.7 MB stays under the cap with headroom. LocalDb__Replication__MaxBatchSize: "16" ports: - "4842:4840" volumes: - otopcua-localdb-site-a-1:/app/data site-a-2: <<: *otopcua-host depends_on: sql: { condition: service_healthy } central-1: { condition: service_started } migrator: { condition: service_completed_successfully } # Serialize the SITE-A pair's startup so site-a-1 forms the mesh first and site-a-2 JOINS it — # exactly as central-2 waits on central-1. Both nodes are self-first seeds (seed[0]=self), so BOTH # run Akka's FirstSeedNodeProcess and, if they start simultaneously, EACH forms its own single-node # cluster (split brain — observed in the Phase 6 live gate: both logged "JOINING itself ... forming # a new cluster", both elected themselves Primary → 250/250 instead of 250/240). Waiting on site-a-1 # gives it the head start to be Up before site-a-2's InitJoin, so site-a-2 joins the existing mesh. site-a-1: { condition: service_healthy } environment: OTOPCUA_ROLES: "driver,cluster-SITE-A" # Per-cluster mesh Phase 4: no ConfigDb string on driver-only site nodes (see site-a-1). Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-a-2" # Per-cluster mesh Phase 4: FetchAndCache mandatory for driver-only nodes (see site-a-1) — no # ConfigDb string; fetch the artifact from central over gRPC and read only the LocalDb cache. ConfigSource__Mode: "FetchAndCache" ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055" ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055" ConfigSource__ApiKey: "configserve-docker-dev-key" # Per-cluster mesh Phase 5/6 (gRPC live-telemetry stream) — see central-1/site-a-1. # Telemetry:Mode=Grpc is mandatory as of Phase 6 (cluster-SITE-A + driver); the dedicated # :4056 listener binds regardless. Telemetry__GrpcListenPort: "4056" Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" # Per-cluster mesh Phase 6: site-a-2 seeds its OWN "SITE-A" mesh (itself first, site-a-1 # second) — it no longer seeds off central-1. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-2:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-a-1:4053" Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-A" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. # # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in # silence. Both central nodes are listed so a driver node's client can rotate to whichever one # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: [*secrets-env, *ldap-env] Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # site-a-2 is the passive half of the replicating pair: it binds the sync listener on 9001 # but does NOT dial (no PeerAddress). The replication stream is bidirectional, so a-1's # single dial carries both directions. Same key + MaxBatchSize as a-1 (byte-identical key # is mandatory — the interceptor fail-closes on a mismatch). # Alarm store-and-forward buffer, Phase 2. Enabled on BOTH halves of the pair so the # buffer is a live, replicated table rather than an empty one — the live gate needs a real # queue to watch converge, and needs to see that only the Primary drains it. # # There is no HistorianGateway on this rig, so the endpoint below is deliberately # unresolvable: every drain attempt fails, which is exactly the historian-outage state the # buffer exists for. Note the endpoint must still be a syntactically valid absolute http(s) # URI or the host refuses to start — ServerHistorianOptionsValidator is consumer-gated, so # AlarmHistorian:Enabled=true makes it required even while ServerHistorian:Enabled=false. # # MaxAttempts is raised far above the production default of 10. With a permanently # unreachable gateway the default would dead-letter the whole queue about five minutes into # the outage, turning a buffering test into a dead-letter test. AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" # Required whenever the section is consumed: the gateway client validates its own options at # construction and throws on an empty key, so a keyless node crash-loops during Akka startup # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via # ServerHistorian__ApiKey from the environment. ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER # direction, so an http endpoint with the default UseTls=true crash-loops the host. ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" LocalDb__SyncListenPort: "9001" LocalDb__Replication__ApiKey: "dev-site-a-localdb-sync-key" LocalDb__Replication__MaxBatchSize: "16" ports: - "4843:4840" volumes: - otopcua-localdb-site-a-2:/app/data # ── Site B cluster (2-node driver-only) ───────────────────────────────────── # Per-cluster mesh Phase 6: site-b-1 and site-b-2 form their OWN 2-node "SITE-B" Akka mesh, # separate from MAIN and SITE-A — same shape as site-a-1/site-a-2, see that pair's header. site-b-1: <<: *otopcua-host healthcheck: *akka-founder-healthcheck depends_on: sql: { condition: service_healthy } central-1: { condition: service_started } migrator: { condition: service_completed_successfully } environment: OTOPCUA_ROLES: "driver,cluster-SITE-B" # Per-cluster mesh Phase 4: no ConfigDb string on driver-only site nodes (see site-a-1). Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-b-1" # Per-cluster mesh Phase 4: FetchAndCache mandatory for driver-only nodes (see site-a-1) — no # ConfigDb string; fetch the artifact from central over gRPC and read only the LocalDb cache. ConfigSource__Mode: "FetchAndCache" ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055" ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055" ConfigSource__ApiKey: "configserve-docker-dev-key" # Per-cluster mesh Phase 5/6 (gRPC live-telemetry stream) — see central-1/site-a-1. # Telemetry:Mode=Grpc is mandatory as of Phase 6 (cluster-SITE-B + driver); the dedicated # :4056 listener binds regardless. Telemetry__GrpcListenPort: "4056" Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" # Per-cluster mesh Phase 6: site-b-1 seeds its OWN "SITE-B" mesh (itself first, site-b-2 # second) — it no longer seeds off central-1. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-b-1:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-b-2:4053" Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-B" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. # # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in # silence. Both central nodes are listed so a driver node's client can rotate to whichever one # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: [*secrets-env, *ldap-env] Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # site-b is the default-OFF pin: it gets the pair-local cache but NO SyncListenPort and NO # replication config, so no sync listener binds and ISyncStatus stays disconnected/Healthy. # This is what the live gate's check 8 verifies — the cache works locally with replication off. # Alarm store-and-forward with replication OFF — the other half of the site-b pin. The sink # must work as a plain node-local buffer here, with no sync listener and no peer traffic. AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" # Required whenever the section is consumed: the gateway client validates its own options at # construction and throws on an empty key, so a keyless node crash-loops during Akka startup # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via # ServerHistorian__ApiKey from the environment. ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER # direction, so an http endpoint with the default UseTls=true crash-loops the host. ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4844:4840" volumes: - otopcua-localdb-site-b-1:/app/data site-b-2: <<: *otopcua-host depends_on: sql: { condition: service_healthy } central-1: { condition: service_started } migrator: { condition: service_completed_successfully } # Serialize the SITE-B pair's startup so site-b-1 forms the mesh first and site-b-2 JOINS it — see # the identical comment on site-a-2. Without this both self-first seeds form their own single-node # cluster on simultaneous start (split brain). site-b-1: { condition: service_healthy } environment: OTOPCUA_ROLES: "driver,cluster-SITE-B" # Per-cluster mesh Phase 4: no ConfigDb string on driver-only site nodes (see site-a-1). Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-b-2" # Per-cluster mesh Phase 4: FetchAndCache mandatory for driver-only nodes (see site-a-1) — no # ConfigDb string; fetch the artifact from central over gRPC and read only the LocalDb cache. ConfigSource__Mode: "FetchAndCache" ConfigSource__CentralFetchEndpoints__0: "http://central-1:4055" ConfigSource__CentralFetchEndpoints__1: "http://central-2:4055" ConfigSource__ApiKey: "configserve-docker-dev-key" # Per-cluster mesh Phase 5/6 (gRPC live-telemetry stream) — see central-1/site-a-1. # Telemetry:Mode=Grpc is mandatory as of Phase 6 (cluster-SITE-B + driver); the dedicated # :4056 listener binds regardless. Telemetry__GrpcListenPort: "4056" Telemetry__Mode: "Grpc" Telemetry__ApiKey: "telemetry-docker-dev-key" # Per-cluster mesh Phase 6: site-b-2 seeds its OWN "SITE-B" mesh (itself first, site-b-1 # second) — it no longer seeds off central-1. Cluster__SeedNodes__0: "akka.tcp://otopcua@site-b-2:4053" Cluster__SeedNodes__1: "akka.tcp://otopcua@site-b-1:4053" Cluster__Roles__0: "driver" Cluster__Roles__1: "cluster-SITE-B" # Mesh command transport (per-cluster mesh Phase 2, made mandatory by Phase 6) — see # central-1/site-a-1. # # Contact points are node ADDRESSES ONLY. "/system/receptionist" is appended at construction # time, and MeshTransportOptionsValidator refuses to start a node whose contact carries an # actor-path suffix, because a doubled suffix resolves to nothing and drops every ack in # silence. Both central nodes are listed so a driver node's client can rotate to whichever one # is up — that rotation is why the comm actors are per-node and NOT cluster singletons. MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" <<: [*secrets-env, *ldap-env] Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" # site-b default-OFF pin (see site-b-1): cache on, replication off. # Alarm store-and-forward with replication OFF — the other half of the site-b pin. The sink # must work as a plain node-local buffer here, with no sync listener and no peer traffic. AlarmHistorian__Enabled: "true" AlarmHistorian__MaxAttempts: "1000000" ServerHistorian__Endpoint: "${OTOPCUA_HISTORIAN_ENDPOINT:-http://histgw-absent.invalid:5222}" # Required whenever the section is consumed: the gateway client validates its own options at # construction and throws on an empty key, so a keyless node crash-loops during Akka startup # rather than degrading. DEV-ONLY placeholder — it authenticates nothing, because the endpoint # above resolves nowhere. NEVER reuse this shape for a real gateway key; supply those via # ServerHistorian__ApiKey from the environment. ServerHistorian__ApiKey: "${OTOPCUA_HISTORIAN_APIKEY:-histgw_dev_rig-no-gateway-present}" # Must agree with the endpoint scheme above — the client throws on a mismatch in EITHER # direction, so an http endpoint with the default UseTls=true crash-loops the host. ServerHistorian__UseTls: "false" LocalDb__Path: "/app/data/otopcua-localdb.db" ports: - "4845:4840" volumes: - otopcua-localdb-site-b-2:/app/data traefik: image: traefik:v3.1 command: - --entrypoints.web.address=:80 - --providers.file.filename=/etc/traefik/dynamic.yml - --providers.file.watch=true - --api.insecure=true ports: - "9200:80" # host port 9200 → traefik :80 entrypoint (80 conflicts with scadabridge-traefik) - "8089:8080" # 8080 conflicts with the sister scadalink dev stack volumes: - ./traefik-dynamic.yml:/etc/traefik/dynamic.yml:ro depends_on: - central-1 - central-2 volumes: # SQL Server data dir — persists the OtOpcUa ConfigDb across container recreates. otopcua-mssql-data: # Per-node pair-local config-cache (ZB.MOM.WW.LocalDb) files. One volume per node — the # cache is node-local, not shared; it persists the cached deployment artifact (and, on the # site-a pair, the replicated oplog state) across container recreates so a node can boot # from cache after a restart even with central SQL down. otopcua-localdb-central-1: otopcua-localdb-central-2: otopcua-localdb-site-a-1: otopcua-localdb-site-a-2: otopcua-localdb-site-b-1: otopcua-localdb-site-b-2: