From 5f48f81d5ade6ea4ddcb347d92a944d7a3d3894b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 7 Jun 2026 03:08:17 -0400 Subject: [PATCH] feat(docker-dev): single-mesh hub-and-spoke (central-1/2 + driver-only sites) --- docker-dev/docker-compose.yml | 310 +++++++++++++--------------------- 1 file changed, 122 insertions(+), 188 deletions(-) diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index b2a09071..add0daaf 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -1,40 +1,46 @@ -# docker-dev/ — Mac-friendly multi-cluster fleet for v2 development + manual UI exercise. +# docker-dev/ — Mac-friendly single-mesh hub-and-spoke fleet for v2 development + manual UI exercise. # -# Stack (3 separate Akka clusters — all share the single `OtOpcUa` ConfigDb): -# sql SQL Server 2022 — hosts the one ConfigDb that all three clusters use -# ldap OpenLDAP with the dev users from C:\publish\glauth\auth.md mirrored in +# Topology: ONE Akka mesh seeded by `central-1`. Logical separation between +# tenants is by ServerCluster.ClusterId rows (MAIN / SITE-A / SITE-B) in the one +# shared `OtOpcUa` ConfigDb — NOT by separate meshes. All six host nodes join the +# same gossip ring and the central UI deploys to every cluster over it. # -# Main cluster (existing — split-role admin / driver pair on a single Akka mesh): -# admin-a OtOpcUa.Host with OTOPCUA_ROLES=admin (seed) -# admin-b OtOpcUa.Host with OTOPCUA_ROLES=admin (joins admin-a) -# driver-a OtOpcUa.Host with OTOPCUA_ROLES=driver (joins via admin-a) -# driver-b OtOpcUa.Host with OTOPCUA_ROLES=driver (joins via admin-a) +# Stack: +# sql SQL Server 2022 — hosts the one ConfigDb every node uses +# cluster-seed one-shot mssql-tools job that INSERTs the ServerCluster + +# ClusterNode rows scoping each tenant, then exits (idempotent) # -# Site A cluster (2-node fused admin+driver): -# site-a-1, site-a-2 OTOPCUA_ROLES=admin,driver, seed = site-a-1 +# central-1, central-2 OTOPCUA_ROLES=admin,driver — the ONLY UI + deploy +# singleton, plus the MAIN cluster's OPC UA publishers. +# Reachable at http://localhost:9200 (via Traefik). +# central-1 is the Akka seed node; central-2 joins it. +# site-a-1, site-a-2 OTOPCUA_ROLES=driver — driver-only members of the same +# site-b-1, site-b-2 mesh, scoped to SITE-A / SITE-B by ClusterId. They +# serve no UI and authenticate no users; the central +# cluster manages and deploys to them over the mesh. # -# Site B cluster (2-node fused admin+driver): -# site-b-1, site-b-2 OTOPCUA_ROLES=admin,driver, seed = site-b-1 +# Auth is real LDAP against the shared GLAuth on the Linux Docker host +# (10.100.0.35:3893, dc=zb,dc=local) — there is no LDAP container here. +# Only the admin-role central nodes carry the Security__Ldap__* block. +# Sign in `multi-role` / `password`. # -# traefik PathPrefix → main cluster admin-a/admin-b; Host(`site-a.localhost`) → -# site-a-*; Host(`site-b.localhost`) → site-b-*. Add the two site hosts to -# your /etc/hosts (or rely on macOS `.localhost` auto-resolution). +# traefik PathPrefix(`/`) → central-1 / central-2 (the single UI route). # -# Multi-tenancy: ConfigDb is one schema with a `ServerCluster` table; each Akka cluster -# corresponds to a row in it (ClusterId = "MAIN" / "SITE-A" / "SITE-B"), and each node's -# `ClusterNode.NodeId` points back at the row that owns it. After first boot, sign in to -# any cluster's Admin UI and create the matching ServerCluster + ClusterNode rows via -# /clusters and /hosts so the runtime knows what configuration scope applies. +# OPC UA endpoints (host-side port → container 4840): +# central-1 :4840 central-2 :4841 +# site-a-1 :4842 site-a-2 :4843 +# site-b-1 :4844 site-b-2 :4845 # -# Akka mesh isolation: same system name "otopcua" + same remoting port 4053 inside each -# container's own network namespace, but with disjoint seed-node lists — gossip never -# crosses between the three meshes. +# Headless deploy: POST http://localhost:9200/api/deployments with the +# X-Api-Key header (Security__DeployApiKey = "docker-dev-deploy-key"). +# +# SQL persistence: the otopcua-mssql-data named volume keeps the ConfigDb schema +# + seeded clusters across `docker compose up` cycles; without it a recreate +# silently drops the OtOpcUa database. # # Usage: # docker compose -f docker-dev/docker-compose.yml up -d --build -# open http://localhost # main cluster Blazor admin UI -# open http://site-a.localhost # site A admin UI -# open http://site-b.localhost # site B admin UI +# open http://localhost:9200 # central Blazor admin UI # open http://localhost:8089 # Traefik dashboard (8080 is the sister scadalink stack) # # Tear-down: docker compose -f docker-dev/docker-compose.yml down -v @@ -66,8 +72,8 @@ services: # ── Cluster seed (one-shot) ──────────────────────────────────────────────── # Waits for SQL + the host containers' EF auto-migration, then INSERTs the - # three ServerCluster rows and the six ClusterNode rows that scope each Akka - # mesh inside the shared OtOpcUa ConfigDb. Idempotent — re-runs are no-ops. + # three ServerCluster rows and the six ClusterNode rows that scope each tenant + # inside the shared OtOpcUa ConfigDb. Idempotent — re-runs are no-ops. cluster-seed: image: mcr.microsoft.com/mssql-tools:latest depends_on: @@ -78,115 +84,33 @@ services: entrypoint: ["/bin/bash", "/seed/entrypoint.sh"] restart: "no" - # OpenLDAP was previously here but the bitnami/openldap:2.6 image was retired - # (manifest gone) and bitnamilegacy/openldap:2.6 crashes during LDIF setup with - # exit 68. For the dev compose every host container now runs with - # Security__Ldap__DevStubMode=true, so any non-empty username/password - # signs in as `Administrator`. Restore a real LDAP service when there's a need - # for end-to-end LDAP coverage (the host code path is unchanged). + # A local OpenLDAP container used to live here, but the bitnami/openldap:2.6 + # image was retired (manifest gone) and bitnamilegacy/openldap:2.6 crashes + # during LDIF setup (exit 68). Rather than stub auth, the central (admin-role) + # containers bind the shared GLAuth on the Linux Docker host (Security__Ldap__* + # below: 10.100.0.35:3893, dc=zb,dc=local, DevStubMode=false) — so dev auth + # exercises the real LDAP bind + group→role path. Sign in `multi-role` / + # `password` (all roles) or any shared test user / `password`. - admin-a: &otopcua-host + # ── Central cluster (2-node fused admin+driver) ───────────────────────────── + # The only UI + deploy singleton; also the MAIN cluster's OPC UA publishers. + # central-1 seeds the single Akka mesh that every other node joins. + + central-1: &otopcua-host build: context: .. dockerfile: docker-dev/Dockerfile image: otopcua-host:dev depends_on: sql: { condition: service_healthy } - environment: - OTOPCUA_ROLES: "admin" - ASPNETCORE_URLS: "http://+:9000" - ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" - Cluster__Hostname: "0.0.0.0" - Cluster__Port: "4053" - Cluster__PublicHostname: "admin-a" - Cluster__SeedNodes__0: "akka.tcp://otopcua@admin-a:4053" - Cluster__Roles__0: "admin" - Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" - Security__Jwt__Issuer: "otopcua-dev" - Security__Jwt__Audience: "otopcua-dev" - Security__Ldap__Enabled: "true" - Security__Ldap__DevStubMode: "false" - Security__Ldap__Server: "10.100.0.35" - Security__Ldap__Port: "3893" - Security__Ldap__Transport: "None" - Security__Ldap__AllowInsecure: "true" - Security__Ldap__SearchBase: "dc=zb,dc=local" - Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" - Security__Ldap__ServiceAccountPassword: "serviceaccount123" - Security__DeployApiKey: "docker-dev-deploy-key" - GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" - - admin-b: - <<: *otopcua-host - environment: - OTOPCUA_ROLES: "admin" - ASPNETCORE_URLS: "http://+:9000" - ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" - Cluster__Hostname: "0.0.0.0" - Cluster__Port: "4053" - Cluster__PublicHostname: "admin-b" - Cluster__SeedNodes__0: "akka.tcp://otopcua@admin-a:4053" - Cluster__Roles__0: "admin" - Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" - Security__Jwt__Issuer: "otopcua-dev" - Security__Jwt__Audience: "otopcua-dev" - Security__Ldap__Enabled: "true" - Security__Ldap__DevStubMode: "false" - Security__Ldap__Server: "10.100.0.35" - Security__Ldap__Port: "3893" - Security__Ldap__Transport: "None" - Security__Ldap__AllowInsecure: "true" - Security__Ldap__SearchBase: "dc=zb,dc=local" - Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" - Security__Ldap__ServiceAccountPassword: "serviceaccount123" - Security__DeployApiKey: "docker-dev-deploy-key" - GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" - - driver-a: - <<: *otopcua-host - environment: - OTOPCUA_ROLES: "driver" - ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" - Cluster__Hostname: "0.0.0.0" - Cluster__Port: "4053" - Cluster__PublicHostname: "driver-a" - Cluster__SeedNodes__0: "akka.tcp://otopcua@admin-a:4053" - Cluster__Roles__0: "driver" - # Resolved at runtime by GalaxyDriver.ResolveApiKey when a DriverInstance's - # Gateway.ApiKeySecretRef = "env:GALAXY_MXGW_API_KEY". - GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" - ports: - - "4840:4840" - - driver-b: - <<: *otopcua-host - environment: - OTOPCUA_ROLES: "driver" - ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" - Cluster__Hostname: "0.0.0.0" - Cluster__Port: "4053" - Cluster__PublicHostname: "driver-b" - Cluster__SeedNodes__0: "akka.tcp://otopcua@admin-a:4053" - Cluster__Roles__0: "driver" - GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" - ports: - - "4841:4840" - - # ── Site A cluster (2-node fused admin+driver) ────────────────────────────── - # Shares the OtOpcUa ConfigDb with the main + site-b clusters; multi-tenancy is - # enforced by ServerCluster.ClusterId rows (configure via /clusters after boot). - # Akka isolation comes from the disjoint seed list (seed = site-a-1). - - site-a-1: - <<: *otopcua-host environment: OTOPCUA_ROLES: "admin,driver" ASPNETCORE_URLS: "http://+:9000" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" - Cluster__PublicHostname: "site-a-1" - Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-1:4053" + Cluster__PublicHostname: "central-1" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" Cluster__Roles__0: "admin" Cluster__Roles__1: "driver" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" @@ -203,6 +127,62 @@ services: Security__Ldap__ServiceAccountPassword: "serviceaccount123" Security__DeployApiKey: "docker-dev-deploy-key" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" + ports: + - "4840:4840" + + central-2: + <<: *otopcua-host + depends_on: + sql: { condition: service_healthy } + central-1: { condition: service_started } + environment: + OTOPCUA_ROLES: "admin,driver" + ASPNETCORE_URLS: "http://+:9000" + ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" + Cluster__Hostname: "0.0.0.0" + Cluster__Port: "4053" + Cluster__PublicHostname: "central-2" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__Roles__0: "admin" + Cluster__Roles__1: "driver" + Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" + Security__Jwt__Issuer: "otopcua-dev" + Security__Jwt__Audience: "otopcua-dev" + Security__Ldap__Enabled: "true" + Security__Ldap__DevStubMode: "false" + Security__Ldap__Server: "10.100.0.35" + Security__Ldap__Port: "3893" + Security__Ldap__Transport: "None" + Security__Ldap__AllowInsecure: "true" + Security__Ldap__SearchBase: "dc=zb,dc=local" + Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" + Security__Ldap__ServiceAccountPassword: "serviceaccount123" + Security__DeployApiKey: "docker-dev-deploy-key" + GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" + ports: + - "4841:4840" + + # ── Site A cluster (2-node driver-only) ───────────────────────────────────── + # Driver-only members of the single mesh, scoped to SITE-A by ClusterId. No UI, + # no user auth — managed + deployed to by the central cluster over the mesh. + # All site nodes seed central-1. + + site-a-1: + <<: *otopcua-host + depends_on: + sql: { condition: service_healthy } + central-1: { condition: service_started } + environment: + OTOPCUA_ROLES: "driver" + ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" + Cluster__Hostname: "0.0.0.0" + Cluster__Port: "4053" + Cluster__PublicHostname: "site-a-1" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__Roles__0: "driver" + # Resolved at runtime by GalaxyDriver.ResolveApiKey when a DriverInstance's + # Gateway.ApiKeySecretRef = "env:GALAXY_MXGW_API_KEY". + GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" ports: - "4842:4840" @@ -210,61 +190,34 @@ services: <<: *otopcua-host depends_on: sql: { condition: service_healthy } - site-a-1: { condition: service_started } + central-1: { condition: service_started } environment: - OTOPCUA_ROLES: "admin,driver" - ASPNETCORE_URLS: "http://+:9000" + OTOPCUA_ROLES: "driver" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-a-2" - Cluster__SeedNodes__0: "akka.tcp://otopcua@site-a-1:4053" - Cluster__Roles__0: "admin" - Cluster__Roles__1: "driver" - Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" - Security__Jwt__Issuer: "otopcua-dev" - Security__Jwt__Audience: "otopcua-dev" - Security__Ldap__Enabled: "true" - Security__Ldap__DevStubMode: "false" - Security__Ldap__Server: "10.100.0.35" - Security__Ldap__Port: "3893" - Security__Ldap__Transport: "None" - Security__Ldap__AllowInsecure: "true" - Security__Ldap__SearchBase: "dc=zb,dc=local" - Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" - Security__Ldap__ServiceAccountPassword: "serviceaccount123" - Security__DeployApiKey: "docker-dev-deploy-key" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__Roles__0: "driver" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" ports: - "4843:4840" - # ── Site B cluster (2-node fused admin+driver) ────────────────────────────── + # ── Site B cluster (2-node driver-only) ───────────────────────────────────── site-b-1: <<: *otopcua-host + depends_on: + sql: { condition: service_healthy } + central-1: { condition: service_started } environment: - OTOPCUA_ROLES: "admin,driver" - ASPNETCORE_URLS: "http://+:9000" + OTOPCUA_ROLES: "driver" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-b-1" - Cluster__SeedNodes__0: "akka.tcp://otopcua@site-b-1:4053" - Cluster__Roles__0: "admin" - Cluster__Roles__1: "driver" - Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" - Security__Jwt__Issuer: "otopcua-dev" - Security__Jwt__Audience: "otopcua-dev" - Security__Ldap__Enabled: "true" - Security__Ldap__DevStubMode: "false" - Security__Ldap__Server: "10.100.0.35" - Security__Ldap__Port: "3893" - Security__Ldap__Transport: "None" - Security__Ldap__AllowInsecure: "true" - Security__Ldap__SearchBase: "dc=zb,dc=local" - Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" - Security__Ldap__ServiceAccountPassword: "serviceaccount123" - Security__DeployApiKey: "docker-dev-deploy-key" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__Roles__0: "driver" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" ports: - "4844:4840" @@ -273,30 +226,15 @@ services: <<: *otopcua-host depends_on: sql: { condition: service_healthy } - site-b-1: { condition: service_started } + central-1: { condition: service_started } environment: - OTOPCUA_ROLES: "admin,driver" - ASPNETCORE_URLS: "http://+:9000" + OTOPCUA_ROLES: "driver" ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" Cluster__Hostname: "0.0.0.0" Cluster__Port: "4053" Cluster__PublicHostname: "site-b-2" - Cluster__SeedNodes__0: "akka.tcp://otopcua@site-b-1:4053" - Cluster__Roles__0: "admin" - Cluster__Roles__1: "driver" - Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" - Security__Jwt__Issuer: "otopcua-dev" - Security__Jwt__Audience: "otopcua-dev" - Security__Ldap__Enabled: "true" - Security__Ldap__DevStubMode: "false" - Security__Ldap__Server: "10.100.0.35" - Security__Ldap__Port: "3893" - Security__Ldap__Transport: "None" - Security__Ldap__AllowInsecure: "true" - Security__Ldap__SearchBase: "dc=zb,dc=local" - Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" - Security__Ldap__ServiceAccountPassword: "serviceaccount123" - Security__DeployApiKey: "docker-dev-deploy-key" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__Roles__0: "driver" GALAXY_MXGW_API_KEY: "${GALAXY_MXGW_API_KEY:-mxgw_otopcua2_GI7-tNozYE6cXGUSgEzL3AHDV7bYcYIHdMwKYgyHdX4}" ports: - "4845:4840" @@ -314,12 +252,8 @@ services: volumes: - ./traefik-dynamic.yml:/etc/traefik/dynamic.yml:ro depends_on: - - admin-a - - admin-b - - site-a-1 - - site-a-2 - - site-b-1 - - site-b-2 + - central-1 + - central-2 volumes: # SQL Server data dir — persists the OtOpcUa ConfigDb across container recreates.