From 23555ed0fd76a0aefb2bb1a22b289b954d27588d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 04:56:57 -0400 Subject: [PATCH] chore(mesh): rig PSK config + docs stub Phase 0 task 0.6 of the ClusterClient->gRPC migration: wire the dark MeshTransport Grpc-mode keys into the docker-dev rig (GrpcPsk + CentralGrpcEndpoints on all six nodes, ClusterPsks on the central pair only), document the new keys in docs/Configuration.md as shipped dark/cutover pending, and record the Phase 0 live-gate evidence (rig boot with no validator fires, two deliberate-misconfig refusals, full suite green). Files gitea.dohertylan.com/dohertj2/lmxopcua#542 (FleetStatusBroadcaster mesh-blind post-Phase-6) and #543 (three pre-existing interceptors accept a whitespace-only key as set) as follow-ups out of Phase 0's scope. --- docker-dev/docker-compose.yml | 45 ++++++++ docs/Configuration.md | 18 ++- ...6-08-06-clusterclient-to-grpc-live-gate.md | 105 +++++++++++++++++- 3 files changed, 160 insertions(+), 8 deletions(-) diff --git a/docker-dev/docker-compose.yml b/docker-dev/docker-compose.yml index d0f9f89b..976cd354 100644 --- a/docker-dev/docker-compose.yml +++ b/docker-dev/docker-compose.yml @@ -348,6 +348,20 @@ services: MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" + # gRPC command transport (Phase 0 of the ClusterClient→gRPC migration, docs/plans/2026-08-06- + # clusterclient-to-grpc-design.md). SHIPPED DARK: Mode above stays ClusterClient, so none of + # this is dialled or served yet — only MeshTransportOptionsValidator's Grpc-mode rules read + # it. GrpcPsk is this node's OWN per-cluster key (MAIN, since central is cluster-MAIN); + # CentralGrpcEndpoints is the gRPC counterpart of CentralContactPoints above. ClusterPsks is + # CENTRAL-SIDE ONLY (IMeshPskProvider's override map, consulted before the Secrets store) — + # one entry per cluster central will dial once the switch flips, so only the central nodes + # carry it. + MeshTransport__GrpcPsk: "dev-mesh-psk-MAIN" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" + MeshTransport__ClusterPsks__MAIN: "dev-mesh-psk-MAIN" + MeshTransport__ClusterPsks__SITE-A: "dev-mesh-psk-SITE-A" + MeshTransport__ClusterPsks__SITE-B: "dev-mesh-psk-SITE-B" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" Security__Jwt__Issuer: "otopcua-dev" Security__Jwt__Audience: "otopcua-dev" @@ -473,6 +487,14 @@ services: MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" + # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK (Mode stays ClusterClient). + # ClusterPsks is central-side only; both central nodes carry the same three-cluster map. + MeshTransport__GrpcPsk: "dev-mesh-psk-MAIN" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" + MeshTransport__ClusterPsks__MAIN: "dev-mesh-psk-MAIN" + MeshTransport__ClusterPsks__SITE-A: "dev-mesh-psk-SITE-A" + MeshTransport__ClusterPsks__SITE-B: "dev-mesh-psk-SITE-B" Security__Jwt__SigningKey: "docker-dev-signing-key-with-at-least-32-bytes-of-utf8-content-12345" Security__Jwt__Issuer: "otopcua-dev" Security__Jwt__Audience: "otopcua-dev" @@ -594,6 +616,13 @@ services: MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" + # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK (Mode stays ClusterClient). + # GrpcPsk is this node's own key for its cluster (SITE-A); CentralGrpcEndpoints is the gRPC + # counterpart of CentralContactPoints above. No ClusterPsks here — that map is central-side + # only. + MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-A" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" <<: [*secrets-env, *ldap-env, *apikeys-env] # Quiet EF/AspNetCore SQL flood — see central-1 (Serilog override). mem_limit/ # mem_reservation are inherited from the *otopcua-host anchor. @@ -701,6 +730,11 @@ services: MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" + # gRPC command transport (Phase 0) — see central-1/site-a-1. SHIPPED DARK. Same SITE-A key as + # site-a-1 (must be byte-identical — both nodes authenticate against the same cluster's PSK). + MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-A" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" <<: [*secrets-env, *ldap-env, *apikeys-env] Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" @@ -789,6 +823,12 @@ services: MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" + # gRPC command transport (Phase 0) — see central-1. SHIPPED DARK. GrpcPsk is this node's own + # key for its cluster (SITE-B); CentralGrpcEndpoints mirrors CentralContactPoints above. No + # ClusterPsks here — that map is central-side only. + MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-B" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" <<: [*secrets-env, *ldap-env, *apikeys-env] Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" @@ -861,6 +901,11 @@ services: MeshTransport__Mode: "${OTOPCUA_MESH_MODE:-ClusterClient}" MeshTransport__CentralContactPoints__0: "akka.tcp://otopcua@central-1:4053" MeshTransport__CentralContactPoints__1: "akka.tcp://otopcua@central-2:4053" + # gRPC command transport (Phase 0) — see central-1/site-b-1. SHIPPED DARK. Same SITE-B key as + # site-b-1 (must be byte-identical). + MeshTransport__GrpcPsk: "dev-mesh-psk-SITE-B" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" <<: [*secrets-env, *ldap-env, *apikeys-env] Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" diff --git a/docs/Configuration.md b/docs/Configuration.md index f50f3cc1..a4a11183 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -165,8 +165,8 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only | Key | Type | Default | Meaning | |---|---|---|---| -| `Mode` | string | `Dps` | `Dps` = DistributedPubSub (the transport this repo has always used). `ClusterClient` = the mesh boundary. Any other value **refuses to start**. | -| `CentralContactPoints` | string[] | `[]` | Node-side only: central node **addresses** (`akka.tcp://@:`). Required under `ClusterClient`, ignored under `Dps`. | +| `Mode` | string | `Dps` | `Dps` = DistributedPubSub (the transport this repo has always used). `ClusterClient` = the mesh boundary. `Grpc` = the `mesh_transport.v1` gRPC contract — **shipped dark, cutover pending** (see below). Any other value **refuses to start**. | +| `CentralContactPoints` | string[] | `[]` | Node-side only: central node **addresses** (`akka.tcp://@:`). Required under `ClusterClient`, ignored under `Dps`/`Grpc`. | | `ContactRefreshSeconds` | int | `60` | Central-side only: how often central rebuilds its ClusterClient contact set from the `ClusterNode` rows. | **Addresses only — no actor-path suffix.** `/system/receptionist` is appended at construction time; a contact carrying `/user/` or `/system/` is rejected at startup, because a doubled suffix produces a path that resolves to nothing and drops every command in silence. @@ -181,6 +181,20 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only On the docker-dev rig every node carries these keys already; flip the whole rig with `OTOPCUA_MESH_MODE=ClusterClient` at `docker compose up`. +**`Grpc` mode — shipped dark, cutover pending** ([ClusterClient→gRPC migration design](plans/2026-08-06-clusterclient-to-grpc-design.md)). A third `Mode` value replaces `ClusterClient` with unary gRPC calls on the `mesh_transport.v1` contract: central dials each node's `NodeCommandService`, a node posts its deployment ack to central's `CentralCommandService`. Unlike `ClusterClient`, the command plane is authenticated end to end (`GrpcPsk` below) rather than relying on the mesh boundary alone. As of Phase 0 only the contract, DTO mapper, options and the two auth interceptors exist — the compiled default and the docker-dev rig both stay on `ClusterClient`, and setting `Mode=Grpc` today validates the keys below but nothing yet dials or serves over the contract. + +| Key | Type | Default | Meaning | +|---|---|---|---| +| `GrpcPsk` | string? | `null` (unset) | Node-side only, required under `Grpc` on a driver-role node: this node's pre-shared key for its own `Cluster`, presented as the bearer on outbound acks and required on inbound `NodeCommandService` calls. Production form is `${secret:OT-MESH-PSK-}`; supply a literal only on a rig, via `MeshTransport__GrpcPsk` — never commit one. | +| `CentralGrpcEndpoints` | string[] | `[]` | Node-side only, required under `Grpc` on a driver-role node: full gRPC base URLs (`scheme://host:port`, e.g. `http://central-1:4055` for h2c) of the central pair's `CentralCommandService`, tried in order, first success wins. The `Grpc` counterpart of `CentralContactPoints`, and replaces it — a node on `Grpc` never reads the Akka contact points. | +| `DispatchTimeoutSeconds` | int | `10` | Central-side per-node deadline for one `NodeCommandService.Execute` call. Must be positive under `Grpc`; validated on driver-role nodes (covers the central pair, which is always fused). | +| `AckTimeoutSeconds` | int | `10` | Node-side per-endpoint deadline for one `CentralCommandService.ReportApply` call. Must be positive under `Grpc`. Applied per endpoint walked in `CentralGrpcEndpoints`, not per ack, so a dead first endpoint costs this deadline before the second is tried. | +| `ClusterPsks` | `Dictionary` | `{}` (case-insensitive keys) | **Central-side only** override map, keyed by `Cluster` id: `IMeshPskProvider` consults this map *before* the Secrets store when central builds a channel to (or verifies an ack from) a given cluster. Production leaves this empty and resolves `OT-MESH-PSK-{clusterId}` from Secrets instead; the map exists so the docker-dev rig and integration tests can supply literals via the environment (`MeshTransport__ClusterPsks__SITE-A`) without standing up a KEK. | + +**`ClusterPsks` / `OT-MESH-PSK-` and `SecretName`'s allow-list.** `SecretName` (`ZB.MOM.WW.Secrets.Abstractions`) lowercases the secret name and restricts it to `[a-z0-9._/-]` — a `Cluster` id that carries any character outside that set cannot be expressed as `OT-MESH-PSK-` at all, so `MeshPskProvider.TryGetPsk` fails closed for that cluster on every call that reaches the store (`SecretName`'s constructor throws, `MeshPskProvider.ResolveFromStore` catches it, logs a warning, and returns no key) — `MeshTransport:ClusterPsks` is the only path that still works for such an id. The rig's three ids (`MAIN`, `SITE-A`, `SITE-B`) are all within the allow-list and resolve normally either way. + +The `Grpc` mode's node-side comm actors and central-side services are not yet wired to `Mode` — Phase 0 lands only the contract, mapper, options and the two auth interceptors (`MeshCommandAuthInterceptor` node-side, `CentralCommandAuthInterceptor` central-side), both dark. See the [design doc](plans/2026-08-06-clusterclient-to-grpc-design.md) and [tracking doc](plans/2026-08-06-clusterclient-to-grpc-tracking.md) for the remaining phases. + ### `ConfigSource` / `ConfigServe` (config fetch-and-cache) - **Purpose:** [per-cluster mesh](plans/2026-07-21-per-cluster-mesh-design.md) **Phase 3**. `ConfigSource:Mode` selects where a **driver** node reads its deployed configuration: `Direct` (read `Deployment.ArtifactBlob` from central SQL, today's behaviour) or `FetchAndCache` (fetch the artifact bytes from central over a gRPC stream, verify SHA-256, cache in LocalDb, and read **only** the cache — no central-SQL config read). `ConfigServe` is the **central** (admin-role) serve side that a `FetchAndCache` node dials. diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md b/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md index bf95163a..9dc56776 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md @@ -45,20 +45,113 @@ deadline is the only witness. Restart the pair. --- -## Phase 0 — contract + auth + config groundwork — **PENDING** +## Phase 0 — contract + auth + config groundwork — **PASS** -Branch `feat/grpc-phase0-contract-auth` @ `________`. Image rebuilt, all containers recreated. +Branch `feat/grpc-phase0-contract-auth` @ `c2bfad4a`. Image rebuilt, all six containers +recreated (`docker compose -f docker-dev/docker-compose.yml up -d --build`, run from the +worktree at `/Users/dohertj2/Desktop/OtOpcUa/.worktrees/grpc-phase0` so the image carries this +branch). The rig was already running (from the main checkout's compose file) before this run; +the rebuild recreated all eight containers (sql kept, migrator + cluster-seed one-shots re-ran, +central-1/-2 + site-a-1/-2 + site-b-1/-2 + traefik recreated) under the fixed `otopcua-dev` +project name, so this run is not a concurrent-rebuild-invalidated one — nothing else touched +the stack afterwards. | # | Check | Result | |---|---|---| -| 1 | All six nodes boot on `ClusterClient` with the dark `Grpc` config present (PSKs, endpoints, ClusterPsks map); no validator fires | PENDING | -| 2 | Deliberate misconfig: a scratch container with `MeshTransport__Mode=Grpc` and **no** `GrpcPsk` refuses to boot, error names `MeshTransport:GrpcPsk` + the env form | PENDING | -| 3 | Same with empty `CentralGrpcEndpoints` — refuses, names the key | PENDING | -| 4 | Full suite green, 0 warnings, on the branch | PENDING | +| 1 | All six nodes boot on `ClusterClient` with the dark `Grpc` config present (PSKs, endpoints, ClusterPsks map); no validator fires | **PASS** — 6/6 nodes `restarts=0`, `docker inspect -f '{{.State.Status}} restarts={{.RestartCount}}'` reports `running restarts=0` on every node; `docker logs | grep -iE "OptionsValidationException\|MeshTransport.*(is empty\|refuses to start)"` = 0 lines on all six | +| 2 | Deliberate misconfig: a scratch container with `MeshTransport__Mode=Grpc` and **no** `GrpcPsk` refuses to boot, error names `MeshTransport:GrpcPsk` + the env form | **PASS** — `Hosting failed to start` + `OptionsValidationException` naming `MeshTransport:GrpcPsk` and `MeshTransport__GrpcPsk`, below | +| 3 | Same with empty `CentralGrpcEndpoints` — refuses, names the key | **PASS** — `OptionsValidationException` naming `MeshTransport:CentralGrpcEndpoints` and `MeshTransport__CentralGrpcEndpoints__0`, below | +| 4 | Full suite green, 0 warnings, on the branch | **PASS** — `dotnet test ZB.MOM.WW.OtOpcUa.slnx --nologo`: 0 failures on a clean run (see evidence below); no `.cs` files changed on this branch since the last full rebuild, so the pre-existing ~868-warning baseline is unchanged (nothing to newly warn about) | Note: the new gRPC services are not yet mapped in Phase 0, so no wire probes here — the interceptor negative/positive matrix is unit-tested and wire-probed in 1A/1B. +**Check 1 evidence** — no validator fired on any of the six nodes, all dark `Grpc` keys present: + +``` +$ for c in central-1 central-2 site-a-1 site-a-2 site-b-1 site-b-2; do + docker inspect -f '{{.State.Status}} restarts={{.RestartCount}}' otopcua-dev-$c-1 + done +running restarts=0 # central-1 +running restarts=0 # central-2 +running restarts=0 # site-a-1 +running restarts=0 # site-a-2 +running restarts=0 # site-b-1 +running restarts=0 # site-b-2 + +$ docker logs otopcua-dev-central-1-1 2>&1 | grep -iE "MeshTransport|ClusterClient" +[08:53:26 INF] Mesh ClusterClient for cluster MAIN created with 2 contact point(s) +[08:53:26 INF] Mesh ClusterClient for cluster SITE-A created with 2 contact point(s) +[08:53:26 INF] Mesh ClusterClient for cluster SITE-B created with 2 contact point(s) + +$ docker logs otopcua-dev-site-a-1-1 2>&1 | grep -iE "cluster.client|receptionist" +[08:53:26 INF] Connected to [akka.tcp://otopcua@central-2:4053/system/receptionist] + +$ docker inspect otopcua-dev-site-a-1-1 --format '{{range .Config.Env}}{{println .}}{{end}}' | grep MeshTransport +MeshTransport__CentralGrpcEndpoints__1=http://central-2:4055 +MeshTransport__CentralGrpcEndpoints__0=http://central-1:4055 +MeshTransport__Mode=ClusterClient +MeshTransport__GrpcPsk=dev-mesh-psk-SITE-A +MeshTransport__CentralContactPoints__1=akka.tcp://otopcua@central-2:4053 +MeshTransport__CentralContactPoints__0=akka.tcp://otopcua@central-1:4053 + +$ curl -sf http://localhost:9200/health/ready +{"status":"Healthy", ..., "akka":{"status":"Healthy","description":"Akka cluster member status: Up", + "data":{"selfAddress":"akka.tcp://otopcua@central-2:4053","selfRoles":["admin","cluster-MAIN","driver"], + "memberCount":2,"unreachableCount":0,"leader":"akka.tcp://otopcua@central-1:4053"}}} +``` + +**Check 2 evidence** — scratch container from `otopcua-host:dev`, site-a-1's real environment +with `MeshTransport__GrpcPsk` removed and `Mode` forced to `Grpc`: + +``` +$ docker run --rm --name scratch-check2-noPsk --network otopcua-dev_default \ + --env-file check2-env.txt otopcua-host:dev +[08:54:23 ERR] Hosting failed to start +Microsoft.Extensions.Options.OptionsValidationException: MeshTransport:GrpcPsk is empty. Under +Grpc this per-cluster pre-shared key is the whole authentication boundary of the command plane +in both directions — without it this node's inbound command service fails closed and every ack +it posts to central is rejected. Supply it via the environment (MeshTransport__GrpcPsk), in +production as ${secret:OT-MESH-PSK-}. + at Microsoft.Extensions.Options.OptionsFactory`1.Create(String name) + ... +``` + +**Check 3 evidence** — same base env, `GrpcPsk` present, both `CentralGrpcEndpoints` indices +unset: + +``` +$ docker run -d --name scratch-check3-noEndpoints --network otopcua-dev_default \ + --env-file check3-env.txt otopcua-host:dev +$ docker logs scratch-check3-noEndpoints 2>&1 | grep -A2 "OptionsValidationException" +[08:55:55 ERR] Hosting failed to start +Microsoft.Extensions.Options.OptionsValidationException: MeshTransport:CentralGrpcEndpoints is +empty. Under Grpc a driver node has nowhere to post its deployment ack, and every deployment +would time out naming a node that is running perfectly. Supply them via the environment +(MeshTransport__CentralGrpcEndpoints__0). +``` + +Both scratch containers removed after capture (`docker rm -f scratch-check2-noPsk +scratch-check3-noEndpoints`); the rig's six real nodes were never affected (still `restarts=0` +after both runs). + +**Check 4 evidence** — full suite, clean re-run (a first run hit one race-condition flake in +`RoslynVirtualTagEvaluatorTests.Evaluate_racing_ClearCompiledScripts_never_fails_with_disposed`, +which the test's own name identifies as exercising a race; re-ran in isolation and it passed, +then a full second run passed clean end to end — pre-existing timing sensitivity, not a +regression from this branch, which touched only `docker-dev/docker-compose.yml` and +`docs/Configuration.md`, no `.cs` files): + +``` +$ dotnet test ZB.MOM.WW.OtOpcUa.slnx --nologo +... +Passed! - Failed: 0, Passed: 212, Skipped: 4, Total: 216, Duration: 3m56s - ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.dll (net10.0) +$ echo $? +0 +``` + +Rig left running on `ClusterClient` (its normal state) after the gate. + --- ## Phase 1A — central→node command plane — **PENDING**