diff --git a/docker/verify-site-secrets.sh b/docker/verify-site-secrets.sh index 17d23ebd..1e46d0c8 100755 --- a/docker/verify-site-secrets.sh +++ b/docker/verify-site-secrets.sh @@ -22,19 +22,24 @@ set -euo pipefail # Ciphertext is compared by SHA-256 only; secret VALUES are random throwaways held in shell # variables and handed to the browser through the environment — never printed, never on argv. # -# Checks (each prints PASS/FAIL; the run ends "PASS n/8" and exits non-zero on any FAIL): +# Checks (each prints PASS/FAIL; the run ends "PASS n/9" and exits non-zero on any FAIL): # 1 secret + secret-referencing MxGateway connection on site-a -> site-a Out of date, b/c Current # 2 deploy site-a -> success, secretsDeployed 1, site-a Current # 3 both site-a nodes hold the secret at the central revision + ciphertext (standby <= 30 s) # 4 rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision -# 5 drop the secret: reference -> deploy -> both nodes no longer hold it; an out-of-band secret -# seeded on site-a-a through the library's own write path (the ZB.MOM.WW.Secrets `secret` CLI -# run against that node's store) survives -# 6 reference a non-existent secret -> site-a deploy fails closed with the named message, -# site-a stays Out of date, nothing changed on either site-a node -# 7 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the KekId -# 8 one DeploySiteSecrets audit row per secret-bearing deploy; no plaintext / base64 ciphertext -# in any of the 8 nodes' logs +# 5 standby master-key mismatch reaches central (review M1): site-a-b is recreated with a WRONG +# KEK -> central's site-a health report shows node-b failing to decrypt the secret while +# node-a is clean; site-a-b restored with the right KEK -> node-b's failures clear +# 6 drop the secret: reference -> deploy -> both nodes no longer hold it and the reply names it +# in secretsRemoved; an out-of-band secret seeded on site-a-a through the library's own write +# path (the ZB.MOM.WW.Secrets `secret` CLI run against that node's store) survives +# 7 reference a non-existent secret -> site-a deploy fails closed with the named message, the +# CLI exits 1, site-a stays Out of date, nothing changed on either site-a node +# 8 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the +# KekId, the CLI exits 1 +# 9 one DeploySiteSecrets audit row per deploy that carried, lacked or retired a secret (the +# removal-only deploy of check 6 included, with its removed list); no plaintext / base64 +# ciphertext in any of the 8 nodes' logs # # Teardown (trap EXIT, also run up-front so a rerun starts clean): gate connections deleted, # site-a + site-b redeployed so the removal reaches them, the central gate secret deleted, the @@ -56,6 +61,10 @@ A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" - SECRETS_CLI_PROJECT="${SECRETS_CLI_PROJECT:-$HOME/Desktop/scadaproj/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli}" SECRETS_CLI_DIR="${SECRETS_CLI_DIR:-${TMPDIR:-/tmp}/sb-gate-secrets-cli}" STANDBY_WINDOW_S="${STANDBY_WINDOW_S:-30}" +# Check 5: mirror pass (15 s) + status reporter poll (30 s) + health report (30 s), plus a restart. +HEALTH_WINDOW_S="${HEALTH_WINDOW_S:-240}" +# The site-b/site-c dev KEK (docker-compose.yml) — deliberately NOT site-a's, used by check 5. +WRONG_KEK="esKrmqV2RBmMq/+6T/5zK2BcalRy5gzDXhoG9Qydg6Q=" # DEV-ONLY rig KEK shared by central + site-a (docker/docker-compose.yml, x-secrets-shared-kek-env). SITE_A_KEK="zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4=" @@ -333,6 +342,39 @@ v=None if r is None else r.get(sys.argv[2]) print("" if v is None else (json.dumps(v) if isinstance(v,bool) else v))' "$1" "$2" } badge_of() { jget "$1" "$2"; } +# mirror_failures -> JSON {nodeName: [secretName...]} from central's latest health +# report for the site (SiteHealthReport.SecretMirrorStatuses), or "null" when not reported +mirror_failures() { + "$CLI" health site "${A[@]}" --identifier "$1" 2>/dev/null | python3 -c ' +import json,sys +try: + r=(json.load(sys.stdin).get("latestReport") or {}).get("secretMirrorStatuses") +except Exception: + r=None +print("null" if r is None else json.dumps({n["nodeName"]: sorted(f["secretName"] for f in n["failures"]) for n in r}, sort_keys=True))' +} +# The rig's compose project dir (where the bind-mounted appsettings live), from the running container. +RIG_DIR=$(docker inspect "$STANDBY_A" --format '{{ index .Config.Labels "com.docker.compose.project.working_dir" }}' 2>/dev/null) +# recreate_standby_a [wrong] -> recreates site-a-b from the rig's own compose file, optionally with +# an override that swaps in a mismatched KEK; waits until it is a cluster member again. +recreate_standby_a() { + local files=(-f "$RIG_DIR/docker-compose.yml") + if [ "${1:-}" = "wrong" ]; then + printf 'services:\n site-a-b:\n environment:\n ZB_SECRETS_MASTER_KEY: "%s"\n' "$WRONG_KEK" >"$WORK/wrong-kek.yml" + files+=(-f "$WORK/wrong-kek.yml"); touch "$WORK/standby-kek-swapped" + fi + docker compose "${files[@]}" --project-directory "$RIG_DIR" up -d --no-deps --force-recreate site-a-b >/dev/null 2>&1 || return 1 + [ "${1:-}" = "wrong" ] || rm -f "$WORK/standby-kek-swapped" + local i + for i in $(seq 1 60); do + docker run --rm --network "container:$ACTIVE_A" python:3.13-alpine python3 -c ' +import urllib.request as u,sys +try: sys.exit(0 if u.urlopen("http://scadabridge-site-a-b:8084/health/ready",timeout=3).status==200 else 1) +except Exception: sys.exit(1)' >/dev/null 2>&1 && return 0 + sleep 3 + done + return 1 +} ensure_secrets_cli() { [ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ] && return 0 @@ -365,6 +407,9 @@ teardown() { echo "" echo "=== teardown ===" local id + if [ -f "$WORK/standby-kek-swapped" ]; then + recreate_standby_a && echo " restored site-a-b with site-a's KEK" || echo " WARN could not restore site-a-b's KEK" + fi for c in "$CONN_A" "$CONN_B"; do id=$(conn_id "$c" 2>/dev/null) if [ -n "$id" ]; then @@ -493,7 +538,37 @@ B=$(gp badges "$URL") end_check 4 # ── 5 ──────────────────────────────────────────────────────────────────────────────────────── -begin_check 5 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives" +begin_check 5 "standby master-key mismatch reaches central's health report (site-a-b on a wrong KEK)" +# wait_health -> sets HEALTH_S / LAST_F +wait_health() { + local start now; start=$(python3 -c 'import time; print(time.time())'); HEALTH_S="" + while :; do + LAST_F=$(mirror_failures site-a) + if python3 -c "import json,sys; F=json.loads(sys.argv[1]); sys.exit(0 if F is not None and ($1) else 1)" "$LAST_F"; then + now=$(python3 -c 'import time; print(time.time())'); HEALTH_S=$(python3 -c "print(round($now-$start))"); return 0 + fi + now=$(python3 -c 'import time; print(time.time())') + python3 -c "import sys; sys.exit(0 if $now-$start < $HEALTH_WINDOW_S else 1)" || return 1 + sleep 5 + done +} +if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then ok "baseline: central reports both site-a nodes, no failures ($LAST_F)" +else bad "baseline: site-a health report is not two clean node rows: $LAST_F"; fi +if [ -z "$RIG_DIR" ]; then bad "cannot find the rig's compose directory from $STANDBY_A's labels" +elif recreate_standby_a wrong; then + ok "site-a-b recreated with a mismatched KEK and back in the cluster (standby; site-a-a stays active)" + if wait_health "'$SECRET' in F.get('node-b',[]) and F.get('node-a')==[]"; then MISMATCH_S="$HEALTH_S" + ok "central's site-a report shows node-b failing $SECRET, node-a clean (${HEALTH_S}s): $LAST_F" + else bad "node-b's failure never reached central within ${HEALTH_WINDOW_S}s: $LAST_F"; fi + if recreate_standby_a; then ok "site-a-b restored with site-a's KEK"; else bad "could not restore site-a-b"; fi + if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then CLEARED_S="$HEALTH_S" + ok "node-b's failure cleared on central's report (${HEALTH_S}s)" + else bad "node-b's failure did not clear within ${HEALTH_WINDOW_S}s: $LAST_F"; fi +else bad "could not recreate site-a-b with a mismatched KEK"; fi +end_check 5 + +# ── 6 ──────────────────────────────────────────────────────────────────────────────────────── +begin_check 6 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives" SB_GATE_VALUE="$VALUE_OOB" site_a_secret_set "$OOB" "site-secrets gate out-of-band" >/dev/null 2>&1 || true J=$(gp site-secrets "$ACTIVE_A") if [ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ]; then @@ -503,6 +578,7 @@ upsert_conn "$CONN_A" "$SITE_A_ID" "gate-literal-not-a-secret-ref" && ok "$CONN_ T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())') [ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "deploy succeeded (secretsDeployed='$(site_result site-a secretsDeployed)')" \ || bad "deploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)" +case "$(site_result site-a secretsRemoved)" in *"$SECRET"*) ok "reply names $SECRET in secretsRemoved";; *) bad "secretsRemoved='$(site_result site-a secretsRemoved)', expected $SECRET";; esac GONE="(d.get('$SECRET') is None or d['$SECRET']['deleted'])" if wait_node "$ACTIVE_A" "$GONE" "$T_DEPLOYED"; then ok "site-a-a no longer holds $SECRET (${WAIT_S}s)"; else bad "site-a-a still holds $SECRET: $LAST_NODE_JSON"; fi if wait_node "$STANDBY_A" "$GONE" "$T_DEPLOYED"; then STANDBY_5="$WAIT_S"; ok "site-a-b no longer holds $SECRET (${WAIT_S}s after the deploy returned)" @@ -512,10 +588,10 @@ J=$(gp site-secrets "$ACTIVE_A") && ok "out-of-band $OOB still live on site-a-a" || bad "out-of-band $OOB lost: $J" L=$(gp site-localdb "$ACTIVE_A") [ "$(jget "$L" "deployed.$SECRET.removed")" = "true" ] && ok "deployed_secrets.$SECRET is_removed=1 (kept for the standby)" || bad "deployed_secrets: $L" -end_check 5 +end_check 6 -# ── 6 ──────────────────────────────────────────────────────────────────────────────────────── -begin_check 6 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched" +# ── 7 ──────────────────────────────────────────────────────────────────────────────────────── +begin_check 7 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched" SNAP_BEFORE="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")" upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$MISSING" && ok "$CONN_A now references secret:$MISSING" || bad "update $CONN_A" deploy "$SITE_A_ID" @@ -523,6 +599,7 @@ ERR=$(site_result site-a errorMessage) echo " deploy rc=$DEPLOY_RC error: $ERR" EXPECT="Secret '$MISSING' referenced by data connection(s) $CONN_A is not in the central secret store" [ "$(site_result site-a success)" = "false" ] && ok "site-a deploy reported failure" || bad "site-a deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)" +[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1" case "$ERR" in *"$EXPECT"*) ok "named fail-closed message";; *) bad "message does not contain: $EXPECT";; esac B=$(gp badges "$URL"); echo " badges: $B" [ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a stays Out of date" || bad "site-a is '$(badge_of "$B" site-a)'" @@ -530,10 +607,10 @@ sleep 3 SNAP_AFTER="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")" [ "$SNAP_BEFORE" = "$SNAP_AFTER" ] && ok "nothing changed: stamped hash, both nodes' secret rows, deployed_secrets, ledgers and the $CONN_A definition identical" \ || { bad "site state changed"; echo " before: $SNAP_BEFORE"; echo " after: $SNAP_AFTER"; } -end_check 6 +end_check 7 -# ── 7 ──────────────────────────────────────────────────────────────────────────────────────── -begin_check 7 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId" +# ── 8 ──────────────────────────────────────────────────────────────────────────────────────── +begin_check 8 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId" C=$(gp central-secret "$SECRET") KEK=$(jget "$C" kek) upsert_conn "$CONN_B" "$SITE_B_ID" "secret:$SECRET" && ok "created $CONN_B on site-b (ApiKey=secret:$SECRET)" || bad "create $CONN_B" @@ -541,13 +618,14 @@ deploy "$SITE_B_ID" ERR=$(site_result site-b errorMessage) echo " deploy rc=$DEPLOY_RC error: $ERR" [ "$(site_result site-b success)" = "false" ] && ok "site-b deploy reported failure" || bad "site-b deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)" +[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1" case "$ERR" in *"$SECRET"*"$KEK"*) ok "message names $SECRET and the row's KekId $KEK";; *) bad "message does not name $SECRET + KekId $KEK";; esac B=$(gp badges "$URL"); echo " badges: $B" [ "$(badge_of "$B" site-b)" = "out of date" ] && ok "site-b not stamped (Out of date)" || bad "site-b is '$(badge_of "$B" site-b)'" -end_check 7 +end_check 8 -# ── 8 ──────────────────────────────────────────────────────────────────────────────────────── -begin_check 8 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log" +# ── 9 ──────────────────────────────────────────────────────────────────────────────────────── +begin_check 9 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log" AUD=$("$CLI" audit-config query "${A[@]}" --action DeploySiteSecrets --entity-type SiteSecrets --from "$GATE_START" --page-size 50 2>/dev/null) ROWS=$(printf '%s' "$AUD" | python3 -c ' import json,sys @@ -557,18 +635,20 @@ for e in sorted(d.get("entries",[]), key=lambda e: e["timestamp"]): s=json.loads(e.get("afterStateJson") or "{}") names=",".join("%s@r%s" % (x.get("name"), x.get("revision")) for x in (s.get("secrets") or [])) miss=",".join(s.get("missing") or []) - rows.append("%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", s.get("deploymentId"))) + rem=",".join(s.get("removed") or []) + rows.append("%s|%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", rem or "-", s.get("deploymentId"))) print("\n".join(rows))') echo "$ROWS" | sed 's/^/ audit: /' R2=$(jget "$C1" rev); R4=$(jget "$C2" rev) -EXPECTED="site-a|True|$SECRET@r$R2|- -site-a|True|$SECRET@r$R4|- -site-a|False|-|$MISSING -site-b|False|$SECRET@r$R4|-" -GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-4) -[ "$GOT" = "$EXPECTED" ] && ok "exactly 4 rows, one per secret-bearing deploy (2, 4, 6-fail-closed, 7-kek-mismatch); the no-secret deploy of check 5 wrote none" \ +EXPECTED="site-a|True|$SECRET@r$R2|-|- +site-a|True|$SECRET@r$R4|-|- +site-a|True|-|-|$SECRET +site-a|False|-|$MISSING|- +site-b|False|$SECRET@r$R4|-|-" +GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-5) +[ "$GOT" = "$EXPECTED" ] && ok "exactly 5 rows: deploys 2 and 4, the removal-only deploy of check 6 (removed=$SECRET), 7 fail-closed, 8 kek-mismatch" \ || { bad "audit rows differ from expected"; echo " expected:"; echo "$EXPECTED" | sed 's/^/ /'; } -[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f5 | sort -u | grep -c .)" = "4" ] && ok "4 distinct deploymentIds" || bad "deploymentIds not distinct" +[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f6 | sort -u | grep -c .)" = "5" ] && ok "5 distinct deploymentIds" || bad "deploymentIds not distinct" # Material to hunt for: both plaintext values and both ciphertexts (base64, as they appear in any # JSON serialization). Written to a pattern file so they never reach argv or the output. PAT="$WORK/patterns"; : >"$PAT"; chmod 600 "$PAT" @@ -584,10 +664,11 @@ done AUDH=$(printf '%s' "$AUD" | grep -c -F -f "$PAT" || true) [ "$HITS" = "0" ] && ok "0 hits across all 8 nodes' logs since $LOG_SINCE" || bad "$HITS log line(s) carry secret material" [ "$AUDH" = "0" ] && ok "0 hits in the DeploySiteSecrets audit payloads" || bad "audit payload carries secret material" -end_check 8 +end_check 9 echo "" echo "=== Summary ===" -echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check5=${STANDBY_5:-?}" -echo "PASS $CHECKS_PASSED/8" +echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check6=${STANDBY_5:-?}" +echo "standby mismatch on central's health report (check 5): shown after ${MISMATCH_S:-?}s, cleared after ${CLEARED_S:-?}s" +echo "PASS $CHECKS_PASSED/9" [ "$CHECKS_FAILED" -eq 0 ] || exit 1 diff --git a/docs/plans/2026-07-08-deferred-work-register.md b/docs/plans/2026-07-08-deferred-work-register.md index 97c89b1f..096fb472 100644 --- a/docs/plans/2026-07-08-deferred-work-register.md +++ b/docs/plans/2026-07-08-deferred-work-register.md @@ -120,7 +120,7 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round | 124 | **No template-graph-watermark short-circuit on the pending-updates fleet flatten** *(Low)*. `PendingUpdatesEvaluator.EvaluateAsync` re-flattens every deployed instance through `DeploymentService.GetInstanceStalenessAsync` on every cache-TTL expiry (`PendingUpdatesCacheTtl`, default 60 s) while any Deployer/Admin circuit is open, even when nothing in the template/instance/artifact graph has changed since the last evaluation. | `docs/plans/2026-09-14-pending-updates-design.md` §3.2 ("Not in scope") | The single-flight TTL cache already bounds the cost to one fleet flatten per TTL fleet-wide (not per viewer); a watermark short-circuit is a further optimization, not a correctness need, and needs a cheap "has anything changed" signal that does not exist yet. | The fleet flatten's cost (instance count x template depth) becomes measurable against `PendingUpdatesCacheTtl` on a large deployment. | | 125 | **Pending-updates counts have no Health-dashboard tile or CLI surface** *(Low)*. The aggregate (`PendingUpdatesSnapshot.StaleInstanceCount` / `StaleArtifactSiteCount`) is consumed only by the Central UI Sites page and rail indicator; there is no Health dashboard KPI tile and no `deploy`/`site` CLI command surfacing it — `site list` shows only the raw `artifactRevisionHash`/`artifactsDeployedAt` fields on each site, not whether they are current. | `docs/plans/2026-09-14-pending-updates-design.md` §3.2 ("Not in scope") | Out of scope for the UI-focused ask; the CLI/Health surfaces are a separate consumer of the same `IPendingUpdatesCache`. | An operator wants the pending-updates counts without opening the Central UI (headless monitoring, CLI scripting). | | 126 | ~~**TODO — roll the two outstanding migrations to the QA pair vq03/vq04**~~ **DONE 2026-09-28** — QA vq03/vq04 upgraded `2b9a7c3c` → `b3021e7c`, 55 → 57, artifacts deployed (record `docs/deployment/deployment-records/2026-09-28-wonder-app-vq03-vq04-b3021e7c.md`). *(deployment, closed)*. **DEV vd03 DONE 2026-09-28** (`fe89363b`, 55 → 57, record `docs/deployment/deployment-records/2026-09-28-wonder-app-vd03-fe89363b.md` — the `sb-mig-55to57.sql` there is the same script QA needs). QA still runs `2b9a7c3c` with the DB at migration 55 (`20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration`); `main` (`ac833f5a`, pushed 2026-09-14) carries two more: **56** `20260911135501_AddAlarmMesReportable` (computed-alarm `IsMesReportable` flag) and **57** `20260914173147_AddSiteArtifactRevision` (`Sites.ArtifactRevisionHash`/`ArtifactsDeployedAt`, both nullable). Both are additive `AddColumn`s — no data upgrade, no coordinated step like the 2026-09-10 protocol-config encryption. Recipe (same as the 55 rollout): generate ONE from→to delta script `dotnet ef migrations script EnforceGlobalInstanceNamesAndProtectProtocolConfiguration AddSiteArtifactRevision --project src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase --startup-project src/ZB.MOM.WW.ScadaBridge.Host -o sb-mig-55to57.sql` (NOT idempotent, identical for vd03 and the QA pair), take the pre-upgrade DB backup, stop the pair TOGETHER, apply via `sqlcmd -C -b -I` on the SQL host, publish/swap the `ac833f5a` binaries, start the pair, confirm the Central boot log shows 0 pending migrations and no "Applying migration" (the script already applied them). After upgrade every site reads *Artifacts never deployed* until one *Deploy Artifacts to All Sites* — run it as the last step. Write the deployment record `docs/deployment/deployment-records/-wonder-app-{vd03,vq03-vq04}-ac833f5a.md`. | Pending-updates finish (2026-09-14) | ~~Blocked by the 2026-09-11 remote-access hold~~ — **unblocked 2026-09-28**: `winrm` is re-enabled for deployment activities (CLAUDE.md). vd03 done; QA done 2026-09-28. | Closed 2026-09-28. | -| 127 | **TODO — roll out site secrets via artifact deploy to vd03 + QA (vq03/vq04)**. The gRPC secrets hub is retired in ScadaBridge as of `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` (implemented on branch `feat/site-secrets-artifact-deploy`, rig-only so far); vd03 and QA still run the pre-feature build with `Secrets:Replication:Enabled=false`, so upgrading them will **not** fail the boot (the retirement's `StartupValidator` rejection only fires on `Enabled=true`). **Upgrade impact:** once upgraded, any site whose data connections carry `secret:` references fails its **whole** artifact deploy (scripts, external systems, DB connections too) and reads *Out of date* until every referenced secret exists in the central store — so before upgrading vd03/QA, check whether their connection configs use `secret:` references and whether their site pairs run LocalDb replication, and do the prerequisites in the same window. Prerequisites, in order: (1) provision the `zbsecrets` schema/store in the environment's configuration SQL Server (the same server `ConfigurationDb` already points at); (2) set `Secrets:Store:Provider=SqlServer` on **both** central nodes of the environment; (3) verify the KEK (`ZB_SECRETS_MASTER_KEY` / master-key file) is identical across **every** node — both centrals and every site node — since a mismatch is caught only at deploy time by `DeployedSecretsMirror`'s per-secret verify, not at boot; (4) confirm every site pair that uses secrets runs LocalDb replication — on an unreplicated pair only the node active at deploy time gets the secrets, so a failover leaves the new active node without them until the next *Deploy Artifacts*; (5) recreate, at the new central shared store, any secret an operator previously hand-seeded directly into a site's local `secrets.db` (those never round-tripped through central and would otherwise be silently absent from the referenced-secrets scan). Then run *Deploy Artifacts* against every affected site as the rollout's last step. | `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` §4 *Rollout*; `docs/deployment/topology-guide.md` → *Secrets: shared store, per-site deploy, and the KEK* | Deployment record, not a code change — vd03/QA config is user-held and their rollout schedule is independent of the branch merging to main. | User asks for the vd03/QA secrets window. | +| 127 | **TODO — roll out site secrets via artifact deploy to vd03 + QA (vq03/vq04)**. The gRPC secrets hub is retired in ScadaBridge as of `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` (merged to `main` @ `23a139a4` 2026-09-28, plus the same-day follow-ups: per-site secret-store outage handling, removal audit rows, per-node mirror status on the health report, artifact-deploy CLI exit codes; rig-only so far); vd03 and QA still run the pre-feature build with `Secrets:Replication:Enabled=false`, so upgrading them will **not** fail the boot (the retirement's `StartupValidator` rejection only fires on `Enabled=true`). **Upgrade impact:** once upgraded, any site whose data connections carry `secret:` references fails its **whole** artifact deploy (scripts, external systems, DB connections too) and reads *Out of date* until every referenced secret exists in the central store — so before upgrading vd03/QA, check whether their connection configs use `secret:` references and whether their site pairs run LocalDb replication, and do the prerequisites in the same window. Prerequisites, in order: (1) provision the `zbsecrets` schema/store in the environment's configuration SQL Server (the same server `ConfigurationDb` already points at); (2) set `Secrets:Store:Provider=SqlServer` on **both** central nodes of the environment; (3) verify the KEK (`ZB_SECRETS_MASTER_KEY` / master-key file) is identical across **every** node — both centrals and every site node — since a mismatch is caught only at deploy time by `DeployedSecretsMirror`'s per-secret verify, not at boot (after the first deploy a mismatch on either node — the standby included — shows as a *Secret mirror failing* badge on that site's Health card); (4) confirm every site pair that uses secrets runs LocalDb replication — on an unreplicated pair only the node active at deploy time gets the secrets, so a failover leaves the new active node without them until the next *Deploy Artifacts*; the upgrade also adds two replicated site tables (`deployed_secrets`, `secret_mirror_status`), so stop and start each site pair together as usual; (5) recreate, at the new central shared store, any secret an operator previously hand-seeded directly into a site's local `secrets.db` (those never round-tripped through central and would otherwise be silently absent from the referenced-secrets scan). Then run *Deploy Artifacts* against every affected site as the rollout's last step. | `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` §4 *Rollout*; `docs/deployment/topology-guide.md` → *Secrets: shared store, per-site deploy, and the KEK* | Deployment record, not a code change — vd03/QA config is user-held and their rollout schedule is independent of the branch merging to main. | User asks for the vd03/QA secrets window. | ## Resolved (verified against the code 2026-07-10) Rows removed from the Deferred table above once confirmed shipped. Kept here for traceability. diff --git a/docs/plans/2026-09-28-site-secrets-artifact-deploy-live-gate.md b/docs/plans/2026-09-28-site-secrets-artifact-deploy-live-gate.md index a6293b7f..2ea34522 100644 --- a/docs/plans/2026-09-28-site-secrets-artifact-deploy-live-gate.md +++ b/docs/plans/2026-09-28-site-secrets-artifact-deploy-live-gate.md @@ -151,3 +151,46 @@ rows, 0 plaintext/ciphertext hits across all 8 node logs and the audit payloads. Tests before the rig run: `dotnet build ZB.MOM.WW.ScadaBridge.slnx` 0 errors; Commons.Tests 875/875, SiteRuntime.Tests 892/892, DeploymentManager.Tests 235/235, DataConnectionLayer.Tests 383/383, Host.Tests 593/593. + +## 7. Follow-ups run (2026-09-28) + +Code under test: branch `fix/site-secrets-followups` — per-site secret-store outage handling and +removal-only audit rows (`29049c4e`), artifact-deploy CLI exit codes (`ef03a0dc`), per-node mirror status on +the site health report (`0289d511`, merged in `4e640918`). Image rebuilt from the worktree with +`bash docker/build.sh`, all 8 nodes recreated from the primary checkout's compose file. + +**Hub-era secrets removed from site-a (review M8), before the run.** The retired gRPC hub had left nine rows +dated 2026-08-07 in both site-a nodes' local stores (`/app/data/scadabridge-secrets.db`, table `secret`): +six live (`hub-gate-smoke-2`, `hub-gate-smoke-3`, `shared-gate-051`, `shared-gate-2`, `shared-gate-2b`, +`shared-gate-final`) and three tombstones (`expander-gate-pepper`, `hub-gate-smoke-1`, `shared-gate-1`). +Nothing in the rig config referenced them. With the site-a pair stopped together, each node's file was edited +from a throwaway `python:3.13-alpine` container mounting that node's data volume (`DELETE … WHERE name IN (…) +AND updated_utc < '2026-08-08'`, then `wal_checkpoint(TRUNCATE)`); 9 rows deleted per node, only the `sb-gate-*` +gate tombstones left. Both nodes came back ready, site-a-a active. site-b/site-c never ran the hub and had no +such rows; QA and vd03 never enabled replication. + +**The gate grew to nine checks:** +- new check 5 (M1): site-a-b is recreated with the site-b/site-c dev KEK (a compose override, removed again in + the same check and in teardown), and central's `health site --identifier site-a` must show node-b failing + `sb-gate-mxgw` while node-a is clean, then both clean once site-a-b has site-a's KEK again; +- check 6 (was 5) also asserts the reply's `secretsRemoved` names the dropped secret; +- checks 7 and 8 (were 6 and 7) also assert the CLI exits 1; +- check 9 (was 8) now expects five `DeploySiteSecrets` rows — the removal-only deploy of check 6 writes one, + with `removed: [sb-gate-mxgw]`. + +`bash docker/verify-site-secrets.sh` → **PASS 9/9**. Before the check, central reported both site-a node +rows with no failures; site-b and site-c (unreplicated) report their active node only. With the wrong KEK, +node-b's `sb-gate-mxgw` failure appeared on central's site-a report 57 s after site-a-b was back in the cluster +(mirror pass 15 s + status reporter 30 s + health report 30 s), and cleared 57 s after the KEK was restored. +Standby convergence: check 3 = 4.3 s, check 4 = 4.2 s, check 6 = 15.1 s. Audit: the 5 expected rows, 5 distinct +deployment ids; 0 plaintext/ciphertext hits in all 8 node logs and the audit payloads. Teardown left all three +sites *Current* and site-a-b on site-a's KEK. + +Not exercised live: the central secret-store outage (review M5) — taking the shared SQL store down on the rig +also takes down central's configuration database. Covered by unit tests (`SiteSecretsCollectorTests`, +`ArtifactDeploymentServiceTests.StoreOutage_*`, `ComputeCurrentHashes_StoreOutage_*`, +`RetryForSite_StoreOutage_*`). + +Tests: `dotnet build ZB.MOM.WW.ScadaBridge.slnx` 0 errors / 0 warnings; every non-Playwright test project green +(CLI.Tests 553 after updating `DeploymentDeadlineParityTests`' stub server to reply with a real zero-failure +summary — a bare `{}` now reads as an invalid artifact-deploy summary and exits 1). Playwright not run. diff --git a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/DeploymentDeadlineParityTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/DeploymentDeadlineParityTests.cs index 4c5bbc11..40b38ced 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/DeploymentDeadlineParityTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/DeploymentDeadlineParityTests.cs @@ -50,7 +50,11 @@ public class DeploymentDeadlineParityTests using var payload = JsonDocument.Parse(new string(request)); Assert.Equal(expectedCommand, payload.RootElement.GetProperty("command").GetString()); await Task.Delay(TimeSpan.FromSeconds(31), deadline.Token); - var reply = Encoding.ASCII.GetBytes("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"); + // A zero-failure summary: the artifact-deploy aliases derive their exit code from + // failureCount, so a bare "{}" would read as an invalid summary (exit 1). + const string body = "{\"successCount\":1,\"failureCount\":0}"; + var reply = Encoding.ASCII.GetBytes( + $"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {body.Length}\r\nConnection: close\r\n\r\n{body}"); try { await stream.WriteAsync(reply, deadline.Token); } catch (IOException) { /* Old client closes at 30s; exit assertion diagnoses it. */ } }, deadline.Token);