test(rig): site secrets gate covers the follow-ups — standby KEK mismatch on the health report, removal audit, CLI exit codes — PASS 9/9
Also: DeploymentDeadlineParityTests' stub replies with a real zero-failure summary (the artifact aliases now derive their exit code from failureCount), and register row 127 reflects the merge and the two new replicated site tables.
This commit is contained in:
Joseph Doherty
+111
-30
@@ -22,19 +22,24 @@ set -euo pipefail
|
||||
# Ciphertext is compared by SHA-256 only; secret VALUES are random throwaways held in shell
|
||||
# variables and handed to the browser through the environment — never printed, never on argv.
|
||||
#
|
||||
# Checks (each prints PASS/FAIL; the run ends "PASS n/8" and exits non-zero on any FAIL):
|
||||
# Checks (each prints PASS/FAIL; the run ends "PASS n/9" and exits non-zero on any FAIL):
|
||||
# 1 secret + secret-referencing MxGateway connection on site-a -> site-a Out of date, b/c Current
|
||||
# 2 deploy site-a -> success, secretsDeployed 1, site-a Current
|
||||
# 3 both site-a nodes hold the secret at the central revision + ciphertext (standby <= 30 s)
|
||||
# 4 rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision
|
||||
# 5 drop the secret: reference -> deploy -> both nodes no longer hold it; an out-of-band secret
|
||||
# seeded on site-a-a through the library's own write path (the ZB.MOM.WW.Secrets `secret` CLI
|
||||
# run against that node's store) survives
|
||||
# 6 reference a non-existent secret -> site-a deploy fails closed with the named message,
|
||||
# site-a stays Out of date, nothing changed on either site-a node
|
||||
# 7 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the KekId
|
||||
# 8 one DeploySiteSecrets audit row per secret-bearing deploy; no plaintext / base64 ciphertext
|
||||
# in any of the 8 nodes' logs
|
||||
# 5 standby master-key mismatch reaches central (review M1): site-a-b is recreated with a WRONG
|
||||
# KEK -> central's site-a health report shows node-b failing to decrypt the secret while
|
||||
# node-a is clean; site-a-b restored with the right KEK -> node-b's failures clear
|
||||
# 6 drop the secret: reference -> deploy -> both nodes no longer hold it and the reply names it
|
||||
# in secretsRemoved; an out-of-band secret seeded on site-a-a through the library's own write
|
||||
# path (the ZB.MOM.WW.Secrets `secret` CLI run against that node's store) survives
|
||||
# 7 reference a non-existent secret -> site-a deploy fails closed with the named message, the
|
||||
# CLI exits 1, site-a stays Out of date, nothing changed on either site-a node
|
||||
# 8 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the
|
||||
# KekId, the CLI exits 1
|
||||
# 9 one DeploySiteSecrets audit row per deploy that carried, lacked or retired a secret (the
|
||||
# removal-only deploy of check 6 included, with its removed list); no plaintext / base64
|
||||
# ciphertext in any of the 8 nodes' logs
|
||||
#
|
||||
# Teardown (trap EXIT, also run up-front so a rerun starts clean): gate connections deleted,
|
||||
# site-a + site-b redeployed so the removal reaches them, the central gate secret deleted, the
|
||||
@@ -56,6 +61,10 @@ A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" -
|
||||
SECRETS_CLI_PROJECT="${SECRETS_CLI_PROJECT:-$HOME/Desktop/scadaproj/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli}"
|
||||
SECRETS_CLI_DIR="${SECRETS_CLI_DIR:-${TMPDIR:-/tmp}/sb-gate-secrets-cli}"
|
||||
STANDBY_WINDOW_S="${STANDBY_WINDOW_S:-30}"
|
||||
# Check 5: mirror pass (15 s) + status reporter poll (30 s) + health report (30 s), plus a restart.
|
||||
HEALTH_WINDOW_S="${HEALTH_WINDOW_S:-240}"
|
||||
# The site-b/site-c dev KEK (docker-compose.yml) — deliberately NOT site-a's, used by check 5.
|
||||
WRONG_KEK="esKrmqV2RBmMq/+6T/5zK2BcalRy5gzDXhoG9Qydg6Q="
|
||||
|
||||
# DEV-ONLY rig KEK shared by central + site-a (docker/docker-compose.yml, x-secrets-shared-kek-env).
|
||||
SITE_A_KEK="zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4="
|
||||
@@ -333,6 +342,39 @@ v=None if r is None else r.get(sys.argv[2])
|
||||
print("" if v is None else (json.dumps(v) if isinstance(v,bool) else v))' "$1" "$2"
|
||||
}
|
||||
badge_of() { jget "$1" "$2"; }
|
||||
# mirror_failures <siteIdentifier> -> JSON {nodeName: [secretName...]} from central's latest health
|
||||
# report for the site (SiteHealthReport.SecretMirrorStatuses), or "null" when not reported
|
||||
mirror_failures() {
|
||||
"$CLI" health site "${A[@]}" --identifier "$1" 2>/dev/null | python3 -c '
|
||||
import json,sys
|
||||
try:
|
||||
r=(json.load(sys.stdin).get("latestReport") or {}).get("secretMirrorStatuses")
|
||||
except Exception:
|
||||
r=None
|
||||
print("null" if r is None else json.dumps({n["nodeName"]: sorted(f["secretName"] for f in n["failures"]) for n in r}, sort_keys=True))'
|
||||
}
|
||||
# The rig's compose project dir (where the bind-mounted appsettings live), from the running container.
|
||||
RIG_DIR=$(docker inspect "$STANDBY_A" --format '{{ index .Config.Labels "com.docker.compose.project.working_dir" }}' 2>/dev/null)
|
||||
# recreate_standby_a [wrong] -> recreates site-a-b from the rig's own compose file, optionally with
|
||||
# an override that swaps in a mismatched KEK; waits until it is a cluster member again.
|
||||
recreate_standby_a() {
|
||||
local files=(-f "$RIG_DIR/docker-compose.yml")
|
||||
if [ "${1:-}" = "wrong" ]; then
|
||||
printf 'services:\n site-a-b:\n environment:\n ZB_SECRETS_MASTER_KEY: "%s"\n' "$WRONG_KEK" >"$WORK/wrong-kek.yml"
|
||||
files+=(-f "$WORK/wrong-kek.yml"); touch "$WORK/standby-kek-swapped"
|
||||
fi
|
||||
docker compose "${files[@]}" --project-directory "$RIG_DIR" up -d --no-deps --force-recreate site-a-b >/dev/null 2>&1 || return 1
|
||||
[ "${1:-}" = "wrong" ] || rm -f "$WORK/standby-kek-swapped"
|
||||
local i
|
||||
for i in $(seq 1 60); do
|
||||
docker run --rm --network "container:$ACTIVE_A" python:3.13-alpine python3 -c '
|
||||
import urllib.request as u,sys
|
||||
try: sys.exit(0 if u.urlopen("http://scadabridge-site-a-b:8084/health/ready",timeout=3).status==200 else 1)
|
||||
except Exception: sys.exit(1)' >/dev/null 2>&1 && return 0
|
||||
sleep 3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
ensure_secrets_cli() {
|
||||
[ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ] && return 0
|
||||
@@ -365,6 +407,9 @@ teardown() {
|
||||
echo ""
|
||||
echo "=== teardown ==="
|
||||
local id
|
||||
if [ -f "$WORK/standby-kek-swapped" ]; then
|
||||
recreate_standby_a && echo " restored site-a-b with site-a's KEK" || echo " WARN could not restore site-a-b's KEK"
|
||||
fi
|
||||
for c in "$CONN_A" "$CONN_B"; do
|
||||
id=$(conn_id "$c" 2>/dev/null)
|
||||
if [ -n "$id" ]; then
|
||||
@@ -493,7 +538,37 @@ B=$(gp badges "$URL")
|
||||
end_check 4
|
||||
|
||||
# ── 5 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 5 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
|
||||
begin_check 5 "standby master-key mismatch reaches central's health report (site-a-b on a wrong KEK)"
|
||||
# wait_health <python predicate over F = {node: [failing secret names]}> -> sets HEALTH_S / LAST_F
|
||||
wait_health() {
|
||||
local start now; start=$(python3 -c 'import time; print(time.time())'); HEALTH_S=""
|
||||
while :; do
|
||||
LAST_F=$(mirror_failures site-a)
|
||||
if python3 -c "import json,sys; F=json.loads(sys.argv[1]); sys.exit(0 if F is not None and ($1) else 1)" "$LAST_F"; then
|
||||
now=$(python3 -c 'import time; print(time.time())'); HEALTH_S=$(python3 -c "print(round($now-$start))"); return 0
|
||||
fi
|
||||
now=$(python3 -c 'import time; print(time.time())')
|
||||
python3 -c "import sys; sys.exit(0 if $now-$start < $HEALTH_WINDOW_S else 1)" || return 1
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then ok "baseline: central reports both site-a nodes, no failures ($LAST_F)"
|
||||
else bad "baseline: site-a health report is not two clean node rows: $LAST_F"; fi
|
||||
if [ -z "$RIG_DIR" ]; then bad "cannot find the rig's compose directory from $STANDBY_A's labels"
|
||||
elif recreate_standby_a wrong; then
|
||||
ok "site-a-b recreated with a mismatched KEK and back in the cluster (standby; site-a-a stays active)"
|
||||
if wait_health "'$SECRET' in F.get('node-b',[]) and F.get('node-a')==[]"; then MISMATCH_S="$HEALTH_S"
|
||||
ok "central's site-a report shows node-b failing $SECRET, node-a clean (${HEALTH_S}s): $LAST_F"
|
||||
else bad "node-b's failure never reached central within ${HEALTH_WINDOW_S}s: $LAST_F"; fi
|
||||
if recreate_standby_a; then ok "site-a-b restored with site-a's KEK"; else bad "could not restore site-a-b"; fi
|
||||
if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then CLEARED_S="$HEALTH_S"
|
||||
ok "node-b's failure cleared on central's report (${HEALTH_S}s)"
|
||||
else bad "node-b's failure did not clear within ${HEALTH_WINDOW_S}s: $LAST_F"; fi
|
||||
else bad "could not recreate site-a-b with a mismatched KEK"; fi
|
||||
end_check 5
|
||||
|
||||
# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 6 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
|
||||
SB_GATE_VALUE="$VALUE_OOB" site_a_secret_set "$OOB" "site-secrets gate out-of-band" >/dev/null 2>&1 || true
|
||||
J=$(gp site-secrets "$ACTIVE_A")
|
||||
if [ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ]; then
|
||||
@@ -503,6 +578,7 @@ upsert_conn "$CONN_A" "$SITE_A_ID" "gate-literal-not-a-secret-ref" && ok "$CONN_
|
||||
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
|
||||
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "deploy succeeded (secretsDeployed='$(site_result site-a secretsDeployed)')" \
|
||||
|| bad "deploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
||||
case "$(site_result site-a secretsRemoved)" in *"$SECRET"*) ok "reply names $SECRET in secretsRemoved";; *) bad "secretsRemoved='$(site_result site-a secretsRemoved)', expected $SECRET";; esac
|
||||
GONE="(d.get('$SECRET') is None or d['$SECRET']['deleted'])"
|
||||
if wait_node "$ACTIVE_A" "$GONE" "$T_DEPLOYED"; then ok "site-a-a no longer holds $SECRET (${WAIT_S}s)"; else bad "site-a-a still holds $SECRET: $LAST_NODE_JSON"; fi
|
||||
if wait_node "$STANDBY_A" "$GONE" "$T_DEPLOYED"; then STANDBY_5="$WAIT_S"; ok "site-a-b no longer holds $SECRET (${WAIT_S}s after the deploy returned)"
|
||||
@@ -512,10 +588,10 @@ J=$(gp site-secrets "$ACTIVE_A")
|
||||
&& ok "out-of-band $OOB still live on site-a-a" || bad "out-of-band $OOB lost: $J"
|
||||
L=$(gp site-localdb "$ACTIVE_A")
|
||||
[ "$(jget "$L" "deployed.$SECRET.removed")" = "true" ] && ok "deployed_secrets.$SECRET is_removed=1 (kept for the standby)" || bad "deployed_secrets: $L"
|
||||
end_check 5
|
||||
end_check 6
|
||||
|
||||
# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 6 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
|
||||
# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 7 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
|
||||
SNAP_BEFORE="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
|
||||
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$MISSING" && ok "$CONN_A now references secret:$MISSING" || bad "update $CONN_A"
|
||||
deploy "$SITE_A_ID"
|
||||
@@ -523,6 +599,7 @@ ERR=$(site_result site-a errorMessage)
|
||||
echo " deploy rc=$DEPLOY_RC error: $ERR"
|
||||
EXPECT="Secret '$MISSING' referenced by data connection(s) $CONN_A is not in the central secret store"
|
||||
[ "$(site_result site-a success)" = "false" ] && ok "site-a deploy reported failure" || bad "site-a deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
||||
[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1"
|
||||
case "$ERR" in *"$EXPECT"*) ok "named fail-closed message";; *) bad "message does not contain: $EXPECT";; esac
|
||||
B=$(gp badges "$URL"); echo " badges: $B"
|
||||
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a stays Out of date" || bad "site-a is '$(badge_of "$B" site-a)'"
|
||||
@@ -530,10 +607,10 @@ sleep 3
|
||||
SNAP_AFTER="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
|
||||
[ "$SNAP_BEFORE" = "$SNAP_AFTER" ] && ok "nothing changed: stamped hash, both nodes' secret rows, deployed_secrets, ledgers and the $CONN_A definition identical" \
|
||||
|| { bad "site state changed"; echo " before: $SNAP_BEFORE"; echo " after: $SNAP_AFTER"; }
|
||||
end_check 6
|
||||
end_check 7
|
||||
|
||||
# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 7 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
|
||||
# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 8 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
|
||||
C=$(gp central-secret "$SECRET")
|
||||
KEK=$(jget "$C" kek)
|
||||
upsert_conn "$CONN_B" "$SITE_B_ID" "secret:$SECRET" && ok "created $CONN_B on site-b (ApiKey=secret:$SECRET)" || bad "create $CONN_B"
|
||||
@@ -541,13 +618,14 @@ deploy "$SITE_B_ID"
|
||||
ERR=$(site_result site-b errorMessage)
|
||||
echo " deploy rc=$DEPLOY_RC error: $ERR"
|
||||
[ "$(site_result site-b success)" = "false" ] && ok "site-b deploy reported failure" || bad "site-b deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
||||
[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1"
|
||||
case "$ERR" in *"$SECRET"*"$KEK"*) ok "message names $SECRET and the row's KekId $KEK";; *) bad "message does not name $SECRET + KekId $KEK";; esac
|
||||
B=$(gp badges "$URL"); echo " badges: $B"
|
||||
[ "$(badge_of "$B" site-b)" = "out of date" ] && ok "site-b not stamped (Out of date)" || bad "site-b is '$(badge_of "$B" site-b)'"
|
||||
end_check 7
|
||||
end_check 8
|
||||
|
||||
# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 8 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
|
||||
# ── 9 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 9 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
|
||||
AUD=$("$CLI" audit-config query "${A[@]}" --action DeploySiteSecrets --entity-type SiteSecrets --from "$GATE_START" --page-size 50 2>/dev/null)
|
||||
ROWS=$(printf '%s' "$AUD" | python3 -c '
|
||||
import json,sys
|
||||
@@ -557,18 +635,20 @@ for e in sorted(d.get("entries",[]), key=lambda e: e["timestamp"]):
|
||||
s=json.loads(e.get("afterStateJson") or "{}")
|
||||
names=",".join("%s@r%s" % (x.get("name"), x.get("revision")) for x in (s.get("secrets") or []))
|
||||
miss=",".join(s.get("missing") or [])
|
||||
rows.append("%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", s.get("deploymentId")))
|
||||
rem=",".join(s.get("removed") or [])
|
||||
rows.append("%s|%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", rem or "-", s.get("deploymentId")))
|
||||
print("\n".join(rows))')
|
||||
echo "$ROWS" | sed 's/^/ audit: /'
|
||||
R2=$(jget "$C1" rev); R4=$(jget "$C2" rev)
|
||||
EXPECTED="site-a|True|$SECRET@r$R2|-
|
||||
site-a|True|$SECRET@r$R4|-
|
||||
site-a|False|-|$MISSING
|
||||
site-b|False|$SECRET@r$R4|-"
|
||||
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-4)
|
||||
[ "$GOT" = "$EXPECTED" ] && ok "exactly 4 rows, one per secret-bearing deploy (2, 4, 6-fail-closed, 7-kek-mismatch); the no-secret deploy of check 5 wrote none" \
|
||||
EXPECTED="site-a|True|$SECRET@r$R2|-|-
|
||||
site-a|True|$SECRET@r$R4|-|-
|
||||
site-a|True|-|-|$SECRET
|
||||
site-a|False|-|$MISSING|-
|
||||
site-b|False|$SECRET@r$R4|-|-"
|
||||
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-5)
|
||||
[ "$GOT" = "$EXPECTED" ] && ok "exactly 5 rows: deploys 2 and 4, the removal-only deploy of check 6 (removed=$SECRET), 7 fail-closed, 8 kek-mismatch" \
|
||||
|| { bad "audit rows differ from expected"; echo " expected:"; echo "$EXPECTED" | sed 's/^/ /'; }
|
||||
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f5 | sort -u | grep -c .)" = "4" ] && ok "4 distinct deploymentIds" || bad "deploymentIds not distinct"
|
||||
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f6 | sort -u | grep -c .)" = "5" ] && ok "5 distinct deploymentIds" || bad "deploymentIds not distinct"
|
||||
# Material to hunt for: both plaintext values and both ciphertexts (base64, as they appear in any
|
||||
# JSON serialization). Written to a pattern file so they never reach argv or the output.
|
||||
PAT="$WORK/patterns"; : >"$PAT"; chmod 600 "$PAT"
|
||||
@@ -584,10 +664,11 @@ done
|
||||
AUDH=$(printf '%s' "$AUD" | grep -c -F -f "$PAT" || true)
|
||||
[ "$HITS" = "0" ] && ok "0 hits across all 8 nodes' logs since $LOG_SINCE" || bad "$HITS log line(s) carry secret material"
|
||||
[ "$AUDH" = "0" ] && ok "0 hits in the DeploySiteSecrets audit payloads" || bad "audit payload carries secret material"
|
||||
end_check 8
|
||||
end_check 9
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check5=${STANDBY_5:-?}"
|
||||
echo "PASS $CHECKS_PASSED/8"
|
||||
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check6=${STANDBY_5:-?}"
|
||||
echo "standby mismatch on central's health report (check 5): shown after ${MISMATCH_S:-?}s, cleared after ${CLEARED_S:-?}s"
|
||||
echo "PASS $CHECKS_PASSED/9"
|
||||
[ "$CHECKS_FAILED" -eq 0 ] || exit 1
|
||||
|
||||
@@ -120,7 +120,7 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round
|
||||
| 124 | **No template-graph-watermark short-circuit on the pending-updates fleet flatten** *(Low)*. `PendingUpdatesEvaluator.EvaluateAsync` re-flattens every deployed instance through `DeploymentService.GetInstanceStalenessAsync` on every cache-TTL expiry (`PendingUpdatesCacheTtl`, default 60 s) while any Deployer/Admin circuit is open, even when nothing in the template/instance/artifact graph has changed since the last evaluation. | `docs/plans/2026-09-14-pending-updates-design.md` §3.2 ("Not in scope") | The single-flight TTL cache already bounds the cost to one fleet flatten per TTL fleet-wide (not per viewer); a watermark short-circuit is a further optimization, not a correctness need, and needs a cheap "has anything changed" signal that does not exist yet. | The fleet flatten's cost (instance count x template depth) becomes measurable against `PendingUpdatesCacheTtl` on a large deployment. |
|
||||
| 125 | **Pending-updates counts have no Health-dashboard tile or CLI surface** *(Low)*. The aggregate (`PendingUpdatesSnapshot.StaleInstanceCount` / `StaleArtifactSiteCount`) is consumed only by the Central UI Sites page and rail indicator; there is no Health dashboard KPI tile and no `deploy`/`site` CLI command surfacing it — `site list` shows only the raw `artifactRevisionHash`/`artifactsDeployedAt` fields on each site, not whether they are current. | `docs/plans/2026-09-14-pending-updates-design.md` §3.2 ("Not in scope") | Out of scope for the UI-focused ask; the CLI/Health surfaces are a separate consumer of the same `IPendingUpdatesCache`. | An operator wants the pending-updates counts without opening the Central UI (headless monitoring, CLI scripting). |
|
||||
| 126 | ~~**TODO — roll the two outstanding migrations to the QA pair vq03/vq04**~~ **DONE 2026-09-28** — QA vq03/vq04 upgraded `2b9a7c3c` → `b3021e7c`, 55 → 57, artifacts deployed (record `docs/deployment/deployment-records/2026-09-28-wonder-app-vq03-vq04-b3021e7c.md`). *(deployment, closed)*. **DEV vd03 DONE 2026-09-28** (`fe89363b`, 55 → 57, record `docs/deployment/deployment-records/2026-09-28-wonder-app-vd03-fe89363b.md` — the `sb-mig-55to57.sql` there is the same script QA needs). QA still runs `2b9a7c3c` with the DB at migration 55 (`20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration`); `main` (`ac833f5a`, pushed 2026-09-14) carries two more: **56** `20260911135501_AddAlarmMesReportable` (computed-alarm `IsMesReportable` flag) and **57** `20260914173147_AddSiteArtifactRevision` (`Sites.ArtifactRevisionHash`/`ArtifactsDeployedAt`, both nullable). Both are additive `AddColumn`s — no data upgrade, no coordinated step like the 2026-09-10 protocol-config encryption. Recipe (same as the 55 rollout): generate ONE from→to delta script `dotnet ef migrations script EnforceGlobalInstanceNamesAndProtectProtocolConfiguration AddSiteArtifactRevision --project src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase --startup-project src/ZB.MOM.WW.ScadaBridge.Host -o sb-mig-55to57.sql` (NOT idempotent, identical for vd03 and the QA pair), take the pre-upgrade DB backup, stop the pair TOGETHER, apply via `sqlcmd -C -b -I` on the SQL host, publish/swap the `ac833f5a` binaries, start the pair, confirm the Central boot log shows 0 pending migrations and no "Applying migration" (the script already applied them). After upgrade every site reads *Artifacts never deployed* until one *Deploy Artifacts to All Sites* — run it as the last step. Write the deployment record `docs/deployment/deployment-records/<date>-wonder-app-{vd03,vq03-vq04}-ac833f5a.md`. | Pending-updates finish (2026-09-14) | ~~Blocked by the 2026-09-11 remote-access hold~~ — **unblocked 2026-09-28**: `winrm` is re-enabled for deployment activities (CLAUDE.md). vd03 done; QA done 2026-09-28. | Closed 2026-09-28. |
|
||||
| 127 | **TODO — roll out site secrets via artifact deploy to vd03 + QA (vq03/vq04)**. The gRPC secrets hub is retired in ScadaBridge as of `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` (implemented on branch `feat/site-secrets-artifact-deploy`, rig-only so far); vd03 and QA still run the pre-feature build with `Secrets:Replication:Enabled=false`, so upgrading them will **not** fail the boot (the retirement's `StartupValidator` rejection only fires on `Enabled=true`). **Upgrade impact:** once upgraded, any site whose data connections carry `secret:` references fails its **whole** artifact deploy (scripts, external systems, DB connections too) and reads *Out of date* until every referenced secret exists in the central store — so before upgrading vd03/QA, check whether their connection configs use `secret:` references and whether their site pairs run LocalDb replication, and do the prerequisites in the same window. Prerequisites, in order: (1) provision the `zbsecrets` schema/store in the environment's configuration SQL Server (the same server `ConfigurationDb` already points at); (2) set `Secrets:Store:Provider=SqlServer` on **both** central nodes of the environment; (3) verify the KEK (`ZB_SECRETS_MASTER_KEY` / master-key file) is identical across **every** node — both centrals and every site node — since a mismatch is caught only at deploy time by `DeployedSecretsMirror`'s per-secret verify, not at boot; (4) confirm every site pair that uses secrets runs LocalDb replication — on an unreplicated pair only the node active at deploy time gets the secrets, so a failover leaves the new active node without them until the next *Deploy Artifacts*; (5) recreate, at the new central shared store, any secret an operator previously hand-seeded directly into a site's local `secrets.db` (those never round-tripped through central and would otherwise be silently absent from the referenced-secrets scan). Then run *Deploy Artifacts* against every affected site as the rollout's last step. | `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` §4 *Rollout*; `docs/deployment/topology-guide.md` → *Secrets: shared store, per-site deploy, and the KEK* | Deployment record, not a code change — vd03/QA config is user-held and their rollout schedule is independent of the branch merging to main. | User asks for the vd03/QA secrets window. |
|
||||
| 127 | **TODO — roll out site secrets via artifact deploy to vd03 + QA (vq03/vq04)**. The gRPC secrets hub is retired in ScadaBridge as of `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` (merged to `main` @ `23a139a4` 2026-09-28, plus the same-day follow-ups: per-site secret-store outage handling, removal audit rows, per-node mirror status on the health report, artifact-deploy CLI exit codes; rig-only so far); vd03 and QA still run the pre-feature build with `Secrets:Replication:Enabled=false`, so upgrading them will **not** fail the boot (the retirement's `StartupValidator` rejection only fires on `Enabled=true`). **Upgrade impact:** once upgraded, any site whose data connections carry `secret:` references fails its **whole** artifact deploy (scripts, external systems, DB connections too) and reads *Out of date* until every referenced secret exists in the central store — so before upgrading vd03/QA, check whether their connection configs use `secret:` references and whether their site pairs run LocalDb replication, and do the prerequisites in the same window. Prerequisites, in order: (1) provision the `zbsecrets` schema/store in the environment's configuration SQL Server (the same server `ConfigurationDb` already points at); (2) set `Secrets:Store:Provider=SqlServer` on **both** central nodes of the environment; (3) verify the KEK (`ZB_SECRETS_MASTER_KEY` / master-key file) is identical across **every** node — both centrals and every site node — since a mismatch is caught only at deploy time by `DeployedSecretsMirror`'s per-secret verify, not at boot (after the first deploy a mismatch on either node — the standby included — shows as a *Secret mirror failing* badge on that site's Health card); (4) confirm every site pair that uses secrets runs LocalDb replication — on an unreplicated pair only the node active at deploy time gets the secrets, so a failover leaves the new active node without them until the next *Deploy Artifacts*; the upgrade also adds two replicated site tables (`deployed_secrets`, `secret_mirror_status`), so stop and start each site pair together as usual; (5) recreate, at the new central shared store, any secret an operator previously hand-seeded directly into a site's local `secrets.db` (those never round-tripped through central and would otherwise be silently absent from the referenced-secrets scan). Then run *Deploy Artifacts* against every affected site as the rollout's last step. | `docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md` §4 *Rollout*; `docs/deployment/topology-guide.md` → *Secrets: shared store, per-site deploy, and the KEK* | Deployment record, not a code change — vd03/QA config is user-held and their rollout schedule is independent of the branch merging to main. | User asks for the vd03/QA secrets window. |
|
||||
|
||||
## Resolved (verified against the code 2026-07-10)
|
||||
Rows removed from the Deferred table above once confirmed shipped. Kept here for traceability.
|
||||
|
||||
@@ -151,3 +151,46 @@ rows, 0 plaintext/ciphertext hits across all 8 node logs and the audit payloads.
|
||||
|
||||
Tests before the rig run: `dotnet build ZB.MOM.WW.ScadaBridge.slnx` 0 errors; Commons.Tests 875/875,
|
||||
SiteRuntime.Tests 892/892, DeploymentManager.Tests 235/235, DataConnectionLayer.Tests 383/383, Host.Tests 593/593.
|
||||
|
||||
## 7. Follow-ups run (2026-09-28)
|
||||
|
||||
Code under test: branch `fix/site-secrets-followups` — per-site secret-store outage handling and
|
||||
removal-only audit rows (`29049c4e`), artifact-deploy CLI exit codes (`ef03a0dc`), per-node mirror status on
|
||||
the site health report (`0289d511`, merged in `4e640918`). Image rebuilt from the worktree with
|
||||
`bash docker/build.sh`, all 8 nodes recreated from the primary checkout's compose file.
|
||||
|
||||
**Hub-era secrets removed from site-a (review M8), before the run.** The retired gRPC hub had left nine rows
|
||||
dated 2026-08-07 in both site-a nodes' local stores (`/app/data/scadabridge-secrets.db`, table `secret`):
|
||||
six live (`hub-gate-smoke-2`, `hub-gate-smoke-3`, `shared-gate-051`, `shared-gate-2`, `shared-gate-2b`,
|
||||
`shared-gate-final`) and three tombstones (`expander-gate-pepper`, `hub-gate-smoke-1`, `shared-gate-1`).
|
||||
Nothing in the rig config referenced them. With the site-a pair stopped together, each node's file was edited
|
||||
from a throwaway `python:3.13-alpine` container mounting that node's data volume (`DELETE … WHERE name IN (…)
|
||||
AND updated_utc < '2026-08-08'`, then `wal_checkpoint(TRUNCATE)`); 9 rows deleted per node, only the `sb-gate-*`
|
||||
gate tombstones left. Both nodes came back ready, site-a-a active. site-b/site-c never ran the hub and had no
|
||||
such rows; QA and vd03 never enabled replication.
|
||||
|
||||
**The gate grew to nine checks:**
|
||||
- new check 5 (M1): site-a-b is recreated with the site-b/site-c dev KEK (a compose override, removed again in
|
||||
the same check and in teardown), and central's `health site --identifier site-a` must show node-b failing
|
||||
`sb-gate-mxgw` while node-a is clean, then both clean once site-a-b has site-a's KEK again;
|
||||
- check 6 (was 5) also asserts the reply's `secretsRemoved` names the dropped secret;
|
||||
- checks 7 and 8 (were 6 and 7) also assert the CLI exits 1;
|
||||
- check 9 (was 8) now expects five `DeploySiteSecrets` rows — the removal-only deploy of check 6 writes one,
|
||||
with `removed: [sb-gate-mxgw]`.
|
||||
|
||||
`bash docker/verify-site-secrets.sh` → **PASS 9/9**. Before the check, central reported both site-a node
|
||||
rows with no failures; site-b and site-c (unreplicated) report their active node only. With the wrong KEK,
|
||||
node-b's `sb-gate-mxgw` failure appeared on central's site-a report 57 s after site-a-b was back in the cluster
|
||||
(mirror pass 15 s + status reporter 30 s + health report 30 s), and cleared 57 s after the KEK was restored.
|
||||
Standby convergence: check 3 = 4.3 s, check 4 = 4.2 s, check 6 = 15.1 s. Audit: the 5 expected rows, 5 distinct
|
||||
deployment ids; 0 plaintext/ciphertext hits in all 8 node logs and the audit payloads. Teardown left all three
|
||||
sites *Current* and site-a-b on site-a's KEK.
|
||||
|
||||
Not exercised live: the central secret-store outage (review M5) — taking the shared SQL store down on the rig
|
||||
also takes down central's configuration database. Covered by unit tests (`SiteSecretsCollectorTests`,
|
||||
`ArtifactDeploymentServiceTests.StoreOutage_*`, `ComputeCurrentHashes_StoreOutage_*`,
|
||||
`RetryForSite_StoreOutage_*`).
|
||||
|
||||
Tests: `dotnet build ZB.MOM.WW.ScadaBridge.slnx` 0 errors / 0 warnings; every non-Playwright test project green
|
||||
(CLI.Tests 553 after updating `DeploymentDeadlineParityTests`' stub server to reply with a real zero-failure
|
||||
summary — a bare `{}` now reads as an invalid artifact-deploy summary and exits 1). Playwright not run.
|
||||
|
||||
@@ -50,7 +50,11 @@ public class DeploymentDeadlineParityTests
|
||||
using var payload = JsonDocument.Parse(new string(request));
|
||||
Assert.Equal(expectedCommand, payload.RootElement.GetProperty("command").GetString());
|
||||
await Task.Delay(TimeSpan.FromSeconds(31), deadline.Token);
|
||||
var reply = Encoding.ASCII.GetBytes("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}");
|
||||
// A zero-failure summary: the artifact-deploy aliases derive their exit code from
|
||||
// failureCount, so a bare "{}" would read as an invalid summary (exit 1).
|
||||
const string body = "{\"successCount\":1,\"failureCount\":0}";
|
||||
var reply = Encoding.ASCII.GetBytes(
|
||||
$"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {body.Length}\r\nConnection: close\r\n\r\n{body}");
|
||||
try { await stream.WriteAsync(reply, deadline.Token); }
|
||||
catch (IOException) { /* Old client closes at 30s; exit assertion diagnoses it. */ }
|
||||
}, deadline.Token);
|
||||
|
||||
Reference in New Issue
Block a user