test(rig): site secrets gate covers the follow-ups — standby KEK mismatch on the health report, removal audit, CLI exit codes — PASS 9/9
Also: DeploymentDeadlineParityTests' stub replies with a real zero-failure summary (the artifact aliases now derive their exit code from failureCount), and register row 127 reflects the merge and the two new replicated site tables.
This commit is contained in:
Joseph Doherty
+111
-30
@@ -22,19 +22,24 @@ set -euo pipefail
|
||||
# Ciphertext is compared by SHA-256 only; secret VALUES are random throwaways held in shell
|
||||
# variables and handed to the browser through the environment — never printed, never on argv.
|
||||
#
|
||||
# Checks (each prints PASS/FAIL; the run ends "PASS n/8" and exits non-zero on any FAIL):
|
||||
# Checks (each prints PASS/FAIL; the run ends "PASS n/9" and exits non-zero on any FAIL):
|
||||
# 1 secret + secret-referencing MxGateway connection on site-a -> site-a Out of date, b/c Current
|
||||
# 2 deploy site-a -> success, secretsDeployed 1, site-a Current
|
||||
# 3 both site-a nodes hold the secret at the central revision + ciphertext (standby <= 30 s)
|
||||
# 4 rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision
|
||||
# 5 drop the secret: reference -> deploy -> both nodes no longer hold it; an out-of-band secret
|
||||
# seeded on site-a-a through the library's own write path (the ZB.MOM.WW.Secrets `secret` CLI
|
||||
# run against that node's store) survives
|
||||
# 6 reference a non-existent secret -> site-a deploy fails closed with the named message,
|
||||
# site-a stays Out of date, nothing changed on either site-a node
|
||||
# 7 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the KekId
|
||||
# 8 one DeploySiteSecrets audit row per secret-bearing deploy; no plaintext / base64 ciphertext
|
||||
# in any of the 8 nodes' logs
|
||||
# 5 standby master-key mismatch reaches central (review M1): site-a-b is recreated with a WRONG
|
||||
# KEK -> central's site-a health report shows node-b failing to decrypt the secret while
|
||||
# node-a is clean; site-a-b restored with the right KEK -> node-b's failures clear
|
||||
# 6 drop the secret: reference -> deploy -> both nodes no longer hold it and the reply names it
|
||||
# in secretsRemoved; an out-of-band secret seeded on site-a-a through the library's own write
|
||||
# path (the ZB.MOM.WW.Secrets `secret` CLI run against that node's store) survives
|
||||
# 7 reference a non-existent secret -> site-a deploy fails closed with the named message, the
|
||||
# CLI exits 1, site-a stays Out of date, nothing changed on either site-a node
|
||||
# 8 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the
|
||||
# KekId, the CLI exits 1
|
||||
# 9 one DeploySiteSecrets audit row per deploy that carried, lacked or retired a secret (the
|
||||
# removal-only deploy of check 6 included, with its removed list); no plaintext / base64
|
||||
# ciphertext in any of the 8 nodes' logs
|
||||
#
|
||||
# Teardown (trap EXIT, also run up-front so a rerun starts clean): gate connections deleted,
|
||||
# site-a + site-b redeployed so the removal reaches them, the central gate secret deleted, the
|
||||
@@ -56,6 +61,10 @@ A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" -
|
||||
SECRETS_CLI_PROJECT="${SECRETS_CLI_PROJECT:-$HOME/Desktop/scadaproj/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli}"
|
||||
SECRETS_CLI_DIR="${SECRETS_CLI_DIR:-${TMPDIR:-/tmp}/sb-gate-secrets-cli}"
|
||||
STANDBY_WINDOW_S="${STANDBY_WINDOW_S:-30}"
|
||||
# Check 5: mirror pass (15 s) + status reporter poll (30 s) + health report (30 s), plus a restart.
|
||||
HEALTH_WINDOW_S="${HEALTH_WINDOW_S:-240}"
|
||||
# The site-b/site-c dev KEK (docker-compose.yml) — deliberately NOT site-a's, used by check 5.
|
||||
WRONG_KEK="esKrmqV2RBmMq/+6T/5zK2BcalRy5gzDXhoG9Qydg6Q="
|
||||
|
||||
# DEV-ONLY rig KEK shared by central + site-a (docker/docker-compose.yml, x-secrets-shared-kek-env).
|
||||
SITE_A_KEK="zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4="
|
||||
@@ -333,6 +342,39 @@ v=None if r is None else r.get(sys.argv[2])
|
||||
print("" if v is None else (json.dumps(v) if isinstance(v,bool) else v))' "$1" "$2"
|
||||
}
|
||||
badge_of() { jget "$1" "$2"; }
|
||||
# mirror_failures <siteIdentifier> -> JSON {nodeName: [secretName...]} from central's latest health
|
||||
# report for the site (SiteHealthReport.SecretMirrorStatuses), or "null" when not reported
|
||||
mirror_failures() {
|
||||
"$CLI" health site "${A[@]}" --identifier "$1" 2>/dev/null | python3 -c '
|
||||
import json,sys
|
||||
try:
|
||||
r=(json.load(sys.stdin).get("latestReport") or {}).get("secretMirrorStatuses")
|
||||
except Exception:
|
||||
r=None
|
||||
print("null" if r is None else json.dumps({n["nodeName"]: sorted(f["secretName"] for f in n["failures"]) for n in r}, sort_keys=True))'
|
||||
}
|
||||
# The rig's compose project dir (where the bind-mounted appsettings live), from the running container.
|
||||
RIG_DIR=$(docker inspect "$STANDBY_A" --format '{{ index .Config.Labels "com.docker.compose.project.working_dir" }}' 2>/dev/null)
|
||||
# recreate_standby_a [wrong] -> recreates site-a-b from the rig's own compose file, optionally with
|
||||
# an override that swaps in a mismatched KEK; waits until it is a cluster member again.
|
||||
recreate_standby_a() {
|
||||
local files=(-f "$RIG_DIR/docker-compose.yml")
|
||||
if [ "${1:-}" = "wrong" ]; then
|
||||
printf 'services:\n site-a-b:\n environment:\n ZB_SECRETS_MASTER_KEY: "%s"\n' "$WRONG_KEK" >"$WORK/wrong-kek.yml"
|
||||
files+=(-f "$WORK/wrong-kek.yml"); touch "$WORK/standby-kek-swapped"
|
||||
fi
|
||||
docker compose "${files[@]}" --project-directory "$RIG_DIR" up -d --no-deps --force-recreate site-a-b >/dev/null 2>&1 || return 1
|
||||
[ "${1:-}" = "wrong" ] || rm -f "$WORK/standby-kek-swapped"
|
||||
local i
|
||||
for i in $(seq 1 60); do
|
||||
docker run --rm --network "container:$ACTIVE_A" python:3.13-alpine python3 -c '
|
||||
import urllib.request as u,sys
|
||||
try: sys.exit(0 if u.urlopen("http://scadabridge-site-a-b:8084/health/ready",timeout=3).status==200 else 1)
|
||||
except Exception: sys.exit(1)' >/dev/null 2>&1 && return 0
|
||||
sleep 3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
ensure_secrets_cli() {
|
||||
[ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ] && return 0
|
||||
@@ -365,6 +407,9 @@ teardown() {
|
||||
echo ""
|
||||
echo "=== teardown ==="
|
||||
local id
|
||||
if [ -f "$WORK/standby-kek-swapped" ]; then
|
||||
recreate_standby_a && echo " restored site-a-b with site-a's KEK" || echo " WARN could not restore site-a-b's KEK"
|
||||
fi
|
||||
for c in "$CONN_A" "$CONN_B"; do
|
||||
id=$(conn_id "$c" 2>/dev/null)
|
||||
if [ -n "$id" ]; then
|
||||
@@ -493,7 +538,37 @@ B=$(gp badges "$URL")
|
||||
end_check 4
|
||||
|
||||
# ── 5 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 5 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
|
||||
begin_check 5 "standby master-key mismatch reaches central's health report (site-a-b on a wrong KEK)"
|
||||
# wait_health <python predicate over F = {node: [failing secret names]}> -> sets HEALTH_S / LAST_F
|
||||
wait_health() {
|
||||
local start now; start=$(python3 -c 'import time; print(time.time())'); HEALTH_S=""
|
||||
while :; do
|
||||
LAST_F=$(mirror_failures site-a)
|
||||
if python3 -c "import json,sys; F=json.loads(sys.argv[1]); sys.exit(0 if F is not None and ($1) else 1)" "$LAST_F"; then
|
||||
now=$(python3 -c 'import time; print(time.time())'); HEALTH_S=$(python3 -c "print(round($now-$start))"); return 0
|
||||
fi
|
||||
now=$(python3 -c 'import time; print(time.time())')
|
||||
python3 -c "import sys; sys.exit(0 if $now-$start < $HEALTH_WINDOW_S else 1)" || return 1
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then ok "baseline: central reports both site-a nodes, no failures ($LAST_F)"
|
||||
else bad "baseline: site-a health report is not two clean node rows: $LAST_F"; fi
|
||||
if [ -z "$RIG_DIR" ]; then bad "cannot find the rig's compose directory from $STANDBY_A's labels"
|
||||
elif recreate_standby_a wrong; then
|
||||
ok "site-a-b recreated with a mismatched KEK and back in the cluster (standby; site-a-a stays active)"
|
||||
if wait_health "'$SECRET' in F.get('node-b',[]) and F.get('node-a')==[]"; then MISMATCH_S="$HEALTH_S"
|
||||
ok "central's site-a report shows node-b failing $SECRET, node-a clean (${HEALTH_S}s): $LAST_F"
|
||||
else bad "node-b's failure never reached central within ${HEALTH_WINDOW_S}s: $LAST_F"; fi
|
||||
if recreate_standby_a; then ok "site-a-b restored with site-a's KEK"; else bad "could not restore site-a-b"; fi
|
||||
if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then CLEARED_S="$HEALTH_S"
|
||||
ok "node-b's failure cleared on central's report (${HEALTH_S}s)"
|
||||
else bad "node-b's failure did not clear within ${HEALTH_WINDOW_S}s: $LAST_F"; fi
|
||||
else bad "could not recreate site-a-b with a mismatched KEK"; fi
|
||||
end_check 5
|
||||
|
||||
# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 6 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
|
||||
SB_GATE_VALUE="$VALUE_OOB" site_a_secret_set "$OOB" "site-secrets gate out-of-band" >/dev/null 2>&1 || true
|
||||
J=$(gp site-secrets "$ACTIVE_A")
|
||||
if [ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ]; then
|
||||
@@ -503,6 +578,7 @@ upsert_conn "$CONN_A" "$SITE_A_ID" "gate-literal-not-a-secret-ref" && ok "$CONN_
|
||||
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
|
||||
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "deploy succeeded (secretsDeployed='$(site_result site-a secretsDeployed)')" \
|
||||
|| bad "deploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
||||
case "$(site_result site-a secretsRemoved)" in *"$SECRET"*) ok "reply names $SECRET in secretsRemoved";; *) bad "secretsRemoved='$(site_result site-a secretsRemoved)', expected $SECRET";; esac
|
||||
GONE="(d.get('$SECRET') is None or d['$SECRET']['deleted'])"
|
||||
if wait_node "$ACTIVE_A" "$GONE" "$T_DEPLOYED"; then ok "site-a-a no longer holds $SECRET (${WAIT_S}s)"; else bad "site-a-a still holds $SECRET: $LAST_NODE_JSON"; fi
|
||||
if wait_node "$STANDBY_A" "$GONE" "$T_DEPLOYED"; then STANDBY_5="$WAIT_S"; ok "site-a-b no longer holds $SECRET (${WAIT_S}s after the deploy returned)"
|
||||
@@ -512,10 +588,10 @@ J=$(gp site-secrets "$ACTIVE_A")
|
||||
&& ok "out-of-band $OOB still live on site-a-a" || bad "out-of-band $OOB lost: $J"
|
||||
L=$(gp site-localdb "$ACTIVE_A")
|
||||
[ "$(jget "$L" "deployed.$SECRET.removed")" = "true" ] && ok "deployed_secrets.$SECRET is_removed=1 (kept for the standby)" || bad "deployed_secrets: $L"
|
||||
end_check 5
|
||||
end_check 6
|
||||
|
||||
# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 6 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
|
||||
# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 7 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
|
||||
SNAP_BEFORE="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
|
||||
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$MISSING" && ok "$CONN_A now references secret:$MISSING" || bad "update $CONN_A"
|
||||
deploy "$SITE_A_ID"
|
||||
@@ -523,6 +599,7 @@ ERR=$(site_result site-a errorMessage)
|
||||
echo " deploy rc=$DEPLOY_RC error: $ERR"
|
||||
EXPECT="Secret '$MISSING' referenced by data connection(s) $CONN_A is not in the central secret store"
|
||||
[ "$(site_result site-a success)" = "false" ] && ok "site-a deploy reported failure" || bad "site-a deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
||||
[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1"
|
||||
case "$ERR" in *"$EXPECT"*) ok "named fail-closed message";; *) bad "message does not contain: $EXPECT";; esac
|
||||
B=$(gp badges "$URL"); echo " badges: $B"
|
||||
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a stays Out of date" || bad "site-a is '$(badge_of "$B" site-a)'"
|
||||
@@ -530,10 +607,10 @@ sleep 3
|
||||
SNAP_AFTER="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
|
||||
[ "$SNAP_BEFORE" = "$SNAP_AFTER" ] && ok "nothing changed: stamped hash, both nodes' secret rows, deployed_secrets, ledgers and the $CONN_A definition identical" \
|
||||
|| { bad "site state changed"; echo " before: $SNAP_BEFORE"; echo " after: $SNAP_AFTER"; }
|
||||
end_check 6
|
||||
end_check 7
|
||||
|
||||
# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 7 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
|
||||
# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 8 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
|
||||
C=$(gp central-secret "$SECRET")
|
||||
KEK=$(jget "$C" kek)
|
||||
upsert_conn "$CONN_B" "$SITE_B_ID" "secret:$SECRET" && ok "created $CONN_B on site-b (ApiKey=secret:$SECRET)" || bad "create $CONN_B"
|
||||
@@ -541,13 +618,14 @@ deploy "$SITE_B_ID"
|
||||
ERR=$(site_result site-b errorMessage)
|
||||
echo " deploy rc=$DEPLOY_RC error: $ERR"
|
||||
[ "$(site_result site-b success)" = "false" ] && ok "site-b deploy reported failure" || bad "site-b deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
||||
[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1"
|
||||
case "$ERR" in *"$SECRET"*"$KEK"*) ok "message names $SECRET and the row's KekId $KEK";; *) bad "message does not name $SECRET + KekId $KEK";; esac
|
||||
B=$(gp badges "$URL"); echo " badges: $B"
|
||||
[ "$(badge_of "$B" site-b)" = "out of date" ] && ok "site-b not stamped (Out of date)" || bad "site-b is '$(badge_of "$B" site-b)'"
|
||||
end_check 7
|
||||
end_check 8
|
||||
|
||||
# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 8 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
|
||||
# ── 9 ────────────────────────────────────────────────────────────────────────────────────────
|
||||
begin_check 9 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
|
||||
AUD=$("$CLI" audit-config query "${A[@]}" --action DeploySiteSecrets --entity-type SiteSecrets --from "$GATE_START" --page-size 50 2>/dev/null)
|
||||
ROWS=$(printf '%s' "$AUD" | python3 -c '
|
||||
import json,sys
|
||||
@@ -557,18 +635,20 @@ for e in sorted(d.get("entries",[]), key=lambda e: e["timestamp"]):
|
||||
s=json.loads(e.get("afterStateJson") or "{}")
|
||||
names=",".join("%s@r%s" % (x.get("name"), x.get("revision")) for x in (s.get("secrets") or []))
|
||||
miss=",".join(s.get("missing") or [])
|
||||
rows.append("%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", s.get("deploymentId")))
|
||||
rem=",".join(s.get("removed") or [])
|
||||
rows.append("%s|%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", rem or "-", s.get("deploymentId")))
|
||||
print("\n".join(rows))')
|
||||
echo "$ROWS" | sed 's/^/ audit: /'
|
||||
R2=$(jget "$C1" rev); R4=$(jget "$C2" rev)
|
||||
EXPECTED="site-a|True|$SECRET@r$R2|-
|
||||
site-a|True|$SECRET@r$R4|-
|
||||
site-a|False|-|$MISSING
|
||||
site-b|False|$SECRET@r$R4|-"
|
||||
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-4)
|
||||
[ "$GOT" = "$EXPECTED" ] && ok "exactly 4 rows, one per secret-bearing deploy (2, 4, 6-fail-closed, 7-kek-mismatch); the no-secret deploy of check 5 wrote none" \
|
||||
EXPECTED="site-a|True|$SECRET@r$R2|-|-
|
||||
site-a|True|$SECRET@r$R4|-|-
|
||||
site-a|True|-|-|$SECRET
|
||||
site-a|False|-|$MISSING|-
|
||||
site-b|False|$SECRET@r$R4|-|-"
|
||||
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-5)
|
||||
[ "$GOT" = "$EXPECTED" ] && ok "exactly 5 rows: deploys 2 and 4, the removal-only deploy of check 6 (removed=$SECRET), 7 fail-closed, 8 kek-mismatch" \
|
||||
|| { bad "audit rows differ from expected"; echo " expected:"; echo "$EXPECTED" | sed 's/^/ /'; }
|
||||
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f5 | sort -u | grep -c .)" = "4" ] && ok "4 distinct deploymentIds" || bad "deploymentIds not distinct"
|
||||
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f6 | sort -u | grep -c .)" = "5" ] && ok "5 distinct deploymentIds" || bad "deploymentIds not distinct"
|
||||
# Material to hunt for: both plaintext values and both ciphertexts (base64, as they appear in any
|
||||
# JSON serialization). Written to a pattern file so they never reach argv or the output.
|
||||
PAT="$WORK/patterns"; : >"$PAT"; chmod 600 "$PAT"
|
||||
@@ -584,10 +664,11 @@ done
|
||||
AUDH=$(printf '%s' "$AUD" | grep -c -F -f "$PAT" || true)
|
||||
[ "$HITS" = "0" ] && ok "0 hits across all 8 nodes' logs since $LOG_SINCE" || bad "$HITS log line(s) carry secret material"
|
||||
[ "$AUDH" = "0" ] && ok "0 hits in the DeploySiteSecrets audit payloads" || bad "audit payload carries secret material"
|
||||
end_check 8
|
||||
end_check 9
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check5=${STANDBY_5:-?}"
|
||||
echo "PASS $CHECKS_PASSED/8"
|
||||
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check6=${STANDBY_5:-?}"
|
||||
echo "standby mismatch on central's health report (check 5): shown after ${MISMATCH_S:-?}s, cleared after ${CLEARED_S:-?}s"
|
||||
echo "PASS $CHECKS_PASSED/9"
|
||||
[ "$CHECKS_FAILED" -eq 0 ] || exit 1
|
||||
|
||||
Reference in New Issue
Block a user