dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ef03a0dca784a4b036e267bef85d20132d5cc127
ScadaBridge/docker/verify-site-secrets.sh
T

594 lines
34 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
set -euo pipefail
# Live gate for SITE SECRETS RIDING THE ARTIFACT DEPLOY (design
# docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md, plan task 13). Drives the real
# central paths end to end on the running docker rig:
#
# - secrets are created / rotated / deleted at central through the Central UI's own
# /admin/secrets page (ZB.MOM.WW.Secrets.Ui over the central pair's shared SQL store) — the
# supported operator path; there is no CLI/management verb for central secrets
# - data connections and artifact deploys go through the ScadaBridge CLI
# (CLI -> Traefik -> active central -> ArtifactDeploymentService)
# - the Artifacts badge is read from the Sites page after pressing Re-check (the pending-updates
# evaluation has no CLI/management surface; it is Blazor-interactive, so a headless browser is
# the only honest reader)
# - each site node's own secret store (the library's SQLite `secret` table in
# /app/data/scadabridge-secrets.db) and LocalDb (`deployed_secrets`, `mirrored_secrets`,
# `data_connection_definitions` in /app/data/site-localdb.db) are copied out with `docker cp`
# (the image carries no sqlite3) and read with the host's python3 sqlite3 module
# - the central row is read (never written) from ZbSecretsHub.zbsecrets.secret with sqlcmd
#
# Ciphertext is compared by SHA-256 only; secret VALUES are random throwaways held in shell
# variables and handed to the browser through the environment — never printed, never on argv.
#
# Checks (each prints PASS/FAIL; the run ends "PASS n/8" and exits non-zero on any FAIL):
# 1 secret + secret-referencing MxGateway connection on site-a -> site-a Out of date, b/c Current
# 2 deploy site-a -> success, secretsDeployed 1, site-a Current
# 3 both site-a nodes hold the secret at the central revision + ciphertext (standby <= 30 s)
# 4 rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision
# 5 drop the secret: reference -> deploy -> both nodes no longer hold it; an out-of-band secret
# seeded on site-a-a through the library's own write path (the ZB.MOM.WW.Secrets `secret` CLI
# run against that node's store) survives
# 6 reference a non-existent secret -> site-a deploy fails closed with the named message,
# site-a stays Out of date, nothing changed on either site-a node
# 7 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the KekId
# 8 one DeploySiteSecrets audit row per secret-bearing deploy; no plaintext / base64 ciphertext
# in any of the 8 nodes' logs
#
# Teardown (trap EXIT, also run up-front so a rerun starts clean): gate connections deleted,
# site-a + site-b redeployed so the removal reaches them, the central gate secret deleted, the
# out-of-band secret removed from site-a-a, then a fleet-wide artifact deploy so the rig ends with
# every site Current. The secret library tombstones rather than hard-deletes, so deleted gate
# names remain as is_deleted=1 rows in the stores — that is the library's contract, not residue.
#
# Prereqs: rig up (8 nodes ready), `python3 -m playwright` with chromium, the ScadaBridge CLI
# built, and scadaproj checked out next to this repo (the `secret` CLI is published from it once
# into $SECRETS_CLI_DIR).
#
# Usage: bash docker/verify-site-secrets.sh
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CLI="${SCADABRIDGE_CLI:-$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge}"
URL="${SCADABRIDGE_URL:-http://localhost:9000}"
A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}")
SECRETS_CLI_PROJECT="${SECRETS_CLI_PROJECT:-$HOME/Desktop/scadaproj/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli}"
SECRETS_CLI_DIR="${SECRETS_CLI_DIR:-${TMPDIR:-/tmp}/sb-gate-secrets-cli}"
STANDBY_WINDOW_S="${STANDBY_WINDOW_S:-30}"
# DEV-ONLY rig KEK shared by central + site-a (docker/docker-compose.yml, x-secrets-shared-kek-env).
SITE_A_KEK="zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4="
MSSQL_CONTAINER="${MSSQL_CONTAINER:-scadabridge-mssql}"
MSSQL_USER="scadabridge_app"; MSSQL_PASS='ScadaBridge_Dev1#'
SECRET="sb-gate-mxgw"
MISSING="sb-gate-missing"
OOB="sb-gate-oob"
CONN_A="sb-gate-conn-a"
CONN_B="sb-gate-conn-b"
ACTIVE_A="scadabridge-site-a-a"; STANDBY_A="scadabridge-site-a-b"
NODES=(central-a central-b site-a-a site-a-b site-b-a site-b-b site-c-a site-c-b)
VALUE_1=$(python3 -c 'import secrets; print("gate1-" + secrets.token_hex(16))')
VALUE_2=$(python3 -c 'import secrets; print("gate2-" + secrets.token_hex(16))')
VALUE_OOB=$(python3 -c 'import secrets; print("gateoob-" + secrets.token_hex(16))')
CHECKS_PASSED=0; CHECKS_FAILED=0
CHECK_OK=1
begin_check() { CHECK_OK=1; echo ""; echo "=== Check $1: $2 ==="; }
ok() { echo " ok $*"; }
bad() { CHECK_OK=0; echo " FAIL $*"; }
end_check() {
if [ "$CHECK_OK" -eq 1 ]; then CHECKS_PASSED=$((CHECKS_PASSED+1)); echo " PASS check $1"
else CHECKS_FAILED=$((CHECKS_FAILED+1)); echo " FAIL check $1"; fi
}
[ -x "$CLI" ] || { echo "CLI not found at $CLI — build it: dotnet build src/ZB.MOM.WW.ScadaBridge.CLI"; exit 1; }
python3 -c 'import playwright.sync_api' 2>/dev/null || { echo "python3 playwright missing (pip install playwright && playwright install chromium)"; exit 1; }
WORK=$(mktemp -d "${TMPDIR:-/tmp}/sb-gate-site-secrets.XXXXXX")
HELPER="$WORK/gate.py"
# ── python helper: browser (UI) + read-only store readers ──────────────────────────────────────
cat >"$HELPER" <<'PY'
import hashlib, json, os, sqlite3, subprocess, sys, tempfile, time, shutil
def sha(b):
return hashlib.sha256(b if isinstance(b, (bytes, bytearray)) else b.encode()).hexdigest()
# ---- Central UI -------------------------------------------------------------------------------
def _browser(fn):
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
b = p.chromium.launch()
try:
pg = b.new_page()
return fn(pg)
finally:
b.close()
def _wait_circuit(pg):
# Blazor Server: handlers only work once the circuit is up.
pg.wait_for_function("() => window.Blazor !== undefined", timeout=30000)
time.sleep(2)
def badges(url):
def run(pg):
pg.goto(url + "/admin/sites")
pg.wait_for_selector("button[aria-label='Re-check pending updates']", timeout=30000)
_wait_circuit(pg)
pg.click("button[aria-label='Re-check pending updates']")
time.sleep(1)
pg.wait_for_function("() => { const b=document.querySelector(\"button[aria-label='Re-check pending updates']\"); return b && !b.disabled; }", timeout=120000)
pg.wait_for_function("() => !document.querySelector('[aria-label=\"Artifacts: checking\"]')", timeout=60000)
out = {}
for card in pg.query_selector_all(".card"):
code = card.query_selector("code"); badge = card.query_selector("[aria-label^='Artifacts:']")
if code and badge:
out[code.inner_text().strip()] = badge.get_attribute("aria-label").split(": ", 1)[1]
return out
print(json.dumps(_browser(run)))
def _row_exists(pg, name):
return pg.query_selector(f"button.rotate-secret[data-secret-name='{name}']") is not None
def _secrets_page(pg, url):
pg.goto(url + "/admin/secrets")
pg.wait_for_selector("[data-testid='add-secret']", timeout=30000)
pg.wait_for_function("() => !document.body.innerText.includes('Loading…')", timeout=30000)
_wait_circuit(pg)
def secret_upsert(url, name, desc):
value = os.environ["SB_GATE_VALUE"]
def run(pg):
_secrets_page(pg, url)
mode = "rotate" if _row_exists(pg, name) else "add"
for attempt in range(5):
if mode == "rotate":
pg.click(f"button.rotate-secret[data-secret-name='{name}']")
else:
pg.click("[data-testid='add-secret']")
try:
pg.wait_for_selector("[data-testid='editor-submit']", timeout=3000); break
except Exception:
time.sleep(1)
if mode == "add":
pg.fill("[data-testid='secret-name']", name); pg.press("[data-testid='secret-name']", "Tab")
pg.fill("[data-testid='secret-description']", desc); pg.press("[data-testid='secret-description']", "Tab")
pg.fill("[data-testid='secret-value']", value); pg.press("[data-testid='secret-value']", "Tab")
pg.click("[data-testid='editor-submit']")
deadline = time.time() + 20
while time.time() < deadline:
err = pg.query_selector("[data-testid='editor-error']")
if err:
return "error: " + err.inner_text()
if pg.query_selector("[data-testid='editor-submit']") is None and _row_exists(pg, name):
return mode
time.sleep(0.3)
return "error: editor did not close"
print(_browser(run))
def secret_delete(url, name):
def run(pg):
_secrets_page(pg, url)
if not _row_exists(pg, name):
return "absent"
for attempt in range(5):
pg.click(f"button.delete-secret[data-secret-name='{name}']")
try:
pg.wait_for_selector("[data-testid='confirm-delete']", timeout=3000); break
except Exception:
time.sleep(1)
pg.click("[data-testid='confirm-delete']")
deadline = time.time() + 20
while time.time() < deadline:
err = pg.query_selector("[data-testid='delete-error']")
if err:
return "error: " + err.inner_text()
if not _row_exists(pg, name):
return "deleted"
time.sleep(0.3)
return "error: row still listed"
print(_browser(run))
# ---- read-only store readers ------------------------------------------------------------------
def _copy_db(container, path):
"""docker cp the db + its WAL (NOT the -shm: SQLite rebuilds the wal-index from the WAL on
open, which is safer than trusting a copied live shared-memory file)."""
d = tempfile.mkdtemp(prefix="sbgate-")
base = os.path.basename(path)
for suffix in ("-wal", ""):
r = subprocess.run(["docker", "cp", f"{container}:{path}{suffix}", os.path.join(d, base + suffix)],
capture_output=True)
if r.returncode != 0 and suffix == "":
raise SystemExit(f"docker cp {container}:{path} failed: {r.stderr.decode().strip()}")
return d, os.path.join(d, base)
def _query(container, path, sql):
# Two consecutive reads must agree — guards against a checkpoint racing the two docker cp's.
last = None
for _ in range(4):
d, db = _copy_db(container, path)
try:
con = sqlite3.connect(db); rows = con.execute(sql).fetchall(); con.close()
finally:
shutil.rmtree(d, ignore_errors=True)
if rows == last:
return rows
last = rows
return last
def site_secrets(container):
rows = _query(container, "/app/data/scadabridge-secrets.db",
"SELECT name, revision, kek_id, is_deleted, ciphertext FROM secret WHERE name LIKE 'sb-gate-%' ORDER BY name")
print(json.dumps({n: {"rev": r, "kek": k, "deleted": bool(dl), "sha": sha(c)} for n, r, k, dl, c in rows}, sort_keys=True))
def site_localdb(container):
db = "/app/data/site-localdb.db"
dep = _query(container, db, "SELECT name, revision, is_removed, row_json FROM deployed_secrets WHERE name LIKE 'sb-gate-%' ORDER BY name")
mir = _query(container, db, "SELECT name, revision FROM mirrored_secrets WHERE name LIKE 'sb-gate-%' ORDER BY name")
dcs = _query(container, db, "SELECT name, configuration FROM data_connection_definitions WHERE name LIKE 'sb-gate-%' ORDER BY name")
print(json.dumps({
"deployed": {n: {"rev": r, "removed": bool(x), "sha": sha(j)} for n, r, x, j in dep},
"mirrored": {n: r for n, r in mir},
"connections": {n: sha(c or "") for n, c in dcs},
}, sort_keys=True))
def central_secret(name):
q = ("SET NOCOUNT ON; SELECT name, revision, kek_id, is_deleted, "
"LOWER(CONVERT(varchar(64), HASHBYTES('SHA2_256', ciphertext), 2)), "
"CONVERT(varchar(max), ciphertext, 2) "
f"FROM zbsecrets.secret WHERE name = N'{name}'")
r = subprocess.run(["docker", "exec", os.environ["MSSQL_CONTAINER"], "/opt/mssql-tools18/bin/sqlcmd",
"-S", "localhost", "-U", os.environ["MSSQL_USER"], "-P", os.environ["MSSQL_PASS"],
"-C", "-d", "ZbSecretsHub", "-h", "-1", "-W", "-s", "|", "-Q", q],
capture_output=True, text=True)
if r.returncode != 0:
raise SystemExit("sqlcmd failed: " + r.stderr.strip())
lines = [l for l in r.stdout.splitlines() if l.strip()]
if not lines:
print(json.dumps(None)); return
n, rev, kek, dl, h, hexct = lines[0].split("|")
import base64
b64 = base64.b64encode(bytes.fromhex(hexct)).decode()
if sha(bytes.fromhex(hexct)) != h:
raise SystemExit("ciphertext hex truncated by sqlcmd")
# b64 is only used by check 8's log grep; it is written to a file by the caller, never printed.
print(json.dumps({"rev": int(rev), "kek": kek, "deleted": dl == "1", "sha": h, "b64": b64}))
cmd, args = sys.argv[1], sys.argv[2:]
{"badges": lambda: badges(*args),
"secret-upsert": lambda: secret_upsert(*args),
"secret-delete": lambda: secret_delete(*args),
"site-secrets": lambda: site_secrets(*args),
"site-localdb": lambda: site_localdb(*args),
"central-secret": lambda: central_secret(*args)}[cmd]()
PY
export MSSQL_CONTAINER MSSQL_USER MSSQL_PASS
gp() { python3 "$HELPER" "$@"; }
jget() { python3 -c 'import json,sys
d=json.loads(sys.argv[1])
for k in sys.argv[2].split("."):
if d is None: break
d = d.get(k) if isinstance(d, dict) else None
print("" if d is None else (json.dumps(d) if isinstance(d,(dict,list,bool)) else d))' "$1" "$2"; }
UNWRAP='
import json,sys
d=json.load(sys.stdin)
if isinstance(d, dict) and "data" in d and len(d) <= 2:
d = d["data"]
'
site_id() {
"$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
print(next(str(s["id"]) for s in d if s["siteIdentifier"]==sys.argv[1]))' "$1"
}
site_hash() {
"$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
print(next((s.get("artifactRevisionHash") or "-") for s in d if s["siteIdentifier"]==sys.argv[1]))' "$1"
}
conn_id() { # conn_id <name> -> id or empty
"$CLI" data-connection list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
print(next((str(c["id"]) for c in d if c["name"]==sys.argv[1]), ""))' "$1"
}
mxgw_config() { # mxgw_config <apiKeyValue>
python3 -c 'import json,sys; print(json.dumps({"Endpoint":"http://sb-gate-no-such-gateway:5000","ApiKey":sys.argv[1],"ClientName":"sb-gate","ReadTimeoutMs":5000}))' "$1"
}
upsert_conn() { # upsert_conn <name> <siteId> <apiKeyValue>
local id; id=$(conn_id "$1")
if [ -z "$id" ]; then
"$CLI" data-connection create "${A[@]}" --site-id "$2" --name "$1" --protocol MxGateway \
--primary-config "$(mxgw_config "$3")" >/dev/null
else
"$CLI" data-connection update "${A[@]}" --id "$id" --name "$1" --protocol MxGateway \
--primary-config "$(mxgw_config "$3")" >/dev/null
fi
}
# deploy <siteId|all> -> sets DEPLOY_OUT (raw CLI output) and DEPLOY_RC; never aborts the script
DEPLOY_OUT=""; DEPLOY_RC=0
deploy() {
DEPLOY_RC=0
if [ "$1" = "all" ]; then
DEPLOY_OUT=$("$CLI" site deploy-artifacts "${A[@]}" 2>&1) || DEPLOY_RC=$?
else
DEPLOY_OUT=$("$CLI" site deploy-artifacts "${A[@]}" --site-id "$1" 2>&1) || DEPLOY_RC=$?
fi
}
# site_result <siteIdentifier> <field> -> the field from DEPLOY_OUT's siteResults
site_result() {
printf '%s' "$DEPLOY_OUT" | python3 -c '
import json,sys
raw=sys.stdin.read()
try:
d=json.loads(raw[raw.index("{"):])
except Exception:
print(""); sys.exit(0)
r=next((x for x in d.get("siteResults",[]) if x.get("siteId")==sys.argv[1]), None)
v=None if r is None else r.get(sys.argv[2])
print("" if v is None else (json.dumps(v) if isinstance(v,bool) else v))' "$1" "$2"
}
badge_of() { jget "$1" "$2"; }
ensure_secrets_cli() {
[ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ] && return 0
echo " publishing the ZB.MOM.WW.Secrets 'secret' CLI to $SECRETS_CLI_DIR ..."
dotnet publish "$SECRETS_CLI_PROJECT" -c Release -o "$SECRETS_CLI_DIR" >/dev/null
}
# Runs the library's own `secret` CLI against site-a-a's LOCAL store, in a throwaway container that
# shares that node's data volume and carries that node's KEK — the library's normal SQLite write path.
site_a_secret_cli() {
local verb="$1"; shift
docker run --rm --volumes-from "$ACTIVE_A" -v "$SECRETS_CLI_DIR:/cli:ro" -w /cli \
-e Secrets__SqlitePath=/data/scadabridge-secrets.db -e "ZB_SECRETS_MASTER_KEY=$SITE_A_KEK" \
-e SB_GATE_VALUE --entrypoint sh scadabridge:latest \
-c "exec dotnet /cli/ZB.MOM.WW.Secrets.Cli.dll $verb \"\$@\"" _ "$@"
}
# `secret set <name> <value>` takes the value positionally: hand it over through the container's
# environment (SB_GATE_VALUE) so it never appears on this host's argv; the container's inner sh
# expands it.
site_a_secret_set() { # site_a_secret_set <name> <description> (value in $SB_GATE_VALUE)
docker run --rm --volumes-from "$ACTIVE_A" -v "$SECRETS_CLI_DIR:/cli:ro" -w /cli \
-e Secrets__SqlitePath=/data/scadabridge-secrets.db -e "ZB_SECRETS_MASTER_KEY=$SITE_A_KEK" \
-e SB_GATE_VALUE --entrypoint sh scadabridge:latest \
-c 'exec dotnet /cli/ZB.MOM.WW.Secrets.Cli.dll set "$1" "$SB_GATE_VALUE" --description "$2"' _ "$1" "$2"
}
# ── teardown (idempotent; also the pre-clean) ──────────────────────────────────────────────────
teardown() {
local rc=$?
set +e
echo ""
echo "=== teardown ==="
local id
for c in "$CONN_A" "$CONN_B"; do
id=$(conn_id "$c" 2>/dev/null)
if [ -n "$id" ]; then
"$CLI" data-connection delete "${A[@]}" --id "$id" >/dev/null 2>&1 \
&& echo " removed data connection $c (id=$id)" || echo " WARN could not remove data connection $c"
fi
done
# Removal must reach both sites that ever held the gate secret BEFORE central forgets it.
[ -n "${SITE_A_ID:-}" ] && { deploy "$SITE_A_ID"; echo " site-a redeploy rc=$DEPLOY_RC"; }
[ -n "${SITE_B_ID:-}" ] && { deploy "$SITE_B_ID"; echo " site-b redeploy rc=$DEPLOY_RC"; }
echo " central $SECRET: $(gp secret-delete "$URL" "$SECRET" 2>&1 | tail -1)"
if [ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ]; then
local live
live=$(gp site-secrets "$ACTIVE_A" 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print("1" if d.get(sys.argv[1],{}).get("deleted") is False else "")' "$OOB" 2>/dev/null)
if [ -n "$live" ]; then
site_a_secret_cli rm "$OOB" >/dev/null 2>&1 && echo " removed out-of-band $OOB from site-a-a" \
|| echo " WARN could not remove out-of-band $OOB"
fi
fi
deploy all; echo " fleet-wide artifact redeploy rc=$DEPLOY_RC"
echo " badges after teardown: $(gp badges "$URL" 2>&1 | tail -1)"
[ "${1:-}" = "pre" ] || rm -rf "$WORK"
return $rc
}
echo "=== ScadaBridge site-secrets artifact-deploy live gate ==="
echo "management url: $URL"
GATE_START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
SITE_A_ID=$(site_id site-a); SITE_B_ID=$(site_id site-b); SITE_C_ID=$(site_id site-c)
echo "sites: site-a=$SITE_A_ID site-b=$SITE_B_ID site-c=$SITE_C_ID"
ensure_secrets_cli
echo ""
echo "=== pre-clean (idempotent rerun) ==="
teardown pre || true
trap teardown EXIT
GATE_START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
# Docker log checkpoint for check 8: only lines written during this run are grepped.
LOG_SINCE="$GATE_START"
BASE=$(gp badges "$URL")
echo "baseline badges: $BASE"
for s in site-a site-b site-c; do
[ "$(badge_of "$BASE" "$s")" = "current" ] || { echo "baseline: $s is not Current — the gate needs a Current rig to start"; exit 1; }
done
# ── 1 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 1 "secret + referencing connection on site-a -> site-a Out of date, site-b/c unchanged"
R=$(SB_GATE_VALUE="$VALUE_1" gp secret-upsert "$URL" "$SECRET" "site-secrets live gate (throwaway)")
[ "$R" = "add" ] && ok "created $SECRET at central via /admin/secrets" || bad "create $SECRET: $R"
C1=$(gp central-secret "$SECRET")
[ "$(jget "$C1" deleted)" = "false" ] && ok "central row $SECRET r$(jget "$C1" rev) kek $(jget "$C1" kek) sha $(jget "$C1" sha | cut -c1-12)" \
|| bad "central row for $SECRET not found live"
H_A_BEFORE=$(site_hash site-a)
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$SECRET" && ok "created MxGateway connection $CONN_A on site-a (ApiKey=secret:$SECRET, unbound)" \
|| bad "create $CONN_A"
B=$(gp badges "$URL"); echo " badges: $B"
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a Out of date" || bad "site-a is '$(badge_of "$B" site-a)', expected out of date"
[ "$(badge_of "$B" site-b)" = "current" ] && ok "site-b Current (unchanged)" || bad "site-b is '$(badge_of "$B" site-b)'"
[ "$(badge_of "$B" site-c)" = "current" ] && ok "site-c Current (unchanged)" || bad "site-c is '$(badge_of "$B" site-c)'"
end_check 1
# ── 2 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 2 "deploy artifacts to site-a -> success, secretsDeployed 1, site-a Current"
T0=$(python3 -c 'import time; print(time.time())')
deploy "$SITE_A_ID"
T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
echo " deploy rc=$DEPLOY_RC ($(python3 -c "print(round($T_DEPLOYED-$T0,1))") s): $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "site-a deploy succeeded" || bad "site-a deploy failed (rc=$DEPLOY_RC)"
[ "$(site_result site-a secretsDeployed)" = "1" ] && ok "secretsDeployed: 1" || bad "secretsDeployed='$(site_result site-a secretsDeployed)', expected 1"
B=$(gp badges "$URL"); echo " badges: $B"
[ "$(badge_of "$B" site-a)" = "current" ] && ok "site-a Current" || bad "site-a is '$(badge_of "$B" site-a)', expected current"
end_check 2
# wait_node <container> <python predicate over the site-secrets JSON d> -> sets WAIT_S (or empty on timeout)
WAIT_S=""
wait_node() {
local c="$1" pred="$2" start now j
start=$(python3 -c 'import time; print(time.time())'); WAIT_S=""
while :; do
j=$(gp site-secrets "$c")
if python3 -c "import json,sys; d=json.loads(sys.argv[1]); sys.exit(0 if ($pred) else 1)" "$j"; then
now=$(python3 -c 'import time; print(time.time())')
WAIT_S=$(python3 -c "print(round($now-${3:-$start},1))"); LAST_NODE_JSON="$j"; return 0
fi
now=$(python3 -c 'import time; print(time.time())')
python3 -c "import sys; sys.exit(0 if $now-$start < $STANDBY_WINDOW_S else 1)" || { LAST_NODE_JSON="$j"; return 1; }
sleep 1
done
}
matches_central() { # python predicate: node row == central row C (rev + sha), live
echo "d.get('$SECRET') is not None and not d['$SECRET']['deleted'] and d['$SECRET']['rev']==$(jget "$1" rev) and d['$SECRET']['sha']=='$(jget "$1" sha)' and d['$SECRET']['kek']=='$(jget "$1" kek)'"
}
# ── 3 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 3 "both site-a nodes hold $SECRET at the central revision (standby within ${STANDBY_WINDOW_S}s)"
C=$(gp central-secret "$SECRET")
PRED=$(matches_central "$C")
if wait_node "$ACTIVE_A" "$PRED" "$T_DEPLOYED"; then ok "site-a-a (active) r$(jget "$C" rev), ciphertext sha256 $(jget "$C" sha | cut -c1-16)… == central (seen ${WAIT_S}s after the deploy returned)"
else bad "site-a-a does not match central: $LAST_NODE_JSON"; fi
if wait_node "$STANDBY_A" "$PRED" "$T_DEPLOYED"; then STANDBY_3="$WAIT_S"; ok "site-a-b (standby) converged ${WAIT_S}s after the deploy returned"
else STANDBY_3="timeout"; bad "site-a-b did not converge within ${STANDBY_WINDOW_S}s: $LAST_NODE_JSON"; fi
L=$(gp site-localdb "$STANDBY_A")
[ "$(jget "$L" "mirrored.$SECRET")" = "$(jget "$C" rev)" ] && ok "site-a-b mirrored_secrets ledger r$(jget "$C" rev)" || bad "site-a-b ledger: $L"
end_check 3
# ── 4 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 4 "rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision"
R=$(SB_GATE_VALUE="$VALUE_2" gp secret-upsert "$URL" "$SECRET" "site-secrets live gate (throwaway, rotated)")
[ "$R" = "rotate" ] && ok "rotated $SECRET via /admin/secrets" || bad "rotate: $R"
C2=$(gp central-secret "$SECRET")
if [ "$(jget "$C2" rev)" -gt "$(jget "$C1" rev)" ] && [ "$(jget "$C2" sha)" != "$(jget "$C1" sha)" ]; then
ok "central r$(jget "$C1" rev) -> r$(jget "$C2" rev), ciphertext changed"
else bad "central row did not advance: before r$(jget "$C1" rev) after r$(jget "$C2" rev)"; fi
B=$(gp badges "$URL"); echo " badges: $B"
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a Out of date after the rotation" || bad "site-a is '$(badge_of "$B" site-a)'"
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && [ "$(site_result site-a secretsDeployed)" = "1" ] \
&& ok "redeploy succeeded, secretsDeployed 1" || bad "redeploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
PRED=$(matches_central "$C2")
if wait_node "$ACTIVE_A" "$PRED" "$T_DEPLOYED"; then ok "site-a-a at r$(jget "$C2" rev) (${WAIT_S}s)"; else bad "site-a-a: $LAST_NODE_JSON"; fi
if wait_node "$STANDBY_A" "$PRED" "$T_DEPLOYED"; then STANDBY_4="$WAIT_S"; ok "site-a-b at r$(jget "$C2" rev) (${WAIT_S}s after the deploy returned)"
else STANDBY_4="timeout"; bad "site-a-b: $LAST_NODE_JSON"; fi
B=$(gp badges "$URL")
[ "$(badge_of "$B" site-a)" = "current" ] && ok "site-a Current again" || bad "site-a is '$(badge_of "$B" site-a)'"
end_check 4
# ── 5 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 5 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
SB_GATE_VALUE="$VALUE_OOB" site_a_secret_set "$OOB" "site-secrets gate out-of-band" >/dev/null 2>&1 || true
J=$(gp site-secrets "$ACTIVE_A")
if [ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ]; then
ok "seeded out-of-band $OOB on site-a-a through the library's own write path"
else bad "out-of-band seed not visible on site-a-a: $J"; fi
upsert_conn "$CONN_A" "$SITE_A_ID" "gate-literal-not-a-secret-ref" && ok "$CONN_A no longer references a secret" || bad "update $CONN_A"
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "deploy succeeded (secretsDeployed='$(site_result site-a secretsDeployed)')" \
|| bad "deploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
GONE="(d.get('$SECRET') is None or d['$SECRET']['deleted'])"
if wait_node "$ACTIVE_A" "$GONE" "$T_DEPLOYED"; then ok "site-a-a no longer holds $SECRET (${WAIT_S}s)"; else bad "site-a-a still holds $SECRET: $LAST_NODE_JSON"; fi
if wait_node "$STANDBY_A" "$GONE" "$T_DEPLOYED"; then STANDBY_5="$WAIT_S"; ok "site-a-b no longer holds $SECRET (${WAIT_S}s after the deploy returned)"
else STANDBY_5="timeout"; bad "site-a-b still holds $SECRET: $LAST_NODE_JSON"; fi
J=$(gp site-secrets "$ACTIVE_A")
[ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ] \
&& ok "out-of-band $OOB still live on site-a-a" || bad "out-of-band $OOB lost: $J"
L=$(gp site-localdb "$ACTIVE_A")
[ "$(jget "$L" "deployed.$SECRET.removed")" = "true" ] && ok "deployed_secrets.$SECRET is_removed=1 (kept for the standby)" || bad "deployed_secrets: $L"
end_check 5
# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 6 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
SNAP_BEFORE="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$MISSING" && ok "$CONN_A now references secret:$MISSING" || bad "update $CONN_A"
deploy "$SITE_A_ID"
ERR=$(site_result site-a errorMessage)
echo " deploy rc=$DEPLOY_RC error: $ERR"
EXPECT="Secret '$MISSING' referenced by data connection(s) $CONN_A is not in the central secret store"
[ "$(site_result site-a success)" = "false" ] && ok "site-a deploy reported failure" || bad "site-a deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
case "$ERR" in *"$EXPECT"*) ok "named fail-closed message";; *) bad "message does not contain: $EXPECT";; esac
B=$(gp badges "$URL"); echo " badges: $B"
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a stays Out of date" || bad "site-a is '$(badge_of "$B" site-a)'"
sleep 3
SNAP_AFTER="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
[ "$SNAP_BEFORE" = "$SNAP_AFTER" ] && ok "nothing changed: stamped hash, both nodes' secret rows, deployed_secrets, ledgers and the $CONN_A definition identical" \
|| { bad "site state changed"; echo " before: $SNAP_BEFORE"; echo " after: $SNAP_AFTER"; }
end_check 6
# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 7 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
C=$(gp central-secret "$SECRET")
KEK=$(jget "$C" kek)
upsert_conn "$CONN_B" "$SITE_B_ID" "secret:$SECRET" && ok "created $CONN_B on site-b (ApiKey=secret:$SECRET)" || bad "create $CONN_B"
deploy "$SITE_B_ID"
ERR=$(site_result site-b errorMessage)
echo " deploy rc=$DEPLOY_RC error: $ERR"
[ "$(site_result site-b success)" = "false" ] && ok "site-b deploy reported failure" || bad "site-b deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
case "$ERR" in *"$SECRET"*"$KEK"*) ok "message names $SECRET and the row's KekId $KEK";; *) bad "message does not name $SECRET + KekId $KEK";; esac
B=$(gp badges "$URL"); echo " badges: $B"
[ "$(badge_of "$B" site-b)" = "out of date" ] && ok "site-b not stamped (Out of date)" || bad "site-b is '$(badge_of "$B" site-b)'"
end_check 7
# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 8 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
AUD=$("$CLI" audit-config query "${A[@]}" --action DeploySiteSecrets --entity-type SiteSecrets --from "$GATE_START" --page-size 50 2>/dev/null)
ROWS=$(printf '%s' "$AUD" | python3 -c '
import json,sys
d=json.load(sys.stdin)
rows=[]
for e in sorted(d.get("entries",[]), key=lambda e: e["timestamp"]):
s=json.loads(e.get("afterStateJson") or "{}")
names=",".join("%s@r%s" % (x.get("name"), x.get("revision")) for x in (s.get("secrets") or []))
miss=",".join(s.get("missing") or [])
rows.append("%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", s.get("deploymentId")))
print("\n".join(rows))')
echo "$ROWS" | sed 's/^/ audit: /'
R2=$(jget "$C1" rev); R4=$(jget "$C2" rev)
EXPECTED="site-a|True|$SECRET@r$R2|-
site-a|True|$SECRET@r$R4|-
site-a|False|-|$MISSING
site-b|False|$SECRET@r$R4|-"
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-4)
[ "$GOT" = "$EXPECTED" ] && ok "exactly 4 rows, one per secret-bearing deploy (2, 4, 6-fail-closed, 7-kek-mismatch); the no-secret deploy of check 5 wrote none" \
|| { bad "audit rows differ from expected"; echo " expected:"; echo "$EXPECTED" | sed 's/^/ /'; }
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f5 | sort -u | grep -c .)" = "4" ] && ok "4 distinct deploymentIds" || bad "deploymentIds not distinct"
# Material to hunt for: both plaintext values and both ciphertexts (base64, as they appear in any
# JSON serialization). Written to a pattern file so they never reach argv or the output.
PAT="$WORK/patterns"; : >"$PAT"; chmod 600 "$PAT"
printf '%s\n%s\n%s\n' "$VALUE_1" "$VALUE_2" "$VALUE_OOB" >>"$PAT"
jget "$C1" b64 >>"$PAT"; jget "$C2" b64 >>"$PAT"
[ "$(grep -c . "$PAT")" = "5" ] && [ "$(awk 'length($0) < 20' "$PAT" | grep -c .)" = "0" ] \
&& ok "5 material patterns (3 plaintext, 2 base64 ciphertext) armed" || bad "pattern file malformed"
HITS=0
for n in "${NODES[@]}"; do
h=$(docker logs --since "$LOG_SINCE" "scadabridge-$n" 2>&1 | grep -c -F -f "$PAT" || true)
HITS=$((HITS + h)); [ "$h" = "0" ] || echo " scadabridge-$n: $h hit(s)"
done
AUDH=$(printf '%s' "$AUD" | grep -c -F -f "$PAT" || true)
[ "$HITS" = "0" ] && ok "0 hits across all 8 nodes' logs since $LOG_SINCE" || bad "$HITS log line(s) carry secret material"
[ "$AUDH" = "0" ] && ok "0 hits in the DeploySiteSecrets audit payloads" || bad "audit payload carries secret material"
end_check 8
echo ""
echo "=== Summary ==="
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check5=${STANDBY_5:-?}"
echo "PASS $CHECKS_PASSED/8"
[ "$CHECKS_FAILED" -eq 0 ] || exit 1
Reference in New Issue View Git Blame Copy Permalink