Joseph Doherty
2650f79472
Also: DeploymentDeadlineParityTests' stub replies with a real zero-failure summary (the artifact aliases now derive their exit code from failureCount), and register row 127 reflects the merge and the two new replicated site tables.
675 lines
40 KiB
Bash
Executable File
675 lines
40 KiB
Bash
Executable File
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# Live gate for SITE SECRETS RIDING THE ARTIFACT DEPLOY (design
|
|
# docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md, plan task 13). Drives the real
|
|
# central paths end to end on the running docker rig:
|
|
#
|
|
# - secrets are created / rotated / deleted at central through the Central UI's own
|
|
# /admin/secrets page (ZB.MOM.WW.Secrets.Ui over the central pair's shared SQL store) — the
|
|
# supported operator path; there is no CLI/management verb for central secrets
|
|
# - data connections and artifact deploys go through the ScadaBridge CLI
|
|
# (CLI -> Traefik -> active central -> ArtifactDeploymentService)
|
|
# - the Artifacts badge is read from the Sites page after pressing Re-check (the pending-updates
|
|
# evaluation has no CLI/management surface; it is Blazor-interactive, so a headless browser is
|
|
# the only honest reader)
|
|
# - each site node's own secret store (the library's SQLite `secret` table in
|
|
# /app/data/scadabridge-secrets.db) and LocalDb (`deployed_secrets`, `mirrored_secrets`,
|
|
# `data_connection_definitions` in /app/data/site-localdb.db) are copied out with `docker cp`
|
|
# (the image carries no sqlite3) and read with the host's python3 sqlite3 module
|
|
# - the central row is read (never written) from ZbSecretsHub.zbsecrets.secret with sqlcmd
|
|
#
|
|
# Ciphertext is compared by SHA-256 only; secret VALUES are random throwaways held in shell
|
|
# variables and handed to the browser through the environment — never printed, never on argv.
|
|
#
|
|
# Checks (each prints PASS/FAIL; the run ends "PASS n/9" and exits non-zero on any FAIL):
|
|
# 1 secret + secret-referencing MxGateway connection on site-a -> site-a Out of date, b/c Current
|
|
# 2 deploy site-a -> success, secretsDeployed 1, site-a Current
|
|
# 3 both site-a nodes hold the secret at the central revision + ciphertext (standby <= 30 s)
|
|
# 4 rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision
|
|
# 5 standby master-key mismatch reaches central (review M1): site-a-b is recreated with a WRONG
|
|
# KEK -> central's site-a health report shows node-b failing to decrypt the secret while
|
|
# node-a is clean; site-a-b restored with the right KEK -> node-b's failures clear
|
|
# 6 drop the secret: reference -> deploy -> both nodes no longer hold it and the reply names it
|
|
# in secretsRemoved; an out-of-band secret seeded on site-a-a through the library's own write
|
|
# path (the ZB.MOM.WW.Secrets `secret` CLI run against that node's store) survives
|
|
# 7 reference a non-existent secret -> site-a deploy fails closed with the named message, the
|
|
# CLI exits 1, site-a stays Out of date, nothing changed on either site-a node
|
|
# 8 the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the
|
|
# KekId, the CLI exits 1
|
|
# 9 one DeploySiteSecrets audit row per deploy that carried, lacked or retired a secret (the
|
|
# removal-only deploy of check 6 included, with its removed list); no plaintext / base64
|
|
# ciphertext in any of the 8 nodes' logs
|
|
#
|
|
# Teardown (trap EXIT, also run up-front so a rerun starts clean): gate connections deleted,
|
|
# site-a + site-b redeployed so the removal reaches them, the central gate secret deleted, the
|
|
# out-of-band secret removed from site-a-a, then a fleet-wide artifact deploy so the rig ends with
|
|
# every site Current. The secret library tombstones rather than hard-deletes, so deleted gate
|
|
# names remain as is_deleted=1 rows in the stores — that is the library's contract, not residue.
|
|
#
|
|
# Prereqs: rig up (8 nodes ready), `python3 -m playwright` with chromium, the ScadaBridge CLI
|
|
# built, and scadaproj checked out next to this repo (the `secret` CLI is published from it once
|
|
# into $SECRETS_CLI_DIR).
|
|
#
|
|
# Usage: bash docker/verify-site-secrets.sh
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
CLI="${SCADABRIDGE_CLI:-$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge}"
|
|
URL="${SCADABRIDGE_URL:-http://localhost:9000}"
|
|
A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}")
|
|
SECRETS_CLI_PROJECT="${SECRETS_CLI_PROJECT:-$HOME/Desktop/scadaproj/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli}"
|
|
SECRETS_CLI_DIR="${SECRETS_CLI_DIR:-${TMPDIR:-/tmp}/sb-gate-secrets-cli}"
|
|
STANDBY_WINDOW_S="${STANDBY_WINDOW_S:-30}"
|
|
# Check 5: mirror pass (15 s) + status reporter poll (30 s) + health report (30 s), plus a restart.
|
|
HEALTH_WINDOW_S="${HEALTH_WINDOW_S:-240}"
|
|
# The site-b/site-c dev KEK (docker-compose.yml) — deliberately NOT site-a's, used by check 5.
|
|
WRONG_KEK="esKrmqV2RBmMq/+6T/5zK2BcalRy5gzDXhoG9Qydg6Q="
|
|
|
|
# DEV-ONLY rig KEK shared by central + site-a (docker/docker-compose.yml, x-secrets-shared-kek-env).
|
|
SITE_A_KEK="zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4="
|
|
MSSQL_CONTAINER="${MSSQL_CONTAINER:-scadabridge-mssql}"
|
|
MSSQL_USER="scadabridge_app"; MSSQL_PASS='ScadaBridge_Dev1#'
|
|
|
|
SECRET="sb-gate-mxgw"
|
|
MISSING="sb-gate-missing"
|
|
OOB="sb-gate-oob"
|
|
CONN_A="sb-gate-conn-a"
|
|
CONN_B="sb-gate-conn-b"
|
|
ACTIVE_A="scadabridge-site-a-a"; STANDBY_A="scadabridge-site-a-b"
|
|
NODES=(central-a central-b site-a-a site-a-b site-b-a site-b-b site-c-a site-c-b)
|
|
|
|
VALUE_1=$(python3 -c 'import secrets; print("gate1-" + secrets.token_hex(16))')
|
|
VALUE_2=$(python3 -c 'import secrets; print("gate2-" + secrets.token_hex(16))')
|
|
VALUE_OOB=$(python3 -c 'import secrets; print("gateoob-" + secrets.token_hex(16))')
|
|
|
|
CHECKS_PASSED=0; CHECKS_FAILED=0
|
|
CHECK_OK=1
|
|
begin_check() { CHECK_OK=1; echo ""; echo "=== Check $1: $2 ==="; }
|
|
ok() { echo " ok $*"; }
|
|
bad() { CHECK_OK=0; echo " FAIL $*"; }
|
|
end_check() {
|
|
if [ "$CHECK_OK" -eq 1 ]; then CHECKS_PASSED=$((CHECKS_PASSED+1)); echo " PASS check $1"
|
|
else CHECKS_FAILED=$((CHECKS_FAILED+1)); echo " FAIL check $1"; fi
|
|
}
|
|
|
|
[ -x "$CLI" ] || { echo "CLI not found at $CLI — build it: dotnet build src/ZB.MOM.WW.ScadaBridge.CLI"; exit 1; }
|
|
python3 -c 'import playwright.sync_api' 2>/dev/null || { echo "python3 playwright missing (pip install playwright && playwright install chromium)"; exit 1; }
|
|
|
|
WORK=$(mktemp -d "${TMPDIR:-/tmp}/sb-gate-site-secrets.XXXXXX")
|
|
HELPER="$WORK/gate.py"
|
|
|
|
# ── python helper: browser (UI) + read-only store readers ──────────────────────────────────────
|
|
cat >"$HELPER" <<'PY'
|
|
import hashlib, json, os, sqlite3, subprocess, sys, tempfile, time, shutil
|
|
|
|
def sha(b):
|
|
return hashlib.sha256(b if isinstance(b, (bytes, bytearray)) else b.encode()).hexdigest()
|
|
|
|
# ---- Central UI -------------------------------------------------------------------------------
|
|
def _browser(fn):
|
|
from playwright.sync_api import sync_playwright
|
|
with sync_playwright() as p:
|
|
b = p.chromium.launch()
|
|
try:
|
|
pg = b.new_page()
|
|
return fn(pg)
|
|
finally:
|
|
b.close()
|
|
|
|
def _wait_circuit(pg):
|
|
# Blazor Server: handlers only work once the circuit is up.
|
|
pg.wait_for_function("() => window.Blazor !== undefined", timeout=30000)
|
|
time.sleep(2)
|
|
|
|
def badges(url):
|
|
def run(pg):
|
|
pg.goto(url + "/admin/sites")
|
|
pg.wait_for_selector("button[aria-label='Re-check pending updates']", timeout=30000)
|
|
_wait_circuit(pg)
|
|
pg.click("button[aria-label='Re-check pending updates']")
|
|
time.sleep(1)
|
|
pg.wait_for_function("() => { const b=document.querySelector(\"button[aria-label='Re-check pending updates']\"); return b && !b.disabled; }", timeout=120000)
|
|
pg.wait_for_function("() => !document.querySelector('[aria-label=\"Artifacts: checking\"]')", timeout=60000)
|
|
out = {}
|
|
for card in pg.query_selector_all(".card"):
|
|
code = card.query_selector("code"); badge = card.query_selector("[aria-label^='Artifacts:']")
|
|
if code and badge:
|
|
out[code.inner_text().strip()] = badge.get_attribute("aria-label").split(": ", 1)[1]
|
|
return out
|
|
print(json.dumps(_browser(run)))
|
|
|
|
def _row_exists(pg, name):
|
|
return pg.query_selector(f"button.rotate-secret[data-secret-name='{name}']") is not None
|
|
|
|
def _secrets_page(pg, url):
|
|
pg.goto(url + "/admin/secrets")
|
|
pg.wait_for_selector("[data-testid='add-secret']", timeout=30000)
|
|
pg.wait_for_function("() => !document.body.innerText.includes('Loading…')", timeout=30000)
|
|
_wait_circuit(pg)
|
|
|
|
def secret_upsert(url, name, desc):
|
|
value = os.environ["SB_GATE_VALUE"]
|
|
def run(pg):
|
|
_secrets_page(pg, url)
|
|
mode = "rotate" if _row_exists(pg, name) else "add"
|
|
for attempt in range(5):
|
|
if mode == "rotate":
|
|
pg.click(f"button.rotate-secret[data-secret-name='{name}']")
|
|
else:
|
|
pg.click("[data-testid='add-secret']")
|
|
try:
|
|
pg.wait_for_selector("[data-testid='editor-submit']", timeout=3000); break
|
|
except Exception:
|
|
time.sleep(1)
|
|
if mode == "add":
|
|
pg.fill("[data-testid='secret-name']", name); pg.press("[data-testid='secret-name']", "Tab")
|
|
pg.fill("[data-testid='secret-description']", desc); pg.press("[data-testid='secret-description']", "Tab")
|
|
pg.fill("[data-testid='secret-value']", value); pg.press("[data-testid='secret-value']", "Tab")
|
|
pg.click("[data-testid='editor-submit']")
|
|
deadline = time.time() + 20
|
|
while time.time() < deadline:
|
|
err = pg.query_selector("[data-testid='editor-error']")
|
|
if err:
|
|
return "error: " + err.inner_text()
|
|
if pg.query_selector("[data-testid='editor-submit']") is None and _row_exists(pg, name):
|
|
return mode
|
|
time.sleep(0.3)
|
|
return "error: editor did not close"
|
|
print(_browser(run))
|
|
|
|
def secret_delete(url, name):
|
|
def run(pg):
|
|
_secrets_page(pg, url)
|
|
if not _row_exists(pg, name):
|
|
return "absent"
|
|
for attempt in range(5):
|
|
pg.click(f"button.delete-secret[data-secret-name='{name}']")
|
|
try:
|
|
pg.wait_for_selector("[data-testid='confirm-delete']", timeout=3000); break
|
|
except Exception:
|
|
time.sleep(1)
|
|
pg.click("[data-testid='confirm-delete']")
|
|
deadline = time.time() + 20
|
|
while time.time() < deadline:
|
|
err = pg.query_selector("[data-testid='delete-error']")
|
|
if err:
|
|
return "error: " + err.inner_text()
|
|
if not _row_exists(pg, name):
|
|
return "deleted"
|
|
time.sleep(0.3)
|
|
return "error: row still listed"
|
|
print(_browser(run))
|
|
|
|
# ---- read-only store readers ------------------------------------------------------------------
|
|
def _copy_db(container, path):
|
|
"""docker cp the db + its WAL (NOT the -shm: SQLite rebuilds the wal-index from the WAL on
|
|
open, which is safer than trusting a copied live shared-memory file)."""
|
|
d = tempfile.mkdtemp(prefix="sbgate-")
|
|
base = os.path.basename(path)
|
|
for suffix in ("-wal", ""):
|
|
r = subprocess.run(["docker", "cp", f"{container}:{path}{suffix}", os.path.join(d, base + suffix)],
|
|
capture_output=True)
|
|
if r.returncode != 0 and suffix == "":
|
|
raise SystemExit(f"docker cp {container}:{path} failed: {r.stderr.decode().strip()}")
|
|
return d, os.path.join(d, base)
|
|
|
|
def _query(container, path, sql):
|
|
# Two consecutive reads must agree — guards against a checkpoint racing the two docker cp's.
|
|
last = None
|
|
for _ in range(4):
|
|
d, db = _copy_db(container, path)
|
|
try:
|
|
con = sqlite3.connect(db); rows = con.execute(sql).fetchall(); con.close()
|
|
finally:
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
if rows == last:
|
|
return rows
|
|
last = rows
|
|
return last
|
|
|
|
def site_secrets(container):
|
|
rows = _query(container, "/app/data/scadabridge-secrets.db",
|
|
"SELECT name, revision, kek_id, is_deleted, ciphertext FROM secret WHERE name LIKE 'sb-gate-%' ORDER BY name")
|
|
print(json.dumps({n: {"rev": r, "kek": k, "deleted": bool(dl), "sha": sha(c)} for n, r, k, dl, c in rows}, sort_keys=True))
|
|
|
|
def site_localdb(container):
|
|
db = "/app/data/site-localdb.db"
|
|
dep = _query(container, db, "SELECT name, revision, is_removed, row_json FROM deployed_secrets WHERE name LIKE 'sb-gate-%' ORDER BY name")
|
|
mir = _query(container, db, "SELECT name, revision FROM mirrored_secrets WHERE name LIKE 'sb-gate-%' ORDER BY name")
|
|
dcs = _query(container, db, "SELECT name, configuration FROM data_connection_definitions WHERE name LIKE 'sb-gate-%' ORDER BY name")
|
|
print(json.dumps({
|
|
"deployed": {n: {"rev": r, "removed": bool(x), "sha": sha(j)} for n, r, x, j in dep},
|
|
"mirrored": {n: r for n, r in mir},
|
|
"connections": {n: sha(c or "") for n, c in dcs},
|
|
}, sort_keys=True))
|
|
|
|
def central_secret(name):
|
|
q = ("SET NOCOUNT ON; SELECT name, revision, kek_id, is_deleted, "
|
|
"LOWER(CONVERT(varchar(64), HASHBYTES('SHA2_256', ciphertext), 2)), "
|
|
"CONVERT(varchar(max), ciphertext, 2) "
|
|
f"FROM zbsecrets.secret WHERE name = N'{name}'")
|
|
r = subprocess.run(["docker", "exec", os.environ["MSSQL_CONTAINER"], "/opt/mssql-tools18/bin/sqlcmd",
|
|
"-S", "localhost", "-U", os.environ["MSSQL_USER"], "-P", os.environ["MSSQL_PASS"],
|
|
"-C", "-d", "ZbSecretsHub", "-h", "-1", "-W", "-s", "|", "-Q", q],
|
|
capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
raise SystemExit("sqlcmd failed: " + r.stderr.strip())
|
|
lines = [l for l in r.stdout.splitlines() if l.strip()]
|
|
if not lines:
|
|
print(json.dumps(None)); return
|
|
n, rev, kek, dl, h, hexct = lines[0].split("|")
|
|
import base64
|
|
b64 = base64.b64encode(bytes.fromhex(hexct)).decode()
|
|
if sha(bytes.fromhex(hexct)) != h:
|
|
raise SystemExit("ciphertext hex truncated by sqlcmd")
|
|
# b64 is only used by check 8's log grep; it is written to a file by the caller, never printed.
|
|
print(json.dumps({"rev": int(rev), "kek": kek, "deleted": dl == "1", "sha": h, "b64": b64}))
|
|
|
|
cmd, args = sys.argv[1], sys.argv[2:]
|
|
{"badges": lambda: badges(*args),
|
|
"secret-upsert": lambda: secret_upsert(*args),
|
|
"secret-delete": lambda: secret_delete(*args),
|
|
"site-secrets": lambda: site_secrets(*args),
|
|
"site-localdb": lambda: site_localdb(*args),
|
|
"central-secret": lambda: central_secret(*args)}[cmd]()
|
|
PY
|
|
export MSSQL_CONTAINER MSSQL_USER MSSQL_PASS
|
|
gp() { python3 "$HELPER" "$@"; }
|
|
jget() { python3 -c 'import json,sys
|
|
d=json.loads(sys.argv[1])
|
|
for k in sys.argv[2].split("."):
|
|
if d is None: break
|
|
d = d.get(k) if isinstance(d, dict) else None
|
|
print("" if d is None else (json.dumps(d) if isinstance(d,(dict,list,bool)) else d))' "$1" "$2"; }
|
|
|
|
UNWRAP='
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
if isinstance(d, dict) and "data" in d and len(d) <= 2:
|
|
d = d["data"]
|
|
'
|
|
|
|
site_id() {
|
|
"$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
|
|
import sys
|
|
print(next(str(s["id"]) for s in d if s["siteIdentifier"]==sys.argv[1]))' "$1"
|
|
}
|
|
site_hash() {
|
|
"$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
|
|
import sys
|
|
print(next((s.get("artifactRevisionHash") or "-") for s in d if s["siteIdentifier"]==sys.argv[1]))' "$1"
|
|
}
|
|
conn_id() { # conn_id <name> -> id or empty
|
|
"$CLI" data-connection list "${A[@]}" | python3 -c "$UNWRAP"'
|
|
import sys
|
|
print(next((str(c["id"]) for c in d if c["name"]==sys.argv[1]), ""))' "$1"
|
|
}
|
|
mxgw_config() { # mxgw_config <apiKeyValue>
|
|
python3 -c 'import json,sys; print(json.dumps({"Endpoint":"http://sb-gate-no-such-gateway:5000","ApiKey":sys.argv[1],"ClientName":"sb-gate","ReadTimeoutMs":5000}))' "$1"
|
|
}
|
|
upsert_conn() { # upsert_conn <name> <siteId> <apiKeyValue>
|
|
local id; id=$(conn_id "$1")
|
|
if [ -z "$id" ]; then
|
|
"$CLI" data-connection create "${A[@]}" --site-id "$2" --name "$1" --protocol MxGateway \
|
|
--primary-config "$(mxgw_config "$3")" >/dev/null
|
|
else
|
|
"$CLI" data-connection update "${A[@]}" --id "$id" --name "$1" --protocol MxGateway \
|
|
--primary-config "$(mxgw_config "$3")" >/dev/null
|
|
fi
|
|
}
|
|
# deploy <siteId|all> -> sets DEPLOY_OUT (raw CLI output) and DEPLOY_RC; never aborts the script
|
|
DEPLOY_OUT=""; DEPLOY_RC=0
|
|
deploy() {
|
|
DEPLOY_RC=0
|
|
if [ "$1" = "all" ]; then
|
|
DEPLOY_OUT=$("$CLI" site deploy-artifacts "${A[@]}" 2>&1) || DEPLOY_RC=$?
|
|
else
|
|
DEPLOY_OUT=$("$CLI" site deploy-artifacts "${A[@]}" --site-id "$1" 2>&1) || DEPLOY_RC=$?
|
|
fi
|
|
}
|
|
# site_result <siteIdentifier> <field> -> the field from DEPLOY_OUT's siteResults
|
|
site_result() {
|
|
printf '%s' "$DEPLOY_OUT" | python3 -c '
|
|
import json,sys
|
|
raw=sys.stdin.read()
|
|
try:
|
|
d=json.loads(raw[raw.index("{"):])
|
|
except Exception:
|
|
print(""); sys.exit(0)
|
|
r=next((x for x in d.get("siteResults",[]) if x.get("siteId")==sys.argv[1]), None)
|
|
v=None if r is None else r.get(sys.argv[2])
|
|
print("" if v is None else (json.dumps(v) if isinstance(v,bool) else v))' "$1" "$2"
|
|
}
|
|
badge_of() { jget "$1" "$2"; }
|
|
# mirror_failures <siteIdentifier> -> JSON {nodeName: [secretName...]} from central's latest health
|
|
# report for the site (SiteHealthReport.SecretMirrorStatuses), or "null" when not reported
|
|
mirror_failures() {
|
|
"$CLI" health site "${A[@]}" --identifier "$1" 2>/dev/null | python3 -c '
|
|
import json,sys
|
|
try:
|
|
r=(json.load(sys.stdin).get("latestReport") or {}).get("secretMirrorStatuses")
|
|
except Exception:
|
|
r=None
|
|
print("null" if r is None else json.dumps({n["nodeName"]: sorted(f["secretName"] for f in n["failures"]) for n in r}, sort_keys=True))'
|
|
}
|
|
# The rig's compose project dir (where the bind-mounted appsettings live), from the running container.
|
|
RIG_DIR=$(docker inspect "$STANDBY_A" --format '{{ index .Config.Labels "com.docker.compose.project.working_dir" }}' 2>/dev/null)
|
|
# recreate_standby_a [wrong] -> recreates site-a-b from the rig's own compose file, optionally with
|
|
# an override that swaps in a mismatched KEK; waits until it is a cluster member again.
|
|
recreate_standby_a() {
|
|
local files=(-f "$RIG_DIR/docker-compose.yml")
|
|
if [ "${1:-}" = "wrong" ]; then
|
|
printf 'services:\n site-a-b:\n environment:\n ZB_SECRETS_MASTER_KEY: "%s"\n' "$WRONG_KEK" >"$WORK/wrong-kek.yml"
|
|
files+=(-f "$WORK/wrong-kek.yml"); touch "$WORK/standby-kek-swapped"
|
|
fi
|
|
docker compose "${files[@]}" --project-directory "$RIG_DIR" up -d --no-deps --force-recreate site-a-b >/dev/null 2>&1 || return 1
|
|
[ "${1:-}" = "wrong" ] || rm -f "$WORK/standby-kek-swapped"
|
|
local i
|
|
for i in $(seq 1 60); do
|
|
docker run --rm --network "container:$ACTIVE_A" python:3.13-alpine python3 -c '
|
|
import urllib.request as u,sys
|
|
try: sys.exit(0 if u.urlopen("http://scadabridge-site-a-b:8084/health/ready",timeout=3).status==200 else 1)
|
|
except Exception: sys.exit(1)' >/dev/null 2>&1 && return 0
|
|
sleep 3
|
|
done
|
|
return 1
|
|
}
|
|
|
|
ensure_secrets_cli() {
|
|
[ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ] && return 0
|
|
echo " publishing the ZB.MOM.WW.Secrets 'secret' CLI to $SECRETS_CLI_DIR ..."
|
|
dotnet publish "$SECRETS_CLI_PROJECT" -c Release -o "$SECRETS_CLI_DIR" >/dev/null
|
|
}
|
|
# Runs the library's own `secret` CLI against site-a-a's LOCAL store, in a throwaway container that
|
|
# shares that node's data volume and carries that node's KEK — the library's normal SQLite write path.
|
|
site_a_secret_cli() {
|
|
local verb="$1"; shift
|
|
docker run --rm --volumes-from "$ACTIVE_A" -v "$SECRETS_CLI_DIR:/cli:ro" -w /cli \
|
|
-e Secrets__SqlitePath=/data/scadabridge-secrets.db -e "ZB_SECRETS_MASTER_KEY=$SITE_A_KEK" \
|
|
-e SB_GATE_VALUE --entrypoint sh scadabridge:latest \
|
|
-c "exec dotnet /cli/ZB.MOM.WW.Secrets.Cli.dll $verb \"\$@\"" _ "$@"
|
|
}
|
|
# `secret set <name> <value>` takes the value positionally: hand it over through the container's
|
|
# environment (SB_GATE_VALUE) so it never appears on this host's argv; the container's inner sh
|
|
# expands it.
|
|
site_a_secret_set() { # site_a_secret_set <name> <description> (value in $SB_GATE_VALUE)
|
|
docker run --rm --volumes-from "$ACTIVE_A" -v "$SECRETS_CLI_DIR:/cli:ro" -w /cli \
|
|
-e Secrets__SqlitePath=/data/scadabridge-secrets.db -e "ZB_SECRETS_MASTER_KEY=$SITE_A_KEK" \
|
|
-e SB_GATE_VALUE --entrypoint sh scadabridge:latest \
|
|
-c 'exec dotnet /cli/ZB.MOM.WW.Secrets.Cli.dll set "$1" "$SB_GATE_VALUE" --description "$2"' _ "$1" "$2"
|
|
}
|
|
|
|
# ── teardown (idempotent; also the pre-clean) ──────────────────────────────────────────────────
|
|
teardown() {
|
|
local rc=$?
|
|
set +e
|
|
echo ""
|
|
echo "=== teardown ==="
|
|
local id
|
|
if [ -f "$WORK/standby-kek-swapped" ]; then
|
|
recreate_standby_a && echo " restored site-a-b with site-a's KEK" || echo " WARN could not restore site-a-b's KEK"
|
|
fi
|
|
for c in "$CONN_A" "$CONN_B"; do
|
|
id=$(conn_id "$c" 2>/dev/null)
|
|
if [ -n "$id" ]; then
|
|
"$CLI" data-connection delete "${A[@]}" --id "$id" >/dev/null 2>&1 \
|
|
&& echo " removed data connection $c (id=$id)" || echo " WARN could not remove data connection $c"
|
|
fi
|
|
done
|
|
# Removal must reach both sites that ever held the gate secret BEFORE central forgets it.
|
|
[ -n "${SITE_A_ID:-}" ] && { deploy "$SITE_A_ID"; echo " site-a redeploy rc=$DEPLOY_RC"; }
|
|
[ -n "${SITE_B_ID:-}" ] && { deploy "$SITE_B_ID"; echo " site-b redeploy rc=$DEPLOY_RC"; }
|
|
echo " central $SECRET: $(gp secret-delete "$URL" "$SECRET" 2>&1 | tail -1)"
|
|
if [ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ]; then
|
|
local live
|
|
live=$(gp site-secrets "$ACTIVE_A" 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print("1" if d.get(sys.argv[1],{}).get("deleted") is False else "")' "$OOB" 2>/dev/null)
|
|
if [ -n "$live" ]; then
|
|
site_a_secret_cli rm "$OOB" >/dev/null 2>&1 && echo " removed out-of-band $OOB from site-a-a" \
|
|
|| echo " WARN could not remove out-of-band $OOB"
|
|
fi
|
|
fi
|
|
deploy all; echo " fleet-wide artifact redeploy rc=$DEPLOY_RC"
|
|
echo " badges after teardown: $(gp badges "$URL" 2>&1 | tail -1)"
|
|
[ "${1:-}" = "pre" ] || rm -rf "$WORK"
|
|
return $rc
|
|
}
|
|
|
|
echo "=== ScadaBridge site-secrets artifact-deploy live gate ==="
|
|
echo "management url: $URL"
|
|
GATE_START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
SITE_A_ID=$(site_id site-a); SITE_B_ID=$(site_id site-b); SITE_C_ID=$(site_id site-c)
|
|
echo "sites: site-a=$SITE_A_ID site-b=$SITE_B_ID site-c=$SITE_C_ID"
|
|
ensure_secrets_cli
|
|
|
|
echo ""
|
|
echo "=== pre-clean (idempotent rerun) ==="
|
|
teardown pre || true
|
|
trap teardown EXIT
|
|
GATE_START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
# Docker log checkpoint for check 8: only lines written during this run are grepped.
|
|
LOG_SINCE="$GATE_START"
|
|
|
|
BASE=$(gp badges "$URL")
|
|
echo "baseline badges: $BASE"
|
|
for s in site-a site-b site-c; do
|
|
[ "$(badge_of "$BASE" "$s")" = "current" ] || { echo "baseline: $s is not Current — the gate needs a Current rig to start"; exit 1; }
|
|
done
|
|
|
|
# ── 1 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 1 "secret + referencing connection on site-a -> site-a Out of date, site-b/c unchanged"
|
|
R=$(SB_GATE_VALUE="$VALUE_1" gp secret-upsert "$URL" "$SECRET" "site-secrets live gate (throwaway)")
|
|
[ "$R" = "add" ] && ok "created $SECRET at central via /admin/secrets" || bad "create $SECRET: $R"
|
|
C1=$(gp central-secret "$SECRET")
|
|
[ "$(jget "$C1" deleted)" = "false" ] && ok "central row $SECRET r$(jget "$C1" rev) kek $(jget "$C1" kek) sha $(jget "$C1" sha | cut -c1-12)" \
|
|
|| bad "central row for $SECRET not found live"
|
|
H_A_BEFORE=$(site_hash site-a)
|
|
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$SECRET" && ok "created MxGateway connection $CONN_A on site-a (ApiKey=secret:$SECRET, unbound)" \
|
|
|| bad "create $CONN_A"
|
|
B=$(gp badges "$URL"); echo " badges: $B"
|
|
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a Out of date" || bad "site-a is '$(badge_of "$B" site-a)', expected out of date"
|
|
[ "$(badge_of "$B" site-b)" = "current" ] && ok "site-b Current (unchanged)" || bad "site-b is '$(badge_of "$B" site-b)'"
|
|
[ "$(badge_of "$B" site-c)" = "current" ] && ok "site-c Current (unchanged)" || bad "site-c is '$(badge_of "$B" site-c)'"
|
|
end_check 1
|
|
|
|
# ── 2 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 2 "deploy artifacts to site-a -> success, secretsDeployed 1, site-a Current"
|
|
T0=$(python3 -c 'import time; print(time.time())')
|
|
deploy "$SITE_A_ID"
|
|
T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
|
|
echo " deploy rc=$DEPLOY_RC ($(python3 -c "print(round($T_DEPLOYED-$T0,1))") s): $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
|
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "site-a deploy succeeded" || bad "site-a deploy failed (rc=$DEPLOY_RC)"
|
|
[ "$(site_result site-a secretsDeployed)" = "1" ] && ok "secretsDeployed: 1" || bad "secretsDeployed='$(site_result site-a secretsDeployed)', expected 1"
|
|
B=$(gp badges "$URL"); echo " badges: $B"
|
|
[ "$(badge_of "$B" site-a)" = "current" ] && ok "site-a Current" || bad "site-a is '$(badge_of "$B" site-a)', expected current"
|
|
end_check 2
|
|
|
|
# wait_node <container> <python predicate over the site-secrets JSON d> -> sets WAIT_S (or empty on timeout)
|
|
WAIT_S=""
|
|
wait_node() {
|
|
local c="$1" pred="$2" start now j
|
|
start=$(python3 -c 'import time; print(time.time())'); WAIT_S=""
|
|
while :; do
|
|
j=$(gp site-secrets "$c")
|
|
if python3 -c "import json,sys; d=json.loads(sys.argv[1]); sys.exit(0 if ($pred) else 1)" "$j"; then
|
|
now=$(python3 -c 'import time; print(time.time())')
|
|
WAIT_S=$(python3 -c "print(round($now-${3:-$start},1))"); LAST_NODE_JSON="$j"; return 0
|
|
fi
|
|
now=$(python3 -c 'import time; print(time.time())')
|
|
python3 -c "import sys; sys.exit(0 if $now-$start < $STANDBY_WINDOW_S else 1)" || { LAST_NODE_JSON="$j"; return 1; }
|
|
sleep 1
|
|
done
|
|
}
|
|
matches_central() { # python predicate: node row == central row C (rev + sha), live
|
|
echo "d.get('$SECRET') is not None and not d['$SECRET']['deleted'] and d['$SECRET']['rev']==$(jget "$1" rev) and d['$SECRET']['sha']=='$(jget "$1" sha)' and d['$SECRET']['kek']=='$(jget "$1" kek)'"
|
|
}
|
|
|
|
# ── 3 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 3 "both site-a nodes hold $SECRET at the central revision (standby within ${STANDBY_WINDOW_S}s)"
|
|
C=$(gp central-secret "$SECRET")
|
|
PRED=$(matches_central "$C")
|
|
if wait_node "$ACTIVE_A" "$PRED" "$T_DEPLOYED"; then ok "site-a-a (active) r$(jget "$C" rev), ciphertext sha256 $(jget "$C" sha | cut -c1-16)… == central (seen ${WAIT_S}s after the deploy returned)"
|
|
else bad "site-a-a does not match central: $LAST_NODE_JSON"; fi
|
|
if wait_node "$STANDBY_A" "$PRED" "$T_DEPLOYED"; then STANDBY_3="$WAIT_S"; ok "site-a-b (standby) converged ${WAIT_S}s after the deploy returned"
|
|
else STANDBY_3="timeout"; bad "site-a-b did not converge within ${STANDBY_WINDOW_S}s: $LAST_NODE_JSON"; fi
|
|
L=$(gp site-localdb "$STANDBY_A")
|
|
[ "$(jget "$L" "mirrored.$SECRET")" = "$(jget "$C" rev)" ] && ok "site-a-b mirrored_secrets ledger r$(jget "$C" rev)" || bad "site-a-b ledger: $L"
|
|
end_check 3
|
|
|
|
# ── 4 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 4 "rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision"
|
|
R=$(SB_GATE_VALUE="$VALUE_2" gp secret-upsert "$URL" "$SECRET" "site-secrets live gate (throwaway, rotated)")
|
|
[ "$R" = "rotate" ] && ok "rotated $SECRET via /admin/secrets" || bad "rotate: $R"
|
|
C2=$(gp central-secret "$SECRET")
|
|
if [ "$(jget "$C2" rev)" -gt "$(jget "$C1" rev)" ] && [ "$(jget "$C2" sha)" != "$(jget "$C1" sha)" ]; then
|
|
ok "central r$(jget "$C1" rev) -> r$(jget "$C2" rev), ciphertext changed"
|
|
else bad "central row did not advance: before r$(jget "$C1" rev) after r$(jget "$C2" rev)"; fi
|
|
B=$(gp badges "$URL"); echo " badges: $B"
|
|
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a Out of date after the rotation" || bad "site-a is '$(badge_of "$B" site-a)'"
|
|
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
|
|
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && [ "$(site_result site-a secretsDeployed)" = "1" ] \
|
|
&& ok "redeploy succeeded, secretsDeployed 1" || bad "redeploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
|
PRED=$(matches_central "$C2")
|
|
if wait_node "$ACTIVE_A" "$PRED" "$T_DEPLOYED"; then ok "site-a-a at r$(jget "$C2" rev) (${WAIT_S}s)"; else bad "site-a-a: $LAST_NODE_JSON"; fi
|
|
if wait_node "$STANDBY_A" "$PRED" "$T_DEPLOYED"; then STANDBY_4="$WAIT_S"; ok "site-a-b at r$(jget "$C2" rev) (${WAIT_S}s after the deploy returned)"
|
|
else STANDBY_4="timeout"; bad "site-a-b: $LAST_NODE_JSON"; fi
|
|
B=$(gp badges "$URL")
|
|
[ "$(badge_of "$B" site-a)" = "current" ] && ok "site-a Current again" || bad "site-a is '$(badge_of "$B" site-a)'"
|
|
end_check 4
|
|
|
|
# ── 5 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 5 "standby master-key mismatch reaches central's health report (site-a-b on a wrong KEK)"
|
|
# wait_health <python predicate over F = {node: [failing secret names]}> -> sets HEALTH_S / LAST_F
|
|
wait_health() {
|
|
local start now; start=$(python3 -c 'import time; print(time.time())'); HEALTH_S=""
|
|
while :; do
|
|
LAST_F=$(mirror_failures site-a)
|
|
if python3 -c "import json,sys; F=json.loads(sys.argv[1]); sys.exit(0 if F is not None and ($1) else 1)" "$LAST_F"; then
|
|
now=$(python3 -c 'import time; print(time.time())'); HEALTH_S=$(python3 -c "print(round($now-$start))"); return 0
|
|
fi
|
|
now=$(python3 -c 'import time; print(time.time())')
|
|
python3 -c "import sys; sys.exit(0 if $now-$start < $HEALTH_WINDOW_S else 1)" || return 1
|
|
sleep 5
|
|
done
|
|
}
|
|
if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then ok "baseline: central reports both site-a nodes, no failures ($LAST_F)"
|
|
else bad "baseline: site-a health report is not two clean node rows: $LAST_F"; fi
|
|
if [ -z "$RIG_DIR" ]; then bad "cannot find the rig's compose directory from $STANDBY_A's labels"
|
|
elif recreate_standby_a wrong; then
|
|
ok "site-a-b recreated with a mismatched KEK and back in the cluster (standby; site-a-a stays active)"
|
|
if wait_health "'$SECRET' in F.get('node-b',[]) and F.get('node-a')==[]"; then MISMATCH_S="$HEALTH_S"
|
|
ok "central's site-a report shows node-b failing $SECRET, node-a clean (${HEALTH_S}s): $LAST_F"
|
|
else bad "node-b's failure never reached central within ${HEALTH_WINDOW_S}s: $LAST_F"; fi
|
|
if recreate_standby_a; then ok "site-a-b restored with site-a's KEK"; else bad "could not restore site-a-b"; fi
|
|
if wait_health "F.get('node-a')==[] and F.get('node-b')==[]"; then CLEARED_S="$HEALTH_S"
|
|
ok "node-b's failure cleared on central's report (${HEALTH_S}s)"
|
|
else bad "node-b's failure did not clear within ${HEALTH_WINDOW_S}s: $LAST_F"; fi
|
|
else bad "could not recreate site-a-b with a mismatched KEK"; fi
|
|
end_check 5
|
|
|
|
# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 6 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
|
|
SB_GATE_VALUE="$VALUE_OOB" site_a_secret_set "$OOB" "site-secrets gate out-of-band" >/dev/null 2>&1 || true
|
|
J=$(gp site-secrets "$ACTIVE_A")
|
|
if [ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ]; then
|
|
ok "seeded out-of-band $OOB on site-a-a through the library's own write path"
|
|
else bad "out-of-band seed not visible on site-a-a: $J"; fi
|
|
upsert_conn "$CONN_A" "$SITE_A_ID" "gate-literal-not-a-secret-ref" && ok "$CONN_A no longer references a secret" || bad "update $CONN_A"
|
|
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
|
|
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "deploy succeeded (secretsDeployed='$(site_result site-a secretsDeployed)')" \
|
|
|| bad "deploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
|
case "$(site_result site-a secretsRemoved)" in *"$SECRET"*) ok "reply names $SECRET in secretsRemoved";; *) bad "secretsRemoved='$(site_result site-a secretsRemoved)', expected $SECRET";; esac
|
|
GONE="(d.get('$SECRET') is None or d['$SECRET']['deleted'])"
|
|
if wait_node "$ACTIVE_A" "$GONE" "$T_DEPLOYED"; then ok "site-a-a no longer holds $SECRET (${WAIT_S}s)"; else bad "site-a-a still holds $SECRET: $LAST_NODE_JSON"; fi
|
|
if wait_node "$STANDBY_A" "$GONE" "$T_DEPLOYED"; then STANDBY_5="$WAIT_S"; ok "site-a-b no longer holds $SECRET (${WAIT_S}s after the deploy returned)"
|
|
else STANDBY_5="timeout"; bad "site-a-b still holds $SECRET: $LAST_NODE_JSON"; fi
|
|
J=$(gp site-secrets "$ACTIVE_A")
|
|
[ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ] \
|
|
&& ok "out-of-band $OOB still live on site-a-a" || bad "out-of-band $OOB lost: $J"
|
|
L=$(gp site-localdb "$ACTIVE_A")
|
|
[ "$(jget "$L" "deployed.$SECRET.removed")" = "true" ] && ok "deployed_secrets.$SECRET is_removed=1 (kept for the standby)" || bad "deployed_secrets: $L"
|
|
end_check 6
|
|
|
|
# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 7 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
|
|
SNAP_BEFORE="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
|
|
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$MISSING" && ok "$CONN_A now references secret:$MISSING" || bad "update $CONN_A"
|
|
deploy "$SITE_A_ID"
|
|
ERR=$(site_result site-a errorMessage)
|
|
echo " deploy rc=$DEPLOY_RC error: $ERR"
|
|
EXPECT="Secret '$MISSING' referenced by data connection(s) $CONN_A is not in the central secret store"
|
|
[ "$(site_result site-a success)" = "false" ] && ok "site-a deploy reported failure" || bad "site-a deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
|
[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1"
|
|
case "$ERR" in *"$EXPECT"*) ok "named fail-closed message";; *) bad "message does not contain: $EXPECT";; esac
|
|
B=$(gp badges "$URL"); echo " badges: $B"
|
|
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a stays Out of date" || bad "site-a is '$(badge_of "$B" site-a)'"
|
|
sleep 3
|
|
SNAP_AFTER="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
|
|
[ "$SNAP_BEFORE" = "$SNAP_AFTER" ] && ok "nothing changed: stamped hash, both nodes' secret rows, deployed_secrets, ledgers and the $CONN_A definition identical" \
|
|
|| { bad "site state changed"; echo " before: $SNAP_BEFORE"; echo " after: $SNAP_AFTER"; }
|
|
end_check 7
|
|
|
|
# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 8 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
|
|
C=$(gp central-secret "$SECRET")
|
|
KEK=$(jget "$C" kek)
|
|
upsert_conn "$CONN_B" "$SITE_B_ID" "secret:$SECRET" && ok "created $CONN_B on site-b (ApiKey=secret:$SECRET)" || bad "create $CONN_B"
|
|
deploy "$SITE_B_ID"
|
|
ERR=$(site_result site-b errorMessage)
|
|
echo " deploy rc=$DEPLOY_RC error: $ERR"
|
|
[ "$(site_result site-b success)" = "false" ] && ok "site-b deploy reported failure" || bad "site-b deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
|
|
[ "$DEPLOY_RC" -eq 1 ] && ok "CLI exited 1 (failureCount 1)" || bad "CLI exited $DEPLOY_RC, expected 1"
|
|
case "$ERR" in *"$SECRET"*"$KEK"*) ok "message names $SECRET and the row's KekId $KEK";; *) bad "message does not name $SECRET + KekId $KEK";; esac
|
|
B=$(gp badges "$URL"); echo " badges: $B"
|
|
[ "$(badge_of "$B" site-b)" = "out of date" ] && ok "site-b not stamped (Out of date)" || bad "site-b is '$(badge_of "$B" site-b)'"
|
|
end_check 8
|
|
|
|
# ── 9 ────────────────────────────────────────────────────────────────────────────────────────
|
|
begin_check 9 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
|
|
AUD=$("$CLI" audit-config query "${A[@]}" --action DeploySiteSecrets --entity-type SiteSecrets --from "$GATE_START" --page-size 50 2>/dev/null)
|
|
ROWS=$(printf '%s' "$AUD" | python3 -c '
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
rows=[]
|
|
for e in sorted(d.get("entries",[]), key=lambda e: e["timestamp"]):
|
|
s=json.loads(e.get("afterStateJson") or "{}")
|
|
names=",".join("%s@r%s" % (x.get("name"), x.get("revision")) for x in (s.get("secrets") or []))
|
|
miss=",".join(s.get("missing") or [])
|
|
rem=",".join(s.get("removed") or [])
|
|
rows.append("%s|%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", rem or "-", s.get("deploymentId")))
|
|
print("\n".join(rows))')
|
|
echo "$ROWS" | sed 's/^/ audit: /'
|
|
R2=$(jget "$C1" rev); R4=$(jget "$C2" rev)
|
|
EXPECTED="site-a|True|$SECRET@r$R2|-|-
|
|
site-a|True|$SECRET@r$R4|-|-
|
|
site-a|True|-|-|$SECRET
|
|
site-a|False|-|$MISSING|-
|
|
site-b|False|$SECRET@r$R4|-|-"
|
|
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-5)
|
|
[ "$GOT" = "$EXPECTED" ] && ok "exactly 5 rows: deploys 2 and 4, the removal-only deploy of check 6 (removed=$SECRET), 7 fail-closed, 8 kek-mismatch" \
|
|
|| { bad "audit rows differ from expected"; echo " expected:"; echo "$EXPECTED" | sed 's/^/ /'; }
|
|
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f6 | sort -u | grep -c .)" = "5" ] && ok "5 distinct deploymentIds" || bad "deploymentIds not distinct"
|
|
# Material to hunt for: both plaintext values and both ciphertexts (base64, as they appear in any
|
|
# JSON serialization). Written to a pattern file so they never reach argv or the output.
|
|
PAT="$WORK/patterns"; : >"$PAT"; chmod 600 "$PAT"
|
|
printf '%s\n%s\n%s\n' "$VALUE_1" "$VALUE_2" "$VALUE_OOB" >>"$PAT"
|
|
jget "$C1" b64 >>"$PAT"; jget "$C2" b64 >>"$PAT"
|
|
[ "$(grep -c . "$PAT")" = "5" ] && [ "$(awk 'length($0) < 20' "$PAT" | grep -c .)" = "0" ] \
|
|
&& ok "5 material patterns (3 plaintext, 2 base64 ciphertext) armed" || bad "pattern file malformed"
|
|
HITS=0
|
|
for n in "${NODES[@]}"; do
|
|
h=$(docker logs --since "$LOG_SINCE" "scadabridge-$n" 2>&1 | grep -c -F -f "$PAT" || true)
|
|
HITS=$((HITS + h)); [ "$h" = "0" ] || echo " scadabridge-$n: $h hit(s)"
|
|
done
|
|
AUDH=$(printf '%s' "$AUD" | grep -c -F -f "$PAT" || true)
|
|
[ "$HITS" = "0" ] && ok "0 hits across all 8 nodes' logs since $LOG_SINCE" || bad "$HITS log line(s) carry secret material"
|
|
[ "$AUDH" = "0" ] && ok "0 hits in the DeploySiteSecrets audit payloads" || bad "audit payload carries secret material"
|
|
end_check 9
|
|
|
|
echo ""
|
|
echo "=== Summary ==="
|
|
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check6=${STANDBY_5:-?}"
|
|
echo "standby mismatch on central's health report (check 5): shown after ${MISMATCH_S:-?}s, cleared after ${CLEARED_S:-?}s"
|
|
echo "PASS $CHECKS_PASSED/9"
|
|
[ "$CHECKS_FAILED" -eq 0 ] || exit 1
|