#!/bin/bash
set -euo pipefail

# Live gate for SITE SECRETS RIDING THE ARTIFACT DEPLOY (design
# docs/plans/2026-09-28-site-secrets-artifact-deploy-design.md, plan task 13). Drives the real
# central paths end to end on the running docker rig:
#
#   - secrets are created / rotated / deleted at central through the Central UI's own
#     /admin/secrets page (ZB.MOM.WW.Secrets.Ui over the central pair's shared SQL store) — the
#     supported operator path; there is no CLI/management verb for central secrets
#   - data connections and artifact deploys go through the ScadaBridge CLI
#     (CLI -> Traefik -> active central -> ArtifactDeploymentService)
#   - the Artifacts badge is read from the Sites page after pressing Re-check (the pending-updates
#     evaluation has no CLI/management surface; it is Blazor-interactive, so a headless browser is
#     the only honest reader)
#   - each site node's own secret store (the library's SQLite `secret` table in
#     /app/data/scadabridge-secrets.db) and LocalDb (`deployed_secrets`, `mirrored_secrets`,
#     `data_connection_definitions` in /app/data/site-localdb.db) are copied out with `docker cp`
#     (the image carries no sqlite3) and read with the host's python3 sqlite3 module
#   - the central row is read (never written) from ZbSecretsHub.zbsecrets.secret with sqlcmd
#
# Ciphertext is compared by SHA-256 only; secret VALUES are random throwaways held in shell
# variables and handed to the browser through the environment — never printed, never on argv.
#
# Checks (each prints PASS/FAIL; the run ends "PASS n/8" and exits non-zero on any FAIL):
#   1  secret + secret-referencing MxGateway connection on site-a -> site-a Out of date, b/c Current
#   2  deploy site-a -> success, secretsDeployed 1, site-a Current
#   3  both site-a nodes hold the secret at the central revision + ciphertext (standby <= 30 s)
#   4  rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision
#   5  drop the secret: reference -> deploy -> both nodes no longer hold it; an out-of-band secret
#      seeded on site-a-a through the library's own write path (the ZB.MOM.WW.Secrets `secret` CLI
#      run against that node's store) survives
#   6  reference a non-existent secret -> site-a deploy fails closed with the named message,
#      site-a stays Out of date, nothing changed on either site-a node
#   7  the same reference on site-b (deliberately mismatched dev KEK) -> deploy fails naming the KekId
#   8  one DeploySiteSecrets audit row per secret-bearing deploy; no plaintext / base64 ciphertext
#      in any of the 8 nodes' logs
#
# Teardown (trap EXIT, also run up-front so a rerun starts clean): gate connections deleted,
# site-a + site-b redeployed so the removal reaches them, the central gate secret deleted, the
# out-of-band secret removed from site-a-a, then a fleet-wide artifact deploy so the rig ends with
# every site Current. The secret library tombstones rather than hard-deletes, so deleted gate
# names remain as is_deleted=1 rows in the stores — that is the library's contract, not residue.
#
# Prereqs: rig up (8 nodes ready), `python3 -m playwright` with chromium, the ScadaBridge CLI
# built, and scadaproj checked out next to this repo (the `secret` CLI is published from it once
# into $SECRETS_CLI_DIR).
#
# Usage:  bash docker/verify-site-secrets.sh

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CLI="${SCADABRIDGE_CLI:-$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge}"
URL="${SCADABRIDGE_URL:-http://localhost:9000}"
A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}")
SECRETS_CLI_PROJECT="${SECRETS_CLI_PROJECT:-$HOME/Desktop/scadaproj/ZB.MOM.WW.Secrets/src/ZB.MOM.WW.Secrets.Cli}"
SECRETS_CLI_DIR="${SECRETS_CLI_DIR:-${TMPDIR:-/tmp}/sb-gate-secrets-cli}"
STANDBY_WINDOW_S="${STANDBY_WINDOW_S:-30}"

# DEV-ONLY rig KEK shared by central + site-a (docker/docker-compose.yml, x-secrets-shared-kek-env).
SITE_A_KEK="zZiBWuoaVMbJmGXToLk9Lakw0iJozXoL/7Gxac3GwJ4="
MSSQL_CONTAINER="${MSSQL_CONTAINER:-scadabridge-mssql}"
MSSQL_USER="scadabridge_app"; MSSQL_PASS='ScadaBridge_Dev1#'

SECRET="sb-gate-mxgw"
MISSING="sb-gate-missing"
OOB="sb-gate-oob"
CONN_A="sb-gate-conn-a"
CONN_B="sb-gate-conn-b"
ACTIVE_A="scadabridge-site-a-a"; STANDBY_A="scadabridge-site-a-b"
NODES=(central-a central-b site-a-a site-a-b site-b-a site-b-b site-c-a site-c-b)

VALUE_1=$(python3 -c 'import secrets; print("gate1-" + secrets.token_hex(16))')
VALUE_2=$(python3 -c 'import secrets; print("gate2-" + secrets.token_hex(16))')
VALUE_OOB=$(python3 -c 'import secrets; print("gateoob-" + secrets.token_hex(16))')

CHECKS_PASSED=0; CHECKS_FAILED=0
CHECK_OK=1
begin_check() { CHECK_OK=1; echo ""; echo "=== Check $1: $2 ==="; }
ok()   { echo "    ok    $*"; }
bad()  { CHECK_OK=0; echo "    FAIL  $*"; }
end_check() {
  if [ "$CHECK_OK" -eq 1 ]; then CHECKS_PASSED=$((CHECKS_PASSED+1)); echo "  PASS  check $1"
  else CHECKS_FAILED=$((CHECKS_FAILED+1)); echo "  FAIL  check $1"; fi
}

[ -x "$CLI" ] || { echo "CLI not found at $CLI — build it: dotnet build src/ZB.MOM.WW.ScadaBridge.CLI"; exit 1; }
python3 -c 'import playwright.sync_api' 2>/dev/null || { echo "python3 playwright missing (pip install playwright && playwright install chromium)"; exit 1; }

WORK=$(mktemp -d "${TMPDIR:-/tmp}/sb-gate-site-secrets.XXXXXX")
HELPER="$WORK/gate.py"

# ── python helper: browser (UI) + read-only store readers ──────────────────────────────────────
cat >"$HELPER" <<'PY'
import hashlib, json, os, sqlite3, subprocess, sys, tempfile, time, shutil

def sha(b):
    return hashlib.sha256(b if isinstance(b, (bytes, bytearray)) else b.encode()).hexdigest()

# ---- Central UI -------------------------------------------------------------------------------
def _browser(fn):
    from playwright.sync_api import sync_playwright
    with sync_playwright() as p:
        b = p.chromium.launch()
        try:
            pg = b.new_page()
            return fn(pg)
        finally:
            b.close()

def _wait_circuit(pg):
    # Blazor Server: handlers only work once the circuit is up.
    pg.wait_for_function("() => window.Blazor !== undefined", timeout=30000)
    time.sleep(2)

def badges(url):
    def run(pg):
        pg.goto(url + "/admin/sites")
        pg.wait_for_selector("button[aria-label='Re-check pending updates']", timeout=30000)
        _wait_circuit(pg)
        pg.click("button[aria-label='Re-check pending updates']")
        time.sleep(1)
        pg.wait_for_function("() => { const b=document.querySelector(\"button[aria-label='Re-check pending updates']\"); return b && !b.disabled; }", timeout=120000)
        pg.wait_for_function("() => !document.querySelector('[aria-label=\"Artifacts: checking\"]')", timeout=60000)
        out = {}
        for card in pg.query_selector_all(".card"):
            code = card.query_selector("code"); badge = card.query_selector("[aria-label^='Artifacts:']")
            if code and badge:
                out[code.inner_text().strip()] = badge.get_attribute("aria-label").split(": ", 1)[1]
        return out
    print(json.dumps(_browser(run)))

def _row_exists(pg, name):
    return pg.query_selector(f"button.rotate-secret[data-secret-name='{name}']") is not None

def _secrets_page(pg, url):
    pg.goto(url + "/admin/secrets")
    pg.wait_for_selector("[data-testid='add-secret']", timeout=30000)
    pg.wait_for_function("() => !document.body.innerText.includes('Loading…')", timeout=30000)
    _wait_circuit(pg)

def secret_upsert(url, name, desc):
    value = os.environ["SB_GATE_VALUE"]
    def run(pg):
        _secrets_page(pg, url)
        mode = "rotate" if _row_exists(pg, name) else "add"
        for attempt in range(5):
            if mode == "rotate":
                pg.click(f"button.rotate-secret[data-secret-name='{name}']")
            else:
                pg.click("[data-testid='add-secret']")
            try:
                pg.wait_for_selector("[data-testid='editor-submit']", timeout=3000); break
            except Exception:
                time.sleep(1)
        if mode == "add":
            pg.fill("[data-testid='secret-name']", name); pg.press("[data-testid='secret-name']", "Tab")
        pg.fill("[data-testid='secret-description']", desc); pg.press("[data-testid='secret-description']", "Tab")
        pg.fill("[data-testid='secret-value']", value); pg.press("[data-testid='secret-value']", "Tab")
        pg.click("[data-testid='editor-submit']")
        deadline = time.time() + 20
        while time.time() < deadline:
            err = pg.query_selector("[data-testid='editor-error']")
            if err:
                return "error: " + err.inner_text()
            if pg.query_selector("[data-testid='editor-submit']") is None and _row_exists(pg, name):
                return mode
            time.sleep(0.3)
        return "error: editor did not close"
    print(_browser(run))

def secret_delete(url, name):
    def run(pg):
        _secrets_page(pg, url)
        if not _row_exists(pg, name):
            return "absent"
        for attempt in range(5):
            pg.click(f"button.delete-secret[data-secret-name='{name}']")
            try:
                pg.wait_for_selector("[data-testid='confirm-delete']", timeout=3000); break
            except Exception:
                time.sleep(1)
        pg.click("[data-testid='confirm-delete']")
        deadline = time.time() + 20
        while time.time() < deadline:
            err = pg.query_selector("[data-testid='delete-error']")
            if err:
                return "error: " + err.inner_text()
            if not _row_exists(pg, name):
                return "deleted"
            time.sleep(0.3)
        return "error: row still listed"
    print(_browser(run))

# ---- read-only store readers ------------------------------------------------------------------
def _copy_db(container, path):
    """docker cp the db + its WAL (NOT the -shm: SQLite rebuilds the wal-index from the WAL on
    open, which is safer than trusting a copied live shared-memory file)."""
    d = tempfile.mkdtemp(prefix="sbgate-")
    base = os.path.basename(path)
    for suffix in ("-wal", ""):
        r = subprocess.run(["docker", "cp", f"{container}:{path}{suffix}", os.path.join(d, base + suffix)],
                           capture_output=True)
        if r.returncode != 0 and suffix == "":
            raise SystemExit(f"docker cp {container}:{path} failed: {r.stderr.decode().strip()}")
    return d, os.path.join(d, base)

def _query(container, path, sql):
    # Two consecutive reads must agree — guards against a checkpoint racing the two docker cp's.
    last = None
    for _ in range(4):
        d, db = _copy_db(container, path)
        try:
            con = sqlite3.connect(db); rows = con.execute(sql).fetchall(); con.close()
        finally:
            shutil.rmtree(d, ignore_errors=True)
        if rows == last:
            return rows
        last = rows
    return last

def site_secrets(container):
    rows = _query(container, "/app/data/scadabridge-secrets.db",
                  "SELECT name, revision, kek_id, is_deleted, ciphertext FROM secret WHERE name LIKE 'sb-gate-%' ORDER BY name")
    print(json.dumps({n: {"rev": r, "kek": k, "deleted": bool(dl), "sha": sha(c)} for n, r, k, dl, c in rows}, sort_keys=True))

def site_localdb(container):
    db = "/app/data/site-localdb.db"
    dep = _query(container, db, "SELECT name, revision, is_removed, row_json FROM deployed_secrets WHERE name LIKE 'sb-gate-%' ORDER BY name")
    mir = _query(container, db, "SELECT name, revision FROM mirrored_secrets WHERE name LIKE 'sb-gate-%' ORDER BY name")
    dcs = _query(container, db, "SELECT name, configuration FROM data_connection_definitions WHERE name LIKE 'sb-gate-%' ORDER BY name")
    print(json.dumps({
        "deployed": {n: {"rev": r, "removed": bool(x), "sha": sha(j)} for n, r, x, j in dep},
        "mirrored": {n: r for n, r in mir},
        "connections": {n: sha(c or "") for n, c in dcs},
    }, sort_keys=True))

def central_secret(name):
    q = ("SET NOCOUNT ON; SELECT name, revision, kek_id, is_deleted, "
         "LOWER(CONVERT(varchar(64), HASHBYTES('SHA2_256', ciphertext), 2)), "
         "CONVERT(varchar(max), ciphertext, 2) "
         f"FROM zbsecrets.secret WHERE name = N'{name}'")
    r = subprocess.run(["docker", "exec", os.environ["MSSQL_CONTAINER"], "/opt/mssql-tools18/bin/sqlcmd",
                        "-S", "localhost", "-U", os.environ["MSSQL_USER"], "-P", os.environ["MSSQL_PASS"],
                        "-C", "-d", "ZbSecretsHub", "-h", "-1", "-W", "-s", "|", "-Q", q],
                       capture_output=True, text=True)
    if r.returncode != 0:
        raise SystemExit("sqlcmd failed: " + r.stderr.strip())
    lines = [l for l in r.stdout.splitlines() if l.strip()]
    if not lines:
        print(json.dumps(None)); return
    n, rev, kek, dl, h, hexct = lines[0].split("|")
    import base64
    b64 = base64.b64encode(bytes.fromhex(hexct)).decode()
    if sha(bytes.fromhex(hexct)) != h:
        raise SystemExit("ciphertext hex truncated by sqlcmd")
    # b64 is only used by check 8's log grep; it is written to a file by the caller, never printed.
    print(json.dumps({"rev": int(rev), "kek": kek, "deleted": dl == "1", "sha": h, "b64": b64}))

cmd, args = sys.argv[1], sys.argv[2:]
{"badges": lambda: badges(*args),
 "secret-upsert": lambda: secret_upsert(*args),
 "secret-delete": lambda: secret_delete(*args),
 "site-secrets": lambda: site_secrets(*args),
 "site-localdb": lambda: site_localdb(*args),
 "central-secret": lambda: central_secret(*args)}[cmd]()
PY
export MSSQL_CONTAINER MSSQL_USER MSSQL_PASS
gp() { python3 "$HELPER" "$@"; }
jget() { python3 -c 'import json,sys
d=json.loads(sys.argv[1])
for k in sys.argv[2].split("."):
    if d is None: break
    d = d.get(k) if isinstance(d, dict) else None
print("" if d is None else (json.dumps(d) if isinstance(d,(dict,list,bool)) else d))' "$1" "$2"; }

UNWRAP='
import json,sys
d=json.load(sys.stdin)
if isinstance(d, dict) and "data" in d and len(d) <= 2:
    d = d["data"]
'

site_id() {
  "$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
print(next(str(s["id"]) for s in d if s["siteIdentifier"]==sys.argv[1]))' "$1"
}
site_hash() {
  "$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
print(next((s.get("artifactRevisionHash") or "-") for s in d if s["siteIdentifier"]==sys.argv[1]))' "$1"
}
conn_id() {  # conn_id <name> -> id or empty
  "$CLI" data-connection list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
print(next((str(c["id"]) for c in d if c["name"]==sys.argv[1]), ""))' "$1"
}
mxgw_config() {  # mxgw_config <apiKeyValue>
  python3 -c 'import json,sys; print(json.dumps({"Endpoint":"http://sb-gate-no-such-gateway:5000","ApiKey":sys.argv[1],"ClientName":"sb-gate","ReadTimeoutMs":5000}))' "$1"
}
upsert_conn() {  # upsert_conn <name> <siteId> <apiKeyValue>
  local id; id=$(conn_id "$1")
  if [ -z "$id" ]; then
    "$CLI" data-connection create "${A[@]}" --site-id "$2" --name "$1" --protocol MxGateway \
      --primary-config "$(mxgw_config "$3")" >/dev/null
  else
    "$CLI" data-connection update "${A[@]}" --id "$id" --name "$1" --protocol MxGateway \
      --primary-config "$(mxgw_config "$3")" >/dev/null
  fi
}
# deploy <siteId|all> -> sets DEPLOY_OUT (raw CLI output) and DEPLOY_RC; never aborts the script
DEPLOY_OUT=""; DEPLOY_RC=0
deploy() {
  DEPLOY_RC=0
  if [ "$1" = "all" ]; then
    DEPLOY_OUT=$("$CLI" site deploy-artifacts "${A[@]}" 2>&1) || DEPLOY_RC=$?
  else
    DEPLOY_OUT=$("$CLI" site deploy-artifacts "${A[@]}" --site-id "$1" 2>&1) || DEPLOY_RC=$?
  fi
}
# site_result <siteIdentifier> <field> -> the field from DEPLOY_OUT's siteResults
site_result() {
  printf '%s' "$DEPLOY_OUT" | python3 -c '
import json,sys
raw=sys.stdin.read()
try:
    d=json.loads(raw[raw.index("{"):])
except Exception:
    print(""); sys.exit(0)
r=next((x for x in d.get("siteResults",[]) if x.get("siteId")==sys.argv[1]), None)
v=None if r is None else r.get(sys.argv[2])
print("" if v is None else (json.dumps(v) if isinstance(v,bool) else v))' "$1" "$2"
}
badge_of() { jget "$1" "$2"; }

ensure_secrets_cli() {
  [ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ] && return 0
  echo "  publishing the ZB.MOM.WW.Secrets 'secret' CLI to $SECRETS_CLI_DIR ..."
  dotnet publish "$SECRETS_CLI_PROJECT" -c Release -o "$SECRETS_CLI_DIR" >/dev/null
}
# Runs the library's own `secret` CLI against site-a-a's LOCAL store, in a throwaway container that
# shares that node's data volume and carries that node's KEK — the library's normal SQLite write path.
site_a_secret_cli() {
  local verb="$1"; shift
  docker run --rm --volumes-from "$ACTIVE_A" -v "$SECRETS_CLI_DIR:/cli:ro" -w /cli \
    -e Secrets__SqlitePath=/data/scadabridge-secrets.db -e "ZB_SECRETS_MASTER_KEY=$SITE_A_KEK" \
    -e SB_GATE_VALUE --entrypoint sh scadabridge:latest \
    -c "exec dotnet /cli/ZB.MOM.WW.Secrets.Cli.dll $verb \"\$@\"" _ "$@"
}
# `secret set <name> <value>` takes the value positionally: hand it over through the container's
# environment (SB_GATE_VALUE) so it never appears on this host's argv; the container's inner sh
# expands it.
site_a_secret_set() {  # site_a_secret_set <name> <description>   (value in $SB_GATE_VALUE)
  docker run --rm --volumes-from "$ACTIVE_A" -v "$SECRETS_CLI_DIR:/cli:ro" -w /cli \
    -e Secrets__SqlitePath=/data/scadabridge-secrets.db -e "ZB_SECRETS_MASTER_KEY=$SITE_A_KEK" \
    -e SB_GATE_VALUE --entrypoint sh scadabridge:latest \
    -c 'exec dotnet /cli/ZB.MOM.WW.Secrets.Cli.dll set "$1" "$SB_GATE_VALUE" --description "$2"' _ "$1" "$2"
}

# ── teardown (idempotent; also the pre-clean) ──────────────────────────────────────────────────
teardown() {
  local rc=$?
  set +e
  echo ""
  echo "=== teardown ==="
  local id
  for c in "$CONN_A" "$CONN_B"; do
    id=$(conn_id "$c" 2>/dev/null)
    if [ -n "$id" ]; then
      "$CLI" data-connection delete "${A[@]}" --id "$id" >/dev/null 2>&1 \
        && echo "  removed data connection $c (id=$id)" || echo "  WARN could not remove data connection $c"
    fi
  done
  # Removal must reach both sites that ever held the gate secret BEFORE central forgets it.
  [ -n "${SITE_A_ID:-}" ] && { deploy "$SITE_A_ID"; echo "  site-a redeploy rc=$DEPLOY_RC"; }
  [ -n "${SITE_B_ID:-}" ] && { deploy "$SITE_B_ID"; echo "  site-b redeploy rc=$DEPLOY_RC"; }
  echo "  central $SECRET: $(gp secret-delete "$URL" "$SECRET" 2>&1 | tail -1)"
  if [ -f "$SECRETS_CLI_DIR/ZB.MOM.WW.Secrets.Cli.dll" ]; then
    local live
    live=$(gp site-secrets "$ACTIVE_A" 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print("1" if d.get(sys.argv[1],{}).get("deleted") is False else "")' "$OOB" 2>/dev/null)
    if [ -n "$live" ]; then
      site_a_secret_cli rm "$OOB" >/dev/null 2>&1 && echo "  removed out-of-band $OOB from site-a-a" \
        || echo "  WARN could not remove out-of-band $OOB"
    fi
  fi
  deploy all; echo "  fleet-wide artifact redeploy rc=$DEPLOY_RC"
  echo "  badges after teardown: $(gp badges "$URL" 2>&1 | tail -1)"
  [ "${1:-}" = "pre" ] || rm -rf "$WORK"
  return $rc
}

echo "=== ScadaBridge site-secrets artifact-deploy live gate ==="
echo "management url: $URL"
GATE_START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
SITE_A_ID=$(site_id site-a); SITE_B_ID=$(site_id site-b); SITE_C_ID=$(site_id site-c)
echo "sites: site-a=$SITE_A_ID site-b=$SITE_B_ID site-c=$SITE_C_ID"
ensure_secrets_cli

echo ""
echo "=== pre-clean (idempotent rerun) ==="
teardown pre || true
trap teardown EXIT
GATE_START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
# Docker log checkpoint for check 8: only lines written during this run are grepped.
LOG_SINCE="$GATE_START"

BASE=$(gp badges "$URL")
echo "baseline badges: $BASE"
for s in site-a site-b site-c; do
  [ "$(badge_of "$BASE" "$s")" = "current" ] || { echo "baseline: $s is not Current — the gate needs a Current rig to start"; exit 1; }
done

# ── 1 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 1 "secret + referencing connection on site-a -> site-a Out of date, site-b/c unchanged"
R=$(SB_GATE_VALUE="$VALUE_1" gp secret-upsert "$URL" "$SECRET" "site-secrets live gate (throwaway)")
[ "$R" = "add" ] && ok "created $SECRET at central via /admin/secrets" || bad "create $SECRET: $R"
C1=$(gp central-secret "$SECRET")
[ "$(jget "$C1" deleted)" = "false" ] && ok "central row $SECRET r$(jget "$C1" rev) kek $(jget "$C1" kek) sha $(jget "$C1" sha | cut -c1-12)" \
  || bad "central row for $SECRET not found live"
H_A_BEFORE=$(site_hash site-a)
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$SECRET" && ok "created MxGateway connection $CONN_A on site-a (ApiKey=secret:$SECRET, unbound)" \
  || bad "create $CONN_A"
B=$(gp badges "$URL"); echo "    badges: $B"
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a Out of date" || bad "site-a is '$(badge_of "$B" site-a)', expected out of date"
[ "$(badge_of "$B" site-b)" = "current" ] && ok "site-b Current (unchanged)" || bad "site-b is '$(badge_of "$B" site-b)'"
[ "$(badge_of "$B" site-c)" = "current" ] && ok "site-c Current (unchanged)" || bad "site-c is '$(badge_of "$B" site-c)'"
end_check 1

# ── 2 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 2 "deploy artifacts to site-a -> success, secretsDeployed 1, site-a Current"
T0=$(python3 -c 'import time; print(time.time())')
deploy "$SITE_A_ID"
T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
echo "    deploy rc=$DEPLOY_RC ($(python3 -c "print(round($T_DEPLOYED-$T0,1))") s): $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "site-a deploy succeeded" || bad "site-a deploy failed (rc=$DEPLOY_RC)"
[ "$(site_result site-a secretsDeployed)" = "1" ] && ok "secretsDeployed: 1" || bad "secretsDeployed='$(site_result site-a secretsDeployed)', expected 1"
B=$(gp badges "$URL"); echo "    badges: $B"
[ "$(badge_of "$B" site-a)" = "current" ] && ok "site-a Current" || bad "site-a is '$(badge_of "$B" site-a)', expected current"
end_check 2

# wait_node <container> <python predicate over the site-secrets JSON d> -> sets WAIT_S (or empty on timeout)
WAIT_S=""
wait_node() {
  local c="$1" pred="$2" start now j
  start=$(python3 -c 'import time; print(time.time())'); WAIT_S=""
  while :; do
    j=$(gp site-secrets "$c")
    if python3 -c "import json,sys; d=json.loads(sys.argv[1]); sys.exit(0 if ($pred) else 1)" "$j"; then
      now=$(python3 -c 'import time; print(time.time())')
      WAIT_S=$(python3 -c "print(round($now-${3:-$start},1))"); LAST_NODE_JSON="$j"; return 0
    fi
    now=$(python3 -c 'import time; print(time.time())')
    python3 -c "import sys; sys.exit(0 if $now-$start < $STANDBY_WINDOW_S else 1)" || { LAST_NODE_JSON="$j"; return 1; }
    sleep 1
  done
}
matches_central() {  # python predicate: node row == central row C (rev + sha), live
  echo "d.get('$SECRET') is not None and not d['$SECRET']['deleted'] and d['$SECRET']['rev']==$(jget "$1" rev) and d['$SECRET']['sha']=='$(jget "$1" sha)' and d['$SECRET']['kek']=='$(jget "$1" kek)'"
}

# ── 3 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 3 "both site-a nodes hold $SECRET at the central revision (standby within ${STANDBY_WINDOW_S}s)"
C=$(gp central-secret "$SECRET")
PRED=$(matches_central "$C")
if wait_node "$ACTIVE_A" "$PRED" "$T_DEPLOYED"; then ok "site-a-a (active) r$(jget "$C" rev), ciphertext sha256 $(jget "$C" sha | cut -c1-16)… == central (seen ${WAIT_S}s after the deploy returned)"
else bad "site-a-a does not match central: $LAST_NODE_JSON"; fi
if wait_node "$STANDBY_A" "$PRED" "$T_DEPLOYED"; then STANDBY_3="$WAIT_S"; ok "site-a-b (standby) converged ${WAIT_S}s after the deploy returned"
else STANDBY_3="timeout"; bad "site-a-b did not converge within ${STANDBY_WINDOW_S}s: $LAST_NODE_JSON"; fi
L=$(gp site-localdb "$STANDBY_A")
[ "$(jget "$L" "mirrored.$SECRET")" = "$(jget "$C" rev)" ] && ok "site-a-b mirrored_secrets ledger r$(jget "$C" rev)" || bad "site-a-b ledger: $L"
end_check 3

# ── 4 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 4 "rotate at central -> site-a Out of date -> deploy -> both nodes at the new revision"
R=$(SB_GATE_VALUE="$VALUE_2" gp secret-upsert "$URL" "$SECRET" "site-secrets live gate (throwaway, rotated)")
[ "$R" = "rotate" ] && ok "rotated $SECRET via /admin/secrets" || bad "rotate: $R"
C2=$(gp central-secret "$SECRET")
if [ "$(jget "$C2" rev)" -gt "$(jget "$C1" rev)" ] && [ "$(jget "$C2" sha)" != "$(jget "$C1" sha)" ]; then
  ok "central r$(jget "$C1" rev) -> r$(jget "$C2" rev), ciphertext changed"
else bad "central row did not advance: before r$(jget "$C1" rev) after r$(jget "$C2" rev)"; fi
B=$(gp badges "$URL"); echo "    badges: $B"
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a Out of date after the rotation" || bad "site-a is '$(badge_of "$B" site-a)'"
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && [ "$(site_result site-a secretsDeployed)" = "1" ] \
  && ok "redeploy succeeded, secretsDeployed 1" || bad "redeploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
PRED=$(matches_central "$C2")
if wait_node "$ACTIVE_A" "$PRED" "$T_DEPLOYED"; then ok "site-a-a at r$(jget "$C2" rev) (${WAIT_S}s)"; else bad "site-a-a: $LAST_NODE_JSON"; fi
if wait_node "$STANDBY_A" "$PRED" "$T_DEPLOYED"; then STANDBY_4="$WAIT_S"; ok "site-a-b at r$(jget "$C2" rev) (${WAIT_S}s after the deploy returned)"
else STANDBY_4="timeout"; bad "site-a-b: $LAST_NODE_JSON"; fi
B=$(gp badges "$URL")
[ "$(badge_of "$B" site-a)" = "current" ] && ok "site-a Current again" || bad "site-a is '$(badge_of "$B" site-a)'"
end_check 4

# ── 5 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 5 "drop the secret: reference -> deploy -> both nodes drop $SECRET; out-of-band $OOB survives"
SB_GATE_VALUE="$VALUE_OOB" site_a_secret_set "$OOB" "site-secrets gate out-of-band" >/dev/null 2>&1 || true
J=$(gp site-secrets "$ACTIVE_A")
if [ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ]; then
  ok "seeded out-of-band $OOB on site-a-a through the library's own write path"
else bad "out-of-band seed not visible on site-a-a: $J"; fi
upsert_conn "$CONN_A" "$SITE_A_ID" "gate-literal-not-a-secret-ref" && ok "$CONN_A no longer references a secret" || bad "update $CONN_A"
T0=$(python3 -c 'import time; print(time.time())'); deploy "$SITE_A_ID"; T_DEPLOYED=$(python3 -c 'import time; print(time.time())')
[ "$DEPLOY_RC" -eq 0 ] && [ "$(site_result site-a success)" = "true" ] && ok "deploy succeeded (secretsDeployed='$(site_result site-a secretsDeployed)')" \
  || bad "deploy: rc=$DEPLOY_RC $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
GONE="(d.get('$SECRET') is None or d['$SECRET']['deleted'])"
if wait_node "$ACTIVE_A" "$GONE" "$T_DEPLOYED"; then ok "site-a-a no longer holds $SECRET (${WAIT_S}s)"; else bad "site-a-a still holds $SECRET: $LAST_NODE_JSON"; fi
if wait_node "$STANDBY_A" "$GONE" "$T_DEPLOYED"; then STANDBY_5="$WAIT_S"; ok "site-a-b no longer holds $SECRET (${WAIT_S}s after the deploy returned)"
else STANDBY_5="timeout"; bad "site-a-b still holds $SECRET: $LAST_NODE_JSON"; fi
J=$(gp site-secrets "$ACTIVE_A")
[ "$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get(sys.argv[2],{}).get("deleted"))' "$J" "$OOB")" = "False" ] \
  && ok "out-of-band $OOB still live on site-a-a" || bad "out-of-band $OOB lost: $J"
L=$(gp site-localdb "$ACTIVE_A")
[ "$(jget "$L" "deployed.$SECRET.removed")" = "true" ] && ok "deployed_secrets.$SECRET is_removed=1 (kept for the standby)" || bad "deployed_secrets: $L"
end_check 5

# ── 6 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 6 "reference non-existent $MISSING -> site-a deploy fails closed, stays Out of date, site untouched"
SNAP_BEFORE="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
upsert_conn "$CONN_A" "$SITE_A_ID" "secret:$MISSING" && ok "$CONN_A now references secret:$MISSING" || bad "update $CONN_A"
deploy "$SITE_A_ID"
ERR=$(site_result site-a errorMessage)
echo "    deploy rc=$DEPLOY_RC error: $ERR"
EXPECT="Secret '$MISSING' referenced by data connection(s) $CONN_A is not in the central secret store"
[ "$(site_result site-a success)" = "false" ] && ok "site-a deploy reported failure" || bad "site-a deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
case "$ERR" in *"$EXPECT"*) ok "named fail-closed message";; *) bad "message does not contain: $EXPECT";; esac
B=$(gp badges "$URL"); echo "    badges: $B"
[ "$(badge_of "$B" site-a)" = "out of date" ] && ok "site-a stays Out of date" || bad "site-a is '$(badge_of "$B" site-a)'"
sleep 3
SNAP_AFTER="$(site_hash site-a)|$(gp site-secrets "$ACTIVE_A")|$(gp site-localdb "$ACTIVE_A")|$(gp site-secrets "$STANDBY_A")|$(gp site-localdb "$STANDBY_A")"
[ "$SNAP_BEFORE" = "$SNAP_AFTER" ] && ok "nothing changed: stamped hash, both nodes' secret rows, deployed_secrets, ledgers and the $CONN_A definition identical" \
  || { bad "site state changed"; echo "      before: $SNAP_BEFORE"; echo "      after:  $SNAP_AFTER"; }
end_check 6

# ── 7 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 7 "the same reference on site-b (mismatched dev KEK) -> deploy fails naming the KekId"
C=$(gp central-secret "$SECRET")
KEK=$(jget "$C" kek)
upsert_conn "$CONN_B" "$SITE_B_ID" "secret:$SECRET" && ok "created $CONN_B on site-b (ApiKey=secret:$SECRET)" || bad "create $CONN_B"
deploy "$SITE_B_ID"
ERR=$(site_result site-b errorMessage)
echo "    deploy rc=$DEPLOY_RC error: $ERR"
[ "$(site_result site-b success)" = "false" ] && ok "site-b deploy reported failure" || bad "site-b deploy did not fail: $(printf '%s' "$DEPLOY_OUT" | tr -d '\n' | cut -c1-240)"
case "$ERR" in *"$SECRET"*"$KEK"*) ok "message names $SECRET and the row's KekId $KEK";; *) bad "message does not name $SECRET + KekId $KEK";; esac
B=$(gp badges "$URL"); echo "    badges: $B"
[ "$(badge_of "$B" site-b)" = "out of date" ] && ok "site-b not stamped (Out of date)" || bad "site-b is '$(badge_of "$B" site-b)'"
end_check 7

# ── 8 ────────────────────────────────────────────────────────────────────────────────────────
begin_check 8 "DeploySiteSecrets audit rows; no plaintext / base64 ciphertext in any node log"
AUD=$("$CLI" audit-config query "${A[@]}" --action DeploySiteSecrets --entity-type SiteSecrets --from "$GATE_START" --page-size 50 2>/dev/null)
ROWS=$(printf '%s' "$AUD" | python3 -c '
import json,sys
d=json.load(sys.stdin)
rows=[]
for e in sorted(d.get("entries",[]), key=lambda e: e["timestamp"]):
    s=json.loads(e.get("afterStateJson") or "{}")
    names=",".join("%s@r%s" % (x.get("name"), x.get("revision")) for x in (s.get("secrets") or []))
    miss=",".join(s.get("missing") or [])
    rows.append("%s|%s|%s|%s|%s" % (e["entityId"], s.get("success"), names or "-", miss or "-", s.get("deploymentId")))
print("\n".join(rows))')
echo "$ROWS" | sed 's/^/    audit: /'
R2=$(jget "$C1" rev); R4=$(jget "$C2" rev)
EXPECTED="site-a|True|$SECRET@r$R2|-
site-a|True|$SECRET@r$R4|-
site-a|False|-|$MISSING
site-b|False|$SECRET@r$R4|-"
GOT=$(printf '%s\n' "$ROWS" | cut -d'|' -f1-4)
[ "$GOT" = "$EXPECTED" ] && ok "exactly 4 rows, one per secret-bearing deploy (2, 4, 6-fail-closed, 7-kek-mismatch); the no-secret deploy of check 5 wrote none" \
  || { bad "audit rows differ from expected"; echo "      expected:"; echo "$EXPECTED" | sed 's/^/        /'; }
[ "$(printf '%s\n' "$ROWS" | cut -d'|' -f5 | sort -u | grep -c .)" = "4" ] && ok "4 distinct deploymentIds" || bad "deploymentIds not distinct"
# Material to hunt for: both plaintext values and both ciphertexts (base64, as they appear in any
# JSON serialization). Written to a pattern file so they never reach argv or the output.
PAT="$WORK/patterns"; : >"$PAT"; chmod 600 "$PAT"
printf '%s\n%s\n%s\n' "$VALUE_1" "$VALUE_2" "$VALUE_OOB" >>"$PAT"
jget "$C1" b64 >>"$PAT"; jget "$C2" b64 >>"$PAT"
[ "$(grep -c . "$PAT")" = "5" ] && [ "$(awk 'length($0) < 20' "$PAT" | grep -c .)" = "0" ] \
  && ok "5 material patterns (3 plaintext, 2 base64 ciphertext) armed" || bad "pattern file malformed"
HITS=0
for n in "${NODES[@]}"; do
  h=$(docker logs --since "$LOG_SINCE" "scadabridge-$n" 2>&1 | grep -c -F -f "$PAT" || true)
  HITS=$((HITS + h)); [ "$h" = "0" ] || echo "      scadabridge-$n: $h hit(s)"
done
AUDH=$(printf '%s' "$AUD" | grep -c -F -f "$PAT" || true)
[ "$HITS" = "0" ] && ok "0 hits across all 8 nodes' logs since $LOG_SINCE" || bad "$HITS log line(s) carry secret material"
[ "$AUDH" = "0" ] && ok "0 hits in the DeploySiteSecrets audit payloads" || bad "audit payload carries secret material"
end_check 8

echo ""
echo "=== Summary ==="
echo "standby convergence (site-a-b, seconds after the deploy call returned): check3=${STANDBY_3:-?} check4=${STANDBY_4:-?} check5=${STANDBY_5:-?}"
echo "PASS $CHECKS_PASSED/8"
[ "$CHECKS_FAILED" -eq 0 ] || exit 1