ScadaBridge/docker/verify-pending-updates.sh

#!/bin/bash
set -euo pipefail

# Live gate for the PER-SITE ARTIFACT REVISION that backs the pending-updates indicator
# (design docs/plans/2026-09-14-pending-updates-design.md, plan task 10). Drives the real
# CLI -> Traefik -> active central -> ArtifactDeploymentService path, so every assertion
# below is end-to-end against the running rig.
#
# Steps:
#   A  site list carries artifactRevisionHash / artifactsDeployedAt  (the columns are live)
#      CAVEAT: the management API serializer omits null properties, so on a site that has
#      never had an artifact deploy the two keys are ABSENT rather than present-and-null.
#      A therefore asserts the shape (absent, or a well-formed sha256) and B is the proof.
#   B  deploy artifacts fleet-wide -> every site stamped sha256:<64 hex>, deployed-at is now
#   C  create a throwaway shared script -> hashes UNCHANGED until a deploy, then ALL CHANGED
#   D  delete it, redeploy -> every hash is byte-identical to B again (determinism)
#   E  Central UI /admin/sites answers 200 and serves the Artifacts badge markup (soft)
#
# NOTE on step E: the Sites page badge and the rail-footer indicator are Blazor Server
# interactive content — with login ENABLED the badge text only lands after the circuit's
# post-render LoadUpdatesAsync, which curl never sees. E is therefore a reachability +
# static-markup smoke only; the real browser coverage is
# tests/ZB.MOM.WW.ScadaBridge.CentralUI.PlaywrightTests/PendingUpdatesTests.cs.
#
# EVERY read of the site list is checked for both a failed CLI call and a short row count
# (read_rows below). Without that an empty list would pass the whole gate vacuously: "no
# rows" means no malformed hash to report and no difference between two row sets.
#
# Usage:  bash docker/verify-pending-updates.sh
#         SCADABRIDGE_URL=http://localhost:9000 bash docker/verify-pending-updates.sh

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CLI="${SCADABRIDGE_CLI:-$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge}"
URL="${SCADABRIDGE_URL:-http://localhost:9000}"
A=(--url "$URL" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}")
PROBE_NAME="PendingUpdatesProbe_$(date +%s)"

PASS=0; FAIL=0
pass() { PASS=$((PASS+1)); echo "  PASS  $*"; }
fail() { FAIL=$((FAIL+1)); echo "  FAIL  $*"; }

# Bail out mid-run without losing the tally. Used for the failures that make every later
# assertion meaningless (no sites, an unreachable management API, a short row read).
abort() { echo ""; echo "PASS $PASS / FAIL $FAIL"; exit 1; }

[ -x "$CLI" ] || { echo "CLI not found at $CLI — build it: dotnet build src/ZB.MOM.WW.ScadaBridge.CLI"; exit 1; }

# The CLI's JSON envelope varies by command (bare array vs {"data": [...]}); normalise once.
UNWRAP='
import json,sys
d=json.load(sys.stdin)
if isinstance(d, dict):
    d = d.get("data", d)
'

# Prints one "<siteId>\t<identifier>\t<hash|->\t<deployedAt|->" line per site, id-sorted
# NUMERICALLY. Exits non-zero if the CLI call fails (pipefail) or the list is empty.
site_rows() {
  "$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
if not d:
    sys.exit(2)
for s in sorted(d, key=lambda x: int(x["id"])):
    print("%s\t%s\t%s\t%s" % (s["id"], s.get("siteIdentifier"),
                              s.get("artifactRevisionHash") or "-",
                              s.get("artifactsDeployedAt") or "-"))
'
}

# read_rows <label> -> sets ROWS. Aborts the run on a failed call or a row count that does
# not match the site count established in step A — either would let later steps pass on
# nothing at all.
ROWS=""
read_rows() {
  local label="$1" out="" rc=0 n
  out=$(site_rows) || rc=$?
  if [ "$rc" -ne 0 ]; then
    fail "$label could not read the site list (exit $rc)"
    abort
  fi
  n=$(printf '%s\n' "$out" | grep -c . || true)
  if [ "$n" -eq 0 ] || [ "$n" -ne "$SITE_COUNT" ]; then
    fail "$label expected $SITE_COUNT site rows, got $n"
    abort
  fi
  ROWS="$out"
}

# compare_rows <same|different> <rowsBefore> <rowsAfter> -> "OK" or a description.
# Keyed by site id (numeric), and the id SETS must match too: a site appearing or vanishing
# between two reads invalidates the comparison rather than silently shrinking it. Replaces a
# join(1)/cut/diff pipeline, which assumed lexicographic id order == numeric order.
compare_rows() {
  python3 -c '
import sys
mode, before, after = sys.argv[1], sys.argv[2], sys.argv[3]

def parse(text):
    rows = {}
    for line in text.splitlines():
        if not line.strip():
            continue
        sid, ident, h, at = line.split("\t")
        rows[int(sid)] = (ident, h, at)
    return rows

b, a = parse(before), parse(after)
if set(b) != set(a):
    print("site id sets differ between reads: %s vs %s" % (sorted(b), sorted(a)))
    sys.exit(0)
if not b:
    print("no site rows to compare")
    sys.exit(0)

bad = []
for sid in sorted(b):
    unchanged = b[sid][1] == a[sid][1]
    if mode == "same" and not unchanged:
        bad.append("%s moved %s -> %s" % (a[sid][0], b[sid][1][7:19], a[sid][1][7:19]))
    elif mode == "different" and unchanged:
        bad.append("%s still %s" % (a[sid][0], a[sid][1][7:19]))
print("OK" if not bad else "; ".join(bad))
' "$1" "$2" "$3"
}

deploy_all() {
  local out rc=0
  out=$(mktemp)
  "$CLI" site deploy-artifacts "${A[@]}" >"$out" 2>&1 || rc=$?
  if [ "$rc" -ne 0 ]; then
    echo "      deploy-artifacts exited $rc:"; sed 's/^/      /' "$out"
  fi
  rm -f "$out"
  return "$rc"
}

echo "=== ScadaBridge pending-updates live gate ==="
echo "management url: $URL"

echo ""
echo "=== A: site list exposes the artifact-revision columns ==="
RAW=""; RC=0
RAW=$("$CLI" site list "${A[@]}" 2>&1) || RC=$?
if [ "$RC" -ne 0 ]; then
  fail "A  site list failed (exit $RC): $(printf '%s' "$RAW" | head -2)"
  abort
fi
OUT=$(printf '%s' "$RAW" | python3 -c "$UNWRAP"'
import re
bad=[]
stamped=0
for s in d:
    h=s.get("artifactRevisionHash")
    at=s.get("artifactsDeployedAt")
    if h is None and at is None:
        continue          # never deployed: both keys omitted together
    if h is not None and at is not None and re.fullmatch(r"sha256:[0-9a-f]{64}", h):
        stamped+=1
        continue
    bad.append("%s(hash=%r,at=%r)" % (s.get("siteIdentifier"), h, at))
print("EMPTY" if not d else
      ("OK %d site(s), %d already stamped" % (len(d), stamped) if not bad
       else "malformed on " + ",".join(bad)))
' 2>&1) || OUT="could not parse the site list"
case "$OUT" in
  OK*) pass "A  ${OUT#OK }: artifact-revision fields present and well-formed (or absent = never deployed)" ;;
  EMPTY) fail "A  site list returned no sites — run docker/seed-sites.sh"; abort ;;
  *) fail "A  $OUT"; printf '%s\n' "$RAW" | head -5; abort ;;
esac
SITE_COUNT=$(printf '%s' "$RAW" | python3 -c "$UNWRAP"'
print(len(d))
')
echo "      $SITE_COUNT sites"

echo ""
echo "=== B: fleet-wide deploy stamps every site ==="
deploy_all && pass "B0 site deploy-artifacts (fleet-wide) succeeded" \
           || fail "B0 site deploy-artifacts (fleet-wide) failed"
read_rows "B"; B_ROWS="$ROWS"
OUT=$(printf '%s\n' "$B_ROWS" | python3 -c '
import sys,re,datetime
bad_hash=[]; bad_time=[]
now=datetime.datetime.now(datetime.timezone.utc)
rows=[l.rstrip("\n").split("\t") for l in sys.stdin if l.strip()]
for sid,ident,h,at in rows:
    if not re.fullmatch(r"sha256:[0-9a-f]{64}", h or ""):
        bad_hash.append("%s=%r" % (ident, h))
    try:
        t=datetime.datetime.fromisoformat((at or "").replace("Z","+00:00"))
        if t.tzinfo is None: t=t.replace(tzinfo=datetime.timezone.utc)
        age=(now-t).total_seconds()
        if not (-60 <= age <= 300): bad_time.append("%s=%.0fs old" % (ident, age))
    except Exception:
        bad_time.append("%s=%r" % (ident, at))
print("HASH " + ("OK" if not bad_hash else ",".join(bad_hash)))
print("TIME " + ("OK" if not bad_time else ",".join(bad_time)))
')
HASH_LINE=$(printf '%s\n' "$OUT" | sed -n 's/^HASH //p')
TIME_LINE=$(printf '%s\n' "$OUT" | sed -n 's/^TIME //p')
[ "$HASH_LINE" = "OK" ] && pass "B1 all $SITE_COUNT site hashes are sha256:<64 hex> (71 chars)" \
                        || fail "B1 bad hash(es): $HASH_LINE"
[ "$TIME_LINE" = "OK" ] && pass "B2 all $SITE_COUNT artifactsDeployedAt values are within the last 5 minutes" \
                        || fail "B2 stale/unparseable deployed-at: $TIME_LINE"

echo ""
echo "=== C: a new shared script moves the hash — but only on deploy ==="

# Armed BEFORE the create: a create that succeeds server-side but whose id we fail to read
# must still be cleaned up, so the fallback re-finds the probe by its unique name.
PROBE_ID=""
cleanup() {
  local id="$PROBE_ID"
  if [ -z "$id" ]; then
    id=$("$CLI" shared-script list "${A[@]}" 2>/dev/null | python3 -c "$UNWRAP"'
import sys
print(next((str(s["id"]) for s in d if s.get("name")==sys.argv[1]), ""))
' "$PROBE_NAME" 2>/dev/null) || id=""
  fi
  if [ -n "$id" ]; then
    "$CLI" shared-script delete "${A[@]}" --id "$id" >/dev/null 2>&1 \
      && echo "  cleanup: removed probe shared script $PROBE_NAME (id=$id)" || true
  fi
}
trap cleanup EXIT

CREATE=""; RC=0
CREATE=$("$CLI" shared-script create "${A[@]}" --name "$PROBE_NAME" \
          --code 'var pendingUpdatesProbe = 1; return pendingUpdatesProbe;' 2>&1) || RC=$?
# `shared-script create` echoes the created entity, so the id comes straight back.
PROBE_ID=$(printf '%s' "$CREATE" | python3 -c "$UNWRAP"'
print(d.get("id","") if isinstance(d, dict) else "")
' 2>/dev/null) || PROBE_ID=""
if [ "$RC" -eq 0 ] && [ -n "$PROBE_ID" ]; then
  pass "C0 created shared script $PROBE_NAME (id=$PROBE_ID)"
else
  fail "C0 shared-script create failed (exit $RC): $(printf '%s' "$CREATE" | head -2)"
  abort
fi

read_rows "C1"; C_BEFORE="$ROWS"
OUT=$(compare_rows same "$B_ROWS" "$C_BEFORE")
[ "$OUT" = "OK" ] && pass "C1 all $SITE_COUNT hashes UNCHANGED by the create (the stamp only moves on deploy)" \
                  || fail "C1 a hash moved without a deploy: $OUT"

deploy_all && pass "C2 redeploy after the create succeeded" || fail "C2 redeploy after the create failed"
read_rows "C3"; C_ROWS="$ROWS"
OUT=$(compare_rows different "$B_ROWS" "$C_ROWS")
[ "$OUT" = "OK" ] && pass "C3 EVERY one of the $SITE_COUNT site hashes changed after deploying the new shared script" \
                  || fail "C3 hash did not move for: $OUT"

echo ""
echo "=== D: deleting it again restores the exact B hashes (determinism) ==="
if "$CLI" shared-script delete "${A[@]}" --id "$PROBE_ID" >/dev/null 2>&1; then
  pass "D0 deleted shared script $PROBE_NAME"
  PROBE_ID=""
else
  fail "D0 shared-script delete failed"
fi
deploy_all && pass "D1 redeploy after the delete succeeded" || fail "D1 redeploy after the delete failed"
read_rows "D2"; D_ROWS="$ROWS"
OUT=$(compare_rows same "$B_ROWS" "$D_ROWS")
[ "$OUT" = "OK" ] \
  && pass "D2 all $SITE_COUNT hashes are byte-identical to B — the hash is content-addressed, not deploy-addressed" \
  || fail "D2 hashes did not return to their B values: $OUT"

echo ""
echo "=== E: Central UI Sites page reachability (soft — see the header note) ==="
BODY=""; RC=0
BODY=$(curl -sS -w '\n%{http_code}' "$URL/admin/sites" 2>/dev/null) || RC=$?
CODE=$(printf '%s' "$BODY" | tail -1)
if [ "$RC" -eq 0 ] && [ "$CODE" = "200" ]; then
  pass "E1 GET /admin/sites -> 200"
  if printf '%s' "$BODY" | grep -q 'Re-check pending updates'; then
    pass "E2 prerendered markup carries the Re-check pending updates control"
  else
    echo "  NOTE  E2 the Re-check control is not in the prerendered HTML (interactive render) — covered by PendingUpdatesTests"
  fi
else
  echo "  NOTE  E1 GET /admin/sites -> ${CODE:-curl exit $RC} (login enabled on this rig?) — covered by PendingUpdatesTests"
fi

echo ""
echo "=== Summary ==="
python3 -c '
import sys
def parse(text):
    rows = {}
    for line in text.splitlines():
        if not line.strip():
            continue
        sid, ident, h, at = line.split("\t")
        rows[int(sid)] = (ident, h)
    return rows

b, c, dd = (parse(t) for t in sys.argv[1:4])
print("%-4s %-14s %-14s %-14s %-14s" % ("id", "site", "B hash", "C hash", "D hash"))
for sid in sorted(b):
    print("%-4d %-14s %-14s %-14s %-14s" % (
        sid, b[sid][0], b[sid][1][7:19], c.get(sid, ("", "?"))[1][7:19], dd.get(sid, ("", "?"))[1][7:19]))
' "$B_ROWS" "$C_ROWS" "$D_ROWS"
echo ""
echo "PASS $PASS / FAIL $FAIL"
[ "$FAIL" -eq 0 ] || exit 1