fix(rig): verify-pending-updates.sh cannot pass on an empty site list; numeric-safe comparison; earlier cleanup trap

The gate could pass vacuously. Every site-list read went through a bare
`ROWS=$(site_rows)` with neither the exit status nor the row count checked, and
an empty result satisfied every downstream assertion for free: no rows means no
malformed hash for B1/B2 to report, and a diff of two empty strings is no
difference at all for C1/D2. A down management API would therefore have read as
a clean gate.

Reads now go through read_rows, which fails loudly on a non-zero CLI exit
(site_rows exits 2 on an empty list, and pipefail surfaces a failed call) and
aborts unless the line count equals the site count established in step A —
keeping the running tally so the abort still prints PASS/FAIL. Verified against
an unreachable URL: FAIL 1, exit 1.

The join(1)/cut/diff comparisons assumed lexicographic id order matches numeric
order, which breaks once a site id reaches 10 alongside single-digit ids (and BSD
join silently drops the unmatched rows rather than complaining). Replaced with a
python3 comparison keyed by integer id that also asserts the id SETS match, so a
site appearing or vanishing between two reads invalidates the comparison instead
of quietly shrinking it. Same for the summary table.

Also: set -euo pipefail like every sibling script (the cmd && pass || fail guards
keep -e safe, and -e does not cover var=$(cmd) — hence the explicit checks
above); the cleanup trap is armed BEFORE the shared-script create and falls back
to finding the probe by its unique name, so a create that succeeds server-side
but whose id we fail to parse can no longer orphan a probe; mktemp instead of
/tmp/pu-deploy.$$.

PendingUpdatesTests: the rail-scroll check asserted scrollHeight >= clientHeight,
which holds for every element. It now asserts what the pane is actually for —
the pane is bounded by the viewport and the rail footer holding the indicator
stays on screen, so a nav that grows the rail instead of scrolling fails. Live
values are genuinely overflowing (scrollHeight 1196 vs clientHeight 680, footer
bottom 765 of 900), so the assertion has real content.

Re-run against the rig: PASS 13 / FAIL 0; Playwright 3/3.

Claude-Session: https://claude.ai/code/session_01Y5ef34iUUMjmwnMpL9xgmB
This commit is contained in:
Joseph Doherty
2026-09-14 14:20:15 -04:00
parent af9f14a668
commit 623c6d0fe8
2 changed files with 182 additions and 75 deletions
+162 -70
View File
@@ -1,5 +1,5 @@
#!/bin/bash
set -uo pipefail
set -euo pipefail
# Live gate for the PER-SITE ARTIFACT REVISION that backs the pending-updates indicator
# (design docs/plans/2026-09-14-pending-updates-design.md, plan task 10). Drives the real
@@ -17,11 +17,15 @@ set -uo pipefail
# E Central UI /admin/sites answers 200 and serves the Artifacts badge markup (soft)
#
# NOTE on step E: the Sites page badge and the rail-footer indicator are Blazor Server
# interactive content — the badge text only lands after the circuit's post-render
# LoadUpdatesAsync, which curl never sees. E is therefore a reachability + static-markup
# smoke only; the real browser coverage is
# interactive content — with login ENABLED the badge text only lands after the circuit's
# post-render LoadUpdatesAsync, which curl never sees. E is therefore a reachability +
# static-markup smoke only; the real browser coverage is
# tests/ZB.MOM.WW.ScadaBridge.CentralUI.PlaywrightTests/PendingUpdatesTests.cs.
#
# EVERY read of the site list is checked for both a failed CLI call and a short row count
# (read_rows below). Without that an empty list would pass the whole gate vacuously: "no
# rows" means no malformed hash to report and no difference between two row sets.
#
# Usage: bash docker/verify-pending-updates.sh
# SCADABRIDGE_URL=http://localhost:9000 bash docker/verify-pending-updates.sh
@@ -36,6 +40,10 @@ PASS=0; FAIL=0
pass() { PASS=$((PASS+1)); echo " PASS $*"; }
fail() { FAIL=$((FAIL+1)); echo " FAIL $*"; }
# Bail out mid-run without losing the tally. Used for the failures that make every later
# assertion meaningless (no sites, an unreachable management API, a short row read).
abort() { echo ""; echo "PASS $PASS / FAIL $FAIL"; exit 1; }
[ -x "$CLI" ] || { echo "CLI not found at $CLI — build it: dotnet build src/ZB.MOM.WW.ScadaBridge.CLI"; exit 1; }
# The CLI's JSON envelope varies by command (bare array vs {"data": [...]}); normalise once.
@@ -46,24 +54,85 @@ if isinstance(d, dict):
d = d.get("data", d)
'
# Prints one "<siteId>\t<identifier>\t<hash|->\t<deployedAt|->" line per site, id-sorted.
# Prints one "<siteId>\t<identifier>\t<hash|->\t<deployedAt|->" line per site, id-sorted
# NUMERICALLY. Exits non-zero if the CLI call fails (pipefail) or the list is empty.
site_rows() {
"$CLI" site list "${A[@]}" 2>/dev/null | python3 -c "$UNWRAP"'
for s in sorted(d, key=lambda x: x["id"]):
"$CLI" site list "${A[@]}" | python3 -c "$UNWRAP"'
import sys
if not d:
sys.exit(2)
for s in sorted(d, key=lambda x: int(x["id"])):
print("%s\t%s\t%s\t%s" % (s["id"], s.get("siteIdentifier"),
s.get("artifactRevisionHash") or "-",
s.get("artifactsDeployedAt") or "-"))
'
}
deploy_all() {
"$CLI" site deploy-artifacts "${A[@]}" > /tmp/pu-deploy.$$ 2>&1
local rc=$?
if [ $rc -ne 0 ]; then
echo " deploy-artifacts exited $rc:"; sed 's/^/ /' /tmp/pu-deploy.$$
# read_rows <label> -> sets ROWS. Aborts the run on a failed call or a row count that does
# not match the site count established in step A — either would let later steps pass on
# nothing at all.
ROWS=""
read_rows() {
local label="$1" out="" rc=0 n
out=$(site_rows) || rc=$?
if [ "$rc" -ne 0 ]; then
fail "$label could not read the site list (exit $rc)"
abort
fi
rm -f /tmp/pu-deploy.$$
return $rc
n=$(printf '%s\n' "$out" | grep -c . || true)
if [ "$n" -eq 0 ] || [ "$n" -ne "$SITE_COUNT" ]; then
fail "$label expected $SITE_COUNT site rows, got $n"
abort
fi
ROWS="$out"
}
# compare_rows <same|different> <rowsBefore> <rowsAfter> -> "OK" or a description.
# Keyed by site id (numeric), and the id SETS must match too: a site appearing or vanishing
# between two reads invalidates the comparison rather than silently shrinking it. Replaces a
# join(1)/cut/diff pipeline, which assumed lexicographic id order == numeric order.
compare_rows() {
python3 -c '
import sys
mode, before, after = sys.argv[1], sys.argv[2], sys.argv[3]
def parse(text):
rows = {}
for line in text.splitlines():
if not line.strip():
continue
sid, ident, h, at = line.split("\t")
rows[int(sid)] = (ident, h, at)
return rows
b, a = parse(before), parse(after)
if set(b) != set(a):
print("site id sets differ between reads: %s vs %s" % (sorted(b), sorted(a)))
sys.exit(0)
if not b:
print("no site rows to compare")
sys.exit(0)
bad = []
for sid in sorted(b):
unchanged = b[sid][1] == a[sid][1]
if mode == "same" and not unchanged:
bad.append("%s moved %s -> %s" % (a[sid][0], b[sid][1][7:19], a[sid][1][7:19]))
elif mode == "different" and unchanged:
bad.append("%s still %s" % (a[sid][0], a[sid][1][7:19]))
print("OK" if not bad else "; ".join(bad))
' "$1" "$2" "$3"
}
deploy_all() {
local out rc=0
out=$(mktemp)
"$CLI" site deploy-artifacts "${A[@]}" >"$out" 2>&1 || rc=$?
if [ "$rc" -ne 0 ]; then
echo " deploy-artifacts exited $rc:"; sed 's/^/ /' "$out"
fi
rm -f "$out"
return "$rc"
}
echo "=== ScadaBridge pending-updates live gate ==="
@@ -71,7 +140,12 @@ echo "management url: $URL"
echo ""
echo "=== A: site list exposes the artifact-revision columns ==="
RAW=$("$CLI" site list "${A[@]}" 2>&1)
RAW=""; RC=0
RAW=$("$CLI" site list "${A[@]}" 2>&1) || RC=$?
if [ "$RC" -ne 0 ]; then
fail "A site list failed (exit $RC): $(printf '%s' "$RAW" | head -2)"
abort
fi
OUT=$(printf '%s' "$RAW" | python3 -c "$UNWRAP"'
import re
bad=[]
@@ -88,20 +162,22 @@ for s in d:
print("EMPTY" if not d else
("OK %d site(s), %d already stamped" % (len(d), stamped) if not bad
else "malformed on " + ",".join(bad)))
' 2>&1)
' 2>&1) || OUT="could not parse the site list"
case "$OUT" in
OK*) pass "A ${OUT#OK }: artifact-revision fields present and well-formed (or absent = never deployed)" ;;
EMPTY) fail "A site list returned no sites — run docker/seed-sites.sh"; echo "$RAW" | head -3; exit 1 ;;
*) fail "A $OUT"; echo "$RAW" | head -5; exit 1 ;;
EMPTY) fail "A site list returned no sites — run docker/seed-sites.sh"; abort ;;
*) fail "A $OUT"; printf '%s\n' "$RAW" | head -5; abort ;;
esac
SITE_COUNT=$(site_rows | wc -l | tr -d ' ')
SITE_COUNT=$(printf '%s' "$RAW" | python3 -c "$UNWRAP"'
print(len(d))
')
echo " $SITE_COUNT sites"
echo ""
echo "=== B: fleet-wide deploy stamps every site ==="
deploy_all && pass "B0 site deploy-artifacts (fleet-wide) succeeded" \
|| fail "B0 site deploy-artifacts (fleet-wide) failed"
B_ROWS=$(site_rows)
read_rows "B"; B_ROWS="$ROWS"
OUT=$(printf '%s\n' "$B_ROWS" | python3 -c '
import sys,re,datetime
bad_hash=[]; bad_time=[]
@@ -122,75 +198,78 @@ print("TIME " + ("OK" if not bad_time else ",".join(bad_time)))
')
HASH_LINE=$(printf '%s\n' "$OUT" | sed -n 's/^HASH //p')
TIME_LINE=$(printf '%s\n' "$OUT" | sed -n 's/^TIME //p')
[ "$HASH_LINE" = "OK" ] && pass "B1 every site hash is sha256:<64 hex> (71 chars)" \
[ "$HASH_LINE" = "OK" ] && pass "B1 all $SITE_COUNT site hashes are sha256:<64 hex> (71 chars)" \
|| fail "B1 bad hash(es): $HASH_LINE"
[ "$TIME_LINE" = "OK" ] && pass "B2 every artifactsDeployedAt is within the last 5 minutes" \
[ "$TIME_LINE" = "OK" ] && pass "B2 all $SITE_COUNT artifactsDeployedAt values are within the last 5 minutes" \
|| fail "B2 stale/unparseable deployed-at: $TIME_LINE"
echo ""
echo "=== C: a new shared script moves the hash — but only on deploy ==="
CREATE=$("$CLI" shared-script create "${A[@]}" --name "$PROBE_NAME" \
--code 'var pendingUpdatesProbe = 1; return pendingUpdatesProbe;' 2>&1)
# `shared-script create` echoes the created entity, so the id comes straight back.
PROBE_ID=$(printf '%s' "$CREATE" | python3 -c "$UNWRAP"'
print(d.get("id","") if isinstance(d, dict) else "")
' 2>/dev/null)
if [ -n "$PROBE_ID" ]; then
pass "C0 created shared script $PROBE_NAME (id=$PROBE_ID)"
else
fail "C0 shared-script create failed: $(printf '%s' "$CREATE" | head -2)"
exit 1
fi
# Cleanup even if a later assertion aborts the run: the rig must not keep the probe.
# Armed BEFORE the create: a create that succeeds server-side but whose id we fail to read
# must still be cleaned up, so the fallback re-finds the probe by its unique name.
PROBE_ID=""
cleanup() {
if [ -n "${PROBE_ID:-}" ]; then
"$CLI" shared-script delete "${A[@]}" --id "$PROBE_ID" >/dev/null 2>&1 && PROBE_ID=""
local id="$PROBE_ID"
if [ -z "$id" ]; then
id=$("$CLI" shared-script list "${A[@]}" 2>/dev/null | python3 -c "$UNWRAP"'
import sys
print(next((str(s["id"]) for s in d if s.get("name")==sys.argv[1]), ""))
' "$PROBE_NAME" 2>/dev/null) || id=""
fi
if [ -n "$id" ]; then
"$CLI" shared-script delete "${A[@]}" --id "$id" >/dev/null 2>&1 \
&& echo " cleanup: removed probe shared script $PROBE_NAME (id=$id)" || true
fi
}
trap cleanup EXIT
C_BEFORE=$(site_rows)
# Compare hash columns only — artifactsDeployedAt is untouched here by definition.
if [ "$(printf '%s\n' "$C_BEFORE" | cut -f1,3)" = "$(printf '%s\n' "$B_ROWS" | cut -f1,3)" ]; then
pass "C1 hashes UNCHANGED by the create (the stamp only moves on deploy)"
CREATE=""; RC=0
CREATE=$("$CLI" shared-script create "${A[@]}" --name "$PROBE_NAME" \
--code 'var pendingUpdatesProbe = 1; return pendingUpdatesProbe;' 2>&1) || RC=$?
# `shared-script create` echoes the created entity, so the id comes straight back.
PROBE_ID=$(printf '%s' "$CREATE" | python3 -c "$UNWRAP"'
print(d.get("id","") if isinstance(d, dict) else "")
' 2>/dev/null) || PROBE_ID=""
if [ "$RC" -eq 0 ] && [ -n "$PROBE_ID" ]; then
pass "C0 created shared script $PROBE_NAME (id=$PROBE_ID)"
else
fail "C1 a hash moved without a deploy"
diff <(printf '%s\n' "$B_ROWS" | cut -f1,3) <(printf '%s\n' "$C_BEFORE" | cut -f1,3) | sed 's/^/ /'
fail "C0 shared-script create failed (exit $RC): $(printf '%s' "$CREATE" | head -2)"
abort
fi
read_rows "C1"; C_BEFORE="$ROWS"
OUT=$(compare_rows same "$B_ROWS" "$C_BEFORE")
[ "$OUT" = "OK" ] && pass "C1 all $SITE_COUNT hashes UNCHANGED by the create (the stamp only moves on deploy)" \
|| fail "C1 a hash moved without a deploy: $OUT"
deploy_all && pass "C2 redeploy after the create succeeded" || fail "C2 redeploy after the create failed"
C_ROWS=$(site_rows)
UNCHANGED=$(diff <(printf '%s\n' "$B_ROWS" | cut -f1,3) <(printf '%s\n' "$C_ROWS" | cut -f1,3) >/dev/null && echo yes || echo no)
if [ "$UNCHANGED" = "no" ]; then
# Every site must have moved, not just some.
SAME=$(join -t$'\t' <(printf '%s\n' "$B_ROWS" | cut -f1,2,3) <(printf '%s\n' "$C_ROWS" | cut -f1,3) \
| awk -F'\t' '$3==$4 {print $2}')
[ -z "$SAME" ] && pass "C3 EVERY site's hash changed after deploying the new shared script" \
|| fail "C3 hash did not move for: $(echo $SAME)"
else
fail "C3 no site's hash changed after deploying the new shared script"
fi
read_rows "C3"; C_ROWS="$ROWS"
OUT=$(compare_rows different "$B_ROWS" "$C_ROWS")
[ "$OUT" = "OK" ] && pass "C3 EVERY one of the $SITE_COUNT site hashes changed after deploying the new shared script" \
|| fail "C3 hash did not move for: $OUT"
echo ""
echo "=== D: deleting it again restores the exact B hashes (determinism) ==="
"$CLI" shared-script delete "${A[@]}" --id "$PROBE_ID" >/dev/null 2>&1 \
&& { pass "D0 deleted shared script $PROBE_NAME"; PROBE_ID=""; } \
|| fail "D0 shared-script delete failed"
deploy_all && pass "D1 redeploy after the delete succeeded" || fail "D1 redeploy after the delete failed"
D_ROWS=$(site_rows)
if diff <(printf '%s\n' "$B_ROWS" | cut -f1,3) <(printf '%s\n' "$D_ROWS" | cut -f1,3) >/dev/null; then
pass "D2 every hash is byte-identical to B — the hash is content-addressed, not deploy-addressed"
if "$CLI" shared-script delete "${A[@]}" --id "$PROBE_ID" >/dev/null 2>&1; then
pass "D0 deleted shared script $PROBE_NAME"
PROBE_ID=""
else
fail "D2 hashes did not return to their B values"
diff <(printf '%s\n' "$B_ROWS" | cut -f1,3) <(printf '%s\n' "$D_ROWS" | cut -f1,3) | sed 's/^/ /'
fail "D0 shared-script delete failed"
fi
deploy_all && pass "D1 redeploy after the delete succeeded" || fail "D1 redeploy after the delete failed"
read_rows "D2"; D_ROWS="$ROWS"
OUT=$(compare_rows same "$B_ROWS" "$D_ROWS")
[ "$OUT" = "OK" ] \
&& pass "D2 all $SITE_COUNT hashes are byte-identical to B — the hash is content-addressed, not deploy-addressed" \
|| fail "D2 hashes did not return to their B values: $OUT"
echo ""
echo "=== E: Central UI Sites page reachability (soft — see the header note) ==="
BODY=$(curl -sS -w '\n%{http_code}' "$URL/admin/sites" 2>/dev/null)
BODY=""; RC=0
BODY=$(curl -sS -w '\n%{http_code}' "$URL/admin/sites" 2>/dev/null) || RC=$?
CODE=$(printf '%s' "$BODY" | tail -1)
if [ "$CODE" = "200" ]; then
if [ "$RC" -eq 0 ] && [ "$CODE" = "200" ]; then
pass "E1 GET /admin/sites -> 200"
if printf '%s' "$BODY" | grep -q 'Re-check pending updates'; then
pass "E2 prerendered markup carries the Re-check pending updates control"
@@ -198,15 +277,28 @@ if [ "$CODE" = "200" ]; then
echo " NOTE E2 the Re-check control is not in the prerendered HTML (interactive render) — covered by PendingUpdatesTests"
fi
else
echo " NOTE E1 GET /admin/sites -> $CODE (login enabled on this rig?) — covered by PendingUpdatesTests"
echo " NOTE E1 GET /admin/sites -> ${CODE:-curl exit $RC} (login enabled on this rig?) — covered by PendingUpdatesTests"
fi
echo ""
echo "=== Summary ==="
printf '%-4s %-14s %-14s %-14s %-14s\n' "id" "site" "B hash" "C hash" "D hash"
join -t$'\t' <(join -t$'\t' <(printf '%s\n' "$B_ROWS" | cut -f1,2,3) <(printf '%s\n' "$C_ROWS" | cut -f1,3)) \
<(printf '%s\n' "$D_ROWS" | cut -f1,3) \
| awk -F'\t' '{printf "%-4s %-14s %-14s %-14s %-14s\n", $1, $2, substr($3,8,12), substr($4,8,12), substr($5,8,12)}'
python3 -c '
import sys
def parse(text):
rows = {}
for line in text.splitlines():
if not line.strip():
continue
sid, ident, h, at = line.split("\t")
rows[int(sid)] = (ident, h)
return rows
b, c, dd = (parse(t) for t in sys.argv[1:4])
print("%-4s %-14s %-14s %-14s %-14s" % ("id", "site", "B hash", "C hash", "D hash"))
for sid in sorted(b):
print("%-4d %-14s %-14s %-14s %-14s" % (
sid, b[sid][0], b[sid][1][7:19], c.get(sid, ("", "?"))[1][7:19], dd.get(sid, ("", "?"))[1][7:19]))
' "$B_ROWS" "$C_ROWS" "$D_ROWS"
echo ""
echo "PASS $PASS / FAIL $FAIL"
[ "$FAIL" -eq 0 ] || exit 1
@@ -134,12 +134,27 @@ public class PendingUpdatesTests
var overflowY = await scroll.EvaluateAsync<string>("el => getComputedStyle(el).overflowY");
Assert.Equal("auto", overflowY);
// Informational rather than asserted: scrollHeight >= clientHeight holds for any element,
// and whether the nav actually overflows depends on the signed-in user's visible sections.
// The point of the pane is that a long nav scrolls INSIDE it instead of growing the rail
// and carrying the footer off-screen. Assert that: the pane is bounded by the viewport,
// and the rail footer (which holds the indicator this feature adds) stays within it.
// scrollHeight >= clientHeight is NOT asserted — that holds for every element.
var metrics = await scroll.EvaluateAsync<int[]>(
"el => [el.scrollHeight, el.clientHeight]");
"el => [el.scrollHeight, el.clientHeight, Math.round(el.getBoundingClientRect().bottom)]");
_output.WriteLine(
$"rail-scroll: scrollHeight={metrics[0]} clientHeight={metrics[1]} bottom={metrics[2]} (viewport 900)");
Assert.True(
metrics[0] >= metrics[1],
$"rail scroll pane has scrollHeight {metrics[0]} < clientHeight {metrics[1]}");
metrics[1] > 0 && metrics[1] <= 900,
$"rail scroll pane should be bounded by the 900px viewport, clientHeight was {metrics[1]}");
Assert.True(
metrics[2] <= 900,
$"rail scroll pane overflows the viewport (bottom {metrics[2]} > 900) — the nav is growing the rail instead of scrolling");
// The footer the indicator lives in must be on-screen, below the pane.
var footer = page.Locator(".rail-updates");
var footerBottom = await footer.EvaluateAsync<int>(
"el => Math.round(el.getBoundingClientRect().bottom)");
Assert.True(
footerBottom <= 900,
$"rail footer is off-screen (bottom {footerBottom} > 900) — the scroll pane is not containing the nav");
}
}