Joseph Doherty
74 lines
2.8 KiB
Bash
Executable File
74 lines
2.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# assert-not-all-skipped.sh — arch-review 07/S-2 fail-on-skip guard.
|
|
#
|
|
# A CI test leg that ran but executed ZERO tests reports success, so a whole tier
|
|
# silently skipping (fixtures unreachable, a --filter that matched nothing, a probe
|
|
# that Assert.Skip'd every case) is indistinguishable from a real pass. This gate
|
|
# reads the trx result summary and FAILS when fewer than MIN_EXECUTED tests actually
|
|
# ran across the supplied trx files — turning "green means skipped" into a red build.
|
|
#
|
|
# Usage: scripts/ci/assert-not-all-skipped.sh <trx> [<trx> ...]
|
|
# Env: MIN_EXECUTED minimum executed tests required (default 1).
|
|
# Set MIN_EXECUTED=0 for a report-only pass (never fails) —
|
|
# used by the fixtureless integration leg until its fixtures
|
|
# run as workflow services (S-2 option #2 follow-up).
|
|
#
|
|
set -euo pipefail
|
|
|
|
min_executed="${MIN_EXECUTED:-1}"
|
|
|
|
if [ "$#" -eq 0 ]; then
|
|
echo "assert-not-all-skipped: no trx files given" >&2
|
|
exit 2
|
|
fi
|
|
|
|
total_executed=0
|
|
total_skipped=0
|
|
seen=0
|
|
|
|
for f in "$@"; do
|
|
if [ ! -f "$f" ]; then
|
|
echo "assert-not-all-skipped: trx not found: $f" >&2
|
|
exit 2
|
|
fi
|
|
seen=$((seen + 1))
|
|
# Keep the executed-counter floor unchanged; notExecuted is only a fallback
|
|
# for malformed XML that this guard historically accepted.
|
|
counters="$(grep -oE '<Counters[^>]*/?>' "$f" | head -1)"
|
|
ex="$(printf '%s' "$counters" | grep -oE 'executed="[0-9]+"' | grep -oE '[0-9]+' || true)"
|
|
ne="$(printf '%s' "$counters" | grep -oE 'notExecuted="[0-9]+"' | grep -oE '[0-9]+' || true)"
|
|
ex="${ex:-0}"
|
|
ne="${ne:-0}"
|
|
# VSTest can emit notExecuted="0" despite individual NotExecuted results.
|
|
# Count direct Results/UnitTestResult records in the document's namespace,
|
|
# excluding foreign elements, nested records and escaped diagnostic text.
|
|
ne="$(python3 - "$f" "$ne" <<'PY'
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
try:
|
|
root = ET.parse(sys.argv[1]).getroot()
|
|
except ET.ParseError:
|
|
# Skip reporting must not introduce a new malformed-XML gate failure.
|
|
print(sys.argv[2])
|
|
else:
|
|
namespace = root.tag.split("}", 1)[0] + "}" if root.tag.startswith("{") else ""
|
|
results = root.findall(f"{namespace}Results/{namespace}UnitTestResult")
|
|
print(sum(result.get("outcome") == "NotExecuted" for result in results))
|
|
PY
|
|
)"
|
|
total_executed=$((total_executed + ex))
|
|
total_skipped=$((total_skipped + ne))
|
|
echo " $(basename "$f"): executed=${ex} skipped=${ne}"
|
|
done
|
|
|
|
echo "assert-not-all-skipped: files=${seen} executed=${total_executed} skipped=${total_skipped} (min required=${min_executed})"
|
|
|
|
if [ "$total_executed" -lt "$min_executed" ]; then
|
|
echo "::error::CI executed ${total_executed} test(s) (< ${min_executed}) — the tier ran nothing (all skipped?). Treating as FAILURE, not a pass." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "assert-not-all-skipped: OK"
|