#!/usr/bin/env bash # # assert-not-all-skipped.sh — arch-review 07/S-2 fail-on-skip guard. # # A CI test leg that ran but executed ZERO tests reports success, so a whole tier # silently skipping (fixtures unreachable, a --filter that matched nothing, a probe # that Assert.Skip'd every case) is indistinguishable from a real pass. This gate # reads the trx result summary and FAILS when fewer than MIN_EXECUTED tests actually # ran across the supplied trx files — turning "green means skipped" into a red build. # # Usage: scripts/ci/assert-not-all-skipped.sh [ ...] # Env: MIN_EXECUTED minimum executed tests required (default 1). # Set MIN_EXECUTED=0 for a report-only pass (never fails) — # used by the fixtureless integration leg until its fixtures # run as workflow services (S-2 option #2 follow-up). # set -euo pipefail min_executed="${MIN_EXECUTED:-1}" if [ "$#" -eq 0 ]; then echo "assert-not-all-skipped: no trx files given" >&2 exit 2 fi total_executed=0 total_skipped=0 seen=0 for f in "$@"; do if [ ! -f "$f" ]; then echo "assert-not-all-skipped: trx not found: $f" >&2 exit 2 fi seen=$((seen + 1)) # Keep the executed-counter floor unchanged; notExecuted is only a fallback # for malformed XML that this guard historically accepted. counters="$(grep -oE ']*/?>' "$f" | head -1)" ex="$(printf '%s' "$counters" | grep -oE 'executed="[0-9]+"' | grep -oE '[0-9]+' || true)" ne="$(printf '%s' "$counters" | grep -oE 'notExecuted="[0-9]+"' | grep -oE '[0-9]+' || true)" ex="${ex:-0}" ne="${ne:-0}" # VSTest can emit notExecuted="0" despite individual NotExecuted results. # Count direct Results/UnitTestResult records in the document's namespace, # excluding foreign elements, nested records and escaped diagnostic text. ne="$(python3 - "$f" "$ne" <<'PY' import sys import xml.etree.ElementTree as ET try: root = ET.parse(sys.argv[1]).getroot() except ET.ParseError: # Skip reporting must not introduce a new malformed-XML gate failure. print(sys.argv[2]) else: namespace = root.tag.split("}", 1)[0] + "}" if root.tag.startswith("{") else "" results = root.findall(f"{namespace}Results/{namespace}UnitTestResult") print(sum(result.get("outcome") == "NotExecuted" for result in results)) PY )" total_executed=$((total_executed + ex)) total_skipped=$((total_skipped + ne)) echo " $(basename "$f"): executed=${ex} skipped=${ne}" done echo "assert-not-all-skipped: files=${seen} executed=${total_executed} skipped=${total_skipped} (min required=${min_executed})" if [ "$total_executed" -lt "$min_executed" ]; then echo "::error::CI executed ${total_executed} test(s) (< ${min_executed}) — the tier ran nothing (all skipped?). Treating as FAILURE, not a pass." >&2 exit 1 fi echo "assert-not-all-skipped: OK"