diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 45abcef3..2c1bd1b5 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -41,6 +41,8 @@ jobs: # #606: regenerated review summary must remain in sync with module findings. - name: code-reviews README drift check run: python3 code-reviews/regen-readme.py --check + - name: TRX skip guard fixtures + run: python3 scripts/ci/test-assert-not-all-skipped.py # #707: bound project workers and avoid concurrent NuGet restore work. - name: dotnet restore run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx --disable-parallel -m:2 diff --git a/scripts/ci/README.md b/scripts/ci/README.md index c7f97d5c..d6041ee4 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -28,3 +28,18 @@ AKKA_TEST_TIMEFACTOR=3 dotnet test ZB.MOM.WW.OtOpcUa.slnx \ --logger trx --results-directory artifacts/trx MIN_EXECUTED=12000 bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx ``` + +The guard uses the existing TRX `Counters/@executed` value for the aggregate +floor. Its `skipped` diagnostic counts individual `Results/UnitTestResult` +records whose outcome is `NotExecuted`, using the document's XML namespace. +Some VSTest reports set summary `notExecuted` to zero even when these records +are present. Skip records are counted per file, including repeated runs; they +are not a count of unique test methods and do not affect the executed floor. +For malformed XML, the diagnostic retains the old summary-counter fallback; +this guard does not add XML validation to its existing gate policy. + +Run the deterministic script fixtures without building or running the solution: + +```bash +python3 scripts/ci/test-assert-not-all-skipped.py +``` diff --git a/scripts/ci/assert-not-all-skipped.sh b/scripts/ci/assert-not-all-skipped.sh index fd30856b..e0e1e2c3 100755 --- a/scripts/ci/assert-not-all-skipped.sh +++ b/scripts/ci/assert-not-all-skipped.sh @@ -33,12 +33,31 @@ for f in "$@"; do exit 2 fi seen=$((seen + 1)) - # The trx element in carries executed + notExecuted counts. + # Keep the executed-counter floor unchanged; notExecuted is only a fallback + # for malformed XML that this guard historically accepted. counters="$(grep -oE ']*/?>' "$f" | head -1)" ex="$(printf '%s' "$counters" | grep -oE 'executed="[0-9]+"' | grep -oE '[0-9]+' || true)" ne="$(printf '%s' "$counters" | grep -oE 'notExecuted="[0-9]+"' | grep -oE '[0-9]+' || true)" ex="${ex:-0}" ne="${ne:-0}" + # VSTest can emit notExecuted="0" despite individual NotExecuted results. + # Count direct Results/UnitTestResult records in the document's namespace, + # excluding foreign elements, nested records and escaped diagnostic text. + ne="$(python3 - "$f" "$ne" <<'PY' +import sys +import xml.etree.ElementTree as ET + +try: + root = ET.parse(sys.argv[1]).getroot() +except ET.ParseError: + # Skip reporting must not introduce a new malformed-XML gate failure. + print(sys.argv[2]) +else: + namespace = root.tag.split("}", 1)[0] + "}" if root.tag.startswith("{") else "" + results = root.findall(f"{namespace}Results/{namespace}UnitTestResult") + print(sum(result.get("outcome") == "NotExecuted" for result in results)) +PY +)" total_executed=$((total_executed + ex)) total_skipped=$((total_skipped + ne)) echo " $(basename "$f"): executed=${ex} skipped=${ne}" diff --git a/scripts/ci/test-assert-not-all-skipped.py b/scripts/ci/test-assert-not-all-skipped.py new file mode 100644 index 00000000..e231956b --- /dev/null +++ b/scripts/ci/test-assert-not-all-skipped.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Deterministic subprocess checks for the TRX executed-floor guard.""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).with_name("assert-not-all-skipped.sh") +NS = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010" + + +def trx(executed, outcomes, summary_skipped=0, namespace=NS, extra=""): + declaration = f' xmlns="{namespace}"' if namespace else "" + results = "".join(f'' for outcome in outcomes) + return (f'{results}{extra}' + f'') + + +class SkipGuardTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="trx-skip-fixtures-") + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + + def write(self, name, content): + path = self.root / name + path.write_text(content, encoding="utf-8") + return path + + def run_guard(self, *paths, minimum=None): + env = os.environ.copy() + env.pop("MIN_EXECUTED", None) + if minimum is not None: + env["MIN_EXECUTED"] = str(minimum) + return subprocess.run(["bash", str(SCRIPT), *map(str, paths)], env=env, + text=True, capture_output=True, check=False) + + def assert_report(self, result, status, executed, skipped, files=1): + self.assertEqual(status, result.returncode, result.stdout + result.stderr) + self.assertIn(f"files={files} executed={executed} skipped={skipped} ", result.stdout) + + def test_mixed_results_ignore_zero_summary_skip_counter(self): + # A summary-only skip count misses the three actual NotExecuted records. + path = self.write("mixed.trx", trx(827, ["Passed"] * 827 + ["NotExecuted"] * 3)) + result = self.run_guard(path) + self.assert_report(result, 0, 827, 3) + self.assertIn("mixed.trx: executed=827 skipped=3", result.stdout) + + def test_all_skipped_fails_default_floor_but_report_only_passes(self): + path = self.write("all-skipped.trx", trx(0, ["NotExecuted"] * 188)) + self.assert_report(self.run_guard(path), 1, 0, 188) + self.assert_report(self.run_guard(path, minimum=0), 0, 0, 188) + + def test_zero_tests_fails_default_floor_but_report_only_passes(self): + path = self.write("zero.trx", trx(0, [])) + self.assert_report(self.run_guard(path), 1, 0, 0) + self.assert_report(self.run_guard(path, minimum=0), 0, 0, 0) + + def test_agreeing_summary_and_outcomes_are_not_double_counted(self): + path = self.write("agreeing.trx", trx(1, ["Passed", "NotExecuted"], 1)) + self.assert_report(self.run_guard(path), 0, 1, 1) + + def test_outcomes_are_authoritative_when_summary_overstates_skips(self): + path = self.write("overstated.trx", trx(1, ["Passed", "NotExecuted"], 9)) + self.assert_report(self.run_guard(path), 0, 1, 1) + + def test_namespace_path_and_exact_outcome_exclude_decoys(self): + extra = ('' + '' + '<UnitTestResult outcome="NotExecuted"/>') + content = trx(2, ["Passed", "Failed", "NotExecuted", "notexecuted"], extra=extra) + # Prefixes on Results/UnitTestResult must work just like the default namespace. + content = content.replace('', '').replace('', '') + content = content.replace('') + self.assertEqual(1, self.run_guard(path, minimum=0).returncode) + + def test_malformed_xml_keeps_summary_fallback_and_gate_status(self): + path = self.write("malformed.trx", '') + self.assert_report(self.run_guard(path, minimum=2), 0, 2, 3) + self.assert_report(self.run_guard(path, minimum=3), 1, 2, 3) + + +if __name__ == "__main__": + unittest.main(verbosity=2)