Joseph Doherty
109 lines
5.3 KiB
Python
109 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Deterministic subprocess checks for the TRX executed-floor guard."""
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
SCRIPT = Path(__file__).with_name("assert-not-all-skipped.sh")
|
|
NS = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"
|
|
|
|
|
|
def trx(executed, outcomes, summary_skipped=0, namespace=NS, extra=""):
|
|
declaration = f' xmlns="{namespace}"' if namespace else ""
|
|
results = "".join(f'<UnitTestResult outcome="{outcome}"/>' for outcome in outcomes)
|
|
return (f'<TestRun{declaration}><Results>{results}{extra}</Results>'
|
|
f'<ResultSummary><Counters total="{len(outcomes)}" executed="{executed}" '
|
|
f'notExecuted="{summary_skipped}"/></ResultSummary></TestRun>')
|
|
|
|
|
|
class SkipGuardTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.directory = tempfile.TemporaryDirectory(prefix="trx-skip-fixtures-")
|
|
self.addCleanup(self.directory.cleanup)
|
|
self.root = Path(self.directory.name)
|
|
|
|
def write(self, name, content):
|
|
path = self.root / name
|
|
path.write_text(content, encoding="utf-8")
|
|
return path
|
|
|
|
def run_guard(self, *paths, minimum=None):
|
|
env = os.environ.copy()
|
|
env.pop("MIN_EXECUTED", None)
|
|
if minimum is not None:
|
|
env["MIN_EXECUTED"] = str(minimum)
|
|
return subprocess.run(["bash", str(SCRIPT), *map(str, paths)], env=env,
|
|
text=True, capture_output=True, check=False)
|
|
|
|
def assert_report(self, result, status, executed, skipped, files=1):
|
|
self.assertEqual(status, result.returncode, result.stdout + result.stderr)
|
|
self.assertIn(f"files={files} executed={executed} skipped={skipped} ", result.stdout)
|
|
|
|
def test_mixed_results_ignore_zero_summary_skip_counter(self):
|
|
# A summary-only skip count misses the three actual NotExecuted records.
|
|
path = self.write("mixed.trx", trx(827, ["Passed"] * 827 + ["NotExecuted"] * 3))
|
|
result = self.run_guard(path)
|
|
self.assert_report(result, 0, 827, 3)
|
|
self.assertIn("mixed.trx: executed=827 skipped=3", result.stdout)
|
|
|
|
def test_all_skipped_fails_default_floor_but_report_only_passes(self):
|
|
path = self.write("all-skipped.trx", trx(0, ["NotExecuted"] * 188))
|
|
self.assert_report(self.run_guard(path), 1, 0, 188)
|
|
self.assert_report(self.run_guard(path, minimum=0), 0, 0, 188)
|
|
|
|
def test_zero_tests_fails_default_floor_but_report_only_passes(self):
|
|
path = self.write("zero.trx", trx(0, []))
|
|
self.assert_report(self.run_guard(path), 1, 0, 0)
|
|
self.assert_report(self.run_guard(path, minimum=0), 0, 0, 0)
|
|
|
|
def test_agreeing_summary_and_outcomes_are_not_double_counted(self):
|
|
path = self.write("agreeing.trx", trx(1, ["Passed", "NotExecuted"], 1))
|
|
self.assert_report(self.run_guard(path), 0, 1, 1)
|
|
|
|
def test_outcomes_are_authoritative_when_summary_overstates_skips(self):
|
|
path = self.write("overstated.trx", trx(1, ["Passed", "NotExecuted"], 9))
|
|
self.assert_report(self.run_guard(path), 0, 1, 1)
|
|
|
|
def test_namespace_path_and_exact_outcome_exclude_decoys(self):
|
|
extra = ('<Other><UnitTestResult outcome="NotExecuted"/></Other>'
|
|
'<foreign:UnitTestResult xmlns:foreign="urn:other" outcome="NotExecuted"/>'
|
|
'<Output><Message><UnitTestResult outcome="NotExecuted"/></Message></Output>')
|
|
content = trx(2, ["Passed", "Failed", "NotExecuted", "notexecuted"], extra=extra)
|
|
# Prefixes on Results/UnitTestResult must work just like the default namespace.
|
|
content = content.replace('<TestRun ', f'<TestRun xmlns:t="{NS}" ')
|
|
content = content.replace('<Results>', '<t:Results>').replace('</Results>', '</t:Results>')
|
|
content = content.replace('<UnitTestResult ', '<t:UnitTestResult ')
|
|
path = self.write("namespaced.trx", content)
|
|
self.assert_report(self.run_guard(path), 0, 2, 1)
|
|
|
|
def test_unnamespaced_results_remain_supported(self):
|
|
path = self.write("legacy.trx", trx(1, ["Passed", "NotExecuted"], namespace=""))
|
|
self.assert_report(self.run_guard(path), 0, 1, 1)
|
|
|
|
def test_executed_counter_controls_aggregate_floor_at_boundary(self):
|
|
# Counters retain authority even when result records disagree with executed.
|
|
mixed = self.write("floor.trx", trx(12000, ["Passed", "NotExecuted"]))
|
|
skipped = self.write("skipped.trx", trx(0, ["NotExecuted"]))
|
|
self.assert_report(self.run_guard(mixed, skipped, minimum=12000), 0, 12000, 2, 2)
|
|
self.assert_report(self.run_guard(mixed, skipped, minimum=12001), 1, 12000, 2, 2)
|
|
|
|
def test_missing_arguments_and_file_keep_exit_two(self):
|
|
self.assertEqual(2, self.run_guard().returncode)
|
|
self.assertEqual(2, self.run_guard(self.root / "missing.trx").returncode)
|
|
|
|
def test_missing_counters_keeps_existing_exit_one(self):
|
|
path = self.write("no-counters.trx", '<TestRun><Results/></TestRun>')
|
|
self.assertEqual(1, self.run_guard(path, minimum=0).returncode)
|
|
|
|
def test_malformed_xml_keeps_summary_fallback_and_gate_status(self):
|
|
path = self.write("malformed.trx", '<TestRun><Counters executed="2" notExecuted="3"/>')
|
|
self.assert_report(self.run_guard(path, minimum=2), 0, 2, 3)
|
|
self.assert_report(self.run_guard(path, minimum=3), 1, 2, 3)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|