ci: count individual skipped TRX results

.github/workflows/v2-ci.yml

@@ -41,6 +41,8 @@ jobs:
       # #606: regenerated review summary must remain in sync with module findings.
       - name: code-reviews README drift check
         run: python3 code-reviews/regen-readme.py --check
+      - name: TRX skip guard fixtures
+        run: python3 scripts/ci/test-assert-not-all-skipped.py
       # #707: bound project workers and avoid concurrent NuGet restore work.
       - name: dotnet restore
         run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx --disable-parallel -m:2

scripts/ci/README.md

@@ -28,3 +28,18 @@ AKKA_TEST_TIMEFACTOR=3 dotnet test ZB.MOM.WW.OtOpcUa.slnx \
   --logger trx --results-directory artifacts/trx
 MIN_EXECUTED=12000 bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx
 ```
+
+The guard uses the existing TRX `Counters/@executed` value for the aggregate
+floor. Its `skipped` diagnostic counts individual `Results/UnitTestResult`
+records whose outcome is `NotExecuted`, using the document's XML namespace.
+Some VSTest reports set summary `notExecuted` to zero even when these records
+are present. Skip records are counted per file, including repeated runs; they
+are not a count of unique test methods and do not affect the executed floor.
+For malformed XML, the diagnostic retains the old summary-counter fallback;
+this guard does not add XML validation to its existing gate policy.
+
+Run the deterministic script fixtures without building or running the solution:
+
+```bash
+python3 scripts/ci/test-assert-not-all-skipped.py
+```

scripts/ci/assert-not-all-skipped.sh

@@ -33,12 +33,31 @@ for f in "$@"; do
     exit 2
   fi
   seen=$((seen + 1))
-  # The trx <Counters .../> element in <ResultSummary> carries executed + notExecuted counts.
+  # Keep the executed-counter floor unchanged; notExecuted is only a fallback
+  # for malformed XML that this guard historically accepted.
   counters="$(grep -oE '<Counters[^>]*/?>' "$f" | head -1)"
   ex="$(printf '%s' "$counters" | grep -oE 'executed="[0-9]+"' | grep -oE '[0-9]+' || true)"
   ne="$(printf '%s' "$counters" | grep -oE 'notExecuted="[0-9]+"' | grep -oE '[0-9]+' || true)"
   ex="${ex:-0}"
   ne="${ne:-0}"
+  # VSTest can emit notExecuted="0" despite individual NotExecuted results.
+  # Count direct Results/UnitTestResult records in the document's namespace,
+  # excluding foreign elements, nested records and escaped diagnostic text.
+  ne="$(python3 - "$f" "$ne" <<'PY'
+import sys
+import xml.etree.ElementTree as ET
+
+try:
+    root = ET.parse(sys.argv[1]).getroot()
+except ET.ParseError:
+    # Skip reporting must not introduce a new malformed-XML gate failure.
+    print(sys.argv[2])
+else:
+    namespace = root.tag.split("}", 1)[0] + "}" if root.tag.startswith("{") else ""
+    results = root.findall(f"{namespace}Results/{namespace}UnitTestResult")
+    print(sum(result.get("outcome") == "NotExecuted" for result in results))
+PY
+)"
   total_executed=$((total_executed + ex))
   total_skipped=$((total_skipped + ne))
   echo "  $(basename "$f"): executed=${ex} skipped=${ne}"

scripts/ci/test-assert-not-all-skipped.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Deterministic subprocess checks for the TRX executed-floor guard."""
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+
+SCRIPT = Path(__file__).with_name("assert-not-all-skipped.sh")
+NS = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"
+
+
+def trx(executed, outcomes, summary_skipped=0, namespace=NS, extra=""):
+    declaration = f' xmlns="{namespace}"' if namespace else ""
+    results = "".join(f'<UnitTestResult outcome="{outcome}"/>' for outcome in outcomes)
+    return (f'<TestRun{declaration}><Results>{results}{extra}</Results>'
+            f'<ResultSummary><Counters total="{len(outcomes)}" executed="{executed}" '
+            f'notExecuted="{summary_skipped}"/></ResultSummary></TestRun>')
+
+
+class SkipGuardTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory(prefix="trx-skip-fixtures-")
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+
+    def write(self, name, content):
+        path = self.root / name
+        path.write_text(content, encoding="utf-8")
+        return path
+
+    def run_guard(self, *paths, minimum=None):
+        env = os.environ.copy()
+        env.pop("MIN_EXECUTED", None)
+        if minimum is not None:
+            env["MIN_EXECUTED"] = str(minimum)
+        return subprocess.run(["bash", str(SCRIPT), *map(str, paths)], env=env,
+                              text=True, capture_output=True, check=False)
+
+    def assert_report(self, result, status, executed, skipped, files=1):
+        self.assertEqual(status, result.returncode, result.stdout + result.stderr)
+        self.assertIn(f"files={files} executed={executed} skipped={skipped} ", result.stdout)
+
+    def test_mixed_results_ignore_zero_summary_skip_counter(self):
+        # A summary-only skip count misses the three actual NotExecuted records.
+        path = self.write("mixed.trx", trx(827, ["Passed"] * 827 + ["NotExecuted"] * 3))
+        result = self.run_guard(path)
+        self.assert_report(result, 0, 827, 3)
+        self.assertIn("mixed.trx: executed=827 skipped=3", result.stdout)
+
+    def test_all_skipped_fails_default_floor_but_report_only_passes(self):
+        path = self.write("all-skipped.trx", trx(0, ["NotExecuted"] * 188))
+        self.assert_report(self.run_guard(path), 1, 0, 188)
+        self.assert_report(self.run_guard(path, minimum=0), 0, 0, 188)
+
+    def test_zero_tests_fails_default_floor_but_report_only_passes(self):
+        path = self.write("zero.trx", trx(0, []))
+        self.assert_report(self.run_guard(path), 1, 0, 0)
+        self.assert_report(self.run_guard(path, minimum=0), 0, 0, 0)
+
+    def test_agreeing_summary_and_outcomes_are_not_double_counted(self):
+        path = self.write("agreeing.trx", trx(1, ["Passed", "NotExecuted"], 1))
+        self.assert_report(self.run_guard(path), 0, 1, 1)
+
+    def test_outcomes_are_authoritative_when_summary_overstates_skips(self):
+        path = self.write("overstated.trx", trx(1, ["Passed", "NotExecuted"], 9))
+        self.assert_report(self.run_guard(path), 0, 1, 1)
+
+    def test_namespace_path_and_exact_outcome_exclude_decoys(self):
+        extra = ('<Other><UnitTestResult outcome="NotExecuted"/></Other>'
+                 '<foreign:UnitTestResult xmlns:foreign="urn:other" outcome="NotExecuted"/>'
+                 '<Output><Message>&lt;UnitTestResult outcome="NotExecuted"/&gt;</Message></Output>')
+        content = trx(2, ["Passed", "Failed", "NotExecuted", "notexecuted"], extra=extra)
+        # Prefixes on Results/UnitTestResult must work just like the default namespace.
+        content = content.replace('<TestRun ', f'<TestRun xmlns:t="{NS}" ')
+        content = content.replace('<Results>', '<t:Results>').replace('</Results>', '</t:Results>')
+        content = content.replace('<UnitTestResult ', '<t:UnitTestResult ')
+        path = self.write("namespaced.trx", content)
+        self.assert_report(self.run_guard(path), 0, 2, 1)
+
+    def test_unnamespaced_results_remain_supported(self):
+        path = self.write("legacy.trx", trx(1, ["Passed", "NotExecuted"], namespace=""))
+        self.assert_report(self.run_guard(path), 0, 1, 1)
+
+    def test_executed_counter_controls_aggregate_floor_at_boundary(self):
+        # Counters retain authority even when result records disagree with executed.
+        mixed = self.write("floor.trx", trx(12000, ["Passed", "NotExecuted"]))
+        skipped = self.write("skipped.trx", trx(0, ["NotExecuted"]))
+        self.assert_report(self.run_guard(mixed, skipped, minimum=12000), 0, 12000, 2, 2)
+        self.assert_report(self.run_guard(mixed, skipped, minimum=12001), 1, 12000, 2, 2)
+
+    def test_missing_arguments_and_file_keep_exit_two(self):
+        self.assertEqual(2, self.run_guard().returncode)
+        self.assertEqual(2, self.run_guard(self.root / "missing.trx").returncode)
+
+    def test_missing_counters_keeps_existing_exit_one(self):
+        path = self.write("no-counters.trx", '<TestRun><Results/></TestRun>')
+        self.assertEqual(1, self.run_guard(path, minimum=0).returncode)
+
+    def test_malformed_xml_keeps_summary_fallback_and_gate_status(self):
+        path = self.write("malformed.trx", '<TestRun><Counters executed="2" notExecuted="3"/>')
+        self.assert_report(self.run_guard(path, minimum=2), 0, 2, 3)
+        self.assert_report(self.run_guard(path, minimum=3), 1, 2, 3)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)