Files
lmxopcua/docs/plans/evidence/2026-11-12-799-acceptance/make_verdicts.py
T
Joseph Doherty fea7de0aef
v2-ci / unit-tests (push) Waiting to run
docs(evidence): #799 acceptance runs on master aa60b3e3 - Windows x64 and Linux both fail the unchanged cold RSS cap
Owner scope 2026-11-12 (Windows x64 + Linux, unchanged cycle-3 caps, >=7200 s, SDK 1.5 comparator).

- Linux ARM64 (late-slope, 7200.12 s, 1964 cycles / 491 restarts, shutdown verified, analyzer-validated):
  first rejection RSS at cycle 247 / 900.73 s, 358,682,624 > 358,678,528 (+4 KiB); peak 382.98 MiB vs
  limit 342.06 MiB; late plateau medians 344.4/341.1/341.9 MiB, OLS -1.90 MiB/h. FAIL.
- Windows x64 on windev (same workload, working set as RSS, handle table as descriptors): first rejection RSS
  at cycle 285 / 1130.76 s, 303,505,408 > 302,174,208; peak 319.49 vs limit 288.18 MiB; plateau ~314-315 MiB,
  +0.93 MiB/h. FAIL. Also incomplete: the launcher's wall-clock outer deadline fired early after a +552 s
  w32time step and ended the run at test-phase 7014 s (rejection had already latched); launchers now use a
  Stopwatch.
- SDK 1.5.378.182 comparator (efb9197a publication, Windows, 7201 s): RSS peak +65.8 MiB (never over its cap),
  plateau ~+48 MiB, but the +32 handle cap tripped at cycle 17 (max 674 vs 642).
- Managed flat on both SDK2 runs; growth is the generated-factory JIT burst (+40k/+44k methods) - Linux JIT
  doublemapper ~180 MiB; Windows private bytes grew ~25 MiB of a ~132 MiB working-set rise.
- Retained: b61f3e05 cycle-1 fixture failure (#808 unplaced UNS folder), Windows/SDK1 smoke diagnostics.

No cap raised, no warm baseline, no failed run counted as passing.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-12 14:51:31 -05:00

106 lines
5.5 KiB
Python

#!/usr/bin/env python3
"""Compose the lmxopcua#799 acceptance verdicts from the analyzer reports and raw samples (no thresholds added).
A platform run is an acceptance PASS only if every one of these holds: the analyzer validated the raw evidence;
the canonical test and its two descriptor controls executed and passed (exit 0); the fixed 7,200 s phase completed;
shutdown ownership was verified; no cold-cap rejection latched (+64 MiB managed / +128 MiB RSS / +32 descriptors vs
the cycle-3 sample). Late-phase slopes are descriptive (no pass threshold), exactly as in LATE_SLOPE.md.
"""
import gzip
import json
import math
import sys
from pathlib import Path
MIB = 1 << 20
CAPS = {"RssBytes": 128 << 20, "ManagedBytes": 64 << 20, "Descriptors": 32}
def rows(path):
opener = gzip.open if str(path).endswith(".gz") else open
with opener(path, "rt", encoding="utf-8-sig") as stream:
return [json.loads(line) for line in stream if line.strip()]
def quiescent(raw):
return [r["resources"] for r in raw if r["kind"] in ("quiescent", "quiescent-budget-rejected")]
def ols(samples, field, lo=1800, hi=7200):
late = [s for s in samples if lo <= s["Seconds"] <= hi]
if len(late) < 3:
return None, len(late)
t = [s["Seconds"] for s in late]; v = [s[field] for s in late]
mt = math.fsum(t) / len(t); mv = math.fsum(v) / len(v)
return 3600 * math.fsum((a - mt) * (b - mv) for a, b in zip(t, v)) / math.fsum((a - mt) ** 2 for a in t), len(late)
def raw_windows(samples):
from statistics import median
out = []
for lo, hi in ((1800, 2400), (3600, 4200), (6600, 7200)):
sub = [s for s in samples if lo <= s["Seconds"] <= hi]
if not sub:
out.append({"windowSeconds": [lo, hi], "samples": 0}); continue
t = [s["Seconds"] for s in sub]
out.append({"windowSeconds": [lo, hi], "samples": len(sub), "firstSeconds": t[0], "lastSeconds": t[-1],
"complete": len(sub) >= 10 and t[0] <= lo + 60 and t[-1] >= hi - 60,
"rssMedianMiB": median(s["RssBytes"] for s in sub) / MIB, "rssMinMiB": min(s["RssBytes"] for s in sub) / MIB,
"rssMaxMiB": max(s["RssBytes"] for s in sub) / MIB,
"managedMedianMiB": median(s["ManagedBytes"] for s in sub) / MIB})
return out
def envelope(samples):
base = samples[2]
after = samples[2:]
out = {"baselineCycle": 3}
for field, cap in CAPS.items():
peak = max(after, key=lambda s: s[field])
out[field] = {"baseline": base[field], "limit": base[field] + cap, "peak": peak[field], "peakCycle": peak["Cycle"],
"peakSeconds": peak["Seconds"], "headroom": base[field] + cap - peak[field], "final": samples[-1][field]}
first = next(({"cycle": s["Cycle"], "seconds": s["Seconds"], "exceeded": [f for f, c in CAPS.items() if s[f] > base[f] + c]}
for s in after if any(s[f] > base[f] + c for f, c in CAPS.items())), None)
return out, first
def verdict(platform, analysis_path, raw_path, extra):
analysis = json.loads(Path(analysis_path).read_text()) if analysis_path and Path(analysis_path).exists() else None
raw = rows(raw_path)
samples = quiescent(raw)
env, first = envelope(samples)
result = next((r for r in raw if r["kind"] in ("late-slope-result", "comparator-result")), None)
rss_slope, n = ols(samples, "RssBytes")
managed_slope, _ = ols(samples, "ManagedBytes")
start = raw[0]
checks = {
"analyzerValidated": analysis is not None,
"testsPassed": bool(analysis) and analysis["testExitStatus"] == 0 and analysis["experimentTests"]["passed"] == 1
and analysis["descriptorTests"]["passed"] == 2,
"durationCompleted": bool(result) and result.get("durationCompleted") is True,
"shutdownVerified": bool(result) and result.get("shutdownVerified") is True,
"noColdCapRejection": first is None and bool(result) and result.get("coldBudgetRejected") is False,
}
return {
"issue": "lmxopcua#799", "platform": platform, "kind": extra.pop("kind", "acceptance"),
"sourceRevision": start.get("build"), "sdk": start.get("sdk"), "runtime": start.get("runtime"),
"architecture": start.get("architecture"), **extra,
"cycles": result and result.get("cycles"), "restarts": result and result.get("restarts"),
"phaseSeconds": result and result.get("cycleSeconds"), "shutdownSeconds": result and result.get("shutdownSeconds"),
"envelope": env, "firstColdRejection": first,
"lateSlope": {"rangeSeconds": [1800, 7200], "samples": n, "rssBytesPerHour": rss_slope,
"rssMiBPerHour": None if rss_slope is None else rss_slope / MIB,
"retainedManagedBytesPerHour": managed_slope},
"windowsFromRaw": raw_windows(samples),
"windows": analysis and [{"windowSeconds": w["windowSeconds"], "samples": w["sampleCount"],
"rssMedianMiB": w["RssBytes"]["median"] / MIB, "rssMinMiB": w["RssBytes"]["min"] / MIB,
"rssMaxMiB": w["RssBytes"]["max"] / MIB,
"managedMedianMiB": w["ManagedBytes"]["median"] / MIB} for w in analysis["windows"]],
"checks": checks, "acceptance": all(checks.values()) if extra.get("acceptanceRun", True) else False,
}
if __name__ == "__main__":
platform, analysis, raw, extra = sys.argv[1], sys.argv[2], sys.argv[3], json.loads(sys.argv[4])
print(json.dumps(verdict(platform, None if analysis == "-" else analysis, raw, extra), indent=2))