#!/usr/bin/env python3 """Supplementary warmed, matched native-process experiment; not full Gate G4.""" import argparse import hashlib import json import os from pathlib import Path import secrets import shutil import statistics import subprocess import time def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--baseline-harness', type=Path, required=True) parser.add_argument('--candidate-harness', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--tiered-compilation', choices=['0', '1'], default='0') args = parser.parse_args() output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32), DOTNET_TieredCompilation=args.tiered_compilation, OPCUA_MATRIX_PERF_ROUNDS='1', OPCUA_MATRIX_PERF_WARMUPS='1000', OPCUA_MATRIX_PERF_MEASURED='2000') for key in ('OPCUA_MATRIX_SERVER_SECURITY', 'OPCUA_MATRIX_CLIENT_PKI'): env.pop(key, None) metadata = {'scope': __doc__, 'environment': {k: env[k] for k in ('DOTNET_TieredCompilation', 'OPCUA_MATRIX_PERF_ROUNDS', 'OPCUA_MATRIX_PERF_WARMUPS', 'OPCUA_MATRIX_PERF_MEASURED')}, 'dotnetSdk': subprocess.check_output(['dotnet', '--version'], text=True).strip(), 'p95BudgetPercent': 15, 'dateUtc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'logicalCpuCount': os.cpu_count()} commands, results, hashes = [], [], [] def write(path, data): path.write_text(json.dumps(data, indent=2) + '\n') def run(command, log): commands.append({'argv': list(map(str, command)), 'log': str(log)}) with log.open('w') as stream: result = subprocess.run(list(map(str, command)), stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=240) if result.returncode: raise RuntimeError(f'Process exited {result.returncode}; {log}') def copy_certs(source, destination): destination.mkdir(parents=True, exist_ok=True) files = list(source.glob('*')) if not files: raise RuntimeError('Missing public certificates: ' + str(source)) for path in files: shutil.copy2(path, destination / path.name) try: for pair in range(1, 4): for version, harness in [('baseline', args.baseline_harness.resolve()), ('candidate', args.candidate_harness.resolve())]: leg = output / f'pair-{pair}-{version}' leg.mkdir() client, server = leg / 'client', leg / 'server' client.mkdir() server.mkdir() run(['dotnet', harness, 'prepare', client], leg / 'prepare.log') copy_certs(client / 'client-pki/own/certs', server / 'server-pki/trusted/certs') server_log = (leg / 'server.log').open('w') command = ['dotnet', str(harness), 'server', str(server)] commands.append({'argv': command, 'log': str(leg / 'server.log')}) process = subprocess.Popen(command, env=env, stdout=server_log, stderr=subprocess.STDOUT) try: deadline = time.monotonic() + 60 while not (server / 'ready.json').exists(): if process.poll() is not None or time.monotonic() > deadline: raise RuntimeError('Server failed to start: ' + str(leg / 'server.log')) time.sleep(.1) ready = json.loads((server / 'ready.json').read_text()) copy_certs(server / 'server-pki/own/certs', client / 'client-pki/trusted/certs') run(['dotnet', harness, 'performance', client, ready['endpoint']], leg / 'client.log') result = json.loads((client / 'result.json').read_text()) result.update({'pair': pair, 'version': version, 'server': ready}) finally: (server / 'stop').touch() try: process.wait(timeout=15) except subprocess.TimeoutExpired: process.terminate() process.wait(timeout=10) server_log.close() result['serverResources'] = json.loads((server / 'server-resources.json').read_text()) results.append(result) write(leg / 'measurement.json', result) summary = [] for operation in ('scalar-read', 'scalar-write', 'array-read', 'array-write'): samples = {} for version in ('baseline', 'candidate'): rounds = [o for r in results if r['version'] == version for o in r['observations'] if 'benchmarkRound' in o] samples[version] = {'p95Milliseconds': [r['latencyMilliseconds'][operation]['p95'] for r in rounds], 'allocatedBytesPerOperation': [r['clientAllocatedBytesPerOperation'] for r in rounds], 'clientWorkingSetBytes': [r['clientWorkingSetBytes'] for r in rounds], 'serverWorkingSetBytes': [r['serverResources']['WorkingSet64'] for r in results if r['version'] == version]} ratio = statistics.median(samples['candidate']['p95Milliseconds']) / statistics.median(samples['baseline']['p95Milliseconds']) summary.append({'operation': operation, 'samples': samples, 'medianP95ChangePercent': 100 * (ratio - 1), 'status': 'diagnostic-regression' if ratio > 1.15 else 'diagnostic-within-budget'}) metadata['summary'] = summary metadata['status'] = 'diagnostic-regression' if any(r['status'] == 'diagnostic-regression' for r in summary) else 'diagnostic-within-budget' for row in summary: print(f"{row['operation']}: median p95 {row['medianP95ChangePercent']:+.1f}% ({row['status']})") finally: for harness in (args.baseline_harness.resolve(), args.candidate_harness.resolve()): for path in [harness, *harness.parent.glob('Opc.Ua*.dll'), *harness.parent.glob('ZB.MOM.WW.OtOpcUa*.dll'), *harness.parent.glob('OtOpcUa.Host.dll')]: hashes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'bytes': path.stat().st_size}) write(output / 'artifact-hashes.json', hashes) write(output / 'commands.json', commands) write(output / 'measurements.json', results) write(output / 'summary.json', metadata) return 2 if metadata.get('status') == 'diagnostic-regression' else 0 if __name__ == '__main__': raise SystemExit(main())