test: add isolated cross-version application compatibility harness
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Supplementary warmed, matched native-process experiment; not full Gate G4."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--baseline-harness', type=Path, required=True)
|
||||
parser.add_argument('--candidate-harness', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--tiered-compilation', choices=['0', '1'], default='0')
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32),
|
||||
DOTNET_TieredCompilation=args.tiered_compilation, OPCUA_MATRIX_PERF_ROUNDS='1',
|
||||
OPCUA_MATRIX_PERF_WARMUPS='1000', OPCUA_MATRIX_PERF_MEASURED='2000')
|
||||
for key in ('OPCUA_MATRIX_SERVER_SECURITY', 'OPCUA_MATRIX_CLIENT_PKI'):
|
||||
env.pop(key, None)
|
||||
metadata = {'scope': __doc__, 'environment': {k: env[k] for k in ('DOTNET_TieredCompilation', 'OPCUA_MATRIX_PERF_ROUNDS', 'OPCUA_MATRIX_PERF_WARMUPS', 'OPCUA_MATRIX_PERF_MEASURED')},
|
||||
'dotnetSdk': subprocess.check_output(['dotnet', '--version'], text=True).strip(), 'p95BudgetPercent': 15,
|
||||
'dateUtc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'logicalCpuCount': os.cpu_count()}
|
||||
commands, results, hashes = [], [], []
|
||||
|
||||
def write(path, data):
|
||||
path.write_text(json.dumps(data, indent=2) + '\n')
|
||||
|
||||
def run(command, log):
|
||||
commands.append({'argv': list(map(str, command)), 'log': str(log)})
|
||||
with log.open('w') as stream:
|
||||
result = subprocess.run(list(map(str, command)), stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=240)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'Process exited {result.returncode}; {log}')
|
||||
|
||||
def copy_certs(source, destination):
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
files = list(source.glob('*'))
|
||||
if not files:
|
||||
raise RuntimeError('Missing public certificates: ' + str(source))
|
||||
for path in files:
|
||||
shutil.copy2(path, destination / path.name)
|
||||
|
||||
try:
|
||||
for pair in range(1, 4):
|
||||
for version, harness in [('baseline', args.baseline_harness.resolve()), ('candidate', args.candidate_harness.resolve())]:
|
||||
leg = output / f'pair-{pair}-{version}'
|
||||
leg.mkdir()
|
||||
client, server = leg / 'client', leg / 'server'
|
||||
client.mkdir()
|
||||
server.mkdir()
|
||||
run(['dotnet', harness, 'prepare', client], leg / 'prepare.log')
|
||||
copy_certs(client / 'client-pki/own/certs', server / 'server-pki/trusted/certs')
|
||||
server_log = (leg / 'server.log').open('w')
|
||||
command = ['dotnet', str(harness), 'server', str(server)]
|
||||
commands.append({'argv': command, 'log': str(leg / 'server.log')})
|
||||
process = subprocess.Popen(command, env=env, stdout=server_log, stderr=subprocess.STDOUT)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while not (server / 'ready.json').exists():
|
||||
if process.poll() is not None or time.monotonic() > deadline:
|
||||
raise RuntimeError('Server failed to start: ' + str(leg / 'server.log'))
|
||||
time.sleep(.1)
|
||||
ready = json.loads((server / 'ready.json').read_text())
|
||||
copy_certs(server / 'server-pki/own/certs', client / 'client-pki/trusted/certs')
|
||||
run(['dotnet', harness, 'performance', client, ready['endpoint']], leg / 'client.log')
|
||||
result = json.loads((client / 'result.json').read_text())
|
||||
result.update({'pair': pair, 'version': version, 'server': ready})
|
||||
finally:
|
||||
(server / 'stop').touch()
|
||||
try:
|
||||
process.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
server_log.close()
|
||||
result['serverResources'] = json.loads((server / 'server-resources.json').read_text())
|
||||
results.append(result)
|
||||
write(leg / 'measurement.json', result)
|
||||
summary = []
|
||||
for operation in ('scalar-read', 'scalar-write', 'array-read', 'array-write'):
|
||||
samples = {}
|
||||
for version in ('baseline', 'candidate'):
|
||||
rounds = [o for r in results if r['version'] == version for o in r['observations'] if 'benchmarkRound' in o]
|
||||
samples[version] = {'p95Milliseconds': [r['latencyMilliseconds'][operation]['p95'] for r in rounds],
|
||||
'allocatedBytesPerOperation': [r['clientAllocatedBytesPerOperation'] for r in rounds],
|
||||
'clientWorkingSetBytes': [r['clientWorkingSetBytes'] for r in rounds],
|
||||
'serverWorkingSetBytes': [r['serverResources']['WorkingSet64'] for r in results if r['version'] == version]}
|
||||
ratio = statistics.median(samples['candidate']['p95Milliseconds']) / statistics.median(samples['baseline']['p95Milliseconds'])
|
||||
summary.append({'operation': operation, 'samples': samples, 'medianP95ChangePercent': 100 * (ratio - 1), 'status': 'diagnostic-regression' if ratio > 1.15 else 'diagnostic-within-budget'})
|
||||
metadata['summary'] = summary
|
||||
metadata['status'] = 'diagnostic-regression' if any(r['status'] == 'diagnostic-regression' for r in summary) else 'diagnostic-within-budget'
|
||||
for row in summary:
|
||||
print(f"{row['operation']}: median p95 {row['medianP95ChangePercent']:+.1f}% ({row['status']})")
|
||||
finally:
|
||||
for harness in (args.baseline_harness.resolve(), args.candidate_harness.resolve()):
|
||||
for path in [harness, *harness.parent.glob('Opc.Ua*.dll'), *harness.parent.glob('ZB.MOM.WW.OtOpcUa*.dll'), *harness.parent.glob('OtOpcUa.Host.dll')]:
|
||||
hashes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'bytes': path.stat().st_size})
|
||||
write(output / 'artifact-hashes.json', hashes)
|
||||
write(output / 'commands.json', commands)
|
||||
write(output / 'measurements.json', results)
|
||||
write(output / 'summary.json', metadata)
|
||||
return 2 if metadata.get('status') == 'diagnostic-regression' else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user