Joseph Doherty
116 lines
6.6 KiB
Python
116 lines
6.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Supplementary warmed, matched native-process experiment; not full Gate G4."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import time
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--baseline-harness', type=Path, required=True)
|
|
parser.add_argument('--candidate-harness', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--tiered-compilation', choices=['0', '1'], default='0')
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32),
|
|
DOTNET_TieredCompilation=args.tiered_compilation, OPCUA_MATRIX_PERF_ROUNDS='1',
|
|
OPCUA_MATRIX_PERF_WARMUPS='1000', OPCUA_MATRIX_PERF_MEASURED='2000')
|
|
for key in ('OPCUA_MATRIX_SERVER_SECURITY', 'OPCUA_MATRIX_CLIENT_PKI'):
|
|
env.pop(key, None)
|
|
metadata = {'scope': __doc__, 'environment': {k: env[k] for k in ('DOTNET_TieredCompilation', 'OPCUA_MATRIX_PERF_ROUNDS', 'OPCUA_MATRIX_PERF_WARMUPS', 'OPCUA_MATRIX_PERF_MEASURED')},
|
|
'dotnetSdk': subprocess.check_output(['dotnet', '--version'], text=True).strip(), 'p95BudgetPercent': 15,
|
|
'dateUtc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'logicalCpuCount': os.cpu_count()}
|
|
commands, results, hashes = [], [], []
|
|
|
|
def write(path, data):
|
|
path.write_text(json.dumps(data, indent=2) + '\n')
|
|
|
|
def run(command, log):
|
|
commands.append({'argv': list(map(str, command)), 'log': str(log)})
|
|
with log.open('w') as stream:
|
|
result = subprocess.run(list(map(str, command)), stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=240)
|
|
if result.returncode:
|
|
raise RuntimeError(f'Process exited {result.returncode}; {log}')
|
|
|
|
def copy_certs(source, destination):
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
files = list(source.glob('*'))
|
|
if not files:
|
|
raise RuntimeError('Missing public certificates: ' + str(source))
|
|
for path in files:
|
|
shutil.copy2(path, destination / path.name)
|
|
|
|
try:
|
|
for pair in range(1, 4):
|
|
for version, harness in [('baseline', args.baseline_harness.resolve()), ('candidate', args.candidate_harness.resolve())]:
|
|
leg = output / f'pair-{pair}-{version}'
|
|
leg.mkdir()
|
|
client, server = leg / 'client', leg / 'server'
|
|
client.mkdir()
|
|
server.mkdir()
|
|
run(['dotnet', harness, 'prepare', client], leg / 'prepare.log')
|
|
copy_certs(client / 'client-pki/own/certs', server / 'server-pki/trusted/certs')
|
|
server_log = (leg / 'server.log').open('w')
|
|
command = ['dotnet', str(harness), 'server', str(server)]
|
|
commands.append({'argv': command, 'log': str(leg / 'server.log')})
|
|
process = subprocess.Popen(command, env=env, stdout=server_log, stderr=subprocess.STDOUT)
|
|
try:
|
|
deadline = time.monotonic() + 60
|
|
while not (server / 'ready.json').exists():
|
|
if process.poll() is not None or time.monotonic() > deadline:
|
|
raise RuntimeError('Server failed to start: ' + str(leg / 'server.log'))
|
|
time.sleep(.1)
|
|
ready = json.loads((server / 'ready.json').read_text())
|
|
copy_certs(server / 'server-pki/own/certs', client / 'client-pki/trusted/certs')
|
|
run(['dotnet', harness, 'performance', client, ready['endpoint']], leg / 'client.log')
|
|
result = json.loads((client / 'result.json').read_text())
|
|
result.update({'pair': pair, 'version': version, 'server': ready})
|
|
finally:
|
|
(server / 'stop').touch()
|
|
try:
|
|
process.wait(timeout=15)
|
|
except subprocess.TimeoutExpired:
|
|
process.terminate()
|
|
process.wait(timeout=10)
|
|
server_log.close()
|
|
result['serverResources'] = json.loads((server / 'server-resources.json').read_text())
|
|
results.append(result)
|
|
write(leg / 'measurement.json', result)
|
|
summary = []
|
|
for operation in ('scalar-read', 'scalar-write', 'array-read', 'array-write'):
|
|
samples = {}
|
|
for version in ('baseline', 'candidate'):
|
|
rounds = [o for r in results if r['version'] == version for o in r['observations'] if 'benchmarkRound' in o]
|
|
samples[version] = {'p95Milliseconds': [r['latencyMilliseconds'][operation]['p95'] for r in rounds],
|
|
'allocatedBytesPerOperation': [r['clientAllocatedBytesPerOperation'] for r in rounds],
|
|
'clientWorkingSetBytes': [r['clientWorkingSetBytes'] for r in rounds],
|
|
'serverWorkingSetBytes': [r['serverResources']['WorkingSet64'] for r in results if r['version'] == version]}
|
|
ratio = statistics.median(samples['candidate']['p95Milliseconds']) / statistics.median(samples['baseline']['p95Milliseconds'])
|
|
summary.append({'operation': operation, 'samples': samples, 'medianP95ChangePercent': 100 * (ratio - 1), 'status': 'diagnostic-regression' if ratio > 1.15 else 'diagnostic-within-budget'})
|
|
metadata['summary'] = summary
|
|
metadata['status'] = 'diagnostic-regression' if any(r['status'] == 'diagnostic-regression' for r in summary) else 'diagnostic-within-budget'
|
|
for row in summary:
|
|
print(f"{row['operation']}: median p95 {row['medianP95ChangePercent']:+.1f}% ({row['status']})")
|
|
finally:
|
|
for harness in (args.baseline_harness.resolve(), args.candidate_harness.resolve()):
|
|
for path in [harness, *harness.parent.glob('Opc.Ua*.dll'), *harness.parent.glob('ZB.MOM.WW.OtOpcUa*.dll'), *harness.parent.glob('OtOpcUa.Host.dll')]:
|
|
hashes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'bytes': path.stat().st_size})
|
|
write(output / 'artifact-hashes.json', hashes)
|
|
write(output / 'commands.json', commands)
|
|
write(output / 'measurements.json', results)
|
|
write(output / 'summary.json', metadata)
|
|
return 2 if metadata.get('status') == 'diagnostic-regression' else 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|