test: add isolated cross-version application compatibility harness
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Separate-process production application compatibility gate (no mixed SDK process)."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
HOST = 'src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj'
|
||||
CLIENT = 'src/Client/ZB.MOM.WW.OtOpcUa.Client.Shared/ZB.MOM.WW.OtOpcUa.Client.Shared.csproj'
|
||||
DRIVER = 'src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.csproj'
|
||||
CLI = 'src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI/ZB.MOM.WW.OtOpcUa.Client.CLI.csproj'
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
path.write_text(json.dumps(value, indent=2) + '\n')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--baseline-root', required=True, type=Path)
|
||||
parser.add_argument('--baseline-artifacts', required=True, type=Path, help='Existing immutable host/ and client/ publishes')
|
||||
parser.add_argument('--candidate-root', required=True, type=Path, help='Dedicated candidate checkout; this script builds it')
|
||||
parser.add_argument('--container-image', default='mcr.microsoft.com/dotnet/sdk:10.0.103-noble', help='Disposable local driver/CLI isolation; must supply .NET 10 and bash')
|
||||
parser.add_argument('--output', required=True, type=Path, help='Fresh nonexistent evidence directory')
|
||||
args = parser.parse_args()
|
||||
args.baseline_root = args.baseline_root.resolve()
|
||||
args.candidate_root = args.candidate_root.resolve()
|
||||
args.baseline_artifacts = args.baseline_artifacts.resolve()
|
||||
args.output = args.output.resolve()
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
source_info = {}
|
||||
for name, root in [('baseline', args.baseline_root), ('candidate', args.candidate_root)]:
|
||||
source_info[name] = {'root': str(root), 'commit': subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip(),
|
||||
'sdkPackages': str(root / 'Directory.Packages.props')}
|
||||
source_info['scope'] = 'Loopback application contract and 3 warmed 50-operation rounds per scalar/Int32[3] read/write kind; shared development host, no full load/reconnect/failover/backend/soak performance claim.'
|
||||
source_info['dateUtc'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
write_json(args.output / 'source-info.json', source_info)
|
||||
commands = []
|
||||
matrix = []
|
||||
env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32))
|
||||
env.pop('OPCUA_MATRIX_CLIENT_PKI', None)
|
||||
env.pop('OPCUA_MATRIX_SERVER_SECURITY', None)
|
||||
|
||||
def run(command, log, process_env=env, timeout=300):
|
||||
commands.append({'argv': ['<ephemeral-key>' if x == env['OPCUA_MATRIX_PASSWORD'] else str(x) for x in command], 'log': str(log)})
|
||||
with log.open('w') as stream:
|
||||
result = subprocess.run([str(x) for x in command], stdout=stream, stderr=subprocess.STDOUT, env=process_env, timeout=timeout)
|
||||
return result.returncode
|
||||
|
||||
def build(version):
|
||||
folder = args.output / ('build-' + version)
|
||||
folder.mkdir()
|
||||
project = ET.Element('Project', Sdk='Microsoft.NET.Sdk')
|
||||
props = ET.SubElement(project, 'PropertyGroup')
|
||||
for k, v in {'OutputType': 'Exe', 'TargetFramework': 'net10.0', 'Nullable': 'enable', 'ImplicitUsings': 'enable', 'LangVersion': 'latest', 'DefineConstants': 'SDK2' if version == 'candidate' else ''}.items():
|
||||
ET.SubElement(props, k).text = v
|
||||
items = ET.SubElement(project, 'ItemGroup')
|
||||
ET.SubElement(items, 'FrameworkReference', Include='Microsoft.AspNetCore.App')
|
||||
if version == 'candidate':
|
||||
for path in (HOST, CLIENT, DRIVER):
|
||||
ET.SubElement(items, 'ProjectReference', Include=str(args.candidate_root / path))
|
||||
else:
|
||||
dlls = {}
|
||||
for path in ('host', 'client'):
|
||||
for dll in (args.baseline_artifacts / path).glob('*.dll'):
|
||||
dlls[dll.name] = dll
|
||||
for name, dll in sorted(dlls.items()):
|
||||
ref = ET.SubElement(items, 'Reference', Include=name[:-4])
|
||||
ET.SubElement(ref, 'HintPath').text = str(dll)
|
||||
ET.SubElement(ref, 'Private').text = 'true'
|
||||
ET.SubElement(items, 'Compile', Include=str(HERE / 'ProcessHost.cs'))
|
||||
csproj = folder / 'ProcessHost.csproj'
|
||||
ET.ElementTree(project).write(csproj, encoding='unicode')
|
||||
status = run(['dotnet', 'build', csproj, '-c', 'Release', '-m:1', '--nologo'], folder / 'build.log')
|
||||
if status:
|
||||
raise RuntimeError(f'{version} harness build failed ({status}); see {folder / "build.log"}')
|
||||
return folder / 'bin/Release/net10.0/ProcessHost.dll'
|
||||
|
||||
def certificates(source, target):
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
files = list(source.glob('*'))
|
||||
if not files:
|
||||
raise RuntimeError('No public certificates at ' + str(source))
|
||||
for file in files:
|
||||
shutil.copy2(file, target / file.name)
|
||||
|
||||
def receipt(paths):
|
||||
return [{'path': str(p), 'sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'bytes': p.stat().st_size} for p in sorted(set(paths)) if p.is_file()]
|
||||
|
||||
baseline = candidate = candidate_cli = None
|
||||
try:
|
||||
# Baseline builds only the portable harness against the exact published production DLLs.
|
||||
baseline = build('baseline')
|
||||
candidate = build('candidate')
|
||||
candidate_cli = args.output / 'candidate-cli'
|
||||
status = run(['dotnet', 'publish', args.candidate_root / CLI, '-c', 'Release', '-o', candidate_cli, '-m:1', '--nologo'], args.output / 'candidate-cli-build.log')
|
||||
if status:
|
||||
raise RuntimeError('Candidate CLI publish failed')
|
||||
for name, client, server in [('baseline-client-to-baseline-server', baseline, baseline), ('baseline-client-to-candidate-server', baseline, candidate), ('candidate-client-to-baseline-server', candidate, baseline), ('candidate-client-to-candidate-server', candidate, candidate)]:
|
||||
leg = args.output / name
|
||||
leg.mkdir()
|
||||
server_root = leg / 'server'
|
||||
client_root = leg / 'shared'
|
||||
server_root.mkdir()
|
||||
client_root.mkdir()
|
||||
status = run(['dotnet', client, 'prepare', client_root], leg / 'prepare.log')
|
||||
if status:
|
||||
matrix.append({'name': name, 'status': 'failed', 'stage': 'client-certificate-preparation'})
|
||||
continue
|
||||
certificates(client_root / 'client-pki/own/certs', server_root / 'server-pki/trusted/certs')
|
||||
server_log = (leg / 'server.log').open('w')
|
||||
command = ['dotnet', str(server), 'server', str(server_root)]
|
||||
commands.append({'argv': command, 'log': str(leg / 'server.log')})
|
||||
proc = subprocess.Popen(command, stdout=server_log, stderr=subprocess.STDOUT, env=env)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while not (server_root / 'ready.json').exists():
|
||||
if proc.poll() is not None or time.monotonic() > deadline:
|
||||
raise RuntimeError(f'{name}: server did not become ready; see server.log')
|
||||
time.sleep(.1)
|
||||
ready = json.loads((server_root / 'ready.json').read_text())
|
||||
certificates(server_root / 'server-pki/own/certs', client_root / 'client-pki/trusted/certs')
|
||||
status = run(['dotnet', client, 'client', client_root, ready['endpoint']], leg / 'shared.log')
|
||||
result = json.loads((client_root / 'result.json').read_text())
|
||||
history = [json.loads(x) for x in (server_root / 'history-requests.jsonl').read_text().splitlines()] if (server_root / 'history-requests.jsonl').exists() else []
|
||||
result.update({'name': name, 'server': ready, 'status': 'passed' if status == 0 else 'failed', 'historyBackendRequests': history, 'certificateValidation': 'explicit trusted peers; server and Shared client autoaccept disabled', 'authentication': 'ephemeral API-key-shaped credential, exact fixture authenticator; real authority/ACL/method gates'})
|
||||
write_json(leg / 'leg.json', result)
|
||||
matrix.append(result)
|
||||
except Exception as exc:
|
||||
matrix.append({'name': name, 'status': 'failed', 'error': str(exc)})
|
||||
finally:
|
||||
(server_root / 'stop').touch()
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=10)
|
||||
server_log.close()
|
||||
image_info = subprocess.check_output(['docker', 'image', 'inspect', args.container_image, '--format', '{{json .}}'], text=True)
|
||||
image_data = json.loads(image_info)
|
||||
write_json(args.output / 'container-image.json', {k: image_data.get(k) for k in ('Id', 'RepoDigests', 'Architecture', 'Os')})
|
||||
for server_name, server in [('baseline', baseline), ('candidate', candidate)]:
|
||||
name = 'candidate-driver-cli-to-' + server_name + '-server'
|
||||
leg = args.output / name
|
||||
leg.mkdir()
|
||||
shutil.copy2(HERE / 'container-leg.sh', leg / 'run.sh')
|
||||
container_name = 'otopcua-sdk2-matrix-' + uuid.uuid4().hex[:12]
|
||||
command = ['docker', 'run', '--rm', '--name', container_name, '--network', 'none', '-e', 'OPCUA_MATRIX_PASSWORD',
|
||||
'-v', str(candidate.parent) + ':/candidate:ro', '-v', str(server.parent) + ':/server:ro',
|
||||
'-v', str(candidate_cli) + ':/cli:ro', '-v', str(leg) + ':/task', args.container_image, 'bash', '/task/run.sh']
|
||||
try:
|
||||
status = run(command, leg / 'container.log')
|
||||
finally:
|
||||
subprocess.run(['docker', 'rm', '-f', container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
if (leg / 'driver/result.json').exists():
|
||||
result = json.loads((leg / 'driver/result.json').read_text())
|
||||
else:
|
||||
result = {'cases': [], 'error': 'Driver result missing; inspect preparation/container/server logs'}
|
||||
for operation, expected in [('read', 'Value: 92'), ('raw-history', '25 values returned.'), ('event-history', '25 events returned.')]:
|
||||
log = leg / ('cli-' + operation + '.log')
|
||||
passed = log.exists() and expected in log.read_text()
|
||||
result['cases'].append({'name': 'production-CLI-' + operation, 'status': 'passed' if passed else 'failed', 'log': str(log), 'expectedOutput': expected})
|
||||
result.update({'name': name, 'status': 'passed' if status == 0 and all(x['status'] == 'passed' for x in result['cases']) else 'failed',
|
||||
'container': container_name, 'network': 'none; all peers in task container, separate SDK processes',
|
||||
'driverCertificateValidation': 'explicit trusted peers; autoaccept disabled', 'cliCertificateValidation': 'explicit exchanged peers; CLI always configures AutoAcceptCertificates=true (CommandBase.CreateConnectionSettings)',
|
||||
'server': json.loads((leg / 'server/ready.json').read_text()) if (leg / 'server/ready.json').exists() else None})
|
||||
write_json(leg / 'leg.json', result)
|
||||
matrix.append(result)
|
||||
|
||||
finally:
|
||||
paths = [p for p in (baseline, candidate, args.baseline_root / 'Directory.Packages.props', args.candidate_root / 'Directory.Packages.props', HERE / 'ProcessHost.cs', HERE / 'run.py', HERE / 'container-leg.sh') if p is not None]
|
||||
for folder in (args.baseline_artifacts / 'host', args.baseline_artifacts / 'client', candidate.parent if candidate else None, candidate_cli):
|
||||
if folder is not None:
|
||||
paths.extend(folder.glob('Opc.Ua*.dll'))
|
||||
paths.extend(folder.glob('*.deps.json'))
|
||||
paths.extend(folder.glob('ZB.MOM.WW.OtOpcUa*.dll'))
|
||||
paths.extend(folder.glob('OtOpcUa.Host.dll'))
|
||||
paths.extend(folder.glob('*.runtimeconfig.json'))
|
||||
write_json(args.output / 'artifact-hashes.json', receipt(paths))
|
||||
write_json(args.output / 'commands.json', commands)
|
||||
write_json(args.output / 'matrix.json', matrix)
|
||||
return 1 if not matrix or any(x['status'] != 'passed' for x in matrix) else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user