lmxopcua/scripts/validation/opcua-server-lifecycle-linux/run.py

#!/usr/bin/env python3
"""Run the unchanged SDK2 server lifecycle test in isolated Linux ARM64."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import struct
import subprocess
import time
import uuid
import xml.etree.ElementTree as ET

HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[2]
PROJECT = ROOT / 'tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.csproj'
APPNAME = 'ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests'
TARGET = 'Repeated_lifecycles_return_resources_to_a_steady_baseline'
BASE = '67bc61236a7798e87deb9ae45e54bd29cf3d8ae9'


def write(path, value):
    path.write_text(json.dumps(value, indent=2) + '\n')


def sha(path):
    return hashlib.sha256(path.read_bytes()).hexdigest()


def command(argv, log, timeout, **kwargs):
    with log.open('w') as stream:
        result = subprocess.run([str(x) for x in argv], stdout=stream, stderr=subprocess.STDOUT, timeout=timeout, **kwargs)
    return result.returncode


def trx(path, count):
    doc = ET.parse(path)
    ns = {'t': 'http://microsoft.com/schemas/VisualStudio/TeamTest/2010'}
    counts = doc.find('.//t:Counters', ns)
    results = doc.findall('.//t:UnitTestResult', ns)
    if counts is None or any(int(counts.attrib[k]) != count for k in ('total', 'executed', 'passed')):
        raise RuntimeError('TRX expected exact passed/executed count ' + str(count) + ': ' + str(path))
    if len(results) != count or any(r.attrib['outcome'] != 'Passed' for r in results):
        raise RuntimeError('TRX missing or non-passing result: ' + str(path))
    for key in ('failed', 'error', 'timeout', 'aborted', 'notExecuted'):
        if int(counts.attrib.get(key, '0')):
            raise RuntimeError('TRX has nonzero ' + key)
    return {'counts': counts.attrib, 'tests': [r.attrib['testName'] for r in results]}


def validate_evidence(task, seconds, full):
    rows = [json.loads(line) for line in (task / 'soak.jsonl').read_text().splitlines()]
    starts = [r for r in rows if r['kind'] == 'start']
    complete = [r for r in rows if r['kind'] == 'complete']
    if len(starts) != 1 or len(complete) != 1 or any(r['kind'] == 'failed' for r in rows):
        raise RuntimeError('Expected one start and one complete with no failed evidence row')
    start, done = starts[0], complete[0]
    if start['requestedSeconds'] != seconds or start['minimumFullSeconds'] != 7200:
        raise RuntimeError('Duration header mismatch')
    expected_mode = 'full' if full else 'diagnostic'
    if start['mode'] != expected_mode or done['mode'] != expected_mode or done['seconds'] < seconds:
        raise RuntimeError('Real duration/completion mode mismatch')
    if start['limits'] != {'retainedManagedGrowthBytes': 64 << 20, 'rssGrowthBytes': 128 << 20, 'fdGrowth': 32}:
        raise RuntimeError('Canonical test resource budgets changed')
    if done['satisfiesTwoHourDuration'] != full or (full and done['seconds'] < 7200):
        raise RuntimeError('Two-hour gate mismatch')
    samples = [r for r in rows if r['kind'] == 'quiescent']
    active = [r for r in rows if r['kind'] == 'active']
    stalls = [r for r in rows if r['kind'] == 'network-stall-recovered']
    restarts = [r for r in rows if r['kind'] == 'restart']
    if done['cycles'] < 4 or done['restarts'] != done['cycles'] // 4:
        raise RuntimeError('Missing actual lifecycle/restart coverage')
    if not (len(samples) == len(active) == len(stalls) == done['cycles']) or len(restarts) != done['restarts']:
        raise RuntimeError('Incomplete cycle/stall/restart evidence counts')
    if not start['sdk'].startswith('2.0.1+'):
        raise RuntimeError('Runtime SDK header mismatch')
    return {'start': start, 'complete': done, 'rowCounts': {k: sum(r['kind'] == k for r in rows) for k in sorted({r['kind'] for r in rows})},
            'firstResources': samples[0]['resources'], 'baselineResources': samples[2]['resources'], 'lastResources': samples[-1]['resources']}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', required=True, type=Path, help='Fresh output including task private PKI; do not commit the whole directory')
    parser.add_argument('--duration', type=int, default=90)
    parser.add_argument('--full', action='store_true')
    parser.add_argument('--published-app', type=Path, help='Reuse immutable self-contained publish from this launcher/recorded matching build')
    parser.add_argument('--image', default='mcr.microsoft.com/dotnet/sdk:10.0.103-noble')
    args = parser.parse_args()
    if (args.full and args.duration < 7200) or (not args.full and not 60 <= args.duration <= 120):
        parser.error('full requires duration >=7200; diagnostic requires 60..120 seconds')
    output = args.output.resolve()
    output.mkdir(parents=True, exist_ok=False)
    os.chmod(output, 0o700)
    task = output / 'task'
    task.mkdir()
    revision = subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], text=True).strip()
    production_diff = subprocess.check_output(['git', '-C', str(ROOT), 'diff', BASE, 'HEAD', '--', 'src', 'Directory.Packages.props', 'Directory.Build.props'], text=True)
    if production_diff:
        raise RuntimeError('Production/package source differs from reviewed ' + BASE[:8] + ' source; review required')
    image = json.loads(subprocess.check_output(['docker', 'image', 'inspect', args.image, '--format', '{{json .}}'], text=True))
    if image['Architecture'] != 'arm64' or image['Os'] != 'linux':
        raise RuntimeError('Expected cached official Linux ARM64 image')
    app = args.published_app.resolve() if args.published_app else output / 'app'
    build_cmd = ['dotnet', 'publish', str(PROJECT), '-c', 'Release', '-r', 'linux-arm64', '--self-contained', 'true',
                 '-p:RuntimeFrameworkVersion=10.0.14', '-p:PublishSingleFile=false', '-p:PublishTrimmed=false', '-p:UseAppHost=true',
                 '--artifacts-path', str(output / 'build-artifacts'), '-o', str(app), '-m:1', '--nologo']
    if not args.published_app:
        status = command(build_cmd, output / 'publish.log', 600)
        if status:
            raise RuntimeError('Self-contained Linux publish failed: ' + str(status))
    receipt_path = app.parent / 'publish-receipt.json'
    receipt = json.loads(receipt_path.read_text()) if args.published_app else {
        'sourceRevision': revision, 'command': build_cmd,
        'hostDotnet': subprocess.check_output(['dotnet', '--info'], text=True),
        'testSourceSha256': sha(PROJECT.parent / 'Sdk2LifecycleSoakTests.cs')}
    if receipt['testSourceSha256'] != sha(PROJECT.parent / 'Sdk2LifecycleSoakTests.cs'):
        raise RuntimeError('Reused publish test source differs from current canonical source')
    apphost = app / APPNAME
    magic = apphost.read_bytes()[:20]
    if magic[:4] != b'\x7fELF' or magic[4] != 2 or struct.unpack('<H', magic[18:20])[0] != 183:
        raise RuntimeError('xUnit v3 apphost must be Linux AArch64 ELF64, not macOS Mach-O')
    runtime = json.loads((app / (APPNAME + '.runtimeconfig.json')).read_text())
    frameworks = runtime['runtimeOptions'].get('includedFrameworks', [])
    if not any(f['name'] == 'Microsoft.NETCore.App' and f['version'] == '10.0.14' for f in frameworks):
        raise RuntimeError('Test app must carry self-contained runtime 10.0.14')
    deps = json.loads((app / (APPNAME + '.deps.json')).read_text())
    sdkdeps = {k: v for k, v in deps['libraries'].items() if k.startswith('OPCFoundation.')}
    if not sdkdeps or any(k.rsplit('/', 1)[-1] != '2.0.1' for k in sdkdeps):
        raise RuntimeError('Expected only OPC Foundation SDK 2.0.1 dependencies')
    manifest = [{'name': p.name, 'bytes': p.stat().st_size, 'sha256': sha(p)} for p in sorted(app.glob('*')) if p.is_file()]
    if args.published_app and (receipt['apphostSha256'] != sha(apphost)
                              or receipt['testAssemblySha256'] != sha(app / (APPNAME + '.dll'))):
        raise RuntimeError('Reused publish has changed since its receipt')
    if not args.published_app:
        receipt.update({'apphostSha256': sha(apphost), 'testAssemblySha256': sha(app / (APPNAME + '.dll'))})
        write(receipt_path, receipt)
    write(output / 'publish-receipt.json', receipt)
    write(output / 'artifact-manifest.json', manifest)
    source = {'launcherSourceRevision': revision, 'publishedSourceRevision': receipt['sourceRevision'],
              'productionEquivalentTo': BASE, 'productionDiffEmpty': True,
              'canonicalDescriptorProbeCommit': '642aefe3', 'testSourceSha256': sha(PROJECT.parent / 'Sdk2LifecycleSoakTests.cs'),
              'publishCommand': build_cmd if not args.published_app else 'reused self-contained artifact: ' + str(app),
              'appPath': str(app), 'apphost': {'format': 'ELF64 AArch64', 'sha256': sha(apphost)}, 'runtimeConfig': runtime,
              'sdkDependencies': sorted(sdkdeps), 'image': {k: image.get(k) for k in ('Id', 'RepoDigests', 'Architecture', 'Os')},
              'hostDotnet': subprocess.check_output(['dotnet', '--info'], text=True),
              'network': 'none', 'mode': 'full' if args.full else 'diagnostic', 'durationSeconds': args.duration,
              'startedUtc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
    write(output / 'source.json', source)
    shutil.copy2(HERE / 'container.sh', task / 'container.sh')
    name = 'otopcua-server-linux-soak-' + uuid.uuid4().hex[:12]
    cmd = ['docker', 'run', '--rm', '--name', name, '--network', 'none',
           '-e', 'OTOPCUA_SDK2_SOAK_MODE=' + ('full' if args.full else 'diagnostic'),
           '-e', 'OTOPCUA_SDK2_SOAK_SECONDS=' + str(args.duration), '-e', 'OTOPCUA_SOAK_BUILD_SHA=' + revision,
           '-v', str(app) + ':/app:ro', '-v', str(task) + ':/task', image['Id'], 'bash', '/task/container.sh']
    write(output / 'command.json', {'argv': cmd, 'timeoutSeconds': args.duration + 300})
    started = time.monotonic()
    try:
        status = command(cmd, output / 'container.log', args.duration + 300)
    except BaseException:
        subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20)
        raise
    write(output / 'result.json', {'status': 'failed', 'containerExitCode': status,
                                  'containerElapsedSeconds': time.monotonic() - started,
                                  'validation': 'pending TRX and completion validation'})
    try:
        controls = trx(task / 'results' / 'descriptor-controls.trx', 2)
        test = trx(task / 'results' / 'soak.trx', 1)
        if TARGET not in test['tests'][0] or status:
            raise RuntimeError('Soak test did not complete successfully; container exit=' + str(status))
        evidence = validate_evidence(task, args.duration, args.full)
    except Exception as error:
        write(output / 'result.json', {'status': 'failed', 'containerExitCode': status,
                                      'containerElapsedSeconds': time.monotonic() - started,
                                      'validationError': str(error)})
        raise
    write(output / 'result.json', {'status': 'passed', 'containerExitCode': status, 'containerElapsedSeconds': time.monotonic() - started,
                                  'descriptorControls': controls, 'soakTest': test, 'evidence': evidence})
    print(json.dumps({'output': str(output), 'status': 'passed', 'complete': evidence['complete']}, indent=2))


if __name__ == '__main__':
    main()