Joseph Doherty
153 lines
10 KiB
Python
153 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Smoke actual self-contained Linux Host/CLI artifacts with task-only PKI and LocalDb."""
|
|
import argparse, base64, datetime, hashlib, json, os, re, secrets, shutil, subprocess, time, uuid
|
|
from pathlib import Path
|
|
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--artifacts', required=True, type=Path)
|
|
p.add_argument('--image', default='mcr.microsoft.com/dotnet/sdk:10.0.103-noble')
|
|
a = p.parse_args(); out = a.artifacts.resolve()
|
|
run = out / ('runtime-' + uuid.uuid4().hex); run.mkdir(exist_ok=False)
|
|
state = run / 'state'; state.mkdir(); os.chmod(state, 0o700)
|
|
identifier = uuid.uuid4().hex[:12]
|
|
web, ua, akka = [20000 + secrets.randbelow(10000) for _ in range(3)]
|
|
while len({web, ua, akka}) != 3: web, ua, akka = [20000 + secrets.randbelow(10000) for _ in range(3)]
|
|
key = 'otua_' + secrets.token_hex(8) + '_' + secrets.token_urlsafe(32)
|
|
pepper, fetch = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
|
|
master = base64.b64encode(secrets.token_bytes(32)).decode()
|
|
env = {
|
|
'ASPNETCORE_URLS': f'http://127.0.0.1:{web}', 'ASPNETCORE_ENVIRONMENT': 'Production',
|
|
'Cluster__Roles__0': 'driver',
|
|
'Cluster__SystemName': 'otopcua', 'Cluster__Hostname': '127.0.0.1',
|
|
'Cluster__PublicHostname': '127.0.0.1', 'Cluster__Port': str(akka),
|
|
'Cluster__SeedNodes__0': f'akka.tcp://otopcua@127.0.0.1:{akka}',
|
|
'ConfigSource__Mode': 'FetchAndCache', 'ConfigSource__CentralFetchEndpoints__0': 'http://127.0.0.1:1',
|
|
'ConfigSource__ApiKey': fetch, 'ConfigSource__FetchTimeoutSeconds': '1',
|
|
'LocalDb__Path': '/state/localdb.db', 'Secrets__SqlitePath': '/state/secrets.db',
|
|
'ZB_SECRETS_MASTER_KEY': master, 'Security__ApiKeys__Pepper': pepper,
|
|
'Security__Ldap__Enabled': 'false', 'Security__Acl__Enabled': 'false',
|
|
'OpcUa__ApplicationName': 'PublishSmoke-' + identifier,
|
|
'OpcUa__ApplicationUri': 'urn:localhost:PublishSmoke:' + identifier,
|
|
'OpcUa__PublicHostname': 'localhost', 'OpcUa__OpcUaPort': str(ua),
|
|
'OpcUa__PkiStoreRoot': '/state/server-pki-' + identifier,
|
|
'OpcUa__AllowAnonymous': 'false', 'OpcUa__AutoAcceptUntrustedClientCertificates': 'false',
|
|
'OpcUa__EnabledSecurityProfiles__0': 'Basic256Sha256SignAndEncrypt',
|
|
'SMOKE_API_KEY': key, 'XDG_DATA_HOME': '/state/cli-data', 'HOME': '/state/home',
|
|
}
|
|
envfile = run / 'private.env'; envfile.write_text(''.join(f'{k}={v}\n' for k,v in env.items())); os.chmod(envfile, 0o600)
|
|
redactions = [key, key.split('_', 2)[2], pepper, fetch, master]
|
|
def clean(text):
|
|
for secret in redactions: text = text.replace(secret, '<ephemeral-secret>')
|
|
return text
|
|
records = []
|
|
raw_secret_findings = []
|
|
rolling_root = out / 'linux-arm64/host/logs'
|
|
rolling_offsets = {p: p.stat().st_size for p in rolling_root.glob('*') if p.is_file()}
|
|
def command(args, name, expected=0, timeout=90):
|
|
start = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
result = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)
|
|
raw_secret_findings.append({'log': name, 'occurrences': sum(result.stdout.count(secret) for secret in redactions)})
|
|
(run / (name + '.log')).write_text(clean(result.stdout))
|
|
records.append({'name': name, 'utc': start, 'argv': [clean(str(x)) for x in args], 'exit': result.returncode})
|
|
(run / 'commands.json').write_text(json.dumps(records, indent=2))
|
|
if expected is not None and result.returncode != expected:
|
|
raise RuntimeError(f'{name}: exit {result.returncode}; see {run / (name + ".log")}')
|
|
return result
|
|
mounts = ['--mount', f'type=bind,src={out},dst=/work', '--mount', f'type=bind,src={state},dst=/state']
|
|
base = ['docker','run','--rm','--network','none','--label','codex.task=opcua-publish-smoke','--env-file',str(envfile),*mounts,'-w','/work/linux-arm64/host',a.image]
|
|
probe = ['dotnet','/work/probe/PublishProbe.dll']
|
|
cli = ['/work/linux-arm64/cli/ZB.MOM.WW.OtOpcUa.Client.CLI']
|
|
server_pki = state / ('server-pki-' + identifier)
|
|
container = None; summary = {'run': str(run), 'image': a.image, 'ports': {'http':web,'ua':ua,'akka':akka}, 'boots':[], 'cases':[]}
|
|
def copy_public(source, target):
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
certs = list(source.glob('*.der'))
|
|
if not certs: raise RuntimeError(f'No public certificates at {source}')
|
|
for cert in certs: shutil.copy2(cert, target / cert.name)
|
|
def inspect(): return json.loads(subprocess.check_output(['docker','inspect',container], text=True))[0]
|
|
def save_logs(name):
|
|
r = subprocess.run(['docker','logs',container],text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
|
raw_secret_findings.append({'log': name, 'occurrences': sum(r.stdout.count(secret) for secret in redactions)})
|
|
(run / (name + '.log')).write_text(clean(r.stdout))
|
|
def exec_probe(name, pki, expected, credential=None):
|
|
args = ['docker','exec']
|
|
if credential: args += ['-e','SMOKE_API_KEY=' + credential]
|
|
args += [container,*probe,'probe',pki,f'opc.tcp://localhost:{ua}/OtOpcUa',expected]
|
|
r = command(args,name)
|
|
payload = json.loads(r.stdout.strip().splitlines()[-1]); summary['cases'].append({'name':name,**payload})
|
|
try:
|
|
command([*base,*probe,'seed','/state'],'seed')
|
|
command([*base,*probe,'prepare','/state/client-pki'],'prepare-trusted-client')
|
|
command([*base,*probe,'prepare','/state/untrusted-client-pki'],'prepare-untrusted-client')
|
|
command([*base,*cli,'--help'],'cli-help')
|
|
command([*base,*probe,'prepare-cli','/state/cli-data/OtOpcUaClient/pki'],'prepare-cli-certificate')
|
|
failed = command([*base,*cli,'read','--url','opc.tcp://localhost:1/OtOpcUa','--security','encrypt','--username','apikey','--password',key,'--node','i=2259'],'cli-prepare-on-closed-port',expected=None)
|
|
if failed.returncode == 0: raise RuntimeError('Closed-port CLI fixture preparation unexpectedly succeeded.')
|
|
copy_public(state/'client-pki/own/certs',server_pki/'trusted/certs')
|
|
copy_public(state/'cli-data/OtOpcUaClient/pki/own/certs',server_pki/'trusted/certs')
|
|
name = 'otopcua-sdk2-publish-' + identifier
|
|
result = command(['docker','create','--name',name,'--network','none','--label','codex.task=opcua-publish-smoke','--env-file',str(envfile),*mounts,'-w','/work/linux-arm64/host',a.image,'./OtOpcUa.Host'],'container-create')
|
|
container = result.stdout.strip()
|
|
summary['container'] = container
|
|
for boot in [1,2]:
|
|
command(['docker','start',container],f'boot{boot}-start')
|
|
deadline = time.monotonic() + 90
|
|
health = None
|
|
while time.monotonic() < deadline:
|
|
if not inspect()['State']['Running']:
|
|
save_logs(f'boot{boot}-host'); raise RuntimeError('Full Host exited before health readiness.')
|
|
r = subprocess.run(['docker','exec',container,'curl','-fsS',f'http://127.0.0.1:{web}/health/ready'],text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
|
if r.returncode == 0:
|
|
health = r.stdout; break
|
|
time.sleep(1)
|
|
if health is None:
|
|
save_logs(f'boot{boot}-host'); raise RuntimeError('Full Host readiness did not become healthy.')
|
|
if json.loads(health).get('status') != 'Healthy': raise RuntimeError('Readiness is degraded or unhealthy.')
|
|
(run/f'boot{boot}-ready.json').write_text(health)
|
|
command(['docker','exec',container,'curl','-fsS',f'http://127.0.0.1:{web}/healthz'],f'boot{boot}-live')
|
|
command(['docker','exec',container,'curl','-fsS',f'http://127.0.0.1:{web}/health/active'],f'boot{boot}-active')
|
|
certs = list((server_pki/'own/certs').glob('*.der'))
|
|
if len(certs) != 1: raise RuntimeError('Expected exactly one application certificate.')
|
|
cert_sha256 = hashlib.sha256(certs[0].read_bytes()).hexdigest()
|
|
thumbprint = subprocess.check_output(['openssl','x509','-inform','DER','-in',str(certs[0]),'-noout','-fingerprint','-sha1'],text=True).strip()
|
|
copy_public(server_pki/'own/certs',state/'client-pki/trusted/certs')
|
|
copy_public(server_pki/'own/certs',state/'untrusted-client-pki/trusted/certs')
|
|
exec_probe(f'boot{boot}-trusted-key','/state/client-pki','allow')
|
|
exec_probe(f'boot{boot}-wrong-key','/state/client-pki','key-deny',key+'-wrong')
|
|
if boot == 1:
|
|
hold = state/'hold-server-trust'; shutil.move(str(state/'client-pki/trusted/certs'),hold)
|
|
(state/'client-pki/trusted/certs').mkdir()
|
|
try: exec_probe('untrusted-server','/state/client-pki','peer-deny')
|
|
finally:
|
|
shutil.rmtree(state/'client-pki/trusted/certs'); shutil.move(str(hold),state/'client-pki/trusted/certs')
|
|
exec_probe('untrusted-client','/state/untrusted-client-pki','peer-deny')
|
|
r = command(['docker','exec',container,*cli,'read','--url',f'opc.tcp://localhost:{ua}/OtOpcUa','--security','encrypt','--username','apikey','--password',key,'--node','i=2259'],f'boot{boot}-actual-cli-read')
|
|
if not re.search(r'^Status:\s+Good(?:\s|$)', r.stdout, re.MULTILINE): raise RuntimeError('Actual CLI did not report a Good server status read.')
|
|
command(['docker','stop','--time','30',container],f'boot{boot}-stop',timeout=45)
|
|
stopped = inspect()['State']; save_logs(f'boot{boot}-host')
|
|
if stopped['Running'] or stopped['ExitCode'] != 0 or stopped['OOMKilled']: raise RuntimeError('Full Host did not shut down cleanly.')
|
|
summary['boots'].append({'boot':boot,'certificate_sha256':cert_sha256,'thumbprint':thumbprint,'exit':stopped['ExitCode'],'oomKilled':stopped['OOMKilled'],'health_ready':health})
|
|
if summary['boots'][0]['thumbprint'] != summary['boots'][1]['thumbprint']: raise RuntimeError('Application certificate changed between boots.')
|
|
summary['certificate_preserved'] = True
|
|
for path in rolling_root.glob('*'):
|
|
if not path.is_file(): continue
|
|
text = path.read_bytes()[rolling_offsets.get(path, 0):].decode(errors='replace')
|
|
raw_secret_findings.append({'log': 'rolling-' + path.name, 'occurrences': sum(text.count(secret) for secret in redactions)})
|
|
(run / ('rolling-' + path.name)).write_text(clean(text))
|
|
if len(summary['boots']) != 2 or len(summary['cases']) != 6: raise RuntimeError('Required smoke cases did not all execute.')
|
|
if any(x['occurrences'] for x in raw_secret_findings): raise RuntimeError('Task secret appeared in raw application output; see recorded occurrence counts.')
|
|
summary['result'] = 'passed'
|
|
finally:
|
|
if container:
|
|
save_logs('host-final')
|
|
owned = inspect()
|
|
if owned['Config']['Labels'].get('codex.task') != 'opcua-publish-smoke': raise RuntimeError('Task container ownership changed.')
|
|
command(['docker','rm','-f',container],'container-cleanup')
|
|
logs = '\n'.join(x.read_text(errors='replace') for x in run.glob('*.log'))
|
|
summary['task_secret_occurrences_in_saved_logs'] = sum(logs.count(secret) for secret in redactions)
|
|
summary['raw_log_secret_findings'] = raw_secret_findings
|
|
summary['completed_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
(run/'summary.json').write_text(json.dumps(summary,indent=2))
|
|
envfile.unlink(missing_ok=True)
|
|
print(json.dumps(summary,indent=2))
|