From 35a4612630ec7671d9351599a34371c5fda8aae5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 10 Nov 2026 14:35:05 -0500 Subject: [PATCH] test: validate historian TLS migration against isolated source --- scripts/validation/historiangw-131/README.md | 28 ++++-- scripts/validation/historiangw-131/RESULTS.md | 61 ++++++++++-- .../historiangw-131/consumer-1.0.0.patch | 2 +- .../validation/historiangw-131/validate.py | 97 +++++++++++++++++++ 4 files changed, 168 insertions(+), 20 deletions(-) create mode 100644 scripts/validation/historiangw-131/validate.py diff --git a/scripts/validation/historiangw-131/README.md b/scripts/validation/historiangw-131/README.md index 1e0b2622..acb8207c 100644 --- a/scripts/validation/historiangw-131/README.md +++ b/scripts/validation/historiangw-131/README.md @@ -45,12 +45,22 @@ and guard exits, run the full owning project, and complete the consumer solution build/review before issue 131 is closed. The patch is transport-only and can be applied independently of the OPC UA SDK 2 rollout. -If release availability lags, validate only in an archived staging copy: materialize -exact HG commit e432191 with `git archive`, pack both packages with the local-only -version `1.0.0-validation.131`, set an isolated `NUGET_PACKAGES`, and map only those -two package IDs to a task-local feed. Apply the future patch to a separate consumer -archive, then replace ONLY its two package pins with the validation prerelease. -Never pack/build the user's HG checkout, publish these validation packages, place -them in the normal NuGet cache, or pass prerelease validation off as stable-release -verification. Local validation and release availability results are recorded in -`RESULTS.md` when the run completes. +For isolated validation, use a fresh output directory OUTSIDE both repositories +so the archived HG projects do not inherit the consumer's central package policy: + +```bash +python3 scripts/validation/historiangw-131/validate.py \ + --consumer-root "$PWD" \ + --hg-root /Users/dohertj2/Desktop/HistorianGateway \ + --output /tmp/historiangw-131-fresh-validation +``` + +The script checks both live feed indexes. If both actual stable 1.0.0 packages are +available, it validates them in an archived consumer copy. Otherwise it archives +exact HG commit e432191 and temporarily replaces ONLY the archived Gateway csproj's +two package references with source project references. This equivalent source +validation does not create or publish any fake package/version. Restore uses an +independent copy-on-write/copy cache, isolated HTTP cache and CLI home. Neither +user checkout is built or edited. Full owning-suite TRX, required TLS failure +classification, command exits and execution guard are retained under the task +output. Never treat source validation as stable published-package verification. diff --git a/scripts/validation/historiangw-131/RESULTS.md b/scripts/validation/historiangw-131/RESULTS.md index 9dd18c51..c278c093 100644 --- a/scripts/validation/historiangw-131/RESULTS.md +++ b/scripts/validation/historiangw-131/RESULTS.md @@ -1,13 +1,54 @@ -# Validation status +# HG issue 131 validation evidence -Preparation only: future patch passes `git apply --check` in the SDK 2 candidate -checkout. Published Client/Contracts 1.0.0 package verification is pending. +Future consumer patch validated against exact archived HG source commit +`e4321915ebc5e0704a68ecad21b85c1cc1c80d6b`. Consumer archive base: +`18c22cf7eb042126226e540d9b979907328055c3`. Patch SHA-256: +`9452ee82b2bace7828a384b9f6048e9c931e8b4ec2078e0ee1baa175d0652fac`. -Initial primary feed query returned versions 0.4.0, 0.3.0, 0.2.0, 0.1.0 for both -packages. HG e4321915ebc5e0704a68ecad21b85c1cc1c80d6b contains the new option under -Unreleased; both source package version declarations still read 0.4.0. The cached -published Client 0.4.0 exposes RequireCertificateValidation and does not expose the -new property. Current consumer files remain on 0.4.0 and unchanged. +## Scope and isolation -No issue comment/close, package publication, or HG user-checkout modification has -been performed. Test results will be added after actual execution. +Mode: **unreleased source project references**, not published-package verification. +Both live package feed indexes still listed only 0.4.0, 0.3.0, 0.2.0, 0.1.0 after +validation. Stable Client/Contracts 1.0.0 verification remains pending. + +The user's HG checkout was only read by git archive. Only the disposable archived +consumer Gateway csproj had its two package references replaced by project +references into the exact archived HG source. No package was packed or published; +no fake 1.0.0 was inserted into any feed/cache. Restore used an independent copied +NuGet cache, isolated HTTP cache and CLI home. The normal consumer files remain +on published 0.4.0. + +## Actual execution + +- Full Gateway owning project: **529 total, 523 passed, 0 failed, 6 existing live skips**. +- Test exit **0**; repository nonzero-execution guard exit **0**. +- All three new local wire tests passed; no new skip or expected application-result change. +- Strict default: base HistorianGatewayException -> RpcException Status Internal -> + HttpRequestException -> AuthenticationException, certificate-chain **UntrustedRoot**. + The test explicitly requires the typed TLS authentication failure; RPC calls = **0**. +- Explicit TLS opt-out: actual metadata RPC succeeds; calls = **1**. +- Plaintext with stale opt-out and nonexistent CA path: actual metadata RPC succeeds; + calls = **1**. + +The first source run compiled and executed but failed one new assertion because +it predicted the narrower HistorianGatewayUnavailableException type. The actual +TLS exception is the public base HistorianGatewayException; the corrected assertion +retains the required AuthenticationException graph and zero-RPC check. The fresh +second unfiltered run above passed. The six skipped cases are the existing AVEVA +live fixture tests; they are not a substitute for TLS wire coverage. + +Completed UTC: `2026-11-10T19:33:17.272468+00:00`. +Actual metadata, typed TLS exception output and binary hashes: +`/tmp/historiangw-131-validation-20261110T193106Z/metadata-final.json`. +Fresh second-run TRX: `/tmp/historiangw-131-validation-20261110T193106Z/test-results2`. +Test log: `/tmp/historiangw-131-validation-20261110T193106Z/test2.log`. +Initial failed run and restore logs are retained in the same task directory. + +Archived-source Client DLL SHA-256: +`87e995a43c63debdd8f648a8feb8d19d9110b45eb7b2b6906159077d90109862`. +Archived-source Contracts DLL SHA-256: +`ebaaa43ffc485288054eb864a59ec59264d8d9c4ea6871319fd7995192370751`. + +No stable release was consumed by this run. The real paired 1.0.0 packages must be +restored, the full owning suite/solution built and reviewed, and the consumer change +landed before issue 131 is closed. No issue comment/close was performed here. diff --git a/scripts/validation/historiangw-131/consumer-1.0.0.patch b/scripts/validation/historiangw-131/consumer-1.0.0.patch index 1e855bb0..f06017b0 100644 --- a/scripts/validation/historiangw-131/consumer-1.0.0.patch +++ b/scripts/validation/historiangw-131/consumer-1.0.0.patch @@ -112,7 +112,7 @@ new file mode 100644 + await using var server = await Server.StartAsync(tls: true); + using var adapter = HistorianGatewayClientAdapter.Create(server.Options(), NullLoggerFactory.Instance); + -+ var error = await Should.ThrowAsync( ++ var error = await Should.ThrowAsync( + () => adapter.TagExistsAsync("T", Ct).WaitAsync(TimeSpan.FromSeconds(15), Ct)); + output.WriteLine($"TLS rejection: {error}"); + ContainsTlsAuthenticationFailure(error).ShouldBeTrue("the failure must be TLS authentication, not an arbitrary connection failure"); diff --git a/scripts/validation/historiangw-131/validate.py b/scripts/validation/historiangw-131/validate.py new file mode 100644 index 00000000..51cfc291 --- /dev/null +++ b/scripts/validation/historiangw-131/validate.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Validate the future patch in archives, using published 1.0.0 or exact HG source project references.""" +import argparse, datetime, hashlib, io, json, os, shutil, subprocess, tarfile, urllib.request +import xml.etree.ElementTree as ET +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--consumer-root', type=Path, required=True) +parser.add_argument('--hg-root', type=Path, required=True) +parser.add_argument('--hg-commit', default='e4321915ebc5e0704a68ecad21b85c1cc1c80d6b') +parser.add_argument('--output', type=Path, required=True) +a = parser.parse_args(); root = a.consumer_root.resolve(); out = a.output.resolve() +if out.is_relative_to(root) or out.is_relative_to(a.hg_root.resolve()): + raise ValueError('Use an output directory outside both checkouts so archived HG does not inherit consumer central-package settings.') +out.mkdir(parents=True, exist_ok=False) +patch = Path(__file__).with_name('consumer-1.0.0.patch').resolve() +project = Path('tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.csproj') +sha = subprocess.check_output(['git','-C',str(root),'rev-parse','HEAD'],text=True).strip() +hg_sha = subprocess.check_output(['git','-C',str(a.hg_root),'rev-parse',a.hg_commit],text=True).strip() +def archive(repo, rev, target): + target.mkdir() + data = subprocess.check_output(['git','-C',str(repo),'archive',rev]) + with tarfile.open(fileobj=io.BytesIO(data)) as tar: tar.extractall(target, filter='data') + +versions = {} +for name in ['client','contracts']: + url = 'https://gitea.dohertylan.com/api/packages/dohertj2/nuget/package/zb.mom.ww.historiangateway.' + name + '/index.json' + versions[name] = json.load(urllib.request.urlopen(url,timeout=20))['versions'] +published = all('1.0.0' in v for v in versions.values()) +metadata = {'utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'consumer_commit':sha,'hg_snapshot':hg_sha, + 'mode':'published-1.0.0' if published else 'unreleased-source-project-reference', + 'feed_versions':versions,'patch_sha256':hashlib.sha256(patch.read_bytes()).hexdigest(), + 'published_package_verification':published,'normal_package_cache_modified':False} +(out/'metadata.json').write_text(json.dumps(metadata,indent=2)) +consumer = out/'consumer'; archive(root,sha,consumer) +subprocess.run(['git','init','-q',str(consumer)],check=True) +subprocess.run(['git','-C',str(consumer),'apply',str(patch)],check=True) +if not published: + hg = out/'historiangw-source'; archive(a.hg_root,hg_sha,hg) + adapter = consumer/'src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj' + tree = ET.parse(adapter) + removed = 0 + for group in tree.getroot().findall('ItemGroup'): + for item in list(group): + if item.tag=='PackageReference' and item.get('Include') in ['ZB.MOM.WW.HistorianGateway.Client','ZB.MOM.WW.HistorianGateway.Contracts']: + group.remove(item);removed += 1 + assert removed==2 + group = ET.SubElement(tree.getroot(),'ItemGroup') + for relative in ['clients/dotnet/ZB.MOM.WW.HistorianGateway.Client/ZB.MOM.WW.HistorianGateway.Client.csproj', + 'src/ZB.MOM.WW.HistorianGateway.Contracts/ZB.MOM.WW.HistorianGateway.Contracts.csproj']: + ET.SubElement(group,'ProjectReference',Include=str(hg/relative)) + tree.write(adapter,encoding='unicode') + metadata['temporary_override']='Only archived Gateway csproj: two package references replaced by exact archived HG project references; no package packed or published.' +# Copy cached dependencies independently, preferably APFS copy-on-write, into the isolated package root. +cache = out/'packages'; cache.mkdir() +assets = json.loads((root/project.parent/'obj/project.assets.json').read_text()) +normal = Path(next(iter(assets['packageFolders']))) +for library in assets['libraries'].values(): + if library.get('type')!='package':continue + relative = library['path']; source=normal/relative; target=cache/relative + target.parent.mkdir(parents=True,exist_ok=True) + if target.exists():continue + r=subprocess.run(['cp','-cR',str(source),str(target)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + if r.returncode: shutil.copytree(source,target) +config = ET.parse(root/'NuGet.config') +for source in config.getroot().find('packageSources'): + if source.get('key')=='local-mxgw': + local = root/'nuget-packages' + if not local.exists():local=out/'empty-local';local.mkdir() + source.set('value',str(local)) +config.write(out/'NuGet.config',encoding='unicode') +env=dict(os.environ,NUGET_PACKAGES=str(cache),NUGET_HTTP_CACHE_PATH=str(out/'http-cache'),DOTNET_CLI_HOME=str(out/'dotnet-home'), + DOTNET_SKIP_FIRST_TIME_EXPERIENCE='1',DOTNET_GENERATE_ASPNET_CERTIFICATE='false',DOTNET_CLI_TELEMETRY_OPTOUT='1') +commands=[] +def run(args,name): + with (out/name).open('w') as log:r=subprocess.run(args,cwd=consumer,env=env,stdout=log,stderr=subprocess.STDOUT) + commands.append({'argv':args,'exit':r.returncode,'log':name}) + (out/'commands.json').write_text(json.dumps(commands,indent=2));print(name,'exit',r.returncode,flush=True) + return r.returncode +restore = run(['dotnet','restore',str(project),'--configfile',str(out/'NuGet.config'),'--packages',str(cache)],'restore.log') +metadata['restore_exit']=restore +if restore: + (out/'metadata.json').write_text(json.dumps(metadata,indent=2));raise SystemExit(restore) +results=out/'test-results' +test = run(['dotnet','test',str(project),'-c','Release','--no-restore','--logger','trx','--results-directory',str(results)],'test.log') +guard_env=env.copy();guard_env['MIN_EXECUTED']='1' +with (out/'guard.log').open('w') as log: + guard=subprocess.run(['bash',str(consumer/'scripts/ci/assert-not-all-skipped.sh'),*[str(p) for p in results.glob('*.trx')]],env=guard_env,stdout=log,stderr=subprocess.STDOUT) +metadata.update({'test_exit':test,'guard_exit':guard.returncode}) +ns={'t':'http://microsoft.com/schemas/VisualStudio/TeamTest/2010'} +metadata['results']=[] +for trx in results.glob('*.trx'): + xml=ET.parse(trx);metadata['results'].append(xml.find('.//t:Counters',ns).attrib) + metadata['tls_cases']=[{'name':r.get('testName'),'outcome':r.get('outcome'),'output':'\n'.join(p.text or '' for p in r.findall('.//t:StdOut',ns)), + 'error':r.findtext('.//t:Message','',ns)} for r in xml.findall('.//t:UnitTestResult',ns) if 'HistorianGatewayTlsPolicyWireTests' in r.get('testName','')] +(out/'metadata.json').write_text(json.dumps(metadata,indent=2));print(json.dumps(metadata,indent=2),flush=True) +raise SystemExit(test if test else guard.returncode)