test: add isolated actual Host publish smoke probe

scripts/validation/opcua-publish-smoke/Program.cs

@@ -0,0 +1,93 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Opc.Ua;
+using Opc.Ua.Client;
+using Opc.Ua.Configuration;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using ZB.MOM.WW.OtOpcUa.Host.Configuration;
+using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
+
+using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(45));
+var ct = deadline.Token;
+var mode = args[0];
+var root = Path.GetFullPath(args[1]);
+if (mode == "seed")
+{
+    var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
+    { ["LocalDb:Path"] = Path.Combine(root, "localdb.db") }).Build();
+    await using var provider = new ServiceCollection().AddLogging().AddOtOpcUaLocalDb(config).BuildServiceProvider();
+    var presented = Environment.GetEnvironmentVariable("SMOKE_API_KEY") ?? throw new InvalidOperationException("Missing task credential.");
+    if (!ApiKeySecret.TryParse(presented, out var id, out var secret)) throw new InvalidOperationException("Invalid task credential format.");
+    var pepper = Environment.GetEnvironmentVariable("Security__ApiKeys__Pepper") ?? throw new InvalidOperationException("Missing task pepper.");
+    var artifact = JsonSerializer.SerializeToUtf8Bytes(new
+    {
+        Clusters = new[] { new { ClusterId = "smoke", Name = "Smoke" } },
+        RawFolders = new[] { new { RawFolderId = "smoke", Name = "Smoke", ClusterId = "smoke" } },
+        ApiKeys = new[] { new { ApiKeyId = id, Name = "publish-smoke", Principal = "publish-smoke",
+            Enabled = true, SecretHash = ApiKeySecret.ComputeHash(secret, pepper), CreatedUtc = DateTime.UtcNow } },
+        NodeAcls = new[] { new { NodeAclId = "smoke-read", ClusterId = "smoke", Principal = "publish-smoke",
+            ScopeKind = 0, ScopeId = (string?)null, PermissionFlags = 7 } },
+    });
+    var revision = Convert.ToHexStringLower(SHA256.HashData(artifact));
+    await provider.GetRequiredService<IDeploymentArtifactCache>().StoreAsync("smoke", ZB.MOM.WW.OtOpcUa.Commons.Types.DeploymentId.NewId().ToString(), revision, artifact, ct);
+    Console.WriteLine(JsonSerializer.Serialize(new { mode, result = "seeded-through-public-cache", revision }));
+    return;
+}
+
+var telemetry = DefaultTelemetry.Create(_ => { });
+var clientName = mode == "prepare-cli" ? "OtOpcUaClient" : "PublishSmokeClient";
+var security = new SecurityConfiguration
+{
+    ApplicationCertificate = new CertificateIdentifier { StoreType = CertificateStoreType.Directory,
+        StorePath = Path.Combine(root, "own"), SubjectName = "CN=" + clientName },
+    TrustedIssuerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(root, "issuer") },
+    TrustedPeerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(root, "trusted") },
+    RejectedCertificateStore = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(root, "rejected") },
+    AutoAcceptUntrustedCertificates = false,
+};
+var configuration = new ApplicationConfiguration(telemetry)
+{
+    ApplicationName = clientName, ApplicationUri = "urn:localhost:" + clientName,
+    ApplicationType = ApplicationType.Client, SecurityConfiguration = security,
+    ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 60_000 },
+};
+await using var application = new ApplicationInstance(telemetry)
+{ ApplicationName = configuration.ApplicationName, ApplicationType = ApplicationType.Client, ApplicationConfiguration = configuration };
+await configuration.ValidateAsync(ApplicationType.Client, ct);
+await application.CheckApplicationInstanceCertificatesAsync(true, lifeTimeInMonths: 12, ct);
+if (mode is "prepare" or "prepare-cli")
+{
+    Console.WriteLine(JsonSerializer.Serialize(new { mode, result = "prepared", autoAccept = security.AutoAcceptUntrustedCertificates }));
+    return;
+}
+var endpoint = args[2];
+var expected = args[3];
+var credential = Environment.GetEnvironmentVariable("SMOKE_API_KEY") ?? throw new InvalidOperationException("Missing task credential.");
+try
+{
+    var selected = await CoreClientUtils.SelectEndpointAsync(configuration, endpoint, true, telemetry, ct)
+        ?? throw new InvalidOperationException("No secure endpoint advertised.");
+    if (selected.SecurityMode != MessageSecurityMode.SignAndEncrypt || selected.SecurityPolicyUri != SecurityPolicies.Basic256Sha256)
+        throw new InvalidOperationException("Unexpected negotiated security policy.");
+    using var session = await new DefaultSessionFactory(telemetry).CreateAsync(configuration,
+        new ConfiguredEndpoint(null, selected, EndpointConfiguration.Create(configuration)), false, true,
+        "PublishSmoke", 60_000, new UserIdentity("apikey", Encoding.UTF8.GetBytes(credential)), default, ct);
+    if (expected != "allow") throw new InvalidOperationException("A denied test credential or peer was admitted.");
+    var response = await session.ReadAsync(null, 0, TimestampsToReturn.Both,
+        [new ReadValueId { NodeId = VariableIds.Server_ServerStatus_State, AttributeId = Attributes.Value }], ct);
+    if (response.Results.Count != 1 || StatusCode.IsBad(response.Results[0].StatusCode)) throw new InvalidOperationException("Server status read failed.");
+    Console.WriteLine(JsonSerializer.Serialize(new { mode, expected, result = "allowed", policy = selected.SecurityPolicyUri,
+        securityMode = selected.SecurityMode.ToString(), status = response.Results[0].StatusCode.ToString(), autoAccept = false }));
+    await session.CloseAsync(ct);
+}
+catch (ServiceResultException error) when (expected != "allow")
+{
+    var accepted = expected == "key-deny"
+        ? new[] { StatusCodes.BadUserAccessDenied, StatusCodes.BadIdentityTokenRejected, StatusCodes.BadIdentityTokenInvalid }
+        : new[] { StatusCodes.BadCertificateUntrusted, StatusCodes.BadSecurityChecksFailed, StatusCodes.BadSecureChannelClosed };
+    if (!accepted.Contains(error.StatusCode)) throw;
+    Console.WriteLine(JsonSerializer.Serialize(new { mode, expected, result = "denied", status = error.StatusCode.ToString(), autoAccept = false }));
+}

scripts/validation/opcua-publish-smoke/PublishProbe.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="../../../src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj" />
+  </ItemGroup>
+</Project>

scripts/validation/opcua-publish-smoke/README.md

@@ -0,0 +1,70 @@
+# Actual Host publish smoke
+
+From an isolated candidate checkout, publish the actual Host and CLI for Linux ARM64
+and Windows x64 using the install recipe's self-contained setting. Do not use a
+checkout that another process is building, and choose a new artifact directory.
+
+```bash
+task_output=/absolute/fresh/publish-smoke
+test ! -e "$task_output" || exit 2
+mkdir -p "$task_output"
+for rid in linux-arm64 win-x64; do
+  dotnet publish src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj \
+    -c Release -r "$rid" --self-contained true \
+    -p:PublishSingleFile=false -p:PublishTrimmed=false -m:1 \
+    -o "$task_output/$rid/host" || exit
+  dotnet publish src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI/ZB.MOM.WW.OtOpcUa.Client.CLI.csproj \
+    -c Release -r "$rid" --self-contained true \
+    -p:PublishSingleFile=false -p:PublishTrimmed=false -m:1 \
+    -o "$task_output/$rid/cli" || exit
+done
+dotnet publish scripts/validation/opcua-publish-smoke/PublishProbe.csproj \
+  -c Release -m:1 -o "$task_output/probe" || exit
+python3 scripts/validation/opcua-publish-smoke/runtime.py --artifacts "$task_output"
+```
+
+The cached `mcr.microsoft.com/dotnet/sdk:10.0.103-noble` image must be ARM64 for this
+Linux artifact. The SDK image supplies the small probe's framework; the actual
+Host and CLI are launched through their published self-contained apphosts. There
+is no Native AOT, trimming, or single-file transformation.
+
+The runtime script creates a fresh private state directory, ephemeral keys and
+loopback ports. A temporary preparation container seeds the LocalDb cache through
+public production registration/cache APIs. The artifact contains an empty driver
+configuration, one raw folder, and a real HMAC API key with read/browse/subscribe
+grants. No production authentication seam is replaced. The actual Host uses
+`Cluster:Roles=driver`, `ConfigSource:Mode=FetchAndCache`, an unreachable loopback
+central endpoint, and task-only LocalDb, secrets SQLite and GUID PKI paths. Its
+single-node Akka seed is local. Every container has `--network none`; no fleet
+endpoint, ConfigDb, existing container, volume, or PKI is used.
+
+One task-owned container runs the actual Host as its entrypoint, stops cleanly,
+then starts the same executable with the same state and PKI again. Each boot must
+serve healthy readiness, liveness and active endpoints, admit a trusted
+Basic256Sha256 SignAndEncrypt API-key client, reject a wrong key, and allow an
+actual published CLI read of ServerStatus.State with Good quality. The first boot
+also rejects an untrusted server and an untrusted client. The two certificate
+thumbprints must match. Both stops must exit zero without an OOM kill. An explicit
+execution guard requires two boots and all six strict SDK probe cases.
+
+The probe retains its public SDK ApplicationInstance throughout its session and
+disposes it afterward. Peer autoaccept is false. The CLI's existing configuration
+sets autoaccept true and has no strict-validation flag, so its successful reads
+are not counted as strict peer-validation coverage. Its certificate is prepared
+with its normal application name and URI using public SDK APIs and trusted by the
+server before startup. A closed-port CLI preflight is an expected nonzero setup
+check, not a passing connection case.
+
+`commands.json`, per-case logs, health JSON and `summary.json` retain actual exits,
+results and certificate identity. Secrets are supplied through a mode-0600 task
+environment file, never written into recorded arguments, and the file is removed
+at the end. Raw process output and this run's appended rolling-file output are
+checked for task secrets before redacted copies are saved. Only the captured,
+task-labelled container is removed; private bind-mounted state remains local for
+inspection and must not be redistributed with release artifacts.
+
+Windows cross-publish and PE/native/dependency inspection can run on macOS.
+Windows service installation and start/stop remain a separate operator gate on
+an approved Windows environment. The Wonder-box remote-execution hold still
+applies: no WinRM, psbridge, or replacement remote-execution channel. This smoke
+does not complete backend, load, failover, or two-hour soak validation.

scripts/validation/opcua-publish-smoke/runtime.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Smoke actual self-contained Linux Host/CLI artifacts with task-only PKI and LocalDb."""
+import argparse, base64, datetime, hashlib, json, os, re, secrets, shutil, subprocess, time, uuid
+from pathlib import Path
+
+p = argparse.ArgumentParser(description=__doc__)
+p.add_argument('--artifacts', required=True, type=Path)
+p.add_argument('--image', default='mcr.microsoft.com/dotnet/sdk:10.0.103-noble')
+a = p.parse_args(); out = a.artifacts.resolve()
+run = out / ('runtime-' + uuid.uuid4().hex); run.mkdir(exist_ok=False)
+state = run / 'state'; state.mkdir(); os.chmod(state, 0o700)
+identifier = uuid.uuid4().hex[:12]
+web, ua, akka = [20000 + secrets.randbelow(10000) for _ in range(3)]
+while len({web, ua, akka}) != 3: web, ua, akka = [20000 + secrets.randbelow(10000) for _ in range(3)]
+key = 'otua_' + secrets.token_hex(8) + '_' + secrets.token_urlsafe(32)
+pepper, fetch = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
+master = base64.b64encode(secrets.token_bytes(32)).decode()
+env = {
+ 'ASPNETCORE_URLS': f'http://127.0.0.1:{web}', 'ASPNETCORE_ENVIRONMENT': 'Production',
+ 'Cluster__Roles__0': 'driver',
+ 'Cluster__SystemName': 'otopcua', 'Cluster__Hostname': '127.0.0.1',
+ 'Cluster__PublicHostname': '127.0.0.1', 'Cluster__Port': str(akka),
+ 'Cluster__SeedNodes__0': f'akka.tcp://otopcua@127.0.0.1:{akka}',
+ 'ConfigSource__Mode': 'FetchAndCache', 'ConfigSource__CentralFetchEndpoints__0': 'http://127.0.0.1:1',
+ 'ConfigSource__ApiKey': fetch, 'ConfigSource__FetchTimeoutSeconds': '1',
+ 'LocalDb__Path': '/state/localdb.db', 'Secrets__SqlitePath': '/state/secrets.db',
+ 'ZB_SECRETS_MASTER_KEY': master, 'Security__ApiKeys__Pepper': pepper,
+ 'Security__Ldap__Enabled': 'false', 'Security__Acl__Enabled': 'false',
+ 'OpcUa__ApplicationName': 'PublishSmoke-' + identifier,
+ 'OpcUa__ApplicationUri': 'urn:localhost:PublishSmoke:' + identifier,
+ 'OpcUa__PublicHostname': 'localhost', 'OpcUa__OpcUaPort': str(ua),
+ 'OpcUa__PkiStoreRoot': '/state/server-pki-' + identifier,
+ 'OpcUa__AllowAnonymous': 'false', 'OpcUa__AutoAcceptUntrustedClientCertificates': 'false',
+ 'OpcUa__EnabledSecurityProfiles__0': 'Basic256Sha256SignAndEncrypt',
+ 'SMOKE_API_KEY': key, 'XDG_DATA_HOME': '/state/cli-data', 'HOME': '/state/home',
+}
+envfile = run / 'private.env'; envfile.write_text(''.join(f'{k}={v}\n' for k,v in env.items())); os.chmod(envfile, 0o600)
+redactions = [key, key.split('_', 2)[2], pepper, fetch, master]
+def clean(text):
+ for secret in redactions: text = text.replace(secret, '<ephemeral-secret>')
+ return text
+records = []
+raw_secret_findings = []
+rolling_root = out / 'linux-arm64/host/logs'
+rolling_offsets = {p: p.stat().st_size for p in rolling_root.glob('*') if p.is_file()}
+def command(args, name, expected=0, timeout=90):
+ start = datetime.datetime.now(datetime.timezone.utc).isoformat()
+ result = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)
+ raw_secret_findings.append({'log': name, 'occurrences': sum(result.stdout.count(secret) for secret in redactions)})
+ (run / (name + '.log')).write_text(clean(result.stdout))
+ records.append({'name': name, 'utc': start, 'argv': [clean(str(x)) for x in args], 'exit': result.returncode})
+ (run / 'commands.json').write_text(json.dumps(records, indent=2))
+ if expected is not None and result.returncode != expected:
+  raise RuntimeError(f'{name}: exit {result.returncode}; see {run / (name + ".log")}')
+ return result
+mounts = ['--mount', f'type=bind,src={out},dst=/work', '--mount', f'type=bind,src={state},dst=/state']
+base = ['docker','run','--rm','--network','none','--label','codex.task=opcua-publish-smoke','--env-file',str(envfile),*mounts,'-w','/work/linux-arm64/host',a.image]
+probe = ['dotnet','/work/probe/PublishProbe.dll']
+cli = ['/work/linux-arm64/cli/ZB.MOM.WW.OtOpcUa.Client.CLI']
+server_pki = state / ('server-pki-' + identifier)
+container = None; summary = {'run': str(run), 'image': a.image, 'ports': {'http':web,'ua':ua,'akka':akka}, 'boots':[], 'cases':[]}
+def copy_public(source, target):
+ target.mkdir(parents=True, exist_ok=True)
+ certs = list(source.glob('*.der'))
+ if not certs: raise RuntimeError(f'No public certificates at {source}')
+ for cert in certs: shutil.copy2(cert, target / cert.name)
+def inspect(): return json.loads(subprocess.check_output(['docker','inspect',container], text=True))[0]
+def save_logs(name):
+ r = subprocess.run(['docker','logs',container],text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+ raw_secret_findings.append({'log': name, 'occurrences': sum(r.stdout.count(secret) for secret in redactions)})
+ (run / (name + '.log')).write_text(clean(r.stdout))
+def exec_probe(name, pki, expected, credential=None):
+ args = ['docker','exec']
+ if credential: args += ['-e','SMOKE_API_KEY=' + credential]
+ args += [container,*probe,'probe',pki,f'opc.tcp://localhost:{ua}/OtOpcUa',expected]
+ r = command(args,name)
+ payload = json.loads(r.stdout.strip().splitlines()[-1]); summary['cases'].append({'name':name,**payload})
+try:
+ command([*base,*probe,'seed','/state'],'seed')
+ command([*base,*probe,'prepare','/state/client-pki'],'prepare-trusted-client')
+ command([*base,*probe,'prepare','/state/untrusted-client-pki'],'prepare-untrusted-client')
+ command([*base,*cli,'--help'],'cli-help')
+ command([*base,*probe,'prepare-cli','/state/cli-data/OtOpcUaClient/pki'],'prepare-cli-certificate')
+ failed = command([*base,*cli,'read','--url','opc.tcp://localhost:1/OtOpcUa','--security','encrypt','--username','apikey','--password',key,'--node','i=2259'],'cli-prepare-on-closed-port',expected=None)
+ if failed.returncode == 0: raise RuntimeError('Closed-port CLI fixture preparation unexpectedly succeeded.')
+ copy_public(state/'client-pki/own/certs',server_pki/'trusted/certs')
+ copy_public(state/'cli-data/OtOpcUaClient/pki/own/certs',server_pki/'trusted/certs')
+ name = 'otopcua-sdk2-publish-' + identifier
+ result = command(['docker','create','--name',name,'--network','none','--label','codex.task=opcua-publish-smoke','--env-file',str(envfile),*mounts,'-w','/work/linux-arm64/host',a.image,'./OtOpcUa.Host'],'container-create')
+ container = result.stdout.strip()
+ summary['container'] = container
+ for boot in [1,2]:
+  command(['docker','start',container],f'boot{boot}-start')
+  deadline = time.monotonic() + 90
+  health = None
+  while time.monotonic() < deadline:
+   if not inspect()['State']['Running']:
+    save_logs(f'boot{boot}-host'); raise RuntimeError('Full Host exited before health readiness.')
+   r = subprocess.run(['docker','exec',container,'curl','-fsS',f'http://127.0.0.1:{web}/health/ready'],text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+   if r.returncode == 0:
+    health = r.stdout; break
+   time.sleep(1)
+  if health is None:
+   save_logs(f'boot{boot}-host'); raise RuntimeError('Full Host readiness did not become healthy.')
+  if json.loads(health).get('status') != 'Healthy': raise RuntimeError('Readiness is degraded or unhealthy.')
+  (run/f'boot{boot}-ready.json').write_text(health)
+  command(['docker','exec',container,'curl','-fsS',f'http://127.0.0.1:{web}/healthz'],f'boot{boot}-live')
+  command(['docker','exec',container,'curl','-fsS',f'http://127.0.0.1:{web}/health/active'],f'boot{boot}-active')
+  certs = list((server_pki/'own/certs').glob('*.der'))
+  if len(certs) != 1: raise RuntimeError('Expected exactly one application certificate.')
+  cert_sha256 = hashlib.sha256(certs[0].read_bytes()).hexdigest()
+  thumbprint = subprocess.check_output(['openssl','x509','-inform','DER','-in',str(certs[0]),'-noout','-fingerprint','-sha1'],text=True).strip()
+  copy_public(server_pki/'own/certs',state/'client-pki/trusted/certs')
+  copy_public(server_pki/'own/certs',state/'untrusted-client-pki/trusted/certs')
+  exec_probe(f'boot{boot}-trusted-key','/state/client-pki','allow')
+  exec_probe(f'boot{boot}-wrong-key','/state/client-pki','key-deny',key+'-wrong')
+  if boot == 1:
+   hold = state/'hold-server-trust'; shutil.move(str(state/'client-pki/trusted/certs'),hold)
+   (state/'client-pki/trusted/certs').mkdir()
+   try: exec_probe('untrusted-server','/state/client-pki','peer-deny')
+   finally:
+    shutil.rmtree(state/'client-pki/trusted/certs'); shutil.move(str(hold),state/'client-pki/trusted/certs')
+   exec_probe('untrusted-client','/state/untrusted-client-pki','peer-deny')
+  r = command(['docker','exec',container,*cli,'read','--url',f'opc.tcp://localhost:{ua}/OtOpcUa','--security','encrypt','--username','apikey','--password',key,'--node','i=2259'],f'boot{boot}-actual-cli-read')
+  if not re.search(r'^Status:\s+Good(?:\s|$)', r.stdout, re.MULTILINE): raise RuntimeError('Actual CLI did not report a Good server status read.')
+  command(['docker','stop','--time','30',container],f'boot{boot}-stop',timeout=45)
+  stopped = inspect()['State']; save_logs(f'boot{boot}-host')
+  if stopped['Running'] or stopped['ExitCode'] != 0 or stopped['OOMKilled']: raise RuntimeError('Full Host did not shut down cleanly.')
+  summary['boots'].append({'boot':boot,'certificate_sha256':cert_sha256,'thumbprint':thumbprint,'exit':stopped['ExitCode'],'oomKilled':stopped['OOMKilled'],'health_ready':health})
+ if summary['boots'][0]['thumbprint'] != summary['boots'][1]['thumbprint']: raise RuntimeError('Application certificate changed between boots.')
+ summary['certificate_preserved'] = True
+ for path in rolling_root.glob('*'):
+  if not path.is_file(): continue
+  text = path.read_bytes()[rolling_offsets.get(path, 0):].decode(errors='replace')
+  raw_secret_findings.append({'log': 'rolling-' + path.name, 'occurrences': sum(text.count(secret) for secret in redactions)})
+  (run / ('rolling-' + path.name)).write_text(clean(text))
+ if len(summary['boots']) != 2 or len(summary['cases']) != 6: raise RuntimeError('Required smoke cases did not all execute.')
+ if any(x['occurrences'] for x in raw_secret_findings): raise RuntimeError('Task secret appeared in raw application output; see recorded occurrence counts.')
+ summary['result'] = 'passed'
+finally:
+ if container:
+  save_logs('host-final')
+  owned = inspect()
+  if owned['Config']['Labels'].get('codex.task') != 'opcua-publish-smoke': raise RuntimeError('Task container ownership changed.')
+  command(['docker','rm','-f',container],'container-cleanup')
+ logs = '\n'.join(x.read_text(errors='replace') for x in run.glob('*.log'))
+ summary['task_secret_occurrences_in_saved_logs'] = sum(logs.count(secret) for secret in redactions)
+ summary['raw_log_secret_findings'] = raw_secret_findings
+ summary['completed_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
+ (run/'summary.json').write_text(json.dumps(summary,indent=2))
+ envfile.unlink(missing_ok=True)
+ print(json.dumps(summary,indent=2))