<# .SYNOPSIS lmxopcua#800 Windows service gate: install a known-good OtOpcUa build as the OtOpcUaHost service, upgrade it in place to the SDK 2 candidate with preserved config/LocalDb/PKI, restart it, exercise the ScriptWorker isolation, roll back, and clean up. One phase per invocation so an operator can stop between phases. .DESCRIPTION Run elevated, in PowerShell 7, on an approved Windows node that has no OtOpcUaHost service. Everything lives under -Root (default C:\OtOpcUa800). Inputs are unpacked under \upload (see README.md). Task secrets (API keys, pepper, fetch key, secrets KEK) are generated once into \private\secrets.json, which only SYSTEM and Administrators can read; they reach the service through its registry Environment value and the probes through process environment variables, never through arguments or output. Evidence (no secrets) goes to \evidence. Phases, in order: Prepare dirs + ACLs, secrets, baseline (or -PrepareFrom candidate) staged into \OtOpcUa, Install-Services.ps1, node env Boot -Label L Start-Service, wait for readiness, snapshot PKI/ACLs, health Seed (after the first Stop) seed the deployment into the service-created LocalDb Trust cross-trust the probe/CLI certificates with the server PKI Checks -Label L strict-peer/API-key/anonymous probe matrix + the actual CLI read Stop -Label L Stop-Service, exit code, redaction scan of the logs and event log Upgrade backup, stage the candidate separately, carry config/PKI/LocalDb with ACLs, swap folders IsolationInstall Install-ScriptWorkerIsolation.ps1 (-WhatIf on both PowerShell hosts, apply with -ApplyHost, re-apply with the other host and again with -ApplyHost) + Scripting__Runner=OutOfProcess IsolationVerify (service running) worker processes, restricted token, pipe ACL, firewall rule, PE cache IsolationUninstall -Uninstall with -ApplyHost, then with the other host (idempotent), Scripting__Runner removed Rollback quarantine the candidate, restore the known-good folder, carry current PKI/LocalDb back Compare compare every PKI snapshot and the node environment across labels Cleanup delete the service, isolation leftovers, CLI state, the root folder (evidence copied first) #> [CmdletBinding()] param( [Parameter(Mandatory)] [ValidateSet('Prepare', 'Boot', 'Observe', 'Seed', 'Trust', 'Checks', 'Stop', 'Upgrade', 'IsolationInstall', 'IsolationVerify', 'IsolationUninstall', 'Rollback', 'Compare', 'Cleanup')] [string]$Phase, [string]$Label = 'x', [string]$Root = 'C:\OtOpcUa800', [string]$ServiceAccount = 'NT SERVICE\OtOpcUaHost', [int]$HttpPort = 18800, [int]$OpcUaPort = 14841, [int]$AkkaPort = 18801, [switch]$AllowNotReady, # The server under test is the SDK 1.5 known-good build. The SDK 2 probe's own UserName handler sends a password over # 64 bytes (every production 65-character API key) as RsaEncryptedSecret, which that server rejects; the production # SDK 2 CLI forces the legacy format since lmxopcua#880 and must be admitted. See README.md. [switch]$BaselineServer, # Prepare: which build the service is first installed from. 'baseline' (default) is the upgrade/rollback flow; 'candidate' # installs the SDK 2 build directly, with the install scripts of the same tree (used to re-run only the isolation step). [ValidateSet('baseline', 'candidate')] [string]$PrepareFrom = 'baseline', # IsolationInstall / IsolationUninstall: the PowerShell host that applies / rolls back first ('pwsh' = 7, 'powershell.exe' = 5.1). # The other host does the re-apply and the second uninstall, so running both phases once per value exercises each host as the # applying host and as the re-applying host. Evidence files are suffixed with -Label. [ValidateSet('pwsh', 'powershell.exe')] [string]$ApplyHost = 'pwsh' ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $PSStyle.OutputRendering = 'PlainText' $Service = 'OtOpcUaHost' $Install = Join-Path $Root 'OtOpcUa' $Upload = Join-Path $Root 'upload' $Private = Join-Path $Root 'private' $State = Join-Path $Root 'state' $Evidence = Join-Path $Root 'evidence' $Previous = Join-Path $Root 'OtOpcUa.prev-baseline' $Stage = Join-Path $Root 'stage-candidate' $Quarantine = Join-Path $Root 'quarantine-candidate' $Backup = Join-Path $Root 'backup-baseline' $PeCache = Join-Path $State 'pe-cache' $SvcKey = "HKLM:\SYSTEM\CurrentControlSet\Services\$Service" $Url = "opc.tcp://localhost:$OpcUaPort/OtOpcUa" $Http = "http://127.0.0.1:$HttpPort" $ProbeDll = Join-Path $Upload 'probe\Probe.dll' $CliExe = Join-Path $Upload 'candidate-cli\ZB.MOM.WW.OtOpcUa.Client.CLI.exe' $LegacyCliExe = Join-Path $Upload 'baseline-cli\ZB.MOM.WW.OtOpcUa.Client.CLI.exe' $CliRoot = Join-Path $env:LOCALAPPDATA 'OtOpcUaClient' function Write-Evidence([string]$Name, $Value) { $path = Join-Path $Evidence "$Name.json" $Value | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $path -Encoding utf8 Write-Host "evidence: $path" } function Get-Secrets { Get-Content -LiteralPath (Join-Path $Private 'secrets.json') -Raw | ConvertFrom-Json } function Get-SecretValues { $s = Get-Secrets $values = @($s.ApiKey, $s.ApiKey.Split('_', 3)[2], $s.DisabledKey, $s.DisabledKey.Split('_', 3)[2], $s.Pepper, $s.FetchKey, $s.MasterKey, $s.WrongKey, $s.UnknownKey) foreach ($n in 'ShortKey', 'WrongShortKey', 'UnknownShortKey', 'DisabledShortKey') { if ($s.PSObject.Properties[$n]) { $values += @($s.$n, $s.$n.Split('_', 3)[2]) } } $values | Where-Object { $_ } } function Protect-Text([string]$Text) { foreach ($secret in Get-SecretValues) { $Text = $Text.Replace($secret, '') } $Text } function New-TaskKey { $hex = -join ((1..8) | ForEach-Object { '{0:x2}' -f (Get-Random -Maximum 256) }) $bytes = [byte[]]::new(32); [System.Security.Cryptography.RandomNumberGenerator]::Fill($bytes) $secret = [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') "otua_${hex}_$secret" } function New-ShortTaskKey { # Same otua__ form with a 24-byte secret: 54 characters, under the SDK 2 client's 64-byte # RsaEncryptedSecret threshold. $hex = -join ((1..8) | ForEach-Object { '{0:x2}' -f (Get-Random -Maximum 256) }) $bytes = [byte[]]::new(24); [System.Security.Cryptography.RandomNumberGenerator]::Fill($bytes) "otua_${hex}_" + [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') } function Invoke-Cli([string]$Name, [string]$Exe, [string]$Key, [string]$Security) { $out = & $Exe read --url $Url --security $Security --username apikey --password $Key --node 'i=2259' 2>&1 | Out-String $code = $LASTEXITCODE $raw = 0; foreach ($x in Get-SecretValues) { $raw += ([regex]::Matches($out, [regex]::Escape($x))).Count } [ordered]@{ name = $Name; exe = $Exe; argv = "read --url --security $Security --username apikey --password --node i=2259" exit = $code; good = [bool]($out -match '(?m)^Status:\s+Good'); secretOccurrencesInOutput = $raw; output = (Protect-Text $out) } } function New-RandomB64([int]$Bytes) { $b = [byte[]]::new($Bytes); [System.Security.Cryptography.RandomNumberGenerator]::Fill($b); [Convert]::ToBase64String($b) } function Set-ProtectedAcl([string]$Path, [string[]]$ExtraRules) { # Protected: SYSTEM + Administrators full control, plus any explicit extra rules ("account|rights"). $acl = [System.Security.AccessControl.DirectorySecurity]::new() $acl.SetAccessRuleProtection($true, $false) $inherit = [System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit' $none = [System.Security.AccessControl.PropagationFlags]::None foreach ($id in 'NT AUTHORITY\SYSTEM', 'BUILTIN\Administrators') { $acl.AddAccessRule([System.Security.AccessControl.FileSystemAccessRule]::new($id, 'FullControl', $inherit, $none, 'Allow')) } foreach ($rule in $ExtraRules) { $account, $rights = $rule.Split('|') $acl.AddAccessRule([System.Security.AccessControl.FileSystemAccessRule]::new($account, $rights, $inherit, $none, 'Allow')) } Set-Acl -LiteralPath $Path -AclObject $acl } function Add-AllowRule([string]$Path, [string]$Account, [string]$Rights) { $acl = Get-Acl -LiteralPath $Path $acl.AddAccessRule([System.Security.AccessControl.FileSystemAccessRule]::new($Account, $Rights, [System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit', 'None', 'Allow')) Set-Acl -LiteralPath $Path -AclObject $acl } function Get-EnvNames { @((Get-ItemProperty -Path $SvcKey -Name Environment).Environment | ForEach-Object { $_.Split('=', 2)[0] }) | Sort-Object } function Get-EnvDigest { $joined = (@((Get-ItemProperty -Path $SvcKey -Name Environment).Environment) | Sort-Object) -join "`n" [Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($joined))).ToLowerInvariant() } function Get-PkiSnapshot([string]$InstallDir) { $pki = Join-Path $InstallDir 'pki' $certs = @(Get-ChildItem -LiteralPath (Join-Path $pki 'own\certs') -Filter *.der -File -ErrorAction SilentlyContinue | ForEach-Object { $c = [System.Security.Cryptography.X509Certificates.X509CertificateLoader]::LoadCertificateFromFile($_.FullName) [ordered]@{ file = $_.Name; sha256 = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash; thumbprint = $c.Thumbprint; subject = $c.Subject; notBefore = $c.NotBefore.ToUniversalTime().ToString('o'); notAfter = $c.NotAfter.ToUniversalTime().ToString('o'); keySize = $c.PublicKey.GetRSAPublicKey().KeySize } }) $files = @(Get-ChildItem -LiteralPath $pki -Recurse -Force -ErrorAction SilentlyContinue | Sort-Object FullName | ForEach-Object { $rel = $_.FullName.Substring($InstallDir.Length + 1) [ordered]@{ path = $rel; dir = $_.PSIsContainer sha256 = if ($_.PSIsContainer) { $null } else { (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash } sddl = (Get-Acl -LiteralPath $_.FullName).Sddl } }) $data = @(Get-ChildItem -LiteralPath (Join-Path $InstallDir 'data') -Recurse -Force -ErrorAction SilentlyContinue | Sort-Object FullName | ForEach-Object { [ordered]@{ path = $_.FullName.Substring($InstallDir.Length + 1); dir = $_.PSIsContainer; length = if ($_.PSIsContainer) { $null } else { $_.Length } sddl = (Get-Acl -LiteralPath $_.FullName).Sddl } }) [ordered]@{ certs = $certs; pki = $files; data = $data roots = [ordered]@{ install = (Get-Acl -LiteralPath $InstallDir).Sddl appsettings = @(Get-ChildItem -LiteralPath $InstallDir -Filter 'appsettings*.json' -File | ForEach-Object { [ordered]@{ file = $_.Name; sha256 = (Get-FileHash -LiteralPath $_.FullName).Hash; sddl = (Get-Acl -LiteralPath $_.FullName).Sddl } }) } } } function Get-ServiceRecord { $svc = Get-CimInstance Win32_Service -Filter "Name='$Service'" [ordered]@{ name = $svc.Name; state = $svc.State; startMode = $svc.StartMode; pathName = $svc.PathName; startName = $svc.StartName processId = $svc.ProcessId; exitCode = $svc.ExitCode; serviceSpecificExitCode = $svc.ServiceSpecificExitCode envNames = @(Get-EnvNames); envDigest = Get-EnvDigest exe = [ordered]@{ path = Join-Path $Install 'OtOpcUa.Host.exe'; sha256 = (Get-FileHash -LiteralPath (Join-Path $Install 'OtOpcUa.Host.exe')).Hash productVersion = (Get-Item -LiteralPath (Join-Path $Install 'OtOpcUa.Host.exe')).VersionInfo.ProductVersion } opcUaCore = (Get-Item -LiteralPath (Join-Path $Install 'Opc.Ua.Core.dll')).VersionInfo.FileVersion } } function Invoke-Isolation([string[]]$Extra, [string]$PsHost = 'powershell.exe') { # The repository installer from the candidate tree. (Until lmxopcua#883 was fixed it failed on both hosts, and so did -Uninstall.) $dir = 'install-candidate' $script = Join-Path $Upload "$dir\Install-ScriptWorkerIsolation.ps1" $out = & $PsHost -NoProfile -ExecutionPolicy Bypass -File $script -InstallRoot $Install -HostServiceAccount $ServiceAccount ` -PeCacheDir $PeCache -ConfigureHostService @Extra 2>&1 | Out-String [ordered]@{ script = $dir; host = $PsHost; args = ($Extra -join ' '); exit = $LASTEXITCODE; output = $out } } function Invoke-Http([string]$Path) { try { $r = Invoke-WebRequest -UseBasicParsing -Uri "$Http$Path" -TimeoutSec 10 -SkipHttpErrorCheck [ordered]@{ path = $Path; status = [int]$r.StatusCode; body = $r.Content } } catch { [ordered]@{ path = $Path; status = -1; body = $_.Exception.Message } } } function Invoke-Probe([string]$Name, [string[]]$ProbeArgs, [string]$Key) { $env:GATE_PROBE_KEY = $Key try { $out = & dotnet $ProbeDll @ProbeArgs 2>&1 | Out-String; $code = $LASTEXITCODE } finally { Remove-Item Env:\GATE_PROBE_KEY -ErrorAction SilentlyContinue } $raw = 0; foreach ($s in Get-SecretValues) { $raw += ([regex]::Matches($out, [regex]::Escape($s))).Count } $clean = Protect-Text $out $json = $null $last = ($clean.Trim() -split "`r?`n")[-1] try { $json = $last | ConvertFrom-Json } catch { } [ordered]@{ name = $Name; args = $ProbeArgs; exit = $code; secretOccurrencesInOutput = $raw; result = $json output = if ($json) { $null } else { $clean } } } function Wait-Ready([int]$Seconds = 180) { $deadline = (Get-Date).AddSeconds($Seconds) while ((Get-Date) -lt $deadline) { $svc = Get-Service $Service if ($svc.Status -ne 'Running') { throw "service left Running while waiting for readiness: $($svc.Status)" } $r = Invoke-Http '/health/ready' if ($r.status -eq 200) { return $r } Start-Sleep -Seconds 2 } $last = Invoke-Http '/health/ready' if ($AllowNotReady) { return $last } throw "readiness not 200 within $Seconds s (last: $($last.status))" } function Get-OtOpcUaSid([string]$Name) { # NT SERVICE\ virtual-account SID (same algorithm as ScriptWorkerIsolation.psm1 / sc.exe showsid). $hash = [System.Security.Cryptography.SHA1]::HashData([System.Text.Encoding]::Unicode.GetBytes($Name.ToUpperInvariant())) 'S-1-5-80-' + ((0..4 | ForEach-Object { [BitConverter]::ToUInt32($hash, $_ * 4) }) -join '-') } function Read-SharedText([string]$Path) { # A live log reports a stale length while open (windows-file-length-is-stale-while-open); read through a stream. $fs = [System.IO.FileStream]::new($Path, 'Open', 'Read', 'ReadWrite, Delete') try { $sr = [System.IO.StreamReader]::new($fs); $sr.ReadToEnd() } finally { $fs.Dispose() } } function Get-RedactionScan([string]$Since) { $secrets = @(Get-SecretValues) $rows = @() foreach ($f in Get-ChildItem -LiteralPath (Join-Path $Install 'logs') -File -ErrorAction SilentlyContinue) { $text = Read-SharedText $f.FullName $n = 0; foreach ($s in $secrets) { $n += ([regex]::Matches($text, [regex]::Escape($s))).Count } $rows += [ordered]@{ source = "logs\$($f.Name)"; chars = $text.Length; secretOccurrences = $n } Set-Content -LiteralPath (Join-Path $Evidence "log-$Label-$($f.Name)") -Value (Protect-Text $text) -Encoding utf8 } $events = @(Get-WinEvent -FilterHashtable @{ LogName = 'Application'; StartTime = [datetime]$Since } -ErrorAction SilentlyContinue | Where-Object { $_.ProviderName -match 'OtOpcUa|\.NET Runtime|Application Error' }) $etext = ($events | ForEach-Object { "$($_.TimeCreated.ToString('o')) $($_.ProviderName) $($_.Id) $($_.LevelDisplayName) $($_.Message)" }) -join "`n" $n = 0; foreach ($s in $secrets) { $n += ([regex]::Matches($etext, [regex]::Escape($s))).Count } $rows += [ordered]@{ source = 'EventLog:Application'; events = $events.Count; secretOccurrences = $n } Set-Content -LiteralPath (Join-Path $Evidence "eventlog-$Label.txt") -Value (Protect-Text $etext) -Encoding utf8 $rows } switch ($Phase) { 'Prepare' { if (Get-Service $Service -ErrorAction SilentlyContinue) { throw "$Service already exists on this node - refusing" } foreach ($d in $Private, $State, $Evidence) { New-Item -ItemType Directory -Force -Path $d | Out-Null } Set-ProtectedAcl $Root @() Set-ProtectedAcl $Private @() $secretsPath = Join-Path $Private 'secrets.json' if (-not (Test-Path $secretsPath)) { $key = New-TaskKey $wrong = $key.Substring(0, $key.Length - 4) + ($(if ($key.EndsWith('AAAA')) { 'BBBB' } else { 'AAAA' })) [ordered]@{ ApiKey = $key; WrongKey = $wrong; DisabledKey = New-TaskKey; UnknownKey = New-TaskKey Pepper = New-RandomB64 32; FetchKey = New-RandomB64 32; MasterKey = New-RandomB64 32 } | ConvertTo-Json | Set-Content -LiteralPath $secretsPath -Encoding utf8 } $s = Get-Secrets # Known-good binaries into the service path, with an explicit service-account ACL. $from = $PrepareFrom robocopy (Join-Path $Upload "$from-host") $Install /E /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null if ($LASTEXITCODE -ge 8) { throw 'baseline copy failed' } # The repository-supported install (Install-Services.ps1) creates the service and its 2-entry environment. # It runs first because the NT SERVICE\ virtual account only resolves once its service exists. & (Join-Path $Upload "install-$from\Install-Services.ps1") -InstallRoot $Install -ServiceAccount $ServiceAccount -Roles driver if (-not (Get-Service $Service -ErrorAction SilentlyContinue)) { throw 'Install-Services.ps1 did not create the service' } Add-AllowRule $Install $ServiceAccount 'ReadAndExecute' foreach ($d in 'pki', 'data', 'logs') { $p = Join-Path $Install $d; New-Item -ItemType Directory -Force -Path $p | Out-Null; Add-AllowRule $p $ServiceAccount 'Modify' } $secretsDir = Join-Path $State 'secrets'; New-Item -ItemType Directory -Force -Path $secretsDir | Out-Null Set-ProtectedAcl $secretsDir @("$ServiceAccount|Modify") $base = @((Get-ItemProperty -Path $SvcKey -Name Environment).Environment) $nodeEnv = @( "ASPNETCORE_URLS=$Http", 'Cluster__Roles__0=driver', 'Cluster__SystemName=otopcua', 'Cluster__Hostname=127.0.0.1', 'Cluster__PublicHostname=127.0.0.1', "Cluster__Port=$AkkaPort", "Cluster__SeedNodes__0=akka.tcp://otopcua@127.0.0.1:$AkkaPort", 'ConfigSource__Mode=FetchAndCache', 'ConfigSource__CentralFetchEndpoints__0=http://127.0.0.1:1', "ConfigSource__ApiKey=$($s.FetchKey)", 'ConfigSource__FetchTimeoutSeconds=1', "Secrets__SqlitePath=$secretsDir\secrets.db", "ZB_SECRETS_MASTER_KEY=$($s.MasterKey)", "Security__ApiKeys__Pepper=$($s.Pepper)", 'Security__Ldap__Enabled=false', 'Security__Acl__Enabled=false', 'OpcUa__ApplicationName=OtOpcUaGate800', 'OpcUa__ApplicationUri=urn:localhost:OtOpcUaGate800', 'OpcUa__PublicHostname=localhost', "OpcUa__OpcUaPort=$OpcUaPort", 'OpcUa__AllowAnonymous=false', 'OpcUa__AutoAcceptUntrustedClientCertificates=false', 'OpcUa__EnabledSecurityProfiles__0=Basic256Sha256SignAndEncrypt', 'OpcUa__EnabledSecurityProfiles__1=Basic256Sha256Sign') Set-ItemProperty -Path $SvcKey -Name Environment -Type MultiString -Value ([string[]]($base + $nodeEnv)) # Probe and CLI client identities (the CLI keeps its PKI in this user's LocalAppData: back it up first). if (Test-Path $CliRoot) { robocopy $CliRoot (Join-Path $Private 'cli-backup') /E /COPYALL /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null } Set-Content -LiteralPath (Join-Path $Private 'cli-existed.txt') -Value ([string](Test-Path $CliRoot)) $prep = @( Invoke-Probe 'prepare-trusted-client' @('prepare', (Join-Path $State 'client-pki')) '' Invoke-Probe 'prepare-untrusted-client' @('prepare', (Join-Path $State 'untrusted-client-pki')) '' ) $cliPrep = & $CliExe read --url 'opc.tcp://localhost:1/OtOpcUa' --security encrypt --node 'i=2259' 2>&1 | Out-String Write-Evidence 'prepare' ([ordered]@{ utc = (Get-Date).ToUniversalTime().ToString('o'); probes = $prep cliClosedPortExit = $LASTEXITCODE; cliClosedPortOutput = (Protect-Text $cliPrep) service = Get-ServiceRecord; rootSddl = (Get-Acl $Root).Sddl; privateSddl = (Get-Acl $Private).Sddl secretsDirSddl = (Get-Acl $secretsDir).Sddl }) } 'Boot' { $start = (Get-Date).ToUniversalTime().ToString('o') Set-Content -LiteralPath (Join-Path $Evidence "boot-$Label.since") -Value $start Start-Service -Name $Service (Get-Service $Service).WaitForStatus('Running', [TimeSpan]::FromSeconds(60)) $ready = Wait-Ready $(if ($AllowNotReady) { 90 } else { 180 }) Write-Evidence "boot-$Label" ([ordered]@{ label = $Label; startUtc = $start; readyUtc = (Get-Date).ToUniversalTime().ToString('o') service = Get-ServiceRecord; health = @($ready, (Invoke-Http '/healthz'), (Invoke-Http '/health/active')) snapshot = Get-PkiSnapshot $Install }) } 'Observe' { # A start the gate did not make (e.g. the SCM auto-start after a host reboot): record it like a Boot. $svc = Get-CimInstance Win32_Service -Filter "Name='$Service'" if ($svc.State -ne 'Running') { throw "service is $($svc.State), not Running" } $started = (Get-Process -Id $svc.ProcessId).StartTime.ToUniversalTime().ToString('o') $boot = (Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToUniversalTime().ToString('o') Set-Content -LiteralPath (Join-Path $Evidence "boot-$Label.since") -Value $boot Write-Evidence "boot-$Label" ([ordered]@{ label = $Label; observedOnly = $true; osBootUtc = $boot; processStartUtc = $started service = Get-ServiceRecord; health = @((Invoke-Http '/health/ready'), (Invoke-Http '/healthz'), (Invoke-Http '/health/active')) snapshot = Get-PkiSnapshot $Install }) } 'Seed' { if ((Get-Service $Service).Status -ne 'Stopped') { throw 'stop the service before seeding' } $s = Get-Secrets # 54-character keys keep the SDK 2 client on the legacy UserName-token path, so the SDK 1.5 server's own key # checks (wrong secret, unknown id, disabled) are reached rather than failing on the token format. $changed = $false foreach ($n in 'ShortKey', 'UnknownShortKey', 'DisabledShortKey') { if (-not $s.PSObject.Properties[$n]) { $s | Add-Member -NotePropertyName $n -NotePropertyValue (New-ShortTaskKey); $changed = $true } } if (-not $s.PSObject.Properties['WrongShortKey']) { $w = $s.ShortKey.Substring(0, $s.ShortKey.Length - 4) + $(if ($s.ShortKey.EndsWith('AAAA')) { 'BBBB' } else { 'AAAA' }) $s | Add-Member -NotePropertyName WrongShortKey -NotePropertyValue $w; $changed = $true } if ($changed) { $s | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $Private 'secrets.json') -Encoding utf8 } $env:GATE_API_KEY = $s.ApiKey; $env:GATE_API_KEY_SHORT = $s.ShortKey; $env:GATE_API_KEY_DISABLED = $s.DisabledShortKey; $env:Security__ApiKeys__Pepper = $s.Pepper try { $out = & dotnet (Join-Path $Upload 'seeder-baseline\Seeder.dll') (Join-Path $Install 'data\otopcua-localdb.db') 2>&1 | Out-String; $code = $LASTEXITCODE } finally { Remove-Item Env:\GATE_API_KEY, Env:\GATE_API_KEY_SHORT, Env:\GATE_API_KEY_DISABLED, Env:\Security__ApiKeys__Pepper -ErrorAction SilentlyContinue } Write-Evidence 'seed' ([ordered]@{ exit = $code; output = (Protect-Text $out); data = (Get-PkiSnapshot $Install).data }) if ($code -ne 0) { throw "seed failed: $code" } } 'Trust' { $trusted = Join-Path $Install 'pki\trusted\certs' New-Item -ItemType Directory -Force -Path $trusted | Out-Null $copied = @() foreach ($src in (Join-Path $State 'client-pki\own\certs'), (Join-Path $CliRoot 'pki\own\certs')) { foreach ($c in Get-ChildItem -LiteralPath $src -Filter *.der -File) { Copy-Item -LiteralPath $c.FullName -Destination $trusted -Force; $copied += $c.Name } } foreach ($client in 'client-pki', 'untrusted-client-pki') { $t = Join-Path $State "$client\trusted\certs"; New-Item -ItemType Directory -Force -Path $t | Out-Null Get-ChildItem -LiteralPath (Join-Path $Install 'pki\own\certs') -Filter *.der -File | Copy-Item -Destination $t -Force } Write-Evidence 'trust' ([ordered]@{ serverTrusted = $copied; untrustedClientNotCopied = (Get-ChildItem (Join-Path $State 'untrusted-client-pki\own\certs') -Filter *.der).Name }) } 'Checks' { $s = Get-Secrets $client = Join-Path $State 'client-pki' $untrusted = Join-Path $State 'untrusted-client-pki' $long = if ($BaselineServer) { 'key-deny' } else { 'allow' } $cases = @( Invoke-Probe 'encrypt-valid-key-65' @('probe', $client, $Url, 'encrypt', $long) $s.ApiKey Invoke-Probe 'sign-valid-key-65' @('probe', $client, $Url, 'sign', $long) $s.ApiKey Invoke-Probe 'encrypt-valid-key-54' @('probe', $client, $Url, 'encrypt', 'allow') $s.ShortKey Invoke-Probe 'sign-valid-key-54' @('probe', $client, $Url, 'sign', 'allow') $s.ShortKey Invoke-Probe 'encrypt-wrong-secret-54' @('probe', $client, $Url, 'encrypt', 'key-deny') $s.WrongShortKey Invoke-Probe 'sign-wrong-secret-54' @('probe', $client, $Url, 'sign', 'key-deny') $s.WrongShortKey Invoke-Probe 'encrypt-unknown-key-id-54' @('probe', $client, $Url, 'encrypt', 'key-deny') $s.UnknownShortKey Invoke-Probe 'encrypt-disabled-key-54' @('probe', $client, $Url, 'encrypt', 'key-deny') $s.DisabledShortKey Invoke-Probe 'encrypt-wrong-secret-65' @('probe', $client, $Url, 'encrypt', 'key-deny') $s.WrongKey Invoke-Probe 'encrypt-unknown-key-id-65' @('probe', $client, $Url, 'encrypt', 'key-deny') $s.UnknownKey Invoke-Probe 'encrypt-anonymous' @('probe', $client, $Url, 'encrypt', 'anon-deny') '' Invoke-Probe 'encrypt-untrusted-client' @('probe', $untrusted, $Url, 'encrypt', 'peer-deny') $s.ShortKey Invoke-Probe 'sign-untrusted-client' @('probe', $untrusted, $Url, 'sign', 'peer-deny') $s.ShortKey ) # Untrusted server: the trusted client with its server trust list emptied for one attempt. $t = Join-Path $client 'trusted\certs'; $hold = Join-Path $State 'hold-server-trust' Move-Item -LiteralPath $t -Destination $hold; New-Item -ItemType Directory -Path $t | Out-Null try { $cases += Invoke-Probe 'encrypt-untrusted-server' @('probe', $client, $Url, 'encrypt', 'peer-deny') $s.ShortKey } finally { Remove-Item -LiteralPath $t -Recurse -Force; Move-Item -LiteralPath $hold -Destination $t } # Actual published CLIs (they auto-accept the server certificate by design; not strict-peer coverage). $clis = @( Invoke-Cli 'cli-sdk1-encrypt-key-65' $LegacyCliExe $s.ApiKey 'encrypt' Invoke-Cli 'cli-sdk2-encrypt-key-54' $CliExe $s.ShortKey 'encrypt' ) # lmxopcua#880: gating against both servers. A CLI built before the fix fails this against -BaselineServer. $clis += Invoke-Cli 'cli-sdk2-encrypt-key-65' $CliExe $s.ApiKey 'encrypt' # Findings recorded but not gating here (pre-existing; see README.md): the advertised endpoint set and a session # over the None endpoint. $findings = @( Invoke-Probe 'endpoints-advertised' @('endpoints', $client, $Url) '' Invoke-Probe 'none-endpoint-valid-key-54' @('probe', $client, $Url, 'none', 'allow') $s.ShortKey ) $failed = @($cases | Where-Object { $_.exit -ne 0 }) + @($clis | Where-Object { $_.exit -ne 0 -or -not $_.good }) $leaks = @(@($cases) + @($clis) + @($findings) | Where-Object { $_.secretOccurrencesInOutput -ne 0 }) Write-Evidence "checks-$Label" ([ordered]@{ label = $Label; baselineServer = [bool]$BaselineServer; utc = (Get-Date).ToUniversalTime().ToString('o') cases = $cases; clis = $clis; findings = $findings health = @((Invoke-Http '/health/ready'), (Invoke-Http '/healthz'), (Invoke-Http '/health/active')) failed = @($failed | ForEach-Object { $_.name }); outputsWithSecrets = @($leaks | ForEach-Object { $_.name }) }) if ($failed.Count -or $leaks.Count) { throw "checks failed: $(@($failed + $leaks | ForEach-Object { $_.name }) -join ', ')" } } 'Stop' { $since = Get-Content -LiteralPath (Join-Path $Evidence "boot-$Label.since") -ErrorAction SilentlyContinue if (-not $since) { $since = (Get-Date).AddHours(-1).ToUniversalTime().ToString('o') } $svcPid = (Get-CimInstance Win32_Service -Filter "Name='$Service'").ProcessId $sw = [Diagnostics.Stopwatch]::StartNew() Stop-Service -Name $Service (Get-Service $Service).WaitForStatus('Stopped', [TimeSpan]::FromSeconds(60)) $sw.Stop() Start-Sleep -Seconds 2 $workers = @(Get-Process -Name 'ZB.MOM.WW.OtOpcUa.ScriptWorker' -ErrorAction SilentlyContinue) Write-Evidence "stop-$Label" ([ordered]@{ label = $Label; utc = (Get-Date).ToUniversalTime().ToString('o'); stopSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1) hostProcessGone = -not (Get-Process -Id $svcPid -ErrorAction SilentlyContinue); leftoverWorkers = $workers.Count service = Get-ServiceRecord; redaction = @(Get-RedactionScan $since) }) } 'Upgrade' { if ((Get-Service $Service).Status -ne 'Stopped') { throw 'stop the service first' } $before = Get-ServiceRecord reg.exe export "HKLM\SYSTEM\CurrentControlSet\Services\$Service" (Join-Path $Private 'service-baseline.reg') /y | Out-Null robocopy $Install $Backup /E /COPYALL /DCOPY:DAT /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null if ($LASTEXITCODE -ge 8) { throw 'known-good backup failed' } # Stage the candidate separately, give it the install folder's DACL, then carry the node's own state over. robocopy (Join-Path $Upload 'candidate-host') $Stage /E /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null if ($LASTEXITCODE -ge 8) { throw 'candidate stage failed' } Set-Acl -LiteralPath $Stage -AclObject (Get-Acl -LiteralPath $Install) $appsettingsDiff = @(Get-ChildItem -LiteralPath $Install -Filter 'appsettings*.json' -File | ForEach-Object { $cand = Join-Path $Stage $_.Name [ordered]@{ file = $_.Name; node = (Get-FileHash $_.FullName).Hash; publisher = if (Test-Path $cand) { (Get-FileHash $cand).Hash } else { $null } } }) foreach ($f in Get-ChildItem -LiteralPath $Install -Filter 'appsettings*.json' -File) { Copy-Item -LiteralPath $f.FullName -Destination $Stage -Force } foreach ($d in 'pki', 'data', 'logs') { robocopy (Join-Path $Install $d) (Join-Path $Stage $d) /E /COPYALL /DCOPY:DAT /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null if ($LASTEXITCODE -ge 8) { throw "carry $d failed" } } $stagedSnapshot = Get-PkiSnapshot $Stage Move-Item -LiteralPath $Install -Destination $Previous Move-Item -LiteralPath $Stage -Destination $Install Write-Evidence 'upgrade' ([ordered]@{ utc = (Get-Date).ToUniversalTime().ToString('o'); before = $before; after = Get-ServiceRecord appsettingsNodeVsPublisher = $appsettingsDiff; stagedSnapshot = $stagedSnapshot backup = [ordered]@{ path = $Backup; files = (Get-ChildItem $Backup -Recurse -File).Count }; previous = $Previous }) } 'IsolationInstall' { if ((Get-Service $Service).Status -ne 'Stopped') { throw 'stop the service first' } $envBefore = @(Get-EnvNames) $aclBefore = (& icacls $Install | Out-String) $otherHost = if ($ApplyHost -eq 'pwsh') { 'powershell.exe' } else { 'pwsh' } $whatIf = Invoke-Isolation @('-WhatIf') $ApplyHost $whatIfOther = Invoke-Isolation @('-WhatIf') $otherHost $whatIfChanged = ((& icacls $Install | Out-String) -ne $aclBefore) -or [bool](Get-NetFirewallRule -Name 'OtOpcUa-ScriptWorker-BlockOutbound' -ErrorAction SilentlyContinue) -or ((@(Get-EnvNames) -join ',') -ne ($envBefore -join ',')) $apply = Invoke-Isolation @() $ApplyHost $again = Invoke-Isolation @() $otherHost $againSame = Invoke-Isolation @() $ApplyHost foreach ($r in $whatIf, $whatIfOther, $apply, $again, $againSame) { if ($r.exit -ne 0) { throw "isolation install failed ($($r.host) $($r.args)): $($r.output)" } } $env = @((Get-ItemProperty -Path $SvcKey -Name Environment).Environment) + 'Scripting__Runner=OutOfProcess' Set-ItemProperty -Path $SvcKey -Name Environment -Type MultiString -Value ([string[]]$env) $rule = Get-NetFirewallRule -Name 'OtOpcUa-ScriptWorker-BlockOutbound' Write-Evidence "isolation-install-$Label" ([ordered]@{ applyHost = $ApplyHost; whatIf = $whatIf; whatIfOtherHost = $whatIfOther; whatIfChangedState = $whatIfChanged apply = $apply; reapply = $again; reapplySameHost = $againSame reapplyAllUnchanged = (-not ($again.output -match '\[(?!unchanged)[a-z]+\s*\]')) -and (-not ($againSame.output -match '\[(?!unchanged)[a-z]+\s*\]')) reapplyStepCounts = @(([regex]::Matches($again.output, '\[unchanged\s*\]')).Count, ([regex]::Matches($againSame.output, '\[unchanged\s*\]')).Count) envBefore = $envBefore; envAfter = @(Get-EnvNames) anchor = (& sc.exe qc OtOpcUaScriptWorker | Out-String); anchorSid = (& sc.exe showsid OtOpcUaScriptWorker | Out-String) installAcl = (& icacls $Install | Out-String); workerDirAcl = (& icacls (Join-Path $Install 'script-worker') | Out-String) peCacheAcl = (& icacls $PeCache | Out-String); peAcl = (& icacls (Join-Path $PeCache 'pe') | Out-String) firewall = [ordered]@{ name = $rule.Name; direction = "$($rule.Direction)"; action = "$($rule.Action)"; enabled = "$($rule.Enabled)"; profile = "$($rule.Profile)" program = ($rule | Get-NetFirewallApplicationFilter).Program } workerExeSha256 = (Get-FileHash (Join-Path $Install 'script-worker\ZB.MOM.WW.OtOpcUa.ScriptWorker.exe')).Hash workerFiles = (Get-ChildItem (Join-Path $Install 'script-worker') -Recurse -File).Count }) } 'IsolationVerify' { Add-Type -TypeDefinition @' using System; using System.Runtime.InteropServices; public static class Tok { [DllImport("kernel32.dll", SetLastError=true)] public static extern IntPtr OpenProcess(uint a, bool i, int pid); [DllImport("advapi32.dll", SetLastError=true)] public static extern bool OpenProcessToken(IntPtr p, uint a, out IntPtr t); [DllImport("advapi32.dll", SetLastError=true)] public static extern bool IsTokenRestricted(IntPtr t); [DllImport("advapi32.dll", SetLastError=true)] public static extern bool GetTokenInformation(IntPtr t, int c, IntPtr b, int l, out int r); [DllImport("kernel32.dll")] public static extern bool CloseHandle(IntPtr h); public static string Inspect(int pid) { IntPtr p = OpenProcess(0x1000, false, pid); if (p == IntPtr.Zero) return "OpenProcess failed " + Marshal.GetLastWin32Error(); IntPtr t; if (!OpenProcessToken(p, 0x0008, out t)) { int e = Marshal.GetLastWin32Error(); CloseHandle(p); return "OpenProcessToken failed " + e; } bool r = IsTokenRestricted(t); int len; GetTokenInformation(t, 3, IntPtr.Zero, 0, out len); IntPtr b = Marshal.AllocHGlobal(len); int privs = -1; if (GetTokenInformation(t, 3, b, len, out len)) privs = Marshal.ReadInt32(b); Marshal.FreeHGlobal(b); CloseHandle(t); CloseHandle(p); return "restricted=" + r + " privilegeCount=" + privs; } } '@ $hostPid = (Get-CimInstance Win32_Service -Filter "Name='$Service'").ProcessId $workers = @(Get-CimInstance Win32_Process -Filter "Name='ZB.MOM.WW.OtOpcUa.ScriptWorker.exe'") $procs = @($workers | ForEach-Object { [ordered]@{ pid = $_.ProcessId; parent = $_.ParentProcessId; parentIsHost = ($_.ParentProcessId -eq $hostPid) path = $_.ExecutablePath; owner = (Invoke-CimMethod -InputObject $_ -MethodName GetOwner | ForEach-Object { "$($_.Domain)\$($_.User)" }) token = [Tok]::Inspect([int]$_.ProcessId) } }) $pipes = @([System.IO.Directory]::GetFiles('\\.\pipe\') | Where-Object { $_ -match 'otopcua-sw-' }) $pipeChecks = @($pipes | ForEach-Object { $name = $_.Substring(9) $c = [System.IO.Pipes.NamedPipeClientStream]::new('.', $name, 'InOut') $outcome = try { $c.Connect(2000); 'CONNECTED (unexpected)' } catch [UnauthorizedAccessException] { 'denied (UnauthorizedAccessException)' } catch { "other: $($_.Exception.GetType().Name): $($_.Exception.Message)" } finally { $c.Dispose() } [ordered]@{ pipe = 'otopcua-sw-'; adminConnect = $outcome } }) $ready = Invoke-Http '/health/ready' Write-Evidence "isolation-verify-$Label" ([ordered]@{ hostPid = $hostPid; workers = $procs; pipes = $pipeChecks; ready = $ready peFiles = @(Get-ChildItem (Join-Path $PeCache 'pe') -File -ErrorAction SilentlyContinue).Count hostLogWorkerLines = @(Get-ChildItem (Join-Path $Install 'logs') -File | ForEach-Object { (Read-SharedText $_.FullName) -split "`r?`n" } | Where-Object { $_ -match 'ScriptWorker|script-runner|OutOfProcess' } | Select-Object -Last 30 | ForEach-Object { Protect-Text $_ }) }) } 'IsolationUninstall' { if ((Get-Service $Service).Status -ne 'Stopped') { throw 'stop the service first' } $otherHost = if ($ApplyHost -eq 'pwsh') { 'powershell.exe' } else { 'pwsh' } $first = Invoke-Isolation @('-Uninstall') $ApplyHost $second = Invoke-Isolation @('-Uninstall') $otherHost foreach ($r in $first, $second) { if ($r.exit -ne 0) { throw "isolation uninstall failed ($($r.host)): $($r.output)" } } $env = @((Get-ItemProperty -Path $SvcKey -Name Environment).Environment | Where-Object { $_ -notlike 'Scripting__Runner=*' }) Set-ItemProperty -Path $SvcKey -Name Environment -Type MultiString -Value ([string[]]$env) Write-Evidence "isolation-uninstall-$Label" ([ordered]@{ applyHost = $ApplyHost; first = $first; second = $second secondAllUnchanged = -not ($second.output -match '\[(?!unchanged)[a-z]+\s*\]') anchorGone = -not (Get-Service OtOpcUaScriptWorker -ErrorAction SilentlyContinue) firewallGone = -not (Get-NetFirewallRule -Name 'OtOpcUa-ScriptWorker-BlockOutbound' -ErrorAction SilentlyContinue) installAcl = (& icacls $Install | Out-String); envNames = @(Get-EnvNames); envDigest = Get-EnvDigest }) } 'Rollback' { if ((Get-Service $Service).Status -ne 'Stopped') { throw 'stop the service first' } $before = Get-PkiSnapshot $Install Move-Item -LiteralPath $Install -Destination $Quarantine Move-Item -LiteralPath $Previous -Destination $Install foreach ($d in 'pki', 'data') { robocopy (Join-Path $Quarantine $d) (Join-Path $Install $d) /E /COPYALL /DCOPY:DAT /IS /IT /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null if ($LASTEXITCODE -ge 8) { throw "restore $d failed" } } Write-Evidence 'rollback' ([ordered]@{ utc = (Get-Date).ToUniversalTime().ToString('o'); candidateSnapshot = $before restored = Get-ServiceRecord; restoredSnapshot = Get-PkiSnapshot $Install; quarantine = $Quarantine }) } 'Compare' { $labels = $Label.Split(',') $rows = foreach ($l in $labels) { $b = Get-Content (Join-Path $Evidence "boot-$l.json") -Raw | ConvertFrom-Json [ordered]@{ label = $l; exe = $b.service.exe.productVersion; opcUaCore = $b.service.opcUaCore; envDigest = $b.service.envDigest certs = @($b.snapshot.certs | ForEach-Object { "$($_.thumbprint) $($_.sha256)" }) pkiDigest = [Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes((($b.snapshot.pki | ForEach-Object { "$($_.path)|$($_.sha256)|$($_.sddl)" }) -join "`n")))).ToLowerInvariant() pkiEntries = @($b.snapshot.pki).Count dataAcl = @($b.snapshot.data | ForEach-Object { "$($_.path)|$($_.sddl)" }) } } $first = $rows[0] $same = [ordered]@{ certificate = @($rows | Where-Object { ($_.certs -join ';') -ne ($first.certs -join ';') }).Count -eq 0 pkiFilesAndAcls = @($rows | Where-Object { $_.pkiDigest -ne $first.pkiDigest }).Count -eq 0 environment = @($rows | Where-Object { $_.envDigest -ne $first.envDigest }).Count -eq 0 } Write-Evidence "compare-$($labels -join '-')" ([ordered]@{ rows = $rows; identical = $same }) } 'Cleanup' { $out = [ordered]@{} if (Get-Service $Service -ErrorAction SilentlyContinue) { Stop-Service $Service -ErrorAction SilentlyContinue (Get-Service $Service).WaitForStatus('Stopped', [TimeSpan]::FromSeconds(60)) $out.delete = (& sc.exe delete $Service | Out-String) # NOT Uninstall-Services.ps1: it also removes OtOpcUaWonderwareHistorian } if (Get-Service OtOpcUaScriptWorker -ErrorAction SilentlyContinue) { $out.anchorDelete = (& sc.exe delete OtOpcUaScriptWorker | Out-String) } Get-NetFirewallRule -Name 'OtOpcUa-ScriptWorker-BlockOutbound' -ErrorAction SilentlyContinue | Remove-NetFirewallRule Remove-Item -LiteralPath $CliRoot -Recurse -Force -ErrorAction SilentlyContinue if ((Get-Content (Join-Path $Private 'cli-existed.txt')) -eq 'True') { robocopy (Join-Path $Private 'cli-backup') $CliRoot /E /COPYALL /R:1 /W:1 /NFL /NDL /NJH /NP | Out-Null } $out.cliRestored = @(Get-ChildItem $CliRoot -Recurse -File -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName }) foreach ($sid in @((Get-OtOpcUaSid 'OtOpcUaHost'), (Get-OtOpcUaSid 'OtOpcUaScriptWorker'))) { $p = Get-CimInstance Win32_UserProfile | Where-Object { $_.SID -eq $sid } if ($p) { $out["profile-$sid"] = $p.LocalPath; $p | Remove-CimInstance } } $out | ConvertTo-Json -Depth 5 } }