4890a3d16e
The process guards matched only ZB.MOM.WW.MxGateway.*; the wonder boxes run MxGateway.Server.exe / MxGateway.Worker.exe, so the one-gateway guard, the pre-start OrphanWorkerTerminator guard and the STOP_PENDING kill passed vacuously. - Image names come from the installed exes (NSSM Application, Worker env path) plus both known schemes; processes are also matched by exact path (Win32_Process). - One-gateway guard fails closed: a RUNNING service whose gateway is not visible, at the Application path, as the service host's child, is an abort. - STOP_PENDING kill is scoped to this install's processes. - Staged checks use the installed exe names; staged Worker must be x86; a staged self-contained runtime older than the live one is refused (-AllowRuntimeDowngrade). - Carry-over: appsettings*.json and certificates/keys (dashboard.pfx) always keep the live copy; other live-only non-binary top-level files are carried; hash-verified. - Env block read from the registry and fingerprinted (names + SHA-256 prefixes, never values) after stop, swap, start and the stability window; any change aborts. Sensitive values are redacted from all output and notes. - File-based master key (vd03) accepted and backed up; a missing env master key (windev) is a warning. Backup dir is ACL-restricted before anything is written. - Rollback accepts .bak-<stamp> and .bak.<yyyyMMddTHHmmss> sets and refuses a mixed Server/Worker stamp; auto-rollback no longer fails on the old SHA. - -WhatIf now works on Windows PowerShell 5.1 (Get-FileHash returned nothing under WhatIfPreference); an empty Win32_Service result no longer reads as "unavailable". - Install-MxGatewayService.ps1 process guard matches both schemes. - Dry-run harness scripts/deploy/tests (fake box in both layouts, 140 assertions, passes on pwsh/macOS and Windows PowerShell 5.1) plus a mutation runner (12/12 killed). - Runbook updated. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
383 lines
25 KiB
PowerShell
383 lines
25 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Dry-run harness for scripts/deploy/Upgrade-MxGatewayService.ps1 (mxaccessgw#230). Runs the script's real logic
|
|
against a fake box: a temp directory laid out like wonder-app-vd03 (MxGateway.Server.exe / MxGateway.Worker.exe,
|
|
dashboard.pfx, file-based master key) or like windev (ZB.MOM.WW.MxGateway.*.exe, dated Server dir), with a fake
|
|
nssm, process table, registry env block, health endpoint and clock (tests/UpgradeFakes.ps1). The repo has no
|
|
Pester; this is a plain script that exits non-zero when any assertion fails.
|
|
|
|
Besides -DryRun scenarios it runs full upgrades and rollbacks for real against the temp directories (renames,
|
|
copies, carry-over, hash checks, auto-rollback), because only the service manager and processes are faked.
|
|
|
|
.PARAMETER ScriptPath The upgrade script under test (default: ../Upgrade-MxGatewayService.ps1). The mutation runner
|
|
points this at mutated copies to prove the assertions can fail.
|
|
|
|
.EXAMPLE
|
|
pwsh -NoProfile -File scripts/deploy/tests/Test-UpgradeMxGatewayService.ps1
|
|
#>
|
|
param([string]$ScriptPath)
|
|
Set-StrictMode -Off
|
|
$ErrorActionPreference = 'Stop'
|
|
$here = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
if (-not $ScriptPath) { $ScriptPath = Join-Path (Split-Path -Parent $here) 'Upgrade-MxGatewayService.ps1' }
|
|
$FakesPath = Join-Path $here 'UpgradeFakes.ps1'
|
|
|
|
$script:failures = 0
|
|
$script:passes = 0
|
|
$script:group = ''
|
|
function Assert([bool]$Condition, [string]$Message) {
|
|
if ($Condition) { $script:passes++ } else { $script:failures++; Write-Host " FAIL [$($script:group)]: $Message" -ForegroundColor Red }
|
|
}
|
|
# Not 'Group': that is a built-in alias of Group-Object, and aliases win over functions.
|
|
# An exception in a section is a failed assertion, and the run still reaches its verdict.
|
|
trap { $script:failures++; Write-Host " FAIL [$($script:group)]: exception: $($_.Exception.Message)" -ForegroundColor Red; continue }
|
|
function Section([string]$Name) { $script:group = $Name; Write-Host $Name }
|
|
|
|
$Pepper = 'pepper-VALUE-do-not-print-7f3a'
|
|
$LdapPw = 'ldap-PASSWORD-do-not-print-91c2'
|
|
$OldSha = '2b13564e'
|
|
$NewSha = '76a71bc8'
|
|
|
|
function New-File([string]$Path, [string]$Content) {
|
|
[void][IO.Directory]::CreateDirectory((Split-Path -Parent $Path)) # New-Item -Force costs ~70 ms per call on macOS
|
|
[IO.File]::WriteAllText($Path, $Content)
|
|
}
|
|
|
|
# Scheme 'wonder' = vd03 (MxGateway.*.exe, dashboard.pfx in Server\, file KEK in data\); 'windev' = ZB.MOM.WW.* in a dated dir.
|
|
function New-FakeBox([string]$Scheme) {
|
|
$root = Join-Path ([IO.Path]::GetTempPath()) ("mxgw230-" + [guid]::NewGuid().ToString('n').Substring(0, 8))
|
|
if ($Scheme -eq 'wonder') { $srvName = 'MxGateway.Server.exe'; $wrkName = 'MxGateway.Worker.exe'; $srvDir = "$root/Server" }
|
|
else { $srvName = 'ZB.MOM.WW.MxGateway.Server.exe'; $wrkName = 'ZB.MOM.WW.MxGateway.Worker.exe'; $srvDir = "$root/Server-20261112-$OldSha" }
|
|
$wrkDir = "$root/Worker"
|
|
# live install
|
|
New-File "$srvDir/$srvName" "sha=$OldSha;arch=Amd64"
|
|
New-File "$srvDir/ZB.MOM.WW.MxGateway.Server.dll" "old server dll $OldSha"
|
|
New-File "$srvDir/Old.Removed.Dependency.dll" 'binary only in the old build'
|
|
New-File "$srvDir/appsettings.json" '{ "live": "host config with no pepper" }'
|
|
New-File "$srvDir/appsettings.Development.json" '{ "live": "dev overlay" }'
|
|
New-File "$srvDir/mscordaccore_amd64_amd64_10.0.1426.12345.dll" 'runtime 10.0.14'
|
|
if ($Scheme -eq 'wonder') { New-File "$srvDir/dashboard.pfx" 'PFX-BYTES-live-dashboard-cert' }
|
|
New-File "$srvDir/notes-left-by-an-operator.txt" 'live-only text file'
|
|
[void][IO.Directory]::CreateDirectory("$srvDir/logs")
|
|
New-File "$wrkDir/$wrkName" "sha=$OldSha;arch=X86"
|
|
New-File "$wrkDir/$wrkName.config" '<configuration old/>'
|
|
New-File "$root/data/gateway-auth.db" 'auth-db'; New-File "$root/data/gateway-auth.db-wal" 'auth-wal'
|
|
New-File "$root/data/mxgateway-secrets.db" 'secrets-db'; New-File "$root/data/kek.key" 'KEK-BYTES'
|
|
# staged new build (the repo appsettings.json, no pfx)
|
|
New-File "$root/stage/Server/$srvName" "sha=$NewSha;arch=Amd64"
|
|
New-File "$root/stage/Server/ZB.MOM.WW.MxGateway.Server.dll" "new server dll $NewSha"
|
|
New-File "$root/stage/Server/appsettings.json" '{ "repo": "default config" }'
|
|
New-File "$root/stage/Server/mscordaccore_amd64_amd64_10.0.1426.12345.dll" 'runtime 10.0.14'
|
|
New-File "$root/stage/Worker/$wrkName" "sha=$NewSha;arch=X86"
|
|
New-File "$root/stage/Worker/$wrkName.config" '<configuration new/>'
|
|
|
|
$app = "$srvDir/$srvName"; $wexe = "$wrkDir/$wrkName"
|
|
$env = @(
|
|
'ASPNETCORE_ENVIRONMENT=Production',
|
|
"MxGateway__Worker__ExecutablePath=$wexe",
|
|
"MxGateway__Authentication__SqlitePath=$root/data/gateway-auth.db",
|
|
"Secrets__SqlitePath=$root/data/mxgateway-secrets.db",
|
|
"MxGateway__ApiKeyPepper=$Pepper",
|
|
"MxGateway__Ldap__ServiceAccountPassword=$LdapPw")
|
|
if ($Scheme -eq 'wonder') { $env += @("Secrets__MasterKey__FilePath=$root/data/kek.key", 'Secrets__MasterKey__Source=File') }
|
|
$hostPid = 4100
|
|
$box = @{
|
|
Scheme = $Scheme; Root = $root; ServerDir = $srvDir; WorkerDir = $wrkDir; Application = $app; AppDirectory = $srvDir; WorkerExe = $wexe
|
|
State = 'SERVICE_RUNNING'; Env = $env; HostPid = $hostPid; Health = 200; StopHangs = $false; Starts = 0; OnStart = $null
|
|
OtherServices = @(); Survivors = @()
|
|
Procs = @(
|
|
[pscustomobject]@{ Id = 4200; ParentId = $hostPid; Name = [IO.Path]::GetFileNameWithoutExtension($srvName); Path = $app; Role = 'Server' },
|
|
[pscustomobject]@{ Id = 4201; ParentId = 4200; Name = [IO.Path]::GetFileNameWithoutExtension($wrkName); Path = $wexe; Role = 'Worker' },
|
|
[pscustomobject]@{ Id = 4202; ParentId = 4200; Name = [IO.Path]::GetFileNameWithoutExtension($wrkName); Path = $wexe; Role = 'Worker' },
|
|
[pscustomobject]@{ Id = 900; ParentId = 1; Name = 'svchost'; Path = 'C:/Windows/System32/svchost.exe'; Role = 'Other' })
|
|
NssmCalls = (New-Object System.Collections.Generic.List[string]); Killed = (New-Object System.Collections.Generic.List[int])
|
|
HttpCalls = (New-Object System.Collections.Generic.List[string]); Protected = (New-Object System.Collections.Generic.List[string])
|
|
StartedWith = (New-Object System.Collections.Generic.List[string])
|
|
}
|
|
$box
|
|
}
|
|
|
|
function Get-TreeHash([string]$Root) {
|
|
$d = New-Object IO.DirectoryInfo $Root
|
|
((@($d.GetDirectories('*', 'AllDirectories') | ForEach-Object { "D $($_.FullName)" }) + @($d.GetFiles('*', 'AllDirectories') | ForEach-Object { "F $($_.FullName) $((Get-FileHash -LiteralPath $_.FullName).Hash)" })) | Sort-Object) -join "`n"
|
|
}
|
|
|
|
# Runs Invoke-Main of the script under test against $Box with the fakes loaded. Returns rc + all output text.
|
|
function Invoke-Scenario([hashtable]$Box, [hashtable]$P) {
|
|
$script:Box = $Box
|
|
$script:FakeNow = [datetime]'2026-11-12T12:00:00'
|
|
$base = @{ ServerSource = "$($Box.Root)/stage/Server"; WorkerSource = "$($Box.Root)/stage/Worker"; ExpectedSha = $NewSha; MinFreeGB = 0.001; StableSeconds = 10 }
|
|
foreach ($k in $P.Keys) { $base[$k] = $P[$k] }
|
|
foreach ($k in @($base.Keys)) { if ($null -eq $base[$k]) { $base.Remove($k) } }
|
|
. $ScriptPath @base
|
|
. $FakesPath
|
|
$records = @(Invoke-Main 6>&1 3>&1)
|
|
$rc = @($records | Where-Object { $_ -is [int] })[-1]
|
|
$text = (@($records | Where-Object { $_ -isnot [int] } | ForEach-Object { "$_" }) -join "`n")
|
|
[pscustomobject]@{ Rc = $rc; Text = $text }
|
|
}
|
|
|
|
function Assert-NoSecrets([string]$Text, [string]$Where) {
|
|
Assert (-not $Text.Contains($Pepper)) "the pepper value does not appear in $Where"
|
|
Assert (-not $Text.Contains($LdapPw)) "the LDAP password does not appear in $Where"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------------------------------------------
|
|
foreach ($scheme in 'wonder', 'windev') {
|
|
Section "Dry run on a $scheme layout sees the live gateway and changes nothing"
|
|
$b = New-FakeBox $scheme
|
|
$before = Get-TreeHash $b.Root
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 0) "dry run exits 0 (rc=$($r.Rc)) $(if ($r.Rc -ne 0) { $r.Text })"
|
|
$stem = [IO.Path]::GetFileNameWithoutExtension($b.Application)
|
|
Assert ($r.Text -match "gateway Server #4200 is the service's own") 'the guard found the service gateway by PID/parent'
|
|
Assert ($r.Text -match [regex]::Escape("$stem#4200")) "the process listing shows $stem#4200"
|
|
Assert ($r.Text -match "MxGateway.Worker.exe#4201|$([regex]::Escape([IO.Path]::GetFileNameWithoutExtension($b.WorkerExe)))#4201") 'the Worker processes are seen'
|
|
Assert ((Get-TreeHash $b.Root) -eq $before) 'no file or directory changed'
|
|
Assert (@($b.NssmCalls | Where-Object { $_ -match '^(stop|start|set)' }).Count -eq 0) "no nssm stop/start/set ($($b.NssmCalls -join '; '))"
|
|
Assert ($b.Killed.Count -eq 0) 'no process killed'
|
|
Assert ($r.Text -match '\[DRYRUN\] nssm stop') 'the stop is printed, not run'
|
|
Assert ($r.Text -match 'no other service launches') 'the other-service cross-check ran (an empty result is not "unavailable")'
|
|
Assert ($r.Text -match 'carry from live -> [^\r\n]*appsettings\.json') 'plan carries appsettings.json'
|
|
Assert ($r.Text -match 'carry from live -> [^\r\n]*notes-left-by-an-operator\.txt') 'plan carries a live-only text file'
|
|
Assert ($r.Text -match 'dropped -> Old\.Removed\.Dependency\.dll') 'plan drops a live-only binary'
|
|
Assert ($r.Text -match 'live-only directories NOT carried[^\r\n]*logs') 'plan warns about the live-only logs dir'
|
|
Assert ($r.Text -match 'MxGateway__ApiKeyPepper=[0-9a-f]{12}') 'the env fingerprint lists the pepper by name + hash prefix'
|
|
Assert-NoSecrets $r.Text 'the dry-run output'
|
|
if ($scheme -eq 'wonder') {
|
|
Assert ($r.Text -match 'carry from live -> [^\r\n]*dashboard\.pfx') 'plan carries dashboard.pfx'
|
|
Assert ($r.Text -match 'master key: file') 'file-based master key accepted'
|
|
} else {
|
|
Assert ($r.Text -match 'carries no master key') 'missing env master key is a warning on windev, not an abort'
|
|
Assert ($r.Text -match 'carries SHA 2b13564e but will hold 76a71bc8') 'dated dir name mismatch is flagged'
|
|
}
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
}
|
|
|
|
Section '-WhatIf is the same as -DryRun'
|
|
$b = New-FakeBox 'windev'
|
|
$before = Get-TreeHash $b.Root
|
|
$r = Invoke-Scenario $b @{ WhatIf = $true }
|
|
Assert ($r.Rc -eq 0 -and $r.Text -match 'dry=True' -and $r.Text -match '== DONE ==') "-WhatIf runs the whole plan (rc=$($r.Rc))"
|
|
Assert ($r.Text -match '\[DRYRUN\] carry live Server\\appsettings\.json forward[^\r\n]*sha256 [0-9A-F]{12}') 'carry-over hashes are computed under -WhatIf'
|
|
Assert ((Get-TreeHash $b.Root) -eq $before -and $b.Starts -eq 0) 'nothing changed'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Fail closed: service RUNNING but no gateway process visible'
|
|
$b = New-FakeBox 'wonder'
|
|
$b.Procs = @($b.Procs | Where-Object { $_.Role -eq 'Other' })
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1) 'refuses (rc 1)'
|
|
Assert ($r.Text -match 'RUNNING but no gateway Server process was found') "says why ($($r.Text -split "`n" | Select-String ERROR))"
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Fail closed: the gateway runs under an image name and path the script does not know'
|
|
$b = New-FakeBox 'wonder'
|
|
foreach ($p in $b.Procs) { if ($p.Role -ne 'Other') { $p.Name = 'Renamed.Gateway'; $p.Path = $null } }
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1) 'refuses (rc 1)'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Path match works when the image name is unexpected'
|
|
$b = New-FakeBox 'wonder'
|
|
foreach ($p in $b.Procs) { if ($p.Role -eq 'Server') { $p.Name = 'Renamed.Gateway' } }
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 0) "passes via the exact Application path (rc=$($r.Rc))"
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
foreach ($scheme in 'wonder', 'windev') {
|
|
Section "Second gateway of the OTHER naming scheme is caught ($scheme box)"
|
|
$b = New-FakeBox $scheme
|
|
$otherName = if ($scheme -eq 'wonder') { 'ZB.MOM.WW.MxGateway.Server' } else { 'MxGateway.Server' }
|
|
$b.Procs = @($b.Procs) + @([pscustomobject]@{ Id = 5500; ParentId = 77; Name = $otherName; Path = "C:/elsewhere/$otherName.exe"; Role = 'Other' })
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1) 'refuses (rc 1)'
|
|
Assert ($r.Text -match '2 gateway Server processes') 'reports two gateways'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
}
|
|
|
|
Section 'Another service launching the gateway exe'
|
|
$b = New-FakeBox 'wonder'
|
|
$b.OtherServices = @('MxAccessGwTest')
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'another service \(MxAccessGwTest\)') 'refuses a second service'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Service STOPPED with a stray Worker of the other scheme'
|
|
$b = New-FakeBox 'wonder'
|
|
$b.State = 'SERVICE_STOPPED'
|
|
$b.Procs = @([pscustomobject]@{ Id = 5600; ParentId = 1; Name = 'ZB.MOM.WW.MxGateway.Worker'; Path = $null; Role = 'Other' })
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1) 'refuses (rc 1)'
|
|
Assert ($r.Text -match 'STOPPED but gateway process') 'says why'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Gateway that is not the service host child'
|
|
$b = New-FakeBox 'wonder'
|
|
$b.Procs[0].ParentId = 1234
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'is not a child of the MxAccessGw service host') 'refuses a gateway with the wrong parent'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Staged Worker must carry the installed exe name'
|
|
$b = New-FakeBox 'wonder'
|
|
Rename-Item -LiteralPath "$($b.Root)/stage/Worker/MxGateway.Worker.exe" -NewName 'ZB.MOM.WW.MxGateway.Worker.exe'
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'does not contain MxGateway\.Worker\.exe') 'refuses a staged Worker with the other exe name'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Staged Worker must be x86'
|
|
$b = New-FakeBox 'wonder'
|
|
Set-Content -LiteralPath "$($b.Root)/stage/Worker/MxGateway.Worker.exe" -Value "sha=$NewSha;arch=Amd64"
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'not X86') 'refuses an x64 Worker'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Self-contained runtime downgrade is refused'
|
|
$b = New-FakeBox 'wonder'
|
|
Rename-Item -LiteralPath "$($b.Root)/stage/Server/mscordaccore_amd64_amd64_10.0.1426.12345.dll" -NewName 'mscordaccore_amd64_amd64_10.0.826.20000.dll'
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'OLDER than the live') 'refuses 10.0.8 over 10.0.14'
|
|
$r = Invoke-Scenario $b @{ DryRun = $true; AllowRuntimeDowngrade = $true }
|
|
Assert ($r.Rc -eq 0) '-AllowRuntimeDowngrade overrides it'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'A store inside the swapped dir is refused'
|
|
$b = New-FakeBox 'wonder'
|
|
New-File "$($b.ServerDir)/stray.db" 'x'
|
|
$r = Invoke-Scenario $b @{ DryRun = $true }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'would be orphaned by the rename') 'refuses a .db under Server'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
# ---------------------------------------------------------------------------------------------------------------
|
|
foreach ($scheme in 'wonder', 'windev') {
|
|
Section "Real upgrade on a fake $scheme box (files really swapped)"
|
|
$b = New-FakeBox $scheme
|
|
$pfxHash = if ($scheme -eq 'wonder') { (Get-FileHash "$($b.ServerDir)/dashboard.pfx").Hash } else { $null }
|
|
$cfgHash = (Get-FileHash "$($b.ServerDir)/appsettings.json").Hash
|
|
$devHash = (Get-FileHash "$($b.ServerDir)/appsettings.Development.json").Hash
|
|
$r = Invoke-Scenario $b @{ Label = 'test' }
|
|
Assert ($r.Rc -eq 0) "upgrade exits 0 (rc=$($r.Rc)) $(if ($r.Rc -ne 0) { $r.Text })"
|
|
Assert ((Get-Content -Raw $b.Application) -match "sha=$NewSha") 'live Server exe is the new build'
|
|
Assert ((Get-Content -Raw $b.WorkerExe) -match "sha=$NewSha") 'live Worker exe is the new build'
|
|
Assert ((Get-FileHash "$($b.ServerDir)/appsettings.json").Hash -eq $cfgHash) 'appsettings.json is the live copy, byte-identical'
|
|
Assert ((Get-FileHash "$($b.ServerDir)/appsettings.Development.json").Hash -eq $devHash) 'appsettings.Development.json carried'
|
|
if ($pfxHash) { Assert ((Test-Path "$($b.ServerDir)/dashboard.pfx") -and (Get-FileHash "$($b.ServerDir)/dashboard.pfx").Hash -eq $pfxHash) 'dashboard.pfx carried, byte-identical' }
|
|
Assert (Test-Path "$($b.ServerDir)/notes-left-by-an-operator.txt") 'live-only text file carried'
|
|
Assert (-not (Test-Path "$($b.ServerDir)/Old.Removed.Dependency.dll")) 'live-only binary not carried'
|
|
Assert ((Get-Content -Raw "$($b.WorkerDir)/$(Split-Path -Leaf $b.WorkerExe).config") -match 'new') 'Worker exe.config is the new build output'
|
|
$sbak = @(Get-ChildItem -LiteralPath $b.Root -Directory | Where-Object { $_.Name -like "$(Split-Path -Leaf $b.ServerDir).bak-*-test" })
|
|
Assert ($sbak.Count -eq 1 -and ((Get-Content -Raw (Join-Path $sbak[0].FullName (Split-Path -Leaf $b.Application))) -match "sha=$OldSha")) 'old Server kept as .bak-<stamp>-test'
|
|
Assert (@(Get-ChildItem -LiteralPath $b.Root -Directory -Filter 'Worker.bak-*-test').Count -eq 1) 'old Worker kept as .bak'
|
|
Assert ((Test-Path "$($b.Root)/data/gateway-auth.db") -and (Get-Content -Raw "$($b.Root)/data/gateway-auth.db") -eq 'auth-db') 'live API-key store untouched'
|
|
Assert (@(Get-ChildItem "$($b.Root)/data" -Filter 'gateway-auth.db*.bak-*').Count -eq 2) 'store db + wal backed up'
|
|
$bk = @(Get-ChildItem -LiteralPath $b.Root -Directory -Filter '_deploy-backup-*')
|
|
Assert ($bk.Count -eq 1 -and $b.Protected -contains $bk[0].FullName) 'backup dir created and ACL-restricted'
|
|
$fp = Get-Content -Raw (Join-Path $bk[0].FullName 'env-fingerprint.txt')
|
|
Assert ($fp -match 'MxGateway__ApiKeyPepper=[0-9a-f]{12}') 'fingerprint file has the pepper hash prefix'
|
|
Assert-NoSecrets $fp 'env-fingerprint.txt'
|
|
$notes = Get-Content -Raw (Get-ChildItem -LiteralPath $b.Root -Filter '_DEPLOY-NOTES-*.txt')[0].FullName
|
|
Assert-NoSecrets $notes '_DEPLOY-NOTES'
|
|
Assert-NoSecrets $r.Text 'the upgrade output'
|
|
Assert ($b.Env -contains "MxGateway__ApiKeyPepper=$Pepper") 'env block unchanged'
|
|
Assert ($b.Starts -eq 1) "started once ($($b.Starts))"
|
|
Assert ($r.Text -match 'env block unchanged after the stability window') 'final env check ran'
|
|
if ($scheme -eq 'wonder') { Assert ((Test-Path "$($bk[0].FullName)/kek.key")) 'file master key copied into the backup dir' }
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
}
|
|
|
|
Section 'Env block changed during the start -> abort + automatic rollback, values never printed'
|
|
$b = New-FakeBox 'wonder'
|
|
$b.OnStart = { param($x) if ($x.Starts -eq 1) { $x.Env = @($x.Env | ForEach-Object { if ($_ -like 'MxGateway__ApiKeyPepper=*') { 'MxGateway__ApiKeyPepper=a-different-pepper' } else { $_ } }) } }
|
|
$r = Invoke-Scenario $b @{}
|
|
Assert ($r.Rc -eq 1) 'upgrade fails (rc 1)'
|
|
Assert ($r.Text -match 'changed MxGateway__ApiKeyPepper \(sha256 [0-9a-f]{12} -> [0-9a-f]{12}\)') 'names the changed entry with hash prefixes'
|
|
Assert (-not $r.Text.Contains('a-different-pepper')) 'the new value is not printed'
|
|
Assert-NoSecrets $r.Text 'the failure output'
|
|
Assert ((Get-Content -Raw $b.Application) -match "sha=$OldSha") 'auto-rollback restored the old Server'
|
|
Assert (($b.StartedWith -join ',') -eq "$NewSha,$OldSha") "started new, then old ($($b.StartedWith -join ','))"
|
|
Assert ((Get-Content -Raw $b.WorkerExe) -match "sha=$OldSha") 'auto-rollback restored the old Worker'
|
|
Assert (Test-Path "$($b.ServerDir)/dashboard.pfx") 'dashboard.pfx present after the rollback'
|
|
Assert (@(Get-ChildItem -LiteralPath $b.Root -Directory -Filter 'Server.failed-*').Count -eq 1) 'failed build kept as Server.failed-<stamp>'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Env entry removed between the stop and the swap check -> abort, the new build is never started'
|
|
$b = New-FakeBox 'wonder'
|
|
# Simulate something rewriting the block mid-deploy (e.g. install-service.bat): the backup's Export-ServiceKey seam drops the pepper.
|
|
$r = & {
|
|
$script:Box = $b
|
|
$script:FakeNow = [datetime]'2026-11-12T12:00:00'
|
|
. $ScriptPath -ServerSource "$($b.Root)/stage/Server" -WorkerSource "$($b.Root)/stage/Worker" -ExpectedSha $NewSha -MinFreeGB 0.001 -StableSeconds 10
|
|
. $FakesPath
|
|
function Export-ServiceKey([string]$Destination) { Set-Content -LiteralPath $Destination -Value 'fake'; $script:Box.Env = @($script:Box.Env | Where-Object { $_ -notlike 'MxGateway__ApiKeyPepper=*' }) }
|
|
$rec = @(Invoke-Main 6>&1 3>&1)
|
|
[pscustomobject]@{ Rc = @($rec | Where-Object { $_ -is [int] })[-1]; Text = (@($rec | Where-Object { $_ -isnot [int] } | ForEach-Object { "$_" }) -join "`n") }
|
|
}
|
|
Assert ($r.Rc -eq 1) 'upgrade fails (rc 1)'
|
|
Assert ($r.Text -match 'env block changed after the swap: removed MxGateway__ApiKeyPepper') 'names the removed entry'
|
|
Assert (@($b.StartedWith | Where-Object { $_ -eq $NewSha }).Count -eq 0) "the new build was never started (started: $($b.StartedWith -join ','))"
|
|
Assert ((Get-Content -Raw $b.Application) -match "sha=$OldSha") 'auto-rollback put the old Server back'
|
|
Assert ($r.Text -match 'env block changed before the start') 'the rollback reports the env difference again (warning)'
|
|
Assert-NoSecrets $r.Text 'the failure output'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'STOP_PENDING: only this install''s processes are killed'
|
|
$b = New-FakeBox 'wonder'
|
|
$b.StopHangs = $true
|
|
$r = Invoke-Scenario $b @{}
|
|
Assert ($r.Rc -eq 0) "upgrade completes (rc=$($r.Rc)) $(if ($r.Rc -ne 0) { $r.Text })"
|
|
Assert ($r.Text -match 'STOP_PENDING trap') 'STOP_PENDING path taken'
|
|
Assert ($b.Killed.Count -ge 1 -and $b.Killed[0] -eq 4200) "the Server is killed first ($($b.Killed -join ','))"
|
|
Assert (-not ($b.Killed -contains 900)) 'an unrelated process is not touched'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Health never ready -> automatic rollback to the old build'
|
|
$b = New-FakeBox 'windev'
|
|
$b.OnStart = { param($x) if ($x.Starts -eq 1) { $x.Health = 503 } else { $x.Health = 200 } }
|
|
$r = Invoke-Scenario $b @{ ReadyTimeoutSeconds = 10 }
|
|
Assert ($r.Rc -eq 1) 'upgrade fails (rc 1)'
|
|
Assert ($r.Text -match 'ROLLED BACK to the previous build') "rollback verified ($(($r.Text -split "`n" | Select-String 'ROLLBACK FAILED') -join ''))"
|
|
Assert ((Get-Content -Raw $b.Application) -match "sha=$OldSha") 'old Server back in place'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Manual rollback finds a .bak.<stamp> set left by the #229 adapted scripts'
|
|
$b = New-FakeBox 'wonder'
|
|
Copy-Item -LiteralPath $b.ServerDir -Destination "$($b.ServerDir).bak.20261112T211945" -Recurse
|
|
Copy-Item -LiteralPath $b.WorkerDir -Destination "$($b.WorkerDir).bak.20261112T211945" -Recurse
|
|
Set-Content -LiteralPath $b.Application -Value "sha=$NewSha;arch=Amd64"
|
|
$r = Invoke-Scenario $b @{ Rollback = $true; DryRun = $true; ServerSource = $null; WorkerSource = $null }
|
|
Assert ($r.Rc -eq 0 -and $r.Text -match 'Server=Server\.bak\.20261112T211945 Worker=Worker\.bak\.20261112T211945') "dry-run picks the matching pair ($($r.Rc))"
|
|
$r = Invoke-Scenario $b @{ Rollback = $true; BackupStamp = '20261112T211945'; ServerSource = $null; WorkerSource = $null }
|
|
Assert ($r.Rc -eq 0) "rollback exits 0 ($(if ($r.Rc -ne 0) { $r.Text }))"
|
|
Assert ((Get-Content -Raw $b.Application) -match "sha=$OldSha") 'old Server restored'
|
|
Assert (Test-Path "$($b.ServerDir)/dashboard.pfx") 'dashboard.pfx is there after the rollback'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Rollback refuses a Server backup with no matching Worker backup'
|
|
$b = New-FakeBox 'wonder'
|
|
Copy-Item -LiteralPath $b.ServerDir -Destination "$($b.ServerDir).bak-20261101-100000" -Recurse
|
|
Copy-Item -LiteralPath $b.WorkerDir -Destination "$($b.WorkerDir).bak-20261001-100000" -Recurse
|
|
$r = Invoke-Scenario $b @{ Rollback = $true; DryRun = $true; ServerSource = $null; WorkerSource = $null }
|
|
Assert ($r.Rc -eq 1 -and $r.Text -match 'refusing a mixed rollback') 'refuses mixed stamps'
|
|
Remove-Item -LiteralPath $b.Root -Recurse -Force
|
|
|
|
Section 'Env fingerprint helpers'
|
|
. $ScriptPath -DryRun
|
|
$a = Get-EnvFingerprint @('A=1', "MxGateway__ApiKeyPepper=$Pepper", 'C=3')
|
|
Assert (-not ((Format-EnvFingerprint $a).Contains($Pepper))) 'the formatted fingerprint carries no value'
|
|
Assert ((Compare-EnvFingerprint $a (Get-EnvFingerprint @('A=1', "MxGateway__ApiKeyPepper=$Pepper", 'C=3'))).Count -eq 0) 'identical blocks compare equal'
|
|
$d = Compare-EnvFingerprint $a (Get-EnvFingerprint @('A=1', 'MxGateway__ApiKeyPepper=other', 'C=3'))
|
|
Assert ($d.Count -eq 1 -and $d[0] -like 'changed MxGateway__ApiKeyPepper*') "a changed value is caught ($d)"
|
|
$d = Compare-EnvFingerprint $a (Get-EnvFingerprint @('A=1', "MxGateway__ApiKeyPepper=$Pepper"))
|
|
Assert ($d.Count -eq 1 -and $d[0] -eq 'removed C') "a removed entry is caught ($d)"
|
|
$d = Compare-EnvFingerprint $a (Get-EnvFingerprint @('A=1', "MxGateway__ApiKeyPepper=$Pepper", 'C=3', 'D=4'))
|
|
Assert ($d.Count -eq 1 -and $d[0] -eq 'added D') "an added entry is caught ($d)"
|
|
$d = Compare-EnvFingerprint $a (Get-EnvFingerprint @('C=3', 'A=1', "MxGateway__ApiKeyPepper=$Pepper"))
|
|
Assert ($d.Count -eq 1 -and $d[0] -like 'block hash*') "a reorder is caught ($d)"
|
|
Assert ((Hide "x $Pepper y") -eq 'x <redacted> y') 'a sensitive value is redacted from output'
|
|
|
|
Write-Host ''
|
|
if ($script:failures -gt 0) { Write-Host "FAILED: $($script:failures) assertion(s) failed, $($script:passes) passed" -ForegroundColor Red; exit 1 }
|
|
Write-Host "OK: $($script:passes) assertions passed" -ForegroundColor Green
|
|
exit 0
|