fix(deploy): Upgrade-MxGatewayService.ps1 safe on the wonder layout, guards fail closed (#230)
ci / windows-x86 (push) Successful in 2m41s
ci / nightly-windev (push) Has been skipped
ci / portable (push) Failing after 12m34s
ci / java (push) Failing after 14m50s

The process guards matched only ZB.MOM.WW.MxGateway.*; the wonder boxes run
MxGateway.Server.exe / MxGateway.Worker.exe, so the one-gateway guard, the
pre-start OrphanWorkerTerminator guard and the STOP_PENDING kill passed vacuously.

- Image names come from the installed exes (NSSM Application, Worker env path) plus
  both known schemes; processes are also matched by exact path (Win32_Process).
- One-gateway guard fails closed: a RUNNING service whose gateway is not visible, at
  the Application path, as the service host's child, is an abort.
- STOP_PENDING kill is scoped to this install's processes.
- Staged checks use the installed exe names; staged Worker must be x86; a staged
  self-contained runtime older than the live one is refused (-AllowRuntimeDowngrade).
- Carry-over: appsettings*.json and certificates/keys (dashboard.pfx) always keep the
  live copy; other live-only non-binary top-level files are carried; hash-verified.
- Env block read from the registry and fingerprinted (names + SHA-256 prefixes, never
  values) after stop, swap, start and the stability window; any change aborts.
  Sensitive values are redacted from all output and notes.
- File-based master key (vd03) accepted and backed up; a missing env master key
  (windev) is a warning. Backup dir is ACL-restricted before anything is written.
- Rollback accepts .bak-<stamp> and .bak.<yyyyMMddTHHmmss> sets and refuses a
  mixed Server/Worker stamp; auto-rollback no longer fails on the old SHA.
- -WhatIf now works on Windows PowerShell 5.1 (Get-FileHash returned nothing under
  WhatIfPreference); an empty Win32_Service result no longer reads as "unavailable".
- Install-MxGatewayService.ps1 process guard matches both schemes.
- Dry-run harness scripts/deploy/tests (fake box in both layouts, 140 assertions,
  passes on pwsh/macOS and Windows PowerShell 5.1) plus a mutation runner (12/12 killed).
- Runbook updated.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-12 22:23:14 -05:00
parent 453fe01525
commit 4890a3d16e
7 changed files with 1192 additions and 218 deletions
@@ -0,0 +1,77 @@
<#
.SYNOPSIS
Mutation check for Test-UpgradeMxGatewayService.ps1 (mxaccessgw#230): re-introduces each guarded defect into a
temp copy of Upgrade-MxGatewayService.ps1 and requires the harness to FAIL on it. A mutation the harness does not
catch means an assertion is vacuous. Exits non-zero if any mutation survives (or does not apply).
.EXAMPLE
pwsh -NoProfile -File scripts/deploy/tests/Invoke-UpgradeMutations.ps1
#>
$ErrorActionPreference = 'Stop'
$here = Split-Path -Parent $MyInvocation.MyCommand.Path
$src = Get-Content -Raw (Join-Path (Split-Path -Parent $here) 'Upgrade-MxGatewayService.ps1')
$harness = Join-Path $here 'Test-UpgradeMxGatewayService.ps1'
$mutations = [ordered]@{
# The #230 defect: only the ZB.MOM.WW.* names, nothing derived from the installed exes.
'only ZB.MOM.WW.* image names (#230 original)' = @(
@("`$script:KnownServerImages = @('ZB.MOM.WW.MxGateway.Server', 'MxGateway.Server')", "`$script:KnownServerImages = @('ZB.MOM.WW.MxGateway.Server')"),
@("`$script:KnownWorkerImages = @('ZB.MOM.WW.MxGateway.Worker', 'MxGateway.Worker')", "`$script:KnownWorkerImages = @('ZB.MOM.WW.MxGateway.Worker')"),
@('$script:ServerImages = @(@(Get-Stem $ctx.Application) + $script:KnownServerImages', '$script:ServerImages = @(@() + $script:KnownServerImages'),
@('$script:WorkerImages = @(@(Get-Stem $wexe) + $script:KnownWorkerImages', '$script:WorkerImages = @(@() + $script:KnownWorkerImages'),
@('function Test-GatewayServer($p, $ctx) { ($script:ServerImages -contains $p.Name) -or (Test-SamePath $p.Path $ctx.Application) }', 'function Test-GatewayServer($p, $ctx) { ($script:ServerImages -contains $p.Name) }'),
@('function Test-GatewayWorker($p, $ctx) { ($script:WorkerImages -contains $p.Name) -or (Test-SamePath $p.Path $ctx.WorkerExe) }', 'function Test-GatewayWorker($p, $ctx) { ($script:WorkerImages -contains $p.Name) }'))
'guard passes when no gateway is visible' = @(
@(' if ($servers.Count -eq 0) {', ' if ($false) {'))
'other naming scheme not matched' = @(
@("`$script:KnownServerImages = @('ZB.MOM.WW.MxGateway.Server', 'MxGateway.Server')", "`$script:KnownServerImages = @()"),
@("`$script:KnownWorkerImages = @('ZB.MOM.WW.MxGateway.Worker', 'MxGateway.Worker')", "`$script:KnownWorkerImages = @()"))
'no parent check' = @(
@('if ($s.ParentId -ne $hostPid) { Fail', 'if ($false) { Fail'))
'pfx and live-only files not carried (#230 original)' = @(
@(' if (Test-Preserved $f.Name) { $carry.Add($f.Name); continue }', ''),
@('if ($script:BinaryExtensions -contains $f.Extension.ToLowerInvariant()) { $dropped.Add($f.Name) } else { $carry.Add($f.Name) }', 'if ($script:BinaryExtensions -contains $f.Extension.ToLowerInvariant()) { $dropped.Add($f.Name) }'))
'env check is a no-op' = @(
@(' $diff = Compare-EnvFingerprint $ctx.EnvFp $now', ' $diff = @()'))
'env compare looks at names only' = @(
@('elseif ($After.Entries[$k] -ne $Before.Entries[$k])', 'elseif ($false)'),
@('if ($d.Count -eq 0 -and $Before.BlockHash -ne $After.BlockHash)', 'if ($false)'))
'values not redacted' = @(
@('foreach ($s in $script:Redact) { if ($s) { $m = $m.Replace($s, ''<redacted>'') } }', ''),
@('function Format-EnvFingerprint($Fp) { "$($Fp.Count) entries, block sha256 $($Fp.BlockHash): " + (($Fp.Entries.Keys | ForEach-Object { "$_=$($Fp.Entries[$_])" }) -join '', '') }', 'function Format-EnvFingerprint($Fp) { "$($Fp.Count) entries: " + ((ConvertTo-EnvMap (Read-EnvBlockLines)).GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join '', '' }'))
'STOP_PENDING kill by any name' = @(
@('if ($_.Path) { (Test-Inside $_.Path $ctx.ServerDir) -or (Test-Inside $_.Path $ctx.WorkerDir) } else { $installed -contains $_.Name }', '$true'),
@(' $mine = @(Get-GatewayProcesses $ctx | Where-Object {', ' $mine = @(Get-ProcessTable | Where-Object {'))
'runtime downgrade allowed' = @(
@('if ($liveRt -and $newRt -and $newRt -lt $liveRt) {', 'if ($false) {'))
'mixed rollback allowed' = @(
@('if ($wb -and (Get-BackupKey $wb.Name) -ne $key) { $wb = $null }', ''),
@(' $wb = Find-Backup $ctx.WorkerDir $key', ' $wb = Find-Backup $ctx.WorkerDir $null'))
'.bak.<stamp> sets not found' = @(
@('($_.Name -like "$leaf.bak-*" -or $_.Name -like "$leaf.bak.*")', '($_.Name -like "$leaf.bak-*")'))
}
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("mxgw230-mut-" + [guid]::NewGuid().ToString('n').Substring(0, 8))
New-Item -ItemType Directory -Path $tmp | Out-Null
$survivors = 0
try {
foreach ($name in $mutations.Keys) {
$m = $src
$pairs = @($mutations[$name])
if ($pairs[0] -is [string]) { $pairs = @(, $pairs) } # a single @(@(old, new)) flattens to @(old, new)
foreach ($pair in $pairs) {
if (-not $m.Contains($pair[0])) { Write-Host "MUTATION DID NOT APPLY: $name -> '$($pair[0])'" -ForegroundColor Red; $survivors++; continue }
$m = $m.Replace($pair[0], $pair[1])
}
$path = Join-Path $tmp 'Upgrade-MxGatewayService.ps1'
Set-Content -LiteralPath $path -Value $m -NoNewline
$out = @(& pwsh -NoProfile -File $harness -ScriptPath $path 2>&1)
$rc = $LASTEXITCODE
$fails = @($out | Where-Object { "$_" -match '^\s+FAIL \[' })
# Killed only if the harness ran to its verdict and reported failed assertions (a crash is not a kill).
if ($rc -ne 0 -and $fails.Count -gt 0 -and ($out -match '^FAILED: ')) { Write-Host ("killed {0,-48} ({1} assertion(s) failed, e.g. {2})" -f $name, $fails.Count, ("$($fails | Select-Object -First 1)".Trim())) }
else { Write-Host "SURVIVED $name (rc=$rc, $($fails.Count) failed assertion(s)): $(($out | Select-Object -Last 2) -join ' | ')" -ForegroundColor Red; $survivors++ }
}
} finally { Remove-Item -LiteralPath $tmp -Recurse -Force }
if ($survivors -gt 0) { Write-Host "$survivors mutation(s) survived or did not apply" -ForegroundColor Red; exit 1 }
Write-Host "all $($mutations.Count) mutations killed" -ForegroundColor Green