#requires -Version 5.1 <# .SYNOPSIS On-box upgrade / rollback of the NSSM-wrapped MxAccess Gateway (Server + x86 Worker). Implements docs/runbooks/DeployUpgradeNssmService.md sections 3, 5, 6, 7 and 8 (mxaccessgw#220). .DESCRIPTION RUN THIS ON THE WINDOWS BOX ITSELF (Windows PowerShell 5.1 is enough). The new Server publish and the new x86 Worker build must already be staged on the box (UNC/SMB copy; build both from the same SHA). Upgrade flow: preflight -> stop (STOP_PENDING handled) -> backup -> swap Server + Worker -> start -> verify (/health/ready, VersionInfo SHA, service stable) -> automatic rollback if start/verify fails. Rollback flow (-Rollback): stop -> swap the .bak directories back -> optional store restore -> start -> verify. Safety properties: * Exactly one gateway per box. Refuses to run if a second gateway process / service points at the same exe, if the service is stopped but a gateway process exists, or if a Server/Worker process is still alive when a start is about to happen (OrphanWorkerTerminator trap, runbook section 2.1). * Never calls `nssm set ... AppEnvironmentExtra` (that replaces the whole block). The env block is only READ (saved to the backup dir, entry count compared before/after). The Application/AppDirectory are not changed either: the swap is rename-in-place, so the NSSM config is untouched. * API-key DB, Secrets store and master key live outside the content root and are never moved or deleted. They are only COPIED (db + -wal + -shm) into *.bak- siblings while the service is stopped. The script refuses to run if a store path is inside the content root (the 2026-08-09 incident). -DryRun (or -WhatIf) prints every mutating action and performs none. Read-only probing (nssm get/status, process list, disk space, file existence) still runs so the decision logic is exercised. .PARAMETER ServerSource Staged Server publish output (contains ZB.MOM.WW.MxGateway.Server.exe). .PARAMETER WorkerSource Staged x86 Worker output (contains ZB.MOM.WW.MxGateway.Worker.exe). .PARAMETER ServiceName NSSM service name. Default MxAccessGw. .PARAMETER NssmPath nssm executable. Default 'nssm' (on PATH). .PARAMETER ServerDir Live Server directory. Default: NSSM AppDirectory. .PARAMETER WorkerDir Live Worker directory. Default: parent of env MxGateway__Worker__ExecutablePath. .PARAMETER ExpectedSha Short/long git SHA. Staged Server+Worker (preflight) and the deployed Server (verify) must carry it in ProductVersion ('0.1.2+'). Builds from 2026-07-09..08-10 have a bogus stamp. .PARAMETER Label Free-text suffix for backup names (e.g. 'planwrites'). .PARAMETER HealthBaseUrl Default http://localhost:5130 (dashboard port; /health/ready, /healthz anonymous). .PARAMETER Rollback Roll back instead of upgrading. Uses newest Server.bak-* unless -BackupStamp is given. .PARAMETER BackupStamp Stamp (yyyyMMdd-HHmmss) of the backup set to roll back to. .PARAMETER RestoreStores With -Rollback: also restore gateway-auth.db / secrets db from *.bak- (schema-bump case). .PARAMETER NoAutoRollback Do not roll back automatically when start/verification fails. .PARAMETER TakeNewAppSettings Do not carry the live appsettings*.json over into the new Server dir. .PARAMETER SkipFreeSpaceCheck / MinFreeGB / StopTimeoutSeconds / StableSeconds / ReadyTimeoutSeconds tunables. .PARAMETER DryRun Print, do not act. -WhatIf is equivalent. #> [CmdletBinding(SupportsShouldProcess = $true)] param( [string]$ServerSource, [string]$WorkerSource, [string]$ServiceName = 'MxAccessGw', [string]$NssmPath = 'nssm', [string]$ServerDir, [string]$WorkerDir, [string]$ExpectedSha, [string]$Label, [string]$HealthBaseUrl = 'http://localhost:5130', [switch]$Rollback, [string]$BackupStamp, [switch]$RestoreStores, [switch]$NoAutoRollback, [switch]$TakeNewAppSettings, [switch]$SkipFreeSpaceCheck, [double]$MinFreeGB = 2, [int]$StopTimeoutSeconds = 30, [int]$StableSeconds = 60, [int]$ReadyTimeoutSeconds = 120, [switch]$DryRun ) $ErrorActionPreference = 'Stop' $script:Dry = $DryRun.IsPresent -or $WhatIfPreference $script:Stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $script:ServerProc = 'ZB.MOM.WW.MxGateway.Server' $script:WorkerProc = 'ZB.MOM.WW.MxGateway.Worker' $script:Notes = New-Object System.Collections.Generic.List[string] $script:LockStream = $null function Say([string]$m) { Write-Host $m; $script:Notes.Add($m) } function Info([string]$m) { Say "[info] $m" } function Warn([string]$m) { Write-Warning $m; $script:Notes.Add("[WARN] $m") } function Fail([string]$m) { throw "ABORT: $m" } # Mutating step: print in dry-run, run otherwise. function Step([string]$Description, [scriptblock]$Action) { if ($script:Dry) { Say "[DRYRUN] $Description"; return } Say "[ACTION] $Description" & $Action } # --- helpers -------------------------------------------------------------------------------------------- function Invoke-Nssm { param([string[]]$NssmArgs, [switch]$AllowFail) $raw = & $NssmPath @NssmArgs 2>&1 $code = $LASTEXITCODE $text = (($raw | ForEach-Object { "$_" }) -join "`n") -replace "`0", '' if ($code -ne 0 -and -not $AllowFail) { Fail "nssm $($NssmArgs -join ' ') failed (exit $code): $text" } [pscustomobject]@{ Code = $code; Text = $text.Trim() } } function Get-NssmValue([string]$Name) { (Invoke-Nssm -NssmArgs @('get', $ServiceName, $Name)).Text } function Get-ServiceState { $r = Invoke-Nssm -NssmArgs @('status', $ServiceName) -AllowFail if ($r.Code -ne 0) { return 'MISSING' } ($r.Text -split "`n")[0].Trim() } function Parse-EnvBlock([string]$Text) { $map = [ordered]@{} foreach ($line in ($Text -split "[`r`n]+")) { $t = $line.Trim() if ($t -match '^([^=\s]+)=(.*)$') { $map[$Matches[1]] = $Matches[2] } } $map } function Norm([string]$p) { ($p -replace '\\', '/').TrimEnd('/') } function Test-Rooted([string]$p) { $p -match '^[A-Za-z]:[\\/]' -or $p.StartsWith('/') -or $p.StartsWith('\\') } function Test-Inside([string]$Path, [string]$Root) { $a = (Norm $Path).ToLowerInvariant(); $b = (Norm $Root).ToLowerInvariant() $a -eq $b -or $a.StartsWith("$b/") } function Get-ProductSha([string]$Exe) { try { $pv = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($Exe).ProductVersion if ($pv -match '\+([0-9a-fA-F]{7,40})(\b|$)') { return $Matches[1].ToLowerInvariant() } } catch { } $null } function Test-ShaMatch([string]$Actual, [string]$Expected) { if (-not $Actual -or -not $Expected) { return $false } $e = $Expected.ToLowerInvariant() $Actual.StartsWith($e) -or $e.StartsWith($Actual) } function Get-GatewayProcesses { @(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -in @($script:ServerProc, $script:WorkerProc) }) } # Store file set: db + -wal + -shm function Get-StoreFiles([string]$Db) { @($Db, "$Db-wal", "$Db-shm") | Where-Object { Test-Path -LiteralPath $_ } } function Get-DirSize([string]$p) { if (-not (Test-Path -LiteralPath $p)) { return 0 } $s = (Get-ChildItem -LiteralPath $p -Recurse -File -Force -ErrorAction SilentlyContinue | Measure-Object Length -Sum).Sum if ($s) { [double]$s } else { 0 } } # --- context discovery (read only) ------------------------------------------------------------------------- function Get-Context { $state = Get-ServiceState if ($state -eq 'MISSING') { Fail "service '$ServiceName' does not exist (use Install-MxGatewayService.ps1 for a new box)" } $ctx = [ordered]@{} $ctx.State = $state $ctx.Application = Get-NssmValue 'Application' $ctx.AppDirectory = Get-NssmValue 'AppDirectory' $ctx.EnvText = Get-NssmValue 'AppEnvironmentExtra' $ctx.Env = Parse-EnvBlock $ctx.EnvText $ctx.EnvCount = $ctx.Env.Count if (-not $ServerDir) { $ctx.ServerDir = $ctx.AppDirectory } else { $ctx.ServerDir = $ServerDir } $wexe = $ctx.Env['MxGateway__Worker__ExecutablePath'] $ctx.WorkerExe = $wexe if ($WorkerDir) { $ctx.WorkerDir = $WorkerDir } elseif ($wexe) { $ctx.WorkerDir = Split-Path -Parent $wexe } else { Fail 'cannot determine the Worker directory: env MxGateway__Worker__ExecutablePath is not in the NSSM env block; pass -WorkerDir' } $pd = 'C:\ProgramData\MxGateway' $auth = $ctx.Env['MxGateway__Authentication__SqlitePath']; if (-not $auth) { $auth = "$pd\gateway-auth.db" } $sec = $ctx.Env['Secrets__SqlitePath']; if (-not $sec) { $sec = "$pd\mxgateway-secrets.db" } $ctx.AuthDb = $auth; $ctx.SecretsDb = $sec $ctx.ServerExeName = Split-Path -Leaf $ctx.Application [pscustomobject]$ctx } # --- preflight ----------------------------------------------------------------------------------------------- function Invoke-Preflight($ctx) { Say '== Preflight ==' Info "service '$ServiceName' state: $($ctx.State)" Info "Application : $($ctx.Application)" Info "AppDirectory: $($ctx.AppDirectory)" Info "Server dir : $($ctx.ServerDir)" Info "Worker dir : $($ctx.WorkerDir) (exe env: $($ctx.WorkerExe))" Info "API-key DB : $($ctx.AuthDb)" Info "Secrets DB : $($ctx.SecretsDb)" Info "NSSM env block: $($ctx.EnvCount) entries (names only): $(($ctx.Env.Keys) -join ', ')" if ($ctx.EnvCount -eq 0) { Fail 'NSSM env block is empty - refusing: it holds Kestrel/store paths, the KEK and the pepper' } foreach ($need in 'ZB_SECRETS_MASTER_KEY') { if (-not $ctx.Env.Contains($need)) { Fail "env block has no $need (the Secrets store would be unreadable after the restart)" } } if (-not ($ctx.Env.Keys | Where-Object { $_ -match 'Pepper' })) { Warn 'no *Pepper* entry in the env block; confirm the API-key pepper is supplied another way or issued tokens will not verify' } if ($ctx.Env['ASPNETCORE_ENVIRONMENT'] -ne 'Production' -and $ctx.Env['DOTNET_ENVIRONMENT'] -ne 'Production') { Warn 'neither ASPNETCORE_ENVIRONMENT nor DOTNET_ENVIRONMENT is exactly Production (runbook trap 6) - do not change it during an upgrade' } # State that must survive: rooted and outside the content root. foreach ($pair in @(@('API-key DB', $ctx.AuthDb), @('Secrets DB', $ctx.SecretsDb))) { if (-not (Test-Rooted $pair[1])) { Fail "$($pair[0]) path '$($pair[1])' is not absolute" } foreach ($root in @($ctx.ServerDir, $ctx.WorkerDir)) { if (Test-Inside $pair[1] $root) { Fail "$($pair[0]) '$($pair[1])' is INSIDE '$root' - the swap would lose it. Move it out first (runbook section 1)." } } if (Test-Path -LiteralPath $pair[1]) { Info "$($pair[0]) present ($((Get-StoreFiles $pair[1]).Count) file(s) incl. wal/shm)" } else { Warn "$($pair[0]) '$($pair[1])' does not exist yet (fresh store will be created on start)" } } # Directories foreach ($d in @($ctx.ServerDir, $ctx.WorkerDir)) { if (-not (Test-Path -LiteralPath $d -PathType Container)) { Fail "live directory '$d' does not exist" } } if ((Test-Inside $ctx.WorkerDir $ctx.ServerDir) -or (Test-Inside $ctx.ServerDir $ctx.WorkerDir)) { Fail 'Server and Worker directories are nested; the rename-in-place swap needs sibling directories' } # One-gateway guard $procs = Get-GatewayProcesses $servers = @($procs | Where-Object { $_.ProcessName -eq $script:ServerProc }) if ($servers.Count -gt 1) { Fail "$($servers.Count) gateway Server processes are running (PIDs $(($servers.Id) -join ',')). A second gateway is the OrphanWorkerTerminator trap - resolve manually." } if ($ctx.State -eq 'SERVICE_STOPPED' -and $procs.Count -gt 0) { Fail "service is STOPPED but gateway process(es) exist (PIDs $(($procs.Id) -join ',')): a rogue second gateway. Starting would terminate its live workers." } if ($servers.Count -eq 1 -and $ctx.State -ne 'SERVICE_RUNNING' -and $ctx.State -ne 'SERVICE_STOP_PENDING') { Fail "a Server process exists while the service is $($ctx.State)" } if (Get-Command Get-CimInstance -ErrorAction SilentlyContinue) { $svcs = @(Get-CimInstance Win32_Service | Where-Object { $_.Name -ne $ServiceName -and $_.PathName -and $_.PathName -like "*$($ctx.ServerExeName)*" }) if ($svcs.Count -gt 0) { Fail "another service ($(($svcs.Name) -join ', ')) also launches $($ctx.ServerExeName); two gateways on one box is forbidden" } } else { Info 'Win32_Service cross-check unavailable on this host (skipped)' } if ($Rollback) { return } # Sources if (-not $ServerSource -or -not $WorkerSource) { Fail '-ServerSource and -WorkerSource are required for an upgrade' } foreach ($s in @(@($ServerSource, $ctx.ServerExeName), @($WorkerSource, "$($script:WorkerProc).exe"))) { if (-not (Test-Path -LiteralPath (Join-Path $s[0] $s[1]))) { Fail "staged source '$($s[0])' does not contain $($s[1])" } } if ((Test-Inside $ServerSource $ctx.ServerDir) -or (Test-Inside $WorkerSource $ctx.WorkerDir)) { Fail 'staging directory is inside a live directory' } $sSha = Get-ProductSha (Join-Path $ServerSource $ctx.ServerExeName) $wSha = Get-ProductSha (Join-Path $WorkerSource "$($script:WorkerProc).exe") Info "staged Server SHA: $sSha ; staged Worker SHA: $wSha" if ($sSha -and $wSha -and -not (Test-ShaMatch $sSha $wSha)) { Fail "Server ($sSha) and Worker ($wSha) were built from different commits (wire protocol risk)" } if ($ExpectedSha) { if (-not (Test-ShaMatch $sSha $ExpectedSha)) { if ($script:Dry) { Warn "staged Server SHA '$sSha' does not match -ExpectedSha '$ExpectedSha' (dry-run: continuing)" } else { Fail "staged Server SHA '$sSha' does not match -ExpectedSha '$ExpectedSha'" } } } else { Warn 'no -ExpectedSha given: the post-deploy SHA check will only record the SHA' } # Disk space: 2x (server + worker + stores + payload), at least MinFreeGB if (-not $SkipFreeSpaceCheck) { $need = (Get-DirSize $ctx.ServerDir) + (Get-DirSize $ctx.WorkerDir) + (Get-DirSize $ServerSource) + (Get-DirSize $WorkerSource) foreach ($f in (Get-StoreFiles $ctx.AuthDb) + (Get-StoreFiles $ctx.SecretsDb)) { $need += (Get-Item -LiteralPath $f).Length } $need = [math]::Max($need * 2, $MinFreeGB * 1GB) $root = [IO.Path]::GetPathRoot((Resolve-Path -LiteralPath $ctx.ServerDir).Path) $free = (New-Object IO.DriveInfo $root).AvailableFreeSpace Info ("free on {0}: {1:N2} GB, required {2:N2} GB" -f $root, ($free / 1GB), ($need / 1GB)) if ($free -lt $need) { Fail 'not enough free disk space for the backups' } } Say '== Preflight OK ==' } # --- stop / start --------------------------------------------------------------------------------------------- function Wait-State([string]$Want, [int]$TimeoutSec) { $deadline = (Get-Date).AddSeconds($TimeoutSec) do { $s = Get-ServiceState; if ($s -eq $Want) { return $true }; Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) $false } function Assert-NoGatewayProcess([string]$When) { $left = Get-GatewayProcesses if ($left.Count -gt 0) { Fail "${When}: gateway process(es) still alive: $(($left | ForEach-Object { "$($_.ProcessName)#$($_.Id)" }) -join ', ')" } } function Stop-Gateway { Say '-- Stop --' $state = Get-ServiceState if ($state -eq 'SERVICE_STOPPED') { Info 'already stopped' } else { Step "nssm stop $ServiceName (wait up to $StopTimeoutSeconds s for SERVICE_STOPPED)" { [void](Invoke-Nssm -NssmArgs @('stop', $ServiceName) -AllowFail) if (-not (Wait-State 'SERVICE_STOPPED' $StopTimeoutSeconds)) { # STOP_PENDING trap: workers did not exit in time. Kill the Server (then Workers) by image name - # safe because preflight proved there is exactly one gateway on the box. Warn "service still $(Get-ServiceState) after $StopTimeoutSeconds s (STOP_PENDING trap) - killing gateway processes" foreach ($n in @($script:ServerProc, $script:WorkerProc)) { Get-Process -Name $n -ErrorAction SilentlyContinue | ForEach-Object { Say "[ACTION] Stop-Process -Id $($_.Id) -Force ($n)"; Stop-Process -Id $_.Id -Force } } if (-not (Wait-State 'SERVICE_STOPPED' 20)) { Warn 'still not STOPPED; killing the NSSM service host process' try { $svcPid = (Get-CimInstance Win32_Service -Filter "Name='$ServiceName'").ProcessId if ($svcPid) { Stop-Process -Id $svcPid -Force } } catch { Warn "could not kill the service host: $_" } if (-not (Wait-State 'SERVICE_STOPPED' 20)) { Fail "service '$ServiceName' will not stop; investigate manually (state $(Get-ServiceState))" } } } Start-Sleep -Seconds 2 Assert-NoGatewayProcess 'after stop' } } if (-not $script:Dry) { Assert-NoGatewayProcess 'after stop' } } function Start-Gateway($ctx) { Say '-- Start --' if (-not $script:Dry) { Assert-NoGatewayProcess 'before start (OrphanWorkerTerminator guard)' } else { Say '[DRYRUN] assert no Server/Worker process exists (OrphanWorkerTerminator guard)' } Step "nssm start $ServiceName (once)" { [void](Invoke-Nssm -NssmArgs @('start', $ServiceName)) } Step "wait for SERVICE_RUNNING and compare env entry count (expect $($ctx.EnvCount))" { if (-not (Wait-State 'SERVICE_RUNNING' 60)) { Fail "service did not reach RUNNING (state $(Get-ServiceState))" } $now = (Parse-EnvBlock (Get-NssmValue 'AppEnvironmentExtra')).Count if ($now -ne $ctx.EnvCount) { Fail "NSSM env block entry count changed: $($ctx.EnvCount) -> $now" } } } # --- backup / swap ----------------------------------------------------------------------------------------------- function Get-Suffix { if ($Label) { "$($script:Stamp)-$Label" } else { $script:Stamp } } function Backup-State($ctx) { Say '-- Backup (service stopped) --' $suffix = Get-Suffix $root = Split-Path -Parent $ctx.ServerDir $envFile = Join-Path $root "_env-AppEnvironmentExtra-$($script:Stamp).txt" Step "save the NSSM env block + Application/AppDirectory to $envFile (contains secrets: keep it ACL-restricted)" { $body = @("Application=$($ctx.Application)", "AppDirectory=$($ctx.AppDirectory)", '--- AppEnvironmentExtra ---', $ctx.EnvText) Set-Content -LiteralPath $envFile -Value $body -Encoding UTF8 try { & icacls.exe $envFile /inheritance:r /grant:r 'SYSTEM:(F)' 'BUILTIN\Administrators:(F)' | Out-Null } catch { Warn "could not restrict ACL on ${envFile}: $_" } } foreach ($db in @($ctx.AuthDb, $ctx.SecretsDb)) { foreach ($f in (Get-StoreFiles $db)) { $dst = "$f.bak-$suffix" Step "copy store file $f -> $dst (live file is never moved)" { Copy-Item -LiteralPath $f -Destination $dst -Force } } if (-not (Test-Path -LiteralPath $db)) { Info "no store at $db to back up" } } } function Swap-Directory([string]$Live, [string]$Source, [string]$Name, [bool]$KeepSettings) { $bak = "$Live.bak-$(Get-Suffix)" Step "rename $Name dir: $Live -> $bak" { Rename-Item -LiteralPath $Live -NewName (Split-Path -Leaf $bak) } Step "copy new ${Name}: $Source -> $Live" { Copy-Item -LiteralPath $Source -Destination $Live -Recurse -Force } if ($KeepSettings) { $from = if ($script:Dry) { $Live } else { $bak } # in a dry run nothing was renamed yet foreach ($f in 'appsettings.json', 'appsettings.Development.json') { if (Test-Path -LiteralPath (Join-Path $from $f)) { Step "restore live $f byte-identical from $bak" { Copy-Item -LiteralPath (Join-Path $bak $f) -Destination (Join-Path $Live $f) -Force } } else { Info "$f not present in the old $Name dir (nothing to carry over)" } } } $bak } # --- verification ---------------------------------------------------------------------------------------------------- function Invoke-Verify($ctx) { Say '== Verify ==' if ($script:Dry) { Say "[DRYRUN] GET $HealthBaseUrl/health/ready until 200 (<= $ReadyTimeoutSeconds s)" Say "[DRYRUN] GET $HealthBaseUrl/healthz ; read ProductVersion SHA of $(Join-Path $ctx.ServerDir $ctx.ServerExeName)$(if ($ExpectedSha) { " and require it to match $ExpectedSha" })" Say "[DRYRUN] service must stay SERVICE_RUNNING for $StableSeconds s with env entry count $($ctx.EnvCount)" return } $ready = $false; $deadline = (Get-Date).AddSeconds($ReadyTimeoutSeconds) while (-not $ready -and (Get-Date) -lt $deadline) { try { $r = Invoke-WebRequest -Uri "$HealthBaseUrl/health/ready" -UseBasicParsing -TimeoutSec 10; $ready = ($r.StatusCode -eq 200) } catch { Start-Sleep -Seconds 3 } } if (-not $ready) { Fail "$HealthBaseUrl/health/ready did not return 200 within $ReadyTimeoutSeconds s" } Info '/health/ready = 200' try { Info "/healthz = $((Invoke-WebRequest -Uri "$HealthBaseUrl/healthz" -UseBasicParsing -TimeoutSec 10).StatusCode)" } catch { Fail "/healthz failed: $_" } $sha = Get-ProductSha (Join-Path $ctx.ServerDir $ctx.ServerExeName) Info "deployed Server SHA (ProductVersion): $sha" if ($ExpectedSha -and -not $Rollback -and -not (Test-ShaMatch $sha $ExpectedSha)) { Fail "deployed SHA '$sha' != expected '$ExpectedSha'" } Info "watching the service for $StableSeconds s" $until = (Get-Date).AddSeconds($StableSeconds) while ((Get-Date) -lt $until) { if ((Get-ServiceState) -ne 'SERVICE_RUNNING') { Fail 'service left RUNNING during the stability window (restart loop?)' } Start-Sleep -Seconds 5 } $n = (Parse-EnvBlock (Get-NssmValue 'AppEnvironmentExtra')).Count if ($n -ne $ctx.EnvCount) { Fail "env block entry count $n != preflight $($ctx.EnvCount)" } Info "env block still $n entries" Warn 'health is green with zero sessions; still prove the Worker path: open a session with a PRE-EXISTING API key and read a known tag (runbook 7.3)' } # --- rollback ------------------------------------------------------------------------------------------------------------ function Find-Backup([string]$Live, [string]$Stamp) { $leaf = Split-Path -Leaf $Live; $parent = Split-Path -Parent $Live $cands = @(Get-ChildItem -LiteralPath $parent -Directory -Filter "$leaf.bak-*" -ErrorAction SilentlyContinue) if ($Stamp) { $cands = @($cands | Where-Object { $_.Name -like "$leaf.bak-$Stamp*" }) } $cands | Sort-Object Name -Descending | Select-Object -First 1 } function Invoke-RollbackSwap($ctx, [string]$Stamp, [bool]$Stores) { Say '== Rollback ==' $sb = Find-Backup $ctx.ServerDir $Stamp $wb = Find-Backup $ctx.WorkerDir $Stamp if (-not $sb -or -not $wb) { Fail "no matching Server/Worker backup directories found (stamp '$Stamp')" } Info "rolling back to Server=$($sb.Name) Worker=$($wb.Name)" Stop-Gateway foreach ($pair in @(@($ctx.ServerDir, $sb), @($ctx.WorkerDir, $wb))) { $failed = "$($pair[0]).failed-$($script:Stamp)" if (Test-Path -LiteralPath $pair[0]) { Step "rename $($pair[0]) -> $failed" { Rename-Item -LiteralPath $pair[0] -NewName (Split-Path -Leaf $failed) } } Step "rename $($pair[1].FullName) -> $($pair[0])" { Rename-Item -LiteralPath $pair[1].FullName -NewName (Split-Path -Leaf $pair[0]) } } if ($Stores) { $bstamp = ($sb.Name -replace '^.*\.bak-', '') foreach ($db in @($ctx.AuthDb, $ctx.SecretsDb)) { foreach ($ext in '', '-wal', '-shm') { $live = "$db$ext"; $bak = "$db$ext.bak-$bstamp" if (Test-Path -LiteralPath $bak) { Step "restore store file $bak -> $live" { Copy-Item -LiteralPath $bak -Destination $live -Force } } elseif ($ext -and (Test-Path -LiteralPath $live) -and (Test-Path -LiteralPath "$db.bak-$bstamp")) { # a stale wal/shm must not pair with the restored db: move it aside, never delete Step "move stale $live aside -> $live.stale-$($script:Stamp)" { Rename-Item -LiteralPath $live -NewName ("$(Split-Path -Leaf $live).stale-$($script:Stamp)") } } elseif (-not $ext) { Warn "no backup $bak for $db; live store left untouched" } } } } else { Info 'stores left untouched (they live outside the swap); pass -RestoreStores after a schema bump' } Start-Gateway $ctx } # --- main ------------------------------------------------------------------------------------------------------------------- $exit = 0 try { Say "Upgrade-MxGatewayService stamp=$($script:Stamp) mode=$(if ($Rollback) {'ROLLBACK'} else {'UPGRADE'}) dry=$($script:Dry)" $ctx = Get-Context Invoke-Preflight $ctx if (-not $script:Dry) { $lockPath = Join-Path (Split-Path -Parent $ctx.ServerDir) '.mxgw-deploy.lock' try { $script:LockStream = [IO.File]::Open($lockPath, 'OpenOrCreate', 'ReadWrite', 'None') } catch { Fail "another deploy run holds $lockPath" } } if ($Rollback) { Invoke-RollbackSwap $ctx $BackupStamp $RestoreStores.IsPresent Invoke-Verify $ctx } else { Stop-Gateway Backup-State $ctx $swapped = $false try { $null = Swap-Directory $ctx.ServerDir $ServerSource 'Server' (-not $TakeNewAppSettings) $null = Swap-Directory $ctx.WorkerDir $WorkerSource 'Worker' $false $swapped = $true Start-Gateway $ctx Invoke-Verify $ctx } catch { $err = $_ Say "[FAIL] $($err.Exception.Message)" if ($script:Dry -or $NoAutoRollback) { throw } Warn 'upgrade failed - rolling back automatically' try { if (-not $swapped) { Warn 'swap did not complete; rollback swaps whatever .bak sets exist for this stamp' } Invoke-RollbackSwap $ctx $script:Stamp $false Invoke-Verify $ctx Warn 'ROLLED BACK to the previous build; original failure follows' } catch { Warn "AUTOMATIC ROLLBACK FAILED: $($_.Exception.Message). Manual intervention required (backups: *.bak-$(Get-Suffix))." } throw $err } } Say '== DONE ==' } catch { $exit = 1 Write-Host "ERROR: $($_.Exception.Message)" -ForegroundColor Red } finally { if ($script:LockStream) { $script:LockStream.Dispose() } try { if (-not $script:Dry -and $ctx) { $notes = Join-Path (Split-Path -Parent $ctx.ServerDir) "_DEPLOY-NOTES-$($script:Stamp).txt" Set-Content -LiteralPath $notes -Value $script:Notes -Encoding UTF8 Write-Host "deploy notes: $notes" } } catch { } } exit $exit