#requires -Version 5.1 <# .SYNOPSIS On-box upgrade / rollback of the NSSM-wrapped MxAccess Gateway (Server + x86 Worker). Implements docs/runbooks/DeployUpgradeNssmService.md sections 3, 5, 6, 7 and 8 (mxaccessgw#220, #230). .DESCRIPTION RUN THIS ON THE WINDOWS BOX ITSELF (Windows PowerShell 5.1 is enough). The new Server publish and the new x86 Worker build must already be staged on the box (UNC/SMB copy; build both from the same SHA). Upgrade flow: preflight -> stop (STOP_PENDING handled) -> backup -> swap Server + Worker -> start -> verify (/health/ready, VersionInfo SHA, service stable, env block unchanged) -> automatic rollback if start/verify fails. Rollback flow (-Rollback): stop -> swap the .bak directories back -> optional store restore -> start -> verify. Safety properties: * Exactly one gateway per box. The gateway and Worker image names are taken from the INSTALLED exes (NSSM Application and env MxGateway__Worker__ExecutablePath) plus both known naming schemes (ZB.MOM.WW.MxGateway.* on windev, MxGateway.* on the wonder boxes), and processes are also matched by exact path. The guards FAIL CLOSED: a RUNNING service whose gateway process cannot be found is an abort, not a pass (mxaccessgw#230: the guards used to pass vacuously on the wonder boxes). * Never writes the NSSM env block (AppEnvironmentExtra), Application or AppDirectory: the swap is rename-in-place. The env block is fingerprinted (entry names + SHA-256 prefix of each value; values are never printed) at preflight, after the stop, after the swap, after the start and after the stability window. Any added, removed or changed entry aborts (and triggers the automatic rollback). * Live-only top-level files in Server\ and Worker\ are carried into the new directories (appsettings*.json, dashboard.pfx and other certificates/keys always keep the LIVE copy), hash-verified. * API-key DB, Secrets store and a file-based master key must live outside the swapped directories; they are only COPIED (db + -wal + -shm) into *.bak- siblings while the service is stopped. * A self-contained Server is refused if its .NET runtime is older than the live one (no silent downgrade). -DryRun (or -WhatIf) prints every mutating action and performs none. Read-only probing (service state, registry env block, process list, disk space, file hashes) still runs, so the guards are exercised for real. .PARAMETER ServerSource Staged Server publish output (must contain the same exe name as NSSM Application). .PARAMETER WorkerSource Staged x86 Worker output (must contain the exe name of MxGateway__Worker__ExecutablePath). .PARAMETER ServiceName NSSM service name. Default MxAccessGw. .PARAMETER NssmPath nssm executable. Default 'nssm' (on PATH). .PARAMETER ServerDir Live Server directory. Default: NSSM AppDirectory. .PARAMETER WorkerDir Live Worker directory. Default: parent of env MxGateway__Worker__ExecutablePath. .PARAMETER ExpectedSha Short/long git SHA. Staged Server+Worker (preflight) and the deployed Server (verify) must carry it in ProductVersion ('0.1.2+'). Builds from 2026-07-09..08-10 have a bogus stamp. .PARAMETER Label Free-text suffix for backup names (e.g. 'planwrites'). .PARAMETER HealthBaseUrl Default http://localhost:5130 (dashboard port; /health/ready, /healthz anonymous). .PARAMETER Rollback Roll back instead of upgrading. Uses the newest Server.bak-* / Server.bak.* unless -BackupStamp. .PARAMETER BackupStamp Stamp of the backup set to roll back to (yyyyMMdd-HHmmss, or yyyyMMddTHHmmss for .bak. sets). .PARAMETER RestoreStores With -Rollback: also restore gateway-auth.db / secrets db from *.bak- (schema-bump case). .PARAMETER NoAutoRollback Do not roll back automatically when start/verification fails. .PARAMETER TakeNewAppSettings Do not carry the live appsettings*.json over into the new Server dir. .PARAMETER AllowRuntimeDowngrade Accept a staged self-contained runtime older than the live one. .PARAMETER SkipFreeSpaceCheck / MinFreeGB / StopTimeoutSeconds / StableSeconds / ReadyTimeoutSeconds tunables. .PARAMETER DryRun Print, do not act. -WhatIf is equivalent. #> [CmdletBinding(SupportsShouldProcess = $true)] param( [string]$ServerSource, [string]$WorkerSource, [string]$ServiceName = 'MxAccessGw', [string]$NssmPath = 'nssm', [string]$ServerDir, [string]$WorkerDir, [string]$ExpectedSha, [string]$Label, [string]$HealthBaseUrl = 'http://localhost:5130', [switch]$Rollback, [string]$BackupStamp, [switch]$RestoreStores, [switch]$NoAutoRollback, [switch]$TakeNewAppSettings, [switch]$AllowRuntimeDowngrade, [switch]$SkipFreeSpaceCheck, [double]$MinFreeGB = 2, [int]$StopTimeoutSeconds = 30, [int]$StableSeconds = 60, [int]$ReadyTimeoutSeconds = 120, [switch]$DryRun ) $ErrorActionPreference = 'Stop' $script:Dry = $DryRun.IsPresent -or $WhatIfPreference # -WhatIf is handled by $script:Dry (Step). Leaving the preference on makes Windows PowerShell 5.1 cmdlets used for READ-ONLY # probing misbehave (Get-FileHash returns nothing under -WhatIf), so the plan would abort; found on windev, #230. $WhatIfPreference = $false $script:Stamp = Get-Date -Format 'yyyyMMdd-HHmmss' # Both naming schemes seen in the field: windev runs ZB.MOM.WW.MxGateway.*.exe, the wonder boxes MxGateway.*.exe. # The installed exe names (NSSM Application, MxGateway__Worker__ExecutablePath) are added to these in Get-Context. $script:KnownServerImages = @('ZB.MOM.WW.MxGateway.Server', 'MxGateway.Server') $script:KnownWorkerImages = @('ZB.MOM.WW.MxGateway.Worker', 'MxGateway.Worker') $script:ServerImages = $script:KnownServerImages $script:WorkerImages = $script:KnownWorkerImages $script:EnvWorkerExe = 'MxGateway__Worker__ExecutablePath' # Live copy always wins for these (host config, certificates, keys), even when the staged build ships one. $script:PreservePatterns = @('appsettings*.json', '*.pfx', '*.p12', '*.pem', '*.key', '*.crt', '*.cer') # Never carried from the old build (they are build output; a live-only binary is a leftover of an older build). $script:BinaryExtensions = @('.dll', '.exe', '.pdb', '.so', '.dylib', '.winmd') $script:Notes = New-Object System.Collections.Generic.List[string] $script:Redact = New-Object System.Collections.Generic.List[string] $script:LockStream = $null $script:BackupDir = $null $script:EnvWarnOnly = $false # --- output (secret values are redacted from every line) ---------------------------------------------------- function Hide([string]$m) { if ($null -eq $m) { return '' } foreach ($s in $script:Redact) { if ($s) { $m = $m.Replace($s, '') } } $m } function Say([string]$m) { $m = Hide $m; Write-Host $m; $script:Notes.Add($m) } function Info([string]$m) { Say "[info] $m" } function Warn([string]$m) { $m = Hide $m; Write-Warning $m; $script:Notes.Add("[WARN] $m") } function Fail([string]$m) { throw "ABORT: $(Hide $m)" } # Mutating step: print in dry-run, run otherwise. function Step([string]$Description, [scriptblock]$Action) { if ($script:Dry) { Say "[DRYRUN] $Description"; return } Say "[ACTION] $Description" & $Action } # --- environment seams (the dry-run harness in scripts/deploy/tests replaces these) --------------------------- function Test-IsWindowsHost { $env:OS -eq 'Windows_NT' } function Invoke-Nssm { param([string[]]$NssmArgs, [switch]$AllowFail) $raw = & $NssmPath @NssmArgs 2>&1 $code = $LASTEXITCODE $text = (($raw | ForEach-Object { "$_" }) -join "`n") -replace "`0", '' if ($code -ne 0 -and -not $AllowFail) { Fail "nssm $($NssmArgs -join ' ') failed (exit $code): $text" } [pscustomobject]@{ Code = $code; Text = $text.Trim() } } function Get-NssmValue([string]$Name) { (Invoke-Nssm -NssmArgs @('get', $ServiceName, $Name)).Text } function Get-ServiceState { $r = Invoke-Nssm -NssmArgs @('status', $ServiceName) -AllowFail if ($r.Code -ne 0) { return 'MISSING' } ($r.Text -split "`n")[0].Trim() } # The env block exactly as NSSM stores it (REG_MULTI_SZ), one 'NAME=value' per element. Falls back to nssm get. function Read-EnvBlockLines { $key = "HKLM:\SYSTEM\CurrentControlSet\Services\$ServiceName\Parameters" if ((Test-IsWindowsHost) -and (Test-Path -LiteralPath $key)) { $p = Get-ItemProperty -LiteralPath $key -Name AppEnvironmentExtra -ErrorAction SilentlyContinue if ($p) { return @($p.AppEnvironmentExtra | Where-Object { $_ }) } return @() } @((Get-NssmValue 'AppEnvironmentExtra') -split "[`r`n]+" | Where-Object { $_.Trim() }) } # Id, ParentId, Name (image name without .exe), Path (may be $null when unreadable). function Get-ProcessTable { if (Get-Command Get-CimInstance -ErrorAction SilentlyContinue) { return @(Get-CimInstance Win32_Process | ForEach-Object { [pscustomobject]@{ Id = [int]$_.ProcessId; ParentId = [int]$_.ParentProcessId; Name = [IO.Path]::GetFileNameWithoutExtension([string]$_.Name); Path = $_.ExecutablePath } }) } @(Get-Process -ErrorAction SilentlyContinue | ForEach-Object { $p = $null; try { $p = $_.Path } catch { } [pscustomobject]@{ Id = $_.Id; ParentId = 0; Name = $_.ProcessName; Path = $p } }) } # PID of the NSSM service host (the gateway Server is its child). $null when it cannot be read. function Get-ServiceHostPid { if (-not (Get-Command Get-CimInstance -ErrorAction SilentlyContinue)) { return $null } $s = Get-CimInstance Win32_Service -Filter "Name='$ServiceName'" -ErrorAction SilentlyContinue if ($s -and $s.ProcessId) { return [int]$s.ProcessId } $null } # Other services whose command line launches the gateway exe. $null when the check is unavailable. function Get-OtherGatewayServices([string]$ExeName) { if (-not (Get-Command Get-CimInstance -ErrorAction SilentlyContinue)) { return $null } # The leading comma keeps an EMPTY result an array: a bare @() reaches the caller as $null ('check unavailable'). , @(Get-CimInstance Win32_Service | Where-Object { $_.Name -ne $ServiceName -and $_.PathName -and $_.PathName -like "*$ExeName*" } | ForEach-Object { $_.Name }) } function Stop-ProcessById([int]$Id) { Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue } function Get-HttpStatus([string]$Url) { try { return [int](Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 10).StatusCode } catch { if ($_.Exception.Response) { return [int]$_.Exception.Response.StatusCode }; return 0 } } function Get-ProductSha([string]$Exe) { try { $pv = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($Exe).ProductVersion if ($pv -match '\+([0-9a-fA-F]{7,40})(\b|$)') { return $Matches[1].ToLowerInvariant() } } catch { } $null } # 'X86', 'Amd64', 'MSIL', ... or $null when it cannot be read. function Get-ExeArchitecture([string]$Exe) { try { return "$([Reflection.AssemblyName]::GetAssemblyName($Exe).ProcessorArchitecture)" } catch { return $null } } # Restrict a directory to SYSTEM + Administrators, inheritance off. Throws if that fails. function Protect-Directory([string]$Path) { if (-not (Test-IsWindowsHost)) { Warn "not Windows: $Path is not ACL-restricted"; return } $null = & icacls.exe $Path /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' 2>&1 if ($LASTEXITCODE -ne 0) { Fail "icacls could not restrict $Path (exit $LASTEXITCODE); refusing to write the env block there" } } function Export-ServiceKey([string]$Destination) { if (-not (Test-IsWindowsHost)) { Info 'not Windows: service registry export skipped'; return } $null = & reg.exe export "HKLM\SYSTEM\CurrentControlSet\Services\$ServiceName" $Destination /y 2>&1 if ($LASTEXITCODE -ne 0) { Fail "reg export of the service key failed (exit $LASTEXITCODE)" } } function Get-DirSddl([string]$Path) { if (-not (Test-IsWindowsHost)) { return $null } try { (Get-Acl -LiteralPath $Path).Sddl } catch { $null } } # Give a carried file the old file's ACL when the old one had explicit (non-inherited) entries. function Copy-ExplicitAcl([string]$From, [string]$To) { if (-not (Test-IsWindowsHost)) { return } try { $a = Get-Acl -LiteralPath $From if (@($a.Access | Where-Object { -not $_.IsInherited }).Count -gt 0 -or $a.AreAccessRulesProtected) { Set-Acl -LiteralPath $To -AclObject $a Info " ACL of $(Split-Path -Leaf $From) copied (it had explicit entries)" } } catch { Warn "could not copy the ACL onto ${To}: $($_.Exception.Message)" } } # --- pure helpers --------------------------------------------------------------------------------------------- function Norm([string]$p) { ($p -replace '\\', '/').TrimEnd('/') } function Test-Rooted([string]$p) { $p -match '^[A-Za-z]:[\\/]' -or $p.StartsWith('/') -or $p.StartsWith('\\') } function Test-Inside([string]$Path, [string]$Root) { if (-not $Path -or -not $Root) { return $false } $a = (Norm $Path).ToLowerInvariant(); $b = (Norm $Root).ToLowerInvariant() $a -eq $b -or $a.StartsWith("$b/") } function Test-SamePath([string]$A, [string]$B) { $A -and $B -and ((Norm $A).ToLowerInvariant() -eq (Norm $B).ToLowerInvariant()) } function Get-Leaf([string]$p) { if (-not $p) { return $null }; ((Norm $p) -split '/')[-1] } function Get-Stem([string]$p) { $l = Get-Leaf $p; if (-not $l) { return $null }; [IO.Path]::GetFileNameWithoutExtension($l) } function Test-ShaMatch([string]$Actual, [string]$Expected) { if (-not $Actual -or -not $Expected) { return $false } $e = $Expected.ToLowerInvariant() $Actual.StartsWith($e) -or $e.StartsWith($Actual) } function Get-TextHashPrefix([string]$Text) { $sha = [Security.Cryptography.SHA256]::Create() try { (($sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($Text)) | ForEach-Object { $_.ToString('x2') }) -join '').Substring(0, 12) } finally { $sha.Dispose() } } function Get-FileSha([string]$Path) { $sha = [Security.Cryptography.SHA256]::Create(); $fs = [IO.File]::OpenRead($Path) try { (($sha.ComputeHash($fs) | ForEach-Object { $_.ToString('X2') }) -join '') } finally { $fs.Dispose(); $sha.Dispose() } } function Test-SensitiveName([string]$Name) { ($Name -match 'KEY|PEPPER|PASS|SECRET|TOKEN|PWD|CREDENTIAL') -and ($Name -notmatch '(Path|Source|EnvVarName|Name)$') } # name -> value. Every sensitive value is registered for redaction. function ConvertTo-EnvMap([string[]]$Lines) { $map = [ordered]@{} foreach ($line in $Lines) { $i = $line.IndexOf('=') if ($i -le 0) { continue } $k = $line.Substring(0, $i).Trim(); $v = $line.Substring($i + 1) $map[$k] = $v if ((Test-SensitiveName $k) -and $v.Length -ge 4 -and -not $script:Redact.Contains($v)) { $script:Redact.Add($v) } } $map } # Names + per-value SHA-256 prefixes. Never carries a value. function Get-EnvFingerprint([string[]]$Lines) { $map = ConvertTo-EnvMap $Lines $entries = [ordered]@{} foreach ($k in $map.Keys) { $entries[$k] = Get-TextHashPrefix ([string]$map[$k]) } [pscustomobject]@{ Count = @($Lines).Count; Names = @($map.Keys); Entries = $entries; BlockHash = (Get-TextHashPrefix (@($Lines) -join "`n")) } } # List of differences (names only). Empty when identical. function Compare-EnvFingerprint($Before, $After) { $d = New-Object System.Collections.Generic.List[string] foreach ($k in $Before.Entries.Keys) { if (-not $After.Entries.Contains($k)) { $d.Add("removed $k") } elseif ($After.Entries[$k] -ne $Before.Entries[$k]) { $d.Add("changed $k (sha256 $($Before.Entries[$k]) -> $($After.Entries[$k]))") } } foreach ($k in $After.Entries.Keys) { if (-not $Before.Entries.Contains($k)) { $d.Add("added $k") } } if ($d.Count -eq 0 -and $Before.BlockHash -ne $After.BlockHash) { $d.Add("block hash $($Before.BlockHash) -> $($After.BlockHash) (order, duplicates or blank entries changed)") } , $d.ToArray() } function Format-EnvFingerprint($Fp) { "$($Fp.Count) entries, block sha256 $($Fp.BlockHash): " + (($Fp.Entries.Keys | ForEach-Object { "$_=$($Fp.Entries[$_])" }) -join ', ') } # Re-read the env block and abort on any difference from the preflight fingerprint. function Assert-EnvUnchanged($ctx, [string]$When) { $now = Get-EnvFingerprint (Read-EnvBlockLines) $diff = Compare-EnvFingerprint $ctx.EnvFp $now if ($diff.Count -gt 0) { # During an automatic rollback the difference is already the reported failure; restoring the old build still goes ahead. if ($script:EnvWarnOnly) { Warn "NSSM env block changed ${When}: $($diff -join '; ') (not written by this script; restore it from the backup dir's AppEnvironmentExtra.txt if unintended)"; return } Fail "NSSM env block changed ${When}: $($diff -join '; ')" } Info "env block unchanged $When ($($now.Count) entries, block sha256 $($now.BlockHash))" } # Directory listings via .NET (FileInfo/DirectoryInfo, hidden files included). Get-ChildItem costs ~250 ms per call # under pwsh on macOS (per-file owner lookups), which made the dry-run harness 20x slower; on Windows both are fine. function Get-Files([string]$Dir, [string]$Filter = '*', [switch]$Recurse) { if (-not $Dir -or -not [IO.Directory]::Exists($Dir)) { return } $o = if ($Recurse) { [IO.SearchOption]::AllDirectories } else { [IO.SearchOption]::TopDirectoryOnly } try { (New-Object IO.DirectoryInfo $Dir).GetFiles($Filter, $o) } catch { Warn "cannot list ${Dir}: $($_.Exception.Message)" } } function Get-Dirs([string]$Dir, [string]$Filter = '*') { if (-not $Dir -or -not [IO.Directory]::Exists($Dir)) { return } try { (New-Object IO.DirectoryInfo $Dir).GetDirectories($Filter) } catch { Warn "cannot list ${Dir}: $($_.Exception.Message)" } } function Get-StoreFiles([string]$Db) { @($Db, "$Db-wal", "$Db-shm") | Where-Object { $_ -and (Test-Path -LiteralPath $_) } } function Get-DirSize([string]$p) { if (-not (Test-Path -LiteralPath $p)) { return 0 } $s = (@(Get-Files $p -Recurse) | Measure-Object Length -Sum).Sum if ($s) { [double]$s } else { 0 } } # Version of a self-contained .NET runtime in a directory (from mscordaccore___.dll), or $null (framework-dependent). function Get-BundledRuntimeVersion([string]$Dir) { $f = @(Get-Files $Dir 'mscordaccore_*.dll') foreach ($x in $f) { if ($x.Name -match '_(\d+\.\d+\.\d+(\.\d+)?)\.dll$') { return [version]$Matches[1] } } $null } # --- processes -------------------------------------------------------------------------------------------------- function Test-GatewayServer($p, $ctx) { ($script:ServerImages -contains $p.Name) -or (Test-SamePath $p.Path $ctx.Application) } function Test-GatewayWorker($p, $ctx) { ($script:WorkerImages -contains $p.Name) -or (Test-SamePath $p.Path $ctx.WorkerExe) } function Get-GatewayProcesses($ctx) { @(Get-ProcessTable | Where-Object { (Test-GatewayServer $_ $ctx) -or (Test-GatewayWorker $_ $ctx) } | ForEach-Object { $role = if (Test-GatewayServer $_ $ctx) { 'Server' } else { 'Worker' } $_ | Add-Member -NotePropertyName Role -NotePropertyValue $role -PassThru -Force }) } function Format-Procs($procs) { (@($procs) | ForEach-Object { "$($_.Name)#$($_.Id)<-$($_.ParentId) [$(if ($_.Path) { $_.Path } else { 'path unreadable' })]" }) -join ', ' } function Assert-NoGatewayProcess($ctx, [string]$When) { $left = @(Get-GatewayProcesses $ctx) if ($left.Count -gt 0) { Fail "${When}: gateway process(es) still alive: $(Format-Procs $left)" } } # The one-gateway guard. Fails closed: a RUNNING service whose gateway cannot be seen is an abort. function Assert-SingleGateway($ctx, [string]$State) { $procs = @(Get-GatewayProcesses $ctx) $servers = @($procs | Where-Object { $_.Role -eq 'Server' }) $workers = @($procs | Where-Object { $_.Role -eq 'Worker' }) Info "gateway processes on the box (images $($script:ServerImages -join '|') / $($script:WorkerImages -join '|')): $(if ($procs.Count) { Format-Procs $procs } else { 'none' })" if ($servers.Count -gt 1) { Fail "$($servers.Count) gateway Server processes are running ($(Format-Procs $servers)). A second gateway is the OrphanWorkerTerminator trap - resolve manually." } switch ($State) { 'SERVICE_STOPPED' { if ($procs.Count -gt 0) { Fail "service is STOPPED but gateway process(es) exist ($(Format-Procs $procs)): a rogue second gateway or an unclean stop. Starting would terminate those workers." } } 'SERVICE_RUNNING' { if ($servers.Count -eq 0) { Fail "service is RUNNING but no gateway Server process was found (looked for image names $($script:ServerImages -join ', ') and path $($ctx.Application)). The process guards cannot see this gateway, so they would pass vacuously; refusing." } $s = $servers[0] if ($s.Path -and -not (Test-SamePath $s.Path $ctx.Application)) { Fail "the running gateway is $($s.Path), not the service's Application $($ctx.Application): a second gateway is running and the service's own is not" } if (-not $s.Path) { Warn "gateway Server #$($s.Id) path is unreadable; matched by image name only" } $hostPid = Get-ServiceHostPid if ($hostPid) { if ($s.ParentId -ne $hostPid) { Fail "gateway Server #$($s.Id) is not a child of the $ServiceName service host #${hostPid} (parent #$($s.ParentId)): it is another gateway" } Info "gateway Server #$($s.Id) is the service's own (child of service host #$hostPid)" } else { Warn 'service host PID unavailable; parent check skipped' } $foreign = @($workers | Where-Object { $_.ParentId -and $_.ParentId -ne $s.Id }) if ($foreign.Count -gt 0) { Warn "Worker process(es) not owned by the service's gateway: $(Format-Procs $foreign). They are killed as orphans on the next start." } $wrongPath = @($workers | Where-Object { $_.Path -and $ctx.WorkerExe -and -not (Test-SamePath $_.Path $ctx.WorkerExe) }) if ($wrongPath.Count -gt 0) { Warn "Worker process(es) not running the configured $($ctx.WorkerExe): $(Format-Procs $wrongPath)" } } 'SERVICE_STOP_PENDING' { Warn 'service is STOP_PENDING; the stop step will finish it' } default { if ($servers.Count -gt 0) { Fail "a gateway Server process exists while the service is $State" } } } } # --- context discovery (read only) ------------------------------------------------------------------------- function Get-Context { $state = Get-ServiceState if ($state -eq 'MISSING') { Fail "service '$ServiceName' does not exist (use Install-MxGatewayService.ps1 for a new box)" } $ctx = [ordered]@{} $ctx.State = $state $ctx.Application = Get-NssmValue 'Application' $ctx.AppDirectory = Get-NssmValue 'AppDirectory' $lines = @(Read-EnvBlockLines) $ctx.Env = ConvertTo-EnvMap $lines $ctx.EnvFp = Get-EnvFingerprint $lines $ctx.EnvCount = $ctx.EnvFp.Count if ($ServerDir) { $ctx.ServerDir = $ServerDir } else { $ctx.ServerDir = $ctx.AppDirectory } $wexe = $ctx.Env[$script:EnvWorkerExe] $ctx.WorkerExe = $wexe if ($WorkerDir) { $ctx.WorkerDir = $WorkerDir } elseif ($wexe) { $ctx.WorkerDir = Split-Path -Parent $wexe } else { Fail "cannot determine the Worker directory: env $($script:EnvWorkerExe) is not in the NSSM env block; pass -WorkerDir" } $pd = 'C:\ProgramData\MxGateway' $auth = $ctx.Env['MxGateway__Authentication__SqlitePath']; if (-not $auth) { $auth = "$pd\gateway-auth.db" } $sec = $ctx.Env['Secrets__SqlitePath']; if (-not $sec) { $sec = "$pd\mxgateway-secrets.db" } $ctx.AuthDb = $auth; $ctx.SecretsDb = $sec $ctx.KekFile = $ctx.Env['Secrets__MasterKey__FilePath'] $ctx.ServerExeName = Get-Leaf $ctx.Application $ctx.WorkerExeName = Get-Leaf $wexe if (-not $ctx.WorkerExeName) { $ctx.WorkerExeName = 'ZB.MOM.WW.MxGateway.Worker.exe' } # Process image names: the installed ones first, then both known schemes. $script:ServerImages = @(@(Get-Stem $ctx.Application) + $script:KnownServerImages | Where-Object { $_ } | Select-Object -Unique) $script:WorkerImages = @(@(Get-Stem $wexe) + $script:KnownWorkerImages | Where-Object { $_ } | Select-Object -Unique) [pscustomobject]$ctx } # --- preflight ----------------------------------------------------------------------------------------------- function Test-MasterKey($ctx) { $src = $ctx.Env['Secrets__MasterKey__Source'] $envVar = $ctx.Env['Secrets__MasterKey__EnvVarName']; if (-not $envVar) { $envVar = 'ZB_SECRETS_MASTER_KEY' } if ($ctx.KekFile -or $src -eq 'File') { if (-not $ctx.KekFile) { Fail 'Secrets__MasterKey__Source=File but no Secrets__MasterKey__FilePath in the env block' } if (-not (Test-Rooted $ctx.KekFile)) { Fail "master key file '$($ctx.KekFile)' is not absolute" } foreach ($root in @($ctx.ServerDir, $ctx.WorkerDir)) { if (Test-Inside $ctx.KekFile $root) { Fail "master key file '$($ctx.KekFile)' is INSIDE '$root' - the swap would lose it" } } if (Test-Path -LiteralPath $ctx.KekFile) { Info "master key: file $($ctx.KekFile) (present, outside the swap)" } else { Fail "master key file '$($ctx.KekFile)' does not exist (the Secrets store would be unreadable)" } } elseif ($ctx.Env.Contains($envVar)) { Info "master key: env block entry $envVar" } else { Warn "the env block carries no master key ($envVar or Secrets__MasterKey__FilePath); it comes from elsewhere (appsettings or the account environment). The upgrade does not change that, but confirm the Secrets store opens after the start." } } function Invoke-Preflight($ctx) { Say '== Preflight ==' Info "service '$ServiceName' state: $($ctx.State)" Info "Application : $($ctx.Application)" Info "AppDirectory: $($ctx.AppDirectory)" Info "Server dir : $($ctx.ServerDir)" Info "Worker dir : $($ctx.WorkerDir) (exe env: $($ctx.WorkerExe))" Info "API-key DB : $($ctx.AuthDb)" Info "Secrets DB : $($ctx.SecretsDb)" Info "NSSM env block fingerprint (names + value sha256 prefixes, no values): $(Format-EnvFingerprint $ctx.EnvFp)" if ($ctx.EnvCount -eq 0) { Fail 'NSSM env block is empty - refusing: it holds Kestrel/store paths, the master key and the pepper' } if (-not ($ctx.Env.Keys | Where-Object { $_ -match 'Pepper' })) { Warn 'no *Pepper* entry in the env block; confirm the API-key pepper is supplied another way or issued tokens will not verify' } if ($ctx.Env['ASPNETCORE_ENVIRONMENT'] -ne 'Production' -and $ctx.Env['DOTNET_ENVIRONMENT'] -ne 'Production') { Warn 'neither ASPNETCORE_ENVIRONMENT nor DOTNET_ENVIRONMENT is exactly Production (runbook trap 6) - do not change it during an upgrade' } if (-not (Test-SamePath (Split-Path -Parent $ctx.Application) $ctx.ServerDir)) { Fail "NSSM Application '$($ctx.Application)' is not inside the Server dir '$($ctx.ServerDir)'" } if ($ctx.WorkerExe -and -not (Test-SamePath (Split-Path -Parent $ctx.WorkerExe) $ctx.WorkerDir)) { Fail "$($script:EnvWorkerExe) '$($ctx.WorkerExe)' is not inside the Worker dir '$($ctx.WorkerDir)'" } # State that must survive: rooted and outside the swapped directories. foreach ($pair in @(@('API-key DB', $ctx.AuthDb), @('Secrets DB', $ctx.SecretsDb))) { if (-not (Test-Rooted $pair[1])) { Fail "$($pair[0]) path '$($pair[1])' is not absolute" } foreach ($root in @($ctx.ServerDir, $ctx.WorkerDir)) { if (Test-Inside $pair[1] $root) { Fail "$($pair[0]) '$($pair[1])' is INSIDE '$root' - the swap would lose it. Move it out first (runbook section 1)." } } if (Test-Path -LiteralPath $pair[1]) { Info "$($pair[0]) present ($((Get-StoreFiles $pair[1]).Count) file(s) incl. wal/shm)" } else { Warn "$($pair[0]) '$($pair[1])' does not exist yet (fresh store will be created on start)" } } Test-MasterKey $ctx # Directories foreach ($d in @($ctx.ServerDir, $ctx.WorkerDir)) { if (-not (Test-Path -LiteralPath $d -PathType Container)) { Fail "live directory '$d' does not exist" } } if ((Test-Inside $ctx.WorkerDir $ctx.ServerDir) -or (Test-Inside $ctx.ServerDir $ctx.WorkerDir)) { Fail 'Server and Worker directories are nested; the rename-in-place swap needs sibling directories' } if (-not (Test-Path -LiteralPath $ctx.Application)) { Fail "NSSM Application '$($ctx.Application)' does not exist" } if ($ctx.WorkerExe -and -not (Test-Path -LiteralPath $ctx.WorkerExe)) { Fail "$($script:EnvWorkerExe) '$($ctx.WorkerExe)' does not exist" } $dbs = @(@(Get-Files $ctx.ServerDir -Recurse) + @(Get-Files $ctx.WorkerDir -Recurse) | Where-Object { $_.Name -match '\.db(-wal|-shm)?$' }) if ($dbs.Count -gt 0) { Fail "SQLite file(s) inside the swapped directories would be orphaned by the rename: $(($dbs | ForEach-Object { $_.FullName }) -join ', ')" } # One-gateway guard (fail closed) Assert-SingleGateway $ctx $ctx.State $others = Get-OtherGatewayServices $ctx.ServerExeName if ($null -eq $others) { Info 'Win32_Service cross-check unavailable on this host (skipped)' } elseif (@($others).Count -gt 0) { Fail "another service ($(@($others) -join ', ')) also launches $($ctx.ServerExeName); two gateways on one box is forbidden" } else { Info "no other service launches $($ctx.ServerExeName)" } if ($Rollback) { Say '== Preflight OK =='; return } # Sources if (-not $ServerSource -or -not $WorkerSource) { Fail '-ServerSource and -WorkerSource are required for an upgrade' } $stagedServerExe = Join-Path $ServerSource $ctx.ServerExeName $stagedWorkerExe = Join-Path $WorkerSource $ctx.WorkerExeName foreach ($s in @(@($ServerSource, $stagedServerExe), @($WorkerSource, $stagedWorkerExe))) { if (-not (Test-Path -LiteralPath $s[1])) { Fail "staged source '$($s[0])' does not contain $(Get-Leaf $s[1]) (the installed exe name; NSSM and $($script:EnvWorkerExe) point at it)" } } if ((Test-Inside $ServerSource $ctx.ServerDir) -or (Test-Inside $WorkerSource $ctx.WorkerDir) -or (Test-Inside $ServerSource $ctx.WorkerDir) -or (Test-Inside $WorkerSource $ctx.ServerDir)) { Fail 'staging directory is inside a live directory' } $arch = Get-ExeArchitecture $stagedWorkerExe if ($arch -ne 'X86') { Fail "staged Worker $stagedWorkerExe architecture is '$arch', not X86 (trap 5)" } Info "staged Worker architecture: X86" $sSha = Get-ProductSha $stagedServerExe $wSha = Get-ProductSha $stagedWorkerExe Info "staged Server SHA: $sSha ; staged Worker SHA: $wSha ; live Server SHA: $(Get-ProductSha $ctx.Application)" if ($sSha -and $wSha -and -not (Test-ShaMatch $sSha $wSha)) { Fail "Server ($sSha) and Worker ($wSha) were built from different commits (wire protocol risk)" } if ($ExpectedSha) { if (-not (Test-ShaMatch $sSha $ExpectedSha)) { if ($script:Dry) { Warn "staged Server SHA '$sSha' does not match -ExpectedSha '$ExpectedSha' (dry-run: continuing)" } else { Fail "staged Server SHA '$sSha' does not match -ExpectedSha '$ExpectedSha'" } } } else { Warn 'no -ExpectedSha given: the post-deploy SHA check will only record the SHA' } $leafSha = if ((Get-Leaf $ctx.ServerDir) -match '([0-9a-f]{7,40})$') { $Matches[1] } else { $null } if ($leafSha -and $sSha -and -not (Test-ShaMatch $leafSha $sSha)) { Warn "the Server dir name '$(Get-Leaf $ctx.ServerDir)' carries SHA $leafSha but will hold $sSha after the rename-in-place swap" } # Runtime: never silently downgrade a self-contained Server. $liveRt = Get-BundledRuntimeVersion $ctx.ServerDir; $newRt = Get-BundledRuntimeVersion $ServerSource Info "bundled .NET runtime: live $(if ($liveRt) { $liveRt } else { 'none (framework-dependent)' }), staged $(if ($newRt) { $newRt } else { 'none (framework-dependent)' })" if ($liveRt -and -not $newRt) { Warn 'live Server is self-contained, staged is framework-dependent: the box must have a matching .NET 10 runtime installed' } if ($liveRt -and $newRt -and $newRt -lt $liveRt) { if ($AllowRuntimeDowngrade) { Warn "staged runtime $newRt is OLDER than live $liveRt (-AllowRuntimeDowngrade)" } else { Fail "staged self-contained runtime $newRt is OLDER than the live $liveRt; rebuild with a newer SDK or pass -AllowRuntimeDowngrade" } } # What the swap will carry over / drop (top-level files). foreach ($pair in @(@('Server', $ctx.ServerDir, $ServerSource), @('Worker', $ctx.WorkerDir, $WorkerSource))) { $plan = Get-CarryPlan $pair[1] $pair[2] ($pair[0] -eq 'Server' -and -not $TakeNewAppSettings) Info "$($pair[0]): carry from live -> $(if ($plan.Carry.Count) { $plan.Carry -join ', ' } else { 'nothing' }); live-only binaries dropped -> $(if ($plan.Dropped.Count) { $plan.Dropped -join ', ' } else { 'none' })" if ($plan.Dirs.Count) { Warn "$($pair[0]): live-only directories NOT carried (they stay in the .bak): $($plan.Dirs -join ', ')" } } $lw = @(Get-Files $ctx.WorkerDir | ForEach-Object { $_.Name }); $nw = @(Get-Files $WorkerSource | ForEach-Object { $_.Name }) $onlyNew = @($nw | Where-Object { $lw -notcontains $_ }) if ($onlyNew.Count) { Info "Worker: files only in the staged build: $($onlyNew -join ', ')" } # Disk space: 2x (server + worker + stores + payload), at least MinFreeGB if (-not $SkipFreeSpaceCheck) { $need = (Get-DirSize $ctx.ServerDir) + (Get-DirSize $ctx.WorkerDir) + (Get-DirSize $ServerSource) + (Get-DirSize $WorkerSource) foreach ($f in (Get-StoreFiles $ctx.AuthDb) + (Get-StoreFiles $ctx.SecretsDb)) { $need += (Get-Item -LiteralPath $f).Length } $need = [math]::Max($need * 2, $MinFreeGB * 1GB) $root = [IO.Path]::GetPathRoot((Resolve-Path -LiteralPath $ctx.ServerDir).Path) $free = (New-Object IO.DriveInfo $root).AvailableFreeSpace Info ("free on {0}: {1:N2} GB, required {2:N2} GB" -f $root, ($free / 1GB), ($need / 1GB)) if ($free -lt $need) { Fail 'not enough free disk space for the backups' } } Say '== Preflight OK ==' } # --- stop / start --------------------------------------------------------------------------------------------- function Wait-State([string]$Want, [int]$TimeoutSec) { $deadline = (Get-Date).AddSeconds($TimeoutSec) do { $s = Get-ServiceState; if ($s -eq $Want) { return $true }; Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) $false } function Wait-NoGatewayProcess($ctx, [int]$TimeoutSec) { $deadline = (Get-Date).AddSeconds($TimeoutSec) do { if (@(Get-GatewayProcesses $ctx).Count -eq 0) { return $true }; Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) $false } # Kill only this install's gateway processes: path under the Server/Worker dir, or (path unreadable) the installed image name. function Stop-InstallProcesses($ctx, [string]$Why) { $installed = @((Get-Stem $ctx.Application), (Get-Stem $ctx.WorkerExe)) | Where-Object { $_ } $mine = @(Get-GatewayProcesses $ctx | Where-Object { if ($_.Path) { (Test-Inside $_.Path $ctx.ServerDir) -or (Test-Inside $_.Path $ctx.WorkerDir) } else { $installed -contains $_.Name } } | Sort-Object { if ($_.Role -eq 'Server') { 0 } else { 1 } }) foreach ($p in $mine) { Say "[ACTION] Stop-Process -Id $($p.Id) -Force ($($p.Name), $Why)"; Stop-ProcessById $p.Id } $mine.Count } function Stop-Gateway($ctx) { Say '-- Stop --' $state = Get-ServiceState if ($state -eq 'SERVICE_STOPPED') { Info 'already stopped' } else { Step "nssm stop $ServiceName (wait up to $StopTimeoutSeconds s for SERVICE_STOPPED, then for every gateway/Worker process to exit)" { [void](Invoke-Nssm -NssmArgs @('stop', $ServiceName) -AllowFail) if (-not (Wait-State 'SERVICE_STOPPED' $StopTimeoutSeconds)) { # STOP_PENDING trap: workers did not exit in time. Kill this install's Server, then its Workers. Warn "service still $(Get-ServiceState) after $StopTimeoutSeconds s (STOP_PENDING trap) - killing this install's gateway processes" $n = Stop-InstallProcesses $ctx 'STOP_PENDING' if ($n -eq 0) { Warn 'no process of this install was found to kill' } if (-not (Wait-State 'SERVICE_STOPPED' 20)) { Warn 'still not STOPPED; killing the NSSM service host process' $svcPid = Get-ServiceHostPid if ($svcPid) { Say "[ACTION] Stop-Process -Id $svcPid -Force (service host)"; Stop-ProcessById $svcPid } if (-not (Wait-State 'SERVICE_STOPPED' 20)) { Fail "service '$ServiceName' will not stop; investigate manually (state $(Get-ServiceState))" } } } if (-not (Wait-NoGatewayProcess $ctx 20)) { Warn "gateway/Worker process(es) outlived the service stop (unclean stop): $(Format-Procs (Get-GatewayProcesses $ctx))" [void](Stop-InstallProcesses $ctx 'left over after the stop') [void](Wait-NoGatewayProcess $ctx 10) } } } if (-not $script:Dry) { Assert-NoGatewayProcess $ctx 'after stop' Info 'stopped; no gateway/Worker process on the box' Assert-EnvUnchanged $ctx 'after the stop' } } function Start-Gateway($ctx) { Say '-- Start --' if (-not $script:Dry) { Assert-NoGatewayProcess $ctx 'before start (OrphanWorkerTerminator guard)' Assert-EnvUnchanged $ctx 'before the start' } else { Say "[DRYRUN] assert no process named $($script:ServerImages -join '|') / $($script:WorkerImages -join '|') exists, and the env block fingerprint is unchanged (OrphanWorkerTerminator guard)" } Step "nssm start $ServiceName (once)" { [void](Invoke-Nssm -NssmArgs @('start', $ServiceName)) } Step "wait for SERVICE_RUNNING and compare the env block fingerprint" { if (-not (Wait-State 'SERVICE_RUNNING' 60)) { Fail "service did not reach RUNNING (state $(Get-ServiceState))" } Assert-EnvUnchanged $ctx 'after the start' } } # --- backup / swap ----------------------------------------------------------------------------------------------- function Get-Suffix { if ($Label) { "$($script:Stamp)-$Label" } else { $script:Stamp } } function Backup-State($ctx) { Say '-- Backup (service stopped) --' $suffix = Get-Suffix $root = Split-Path -Parent $ctx.ServerDir $dir = Join-Path $root "_deploy-backup-$suffix" $script:BackupDir = $dir Step "create $dir, restrict it to SYSTEM + Administrators, then save the env block (secrets), service registry key and env fingerprint" { New-Item -ItemType Directory -Path $dir -Force | Out-Null Protect-Directory $dir $body = @("Application=$($ctx.Application)", "AppDirectory=$($ctx.AppDirectory)", '--- AppEnvironmentExtra ---') + @(Read-EnvBlockLines) Set-Content -LiteralPath (Join-Path $dir 'AppEnvironmentExtra.txt') -Value $body -Encoding UTF8 Export-ServiceKey (Join-Path $dir "$ServiceName-servicekey.reg") $fp = @("BlockHash=$($ctx.EnvFp.BlockHash)") + @($ctx.EnvFp.Entries.Keys | ForEach-Object { "$_=$($ctx.EnvFp.Entries[$_])" }) Set-Content -LiteralPath (Join-Path $dir 'env-fingerprint.txt') -Value $fp -Encoding ASCII } foreach ($db in @($ctx.AuthDb, $ctx.SecretsDb)) { foreach ($f in (Get-StoreFiles $db)) { $dst = "$f.bak-$suffix" Step "copy store file $f -> $dst (live file is never moved)" { Copy-Item -LiteralPath $f -Destination $dst -Force if ((Get-FileSha $f) -ne (Get-FileSha $dst)) { Fail "store backup hash mismatch for $f" } } } if (-not (Test-Path -LiteralPath $db)) { Info "no store at $db to back up" } } if ($ctx.KekFile -and (Test-Path -LiteralPath $ctx.KekFile)) { Step "copy master key file $($ctx.KekFile) into $dir (live file is never moved)" { Copy-Item -LiteralPath $ctx.KekFile -Destination (Join-Path $dir (Get-Leaf $ctx.KekFile)) -Force } } } function Test-Preserved([string]$Name) { foreach ($p in $script:PreservePatterns) { if ($Name -like $p) { return $true } }; $false } # Which top-level files of the live dir go into the new one. Carry = names copied from live (live wins). function Get-CarryPlan([string]$Live, [string]$Source, [bool]$KeepSettings) { $liveFiles = @(Get-Files $Live) $srcNames = @(Get-Files $Source | ForEach-Object { $_.Name }) $srcDirs = @(Get-Dirs $Source | ForEach-Object { $_.Name }) $carry = New-Object System.Collections.Generic.List[string]; $dropped = New-Object System.Collections.Generic.List[string] foreach ($f in $liveFiles) { $inSource = $srcNames -contains $f.Name if ($f.Name -like 'appsettings*.json') { if ($KeepSettings) { $carry.Add($f.Name) }; continue } if (Test-Preserved $f.Name) { $carry.Add($f.Name); continue } if ($inSource) { continue } if ($script:BinaryExtensions -contains $f.Extension.ToLowerInvariant()) { $dropped.Add($f.Name) } else { $carry.Add($f.Name) } } $dirs = @(Get-Dirs $Live | Where-Object { $srcDirs -notcontains $_.Name } | ForEach-Object { $_.Name }) [pscustomobject]@{ Carry = $carry.ToArray(); Dropped = $dropped.ToArray(); Dirs = $dirs } } function Swap-Directory([string]$Live, [string]$Source, [string]$Name, [bool]$KeepSettings) { $bak = "$Live.bak-$(Get-Suffix)" $plan = Get-CarryPlan $Live $Source $KeepSettings $hashes = @{} foreach ($f in $plan.Carry) { $hashes[$f] = Get-FileSha (Join-Path $Live $f) } $sddlOld = Get-DirSddl $Live Step "rename $Name dir: $Live -> $bak" { Rename-Item -LiteralPath $Live -NewName (Split-Path -Leaf $bak) } Step "copy new ${Name}: $Source -> $Live" { Copy-Item -LiteralPath $Source -Destination $Live -Recurse -Force } foreach ($f in $plan.Carry) { Step "carry live $Name\$f forward from $bak (sha256 $($hashes[$f].Substring(0, 12)))" { Copy-Item -LiteralPath (Join-Path $bak $f) -Destination (Join-Path $Live $f) -Force if ((Get-FileSha (Join-Path $Live $f)) -ne $hashes[$f]) { Fail "carried $Name\$f does not match the live copy" } Copy-ExplicitAcl (Join-Path $bak $f) (Join-Path $Live $f) } } foreach ($f in $plan.Dropped) { Info "$Name\$f exists only in the old build (binary) and is not carried; it stays in $bak" } if (-not $script:Dry) { $sddlNew = Get-DirSddl $Live if ($sddlOld -and $sddlNew -and $sddlOld -ne $sddlNew) { Warn "$Name dir ACL differs from the old one (new inherits from the parent). old: $sddlOld new: $sddlNew" } } $bak } # --- verification ---------------------------------------------------------------------------------------------------- function Invoke-Verify($ctx, [bool]$AfterRollback = $false) { Say '== Verify ==' if ($script:Dry) { Say "[DRYRUN] GET $HealthBaseUrl/health/ready until 200 (<= $ReadyTimeoutSeconds s)" Say "[DRYRUN] GET $HealthBaseUrl/healthz ; read ProductVersion SHA of $($ctx.Application)$(if ($ExpectedSha -and -not $Rollback) { " and require it to match $ExpectedSha" })" Say "[DRYRUN] exactly one gateway Server ($($ctx.Application)) on the box, same PID, service SERVICE_RUNNING for $StableSeconds s, env fingerprint unchanged" return } $ready = $false; $deadline = (Get-Date).AddSeconds($ReadyTimeoutSeconds) while (-not $ready -and (Get-Date) -lt $deadline) { $ready = ((Get-HttpStatus "$HealthBaseUrl/health/ready") -eq 200) if (-not $ready) { Start-Sleep -Seconds 3 } } if (-not $ready) { Fail "$HealthBaseUrl/health/ready did not return 200 within $ReadyTimeoutSeconds s" } Info '/health/ready = 200' $hz = Get-HttpStatus "$HealthBaseUrl/healthz" if ($hz -ne 200) { Fail "/healthz = $hz" } Info '/healthz = 200' $sha = Get-ProductSha $ctx.Application Info "deployed Server SHA (ProductVersion): $sha" if ($ExpectedSha -and -not $Rollback -and -not $AfterRollback -and -not (Test-ShaMatch $sha $ExpectedSha)) { Fail "deployed SHA '$sha' != expected '$ExpectedSha'" } Assert-SingleGateway $ctx 'SERVICE_RUNNING' $pid0 = @(Get-GatewayProcesses $ctx | Where-Object { $_.Role -eq 'Server' })[0].Id Info "watching the service for $StableSeconds s (gateway #$pid0)" $until = (Get-Date).AddSeconds($StableSeconds) do { if ((Get-ServiceState) -ne 'SERVICE_RUNNING') { Fail 'service left RUNNING during the stability window (restart loop?)' } Start-Sleep -Seconds 5 } while ((Get-Date) -lt $until) $servers = @(Get-GatewayProcesses $ctx | Where-Object { $_.Role -eq 'Server' }) if ($servers.Count -ne 1 -or $servers[0].Id -ne $pid0) { Fail "gateway process changed during the stability window: #$pid0 -> $(Format-Procs $servers) (restart loop or second gateway)" } Assert-EnvUnchanged $ctx 'after the stability window' Warn 'health is green with zero sessions; still prove the Worker path: open a session with a PRE-EXISTING API key and read a known tag (runbook 7.3)' } # --- rollback ------------------------------------------------------------------------------------------------------------ # Backup dirs come in two spellings: .bak-yyyyMMdd-HHmmss[-label] (this script) and .bak.yyyyMMddTHHmmss (manual / adapted scripts). function Get-BackupKey([string]$Name) { if ($Name -match '\.bak[-.](\d{8})[-T]?(\d{6})') { return "$($Matches[1])$($Matches[2])" } $null } function Find-Backup([string]$Live, [string]$Stamp) { $leaf = Split-Path -Leaf $Live; $parent = Split-Path -Parent $Live $cands = @(Get-Dirs $parent | Where-Object { ($_.Name -like "$leaf.bak-*" -or $_.Name -like "$leaf.bak.*") -and (Get-BackupKey $_.Name) }) if ($Stamp) { $key = ($Stamp -replace '[^0-9]', '') $cands = @($cands | Where-Object { (Get-BackupKey $_.Name) -eq $key -or $_.Name -like "$leaf.bak-$Stamp*" -or $_.Name -like "$leaf.bak.$Stamp*" }) } $cands | Sort-Object { Get-BackupKey $_.Name } -Descending | Select-Object -First 1 } function Invoke-RollbackSwap($ctx, [string]$Stamp, [bool]$Stores, [bool]$AllowPartial) { Say '== Rollback ==' $sb = Find-Backup $ctx.ServerDir $Stamp if (-not $sb -and -not $AllowPartial) { Fail "no Server backup directory found (stamp '$Stamp')" } $key = if ($sb) { Get-BackupKey $sb.Name } else { ($Stamp -replace '[^0-9]', '') } $wb = Find-Backup $ctx.WorkerDir $key if ($wb -and (Get-BackupKey $wb.Name) -ne $key) { $wb = $null } if (-not $wb -and -not $AllowPartial) { Fail "no Worker backup matching the Server backup $($sb.Name) (stamp key $key); refusing a mixed rollback" } if (-not $sb -and -not $wb) { Fail "nothing to roll back to (stamp '$Stamp')" } Info "rolling back to Server=$(if ($sb) { $sb.Name } else { '(unchanged)' }) Worker=$(if ($wb) { $wb.Name } else { '(unchanged)' })" $fpFile = @(Get-Dirs (Split-Path -Parent $ctx.ServerDir) '_deploy-backup-*' | Where-Object { ($_.Name -replace '[^0-9]', '').StartsWith($key) } | ForEach-Object { Join-Path $_.FullName 'env-fingerprint.txt' } | Where-Object { Test-Path -LiteralPath $_ }) if ($fpFile.Count) { $saved = Get-Content -LiteralPath $fpFile[0] | Where-Object { $_ -notlike 'BlockHash=*' } $now = @($ctx.EnvFp.Entries.Keys | ForEach-Object { "$_=$($ctx.EnvFp.Entries[$_])" }) if (($saved -join '|') -ne ($now -join '|')) { Warn "the env block differs from the one saved before the upgrade ($($fpFile[0])); restore it from AppEnvironmentExtra.txt there if that was not intended (runbook section 8 step 4)" } else { Info "env block matches the pre-upgrade fingerprint in $($fpFile[0])" } } Stop-Gateway $ctx foreach ($pair in @(@($ctx.ServerDir, $sb), @($ctx.WorkerDir, $wb))) { if (-not $pair[1]) { Info "$($pair[0]) has no backup for this stamp; left unchanged"; continue } $failed = "$($pair[0]).failed-$($script:Stamp)" if (Test-Path -LiteralPath $pair[0]) { Step "rename $($pair[0]) -> $failed" { Rename-Item -LiteralPath $pair[0] -NewName (Split-Path -Leaf $failed) } } Step "rename $($pair[1].FullName) -> $($pair[0])" { Rename-Item -LiteralPath $pair[1].FullName -NewName (Split-Path -Leaf $pair[0]) } } if ($Stores) { $bstamp = ($sb.Name -replace '^.*\.bak[-.]', '') foreach ($db in @($ctx.AuthDb, $ctx.SecretsDb)) { foreach ($ext in '', '-wal', '-shm') { $live = "$db$ext"; $bak = "$db$ext.bak-$bstamp" if (Test-Path -LiteralPath $bak) { Step "restore store file $bak -> $live" { Copy-Item -LiteralPath $bak -Destination $live -Force } } elseif ($ext -and (Test-Path -LiteralPath $live) -and (Test-Path -LiteralPath "$db.bak-$bstamp")) { # a stale wal/shm must not pair with the restored db: move it aside, never delete Step "move stale $live aside -> $live.stale-$($script:Stamp)" { Rename-Item -LiteralPath $live -NewName ("$(Split-Path -Leaf $live).stale-$($script:Stamp)") } } elseif (-not $ext) { Warn "no backup $bak for $db; live store left untouched" } } } } else { Info 'stores left untouched (they live outside the swap); pass -RestoreStores after a schema bump' } Start-Gateway $ctx } # --- main ------------------------------------------------------------------------------------------------------------------- function Invoke-Main { $exit = 0 $ctx = $null try { Say "Upgrade-MxGatewayService stamp=$($script:Stamp) mode=$(if ($Rollback) {'ROLLBACK'} else {'UPGRADE'}) dry=$($script:Dry) host=$env:COMPUTERNAME" $ctx = Get-Context Invoke-Preflight $ctx if (-not $script:Dry) { $lockPath = Join-Path (Split-Path -Parent $ctx.ServerDir) '.mxgw-deploy.lock' try { $script:LockStream = [IO.File]::Open($lockPath, 'OpenOrCreate', 'ReadWrite', 'None') } catch { Fail "another deploy run holds $lockPath" } } if ($Rollback) { Invoke-RollbackSwap $ctx $BackupStamp $RestoreStores.IsPresent $false Invoke-Verify $ctx } else { Stop-Gateway $ctx try { Backup-State $ctx } catch { $err = $_ if (-not $script:Dry) { Warn 'backup failed before anything was swapped - starting the unchanged build again' try { Start-Gateway $ctx } catch { Warn "restart of the unchanged build FAILED: $($_.Exception.Message)" } } throw $err } $touched = $false try { $touched = $true $null = Swap-Directory $ctx.ServerDir $ServerSource 'Server' (-not $TakeNewAppSettings) $null = Swap-Directory $ctx.WorkerDir $WorkerSource 'Worker' $false if (-not $script:Dry) { Assert-EnvUnchanged $ctx 'after the swap' } Start-Gateway $ctx Invoke-Verify $ctx } catch { $err = $_ Say "[FAIL] $(Hide $err.Exception.Message)" if ($script:Dry -or $NoAutoRollback -or -not $touched) { throw } Warn 'upgrade failed - rolling back automatically' $script:EnvWarnOnly = $true try { Invoke-RollbackSwap $ctx $script:Stamp $false $true Invoke-Verify $ctx $true Warn 'ROLLED BACK to the previous build; original failure follows' } catch { Warn "AUTOMATIC ROLLBACK FAILED: $($_.Exception.Message). Manual intervention required (backups: *.bak-$(Get-Suffix))." } throw $err } } Say '== DONE ==' } catch { $exit = 1 $m = Hide $_.Exception.Message $script:Notes.Add("ERROR: $m") Write-Host "ERROR: $m" -ForegroundColor Red } finally { if ($script:LockStream) { $script:LockStream.Dispose(); $script:LockStream = $null } try { if (-not $script:Dry -and $ctx) { $notes = Join-Path (Split-Path -Parent $ctx.ServerDir) "_DEPLOY-NOTES-$($script:Stamp).txt" Set-Content -LiteralPath $notes -Value $script:Notes -Encoding UTF8 Write-Host "deploy notes: $notes" } } catch { } $script:Redact.Clear() } $exit } # Dot-sourcing (the dry-run harness) only defines the functions. if ($MyInvocation.InvocationName -ne '.') { $rc = Invoke-Main | Select-Object -Last 1; exit $rc }