feat(deploy): on-box NSSM upgrade/rollback and new-box install scripts, runbook updated (mxaccessgw#220)
Add scripts/deploy/Upgrade-MxGatewayService.ps1 (preflight, stop with STOP_PENDING handling, backup, Server+Worker swap, start, /health/ready + SHA verify, auto/manual rollback, one-gateway guard, never touches the NSSM env block) and Install-MxGatewayService.ps1 (service install, restart/recovery settings, env block). Both support -DryRun/-WhatIf. Dry-run validated on macOS pwsh against a fake layout; untested on a real box. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,490 @@
|
||||
#requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
On-box upgrade / rollback of the NSSM-wrapped MxAccess Gateway (Server + x86 Worker).
|
||||
Implements docs/runbooks/DeployUpgradeNssmService.md sections 3, 5, 6, 7 and 8 (mxaccessgw#220).
|
||||
|
||||
.DESCRIPTION
|
||||
RUN THIS ON THE WINDOWS BOX ITSELF (Windows PowerShell 5.1 is enough). The new Server publish and the
|
||||
new x86 Worker build must already be staged on the box (UNC/SMB copy; build both from the same SHA).
|
||||
|
||||
Upgrade flow: preflight -> stop (STOP_PENDING handled) -> backup -> swap Server + Worker -> start ->
|
||||
verify (/health/ready, VersionInfo SHA, service stable) -> automatic rollback if start/verify fails.
|
||||
Rollback flow (-Rollback): stop -> swap the .bak directories back -> optional store restore -> start -> verify.
|
||||
|
||||
Safety properties:
|
||||
* Exactly one gateway per box. Refuses to run if a second gateway process / service points at the same
|
||||
exe, if the service is stopped but a gateway process exists, or if a Server/Worker process is still
|
||||
alive when a start is about to happen (OrphanWorkerTerminator trap, runbook section 2.1).
|
||||
* Never calls `nssm set ... AppEnvironmentExtra` (that replaces the whole block). The env block is only
|
||||
READ (saved to the backup dir, entry count compared before/after). The Application/AppDirectory
|
||||
are not changed either: the swap is rename-in-place, so the NSSM config is untouched.
|
||||
* API-key DB, Secrets store and master key live outside the content root and are never moved or
|
||||
deleted. They are only COPIED (db + -wal + -shm) into *.bak-<stamp> siblings while the service is stopped.
|
||||
The script refuses to run if a store path is inside the content root (the 2026-08-09 incident).
|
||||
|
||||
-DryRun (or -WhatIf) prints every mutating action and performs none. Read-only probing (nssm get/status,
|
||||
process list, disk space, file existence) still runs so the decision logic is exercised.
|
||||
|
||||
.PARAMETER ServerSource Staged Server publish output (contains ZB.MOM.WW.MxGateway.Server.exe).
|
||||
.PARAMETER WorkerSource Staged x86 Worker output (contains ZB.MOM.WW.MxGateway.Worker.exe).
|
||||
.PARAMETER ServiceName NSSM service name. Default MxAccessGw.
|
||||
.PARAMETER NssmPath nssm executable. Default 'nssm' (on PATH).
|
||||
.PARAMETER ServerDir Live Server directory. Default: NSSM AppDirectory.
|
||||
.PARAMETER WorkerDir Live Worker directory. Default: parent of env MxGateway__Worker__ExecutablePath.
|
||||
.PARAMETER ExpectedSha Short/long git SHA. Staged Server+Worker (preflight) and the deployed Server (verify)
|
||||
must carry it in ProductVersion ('0.1.2+<sha>'). Builds from 2026-07-09..08-10 have a bogus stamp.
|
||||
.PARAMETER Label Free-text suffix for backup names (e.g. 'planwrites').
|
||||
.PARAMETER HealthBaseUrl Default http://localhost:5130 (dashboard port; /health/ready, /healthz anonymous).
|
||||
.PARAMETER Rollback Roll back instead of upgrading. Uses newest Server.bak-* unless -BackupStamp is given.
|
||||
.PARAMETER BackupStamp Stamp (yyyyMMdd-HHmmss) of the backup set to roll back to.
|
||||
.PARAMETER RestoreStores With -Rollback: also restore gateway-auth.db / secrets db from *.bak-<stamp> (schema-bump case).
|
||||
.PARAMETER NoAutoRollback Do not roll back automatically when start/verification fails.
|
||||
.PARAMETER TakeNewAppSettings Do not carry the live appsettings*.json over into the new Server dir.
|
||||
.PARAMETER SkipFreeSpaceCheck / MinFreeGB / StopTimeoutSeconds / StableSeconds / ReadyTimeoutSeconds tunables.
|
||||
.PARAMETER DryRun Print, do not act. -WhatIf is equivalent.
|
||||
#>
|
||||
[CmdletBinding(SupportsShouldProcess = $true)]
|
||||
param(
|
||||
[string]$ServerSource,
|
||||
[string]$WorkerSource,
|
||||
[string]$ServiceName = 'MxAccessGw',
|
||||
[string]$NssmPath = 'nssm',
|
||||
[string]$ServerDir,
|
||||
[string]$WorkerDir,
|
||||
[string]$ExpectedSha,
|
||||
[string]$Label,
|
||||
[string]$HealthBaseUrl = 'http://localhost:5130',
|
||||
[switch]$Rollback,
|
||||
[string]$BackupStamp,
|
||||
[switch]$RestoreStores,
|
||||
[switch]$NoAutoRollback,
|
||||
[switch]$TakeNewAppSettings,
|
||||
[switch]$SkipFreeSpaceCheck,
|
||||
[double]$MinFreeGB = 2,
|
||||
[int]$StopTimeoutSeconds = 30,
|
||||
[int]$StableSeconds = 60,
|
||||
[int]$ReadyTimeoutSeconds = 120,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$script:Dry = $DryRun.IsPresent -or $WhatIfPreference
|
||||
$script:Stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$script:ServerProc = 'ZB.MOM.WW.MxGateway.Server'
|
||||
$script:WorkerProc = 'ZB.MOM.WW.MxGateway.Worker'
|
||||
$script:Notes = New-Object System.Collections.Generic.List[string]
|
||||
$script:LockStream = $null
|
||||
|
||||
function Say([string]$m) { Write-Host $m; $script:Notes.Add($m) }
|
||||
function Info([string]$m) { Say "[info] $m" }
|
||||
function Warn([string]$m) { Write-Warning $m; $script:Notes.Add("[WARN] $m") }
|
||||
function Fail([string]$m) { throw "ABORT: $m" }
|
||||
|
||||
# Mutating step: print in dry-run, run otherwise.
|
||||
function Step([string]$Description, [scriptblock]$Action) {
|
||||
if ($script:Dry) { Say "[DRYRUN] $Description"; return }
|
||||
Say "[ACTION] $Description"
|
||||
& $Action
|
||||
}
|
||||
|
||||
# --- helpers --------------------------------------------------------------------------------------------
|
||||
|
||||
function Invoke-Nssm {
|
||||
param([string[]]$NssmArgs, [switch]$AllowFail)
|
||||
$raw = & $NssmPath @NssmArgs 2>&1
|
||||
$code = $LASTEXITCODE
|
||||
$text = (($raw | ForEach-Object { "$_" }) -join "`n") -replace "`0", ''
|
||||
if ($code -ne 0 -and -not $AllowFail) { Fail "nssm $($NssmArgs -join ' ') failed (exit $code): $text" }
|
||||
[pscustomobject]@{ Code = $code; Text = $text.Trim() }
|
||||
}
|
||||
|
||||
function Get-NssmValue([string]$Name) { (Invoke-Nssm -NssmArgs @('get', $ServiceName, $Name)).Text }
|
||||
|
||||
function Get-ServiceState {
|
||||
$r = Invoke-Nssm -NssmArgs @('status', $ServiceName) -AllowFail
|
||||
if ($r.Code -ne 0) { return 'MISSING' }
|
||||
($r.Text -split "`n")[0].Trim()
|
||||
}
|
||||
|
||||
function Parse-EnvBlock([string]$Text) {
|
||||
$map = [ordered]@{}
|
||||
foreach ($line in ($Text -split "[`r`n]+")) {
|
||||
$t = $line.Trim()
|
||||
if ($t -match '^([^=\s]+)=(.*)$') { $map[$Matches[1]] = $Matches[2] }
|
||||
}
|
||||
$map
|
||||
}
|
||||
|
||||
function Norm([string]$p) { ($p -replace '\\', '/').TrimEnd('/') }
|
||||
function Test-Rooted([string]$p) { $p -match '^[A-Za-z]:[\\/]' -or $p.StartsWith('/') -or $p.StartsWith('\\') }
|
||||
function Test-Inside([string]$Path, [string]$Root) {
|
||||
$a = (Norm $Path).ToLowerInvariant(); $b = (Norm $Root).ToLowerInvariant()
|
||||
$a -eq $b -or $a.StartsWith("$b/")
|
||||
}
|
||||
|
||||
function Get-ProductSha([string]$Exe) {
|
||||
try {
|
||||
$pv = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($Exe).ProductVersion
|
||||
if ($pv -match '\+([0-9a-fA-F]{7,40})(\b|$)') { return $Matches[1].ToLowerInvariant() }
|
||||
} catch { }
|
||||
$null
|
||||
}
|
||||
function Test-ShaMatch([string]$Actual, [string]$Expected) {
|
||||
if (-not $Actual -or -not $Expected) { return $false }
|
||||
$e = $Expected.ToLowerInvariant()
|
||||
$Actual.StartsWith($e) -or $e.StartsWith($Actual)
|
||||
}
|
||||
|
||||
function Get-GatewayProcesses {
|
||||
@(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -in @($script:ServerProc, $script:WorkerProc) })
|
||||
}
|
||||
|
||||
# Store file set: db + -wal + -shm
|
||||
function Get-StoreFiles([string]$Db) { @($Db, "$Db-wal", "$Db-shm") | Where-Object { Test-Path -LiteralPath $_ } }
|
||||
|
||||
function Get-DirSize([string]$p) {
|
||||
if (-not (Test-Path -LiteralPath $p)) { return 0 }
|
||||
$s = (Get-ChildItem -LiteralPath $p -Recurse -File -Force -ErrorAction SilentlyContinue | Measure-Object Length -Sum).Sum
|
||||
if ($s) { [double]$s } else { 0 }
|
||||
}
|
||||
|
||||
# --- context discovery (read only) -------------------------------------------------------------------------
|
||||
|
||||
function Get-Context {
|
||||
$state = Get-ServiceState
|
||||
if ($state -eq 'MISSING') { Fail "service '$ServiceName' does not exist (use Install-MxGatewayService.ps1 for a new box)" }
|
||||
$ctx = [ordered]@{}
|
||||
$ctx.State = $state
|
||||
$ctx.Application = Get-NssmValue 'Application'
|
||||
$ctx.AppDirectory = Get-NssmValue 'AppDirectory'
|
||||
$ctx.EnvText = Get-NssmValue 'AppEnvironmentExtra'
|
||||
$ctx.Env = Parse-EnvBlock $ctx.EnvText
|
||||
$ctx.EnvCount = $ctx.Env.Count
|
||||
if (-not $ServerDir) { $ctx.ServerDir = $ctx.AppDirectory } else { $ctx.ServerDir = $ServerDir }
|
||||
$wexe = $ctx.Env['MxGateway__Worker__ExecutablePath']
|
||||
$ctx.WorkerExe = $wexe
|
||||
if ($WorkerDir) { $ctx.WorkerDir = $WorkerDir }
|
||||
elseif ($wexe) { $ctx.WorkerDir = Split-Path -Parent $wexe }
|
||||
else { Fail 'cannot determine the Worker directory: env MxGateway__Worker__ExecutablePath is not in the NSSM env block; pass -WorkerDir' }
|
||||
$pd = 'C:\ProgramData\MxGateway'
|
||||
$auth = $ctx.Env['MxGateway__Authentication__SqlitePath']; if (-not $auth) { $auth = "$pd\gateway-auth.db" }
|
||||
$sec = $ctx.Env['Secrets__SqlitePath']; if (-not $sec) { $sec = "$pd\mxgateway-secrets.db" }
|
||||
$ctx.AuthDb = $auth; $ctx.SecretsDb = $sec
|
||||
$ctx.ServerExeName = Split-Path -Leaf $ctx.Application
|
||||
[pscustomobject]$ctx
|
||||
}
|
||||
|
||||
# --- preflight -----------------------------------------------------------------------------------------------
|
||||
|
||||
function Invoke-Preflight($ctx) {
|
||||
Say '== Preflight =='
|
||||
Info "service '$ServiceName' state: $($ctx.State)"
|
||||
Info "Application : $($ctx.Application)"
|
||||
Info "AppDirectory: $($ctx.AppDirectory)"
|
||||
Info "Server dir : $($ctx.ServerDir)"
|
||||
Info "Worker dir : $($ctx.WorkerDir) (exe env: $($ctx.WorkerExe))"
|
||||
Info "API-key DB : $($ctx.AuthDb)"
|
||||
Info "Secrets DB : $($ctx.SecretsDb)"
|
||||
Info "NSSM env block: $($ctx.EnvCount) entries (names only): $(($ctx.Env.Keys) -join ', ')"
|
||||
|
||||
if ($ctx.EnvCount -eq 0) { Fail 'NSSM env block is empty - refusing: it holds Kestrel/store paths, the KEK and the pepper' }
|
||||
foreach ($need in 'ZB_SECRETS_MASTER_KEY') {
|
||||
if (-not $ctx.Env.Contains($need)) { Fail "env block has no $need (the Secrets store would be unreadable after the restart)" }
|
||||
}
|
||||
if (-not ($ctx.Env.Keys | Where-Object { $_ -match 'Pepper' })) {
|
||||
Warn 'no *Pepper* entry in the env block; confirm the API-key pepper is supplied another way or issued tokens will not verify'
|
||||
}
|
||||
if ($ctx.Env['ASPNETCORE_ENVIRONMENT'] -ne 'Production' -and $ctx.Env['DOTNET_ENVIRONMENT'] -ne 'Production') {
|
||||
Warn 'neither ASPNETCORE_ENVIRONMENT nor DOTNET_ENVIRONMENT is exactly Production (runbook trap 6) - do not change it during an upgrade'
|
||||
}
|
||||
|
||||
# State that must survive: rooted and outside the content root.
|
||||
foreach ($pair in @(@('API-key DB', $ctx.AuthDb), @('Secrets DB', $ctx.SecretsDb))) {
|
||||
if (-not (Test-Rooted $pair[1])) { Fail "$($pair[0]) path '$($pair[1])' is not absolute" }
|
||||
foreach ($root in @($ctx.ServerDir, $ctx.WorkerDir)) {
|
||||
if (Test-Inside $pair[1] $root) { Fail "$($pair[0]) '$($pair[1])' is INSIDE '$root' - the swap would lose it. Move it out first (runbook section 1)." }
|
||||
}
|
||||
if (Test-Path -LiteralPath $pair[1]) { Info "$($pair[0]) present ($((Get-StoreFiles $pair[1]).Count) file(s) incl. wal/shm)" }
|
||||
else { Warn "$($pair[0]) '$($pair[1])' does not exist yet (fresh store will be created on start)" }
|
||||
}
|
||||
|
||||
# Directories
|
||||
foreach ($d in @($ctx.ServerDir, $ctx.WorkerDir)) {
|
||||
if (-not (Test-Path -LiteralPath $d -PathType Container)) { Fail "live directory '$d' does not exist" }
|
||||
}
|
||||
if ((Test-Inside $ctx.WorkerDir $ctx.ServerDir) -or (Test-Inside $ctx.ServerDir $ctx.WorkerDir)) {
|
||||
Fail 'Server and Worker directories are nested; the rename-in-place swap needs sibling directories'
|
||||
}
|
||||
|
||||
# One-gateway guard
|
||||
$procs = Get-GatewayProcesses
|
||||
$servers = @($procs | Where-Object { $_.ProcessName -eq $script:ServerProc })
|
||||
if ($servers.Count -gt 1) { Fail "$($servers.Count) gateway Server processes are running (PIDs $(($servers.Id) -join ',')). A second gateway is the OrphanWorkerTerminator trap - resolve manually." }
|
||||
if ($ctx.State -eq 'SERVICE_STOPPED' -and $procs.Count -gt 0) {
|
||||
Fail "service is STOPPED but gateway process(es) exist (PIDs $(($procs.Id) -join ',')): a rogue second gateway. Starting would terminate its live workers."
|
||||
}
|
||||
if ($servers.Count -eq 1 -and $ctx.State -ne 'SERVICE_RUNNING' -and $ctx.State -ne 'SERVICE_STOP_PENDING') {
|
||||
Fail "a Server process exists while the service is $($ctx.State)"
|
||||
}
|
||||
if (Get-Command Get-CimInstance -ErrorAction SilentlyContinue) {
|
||||
$svcs = @(Get-CimInstance Win32_Service | Where-Object { $_.Name -ne $ServiceName -and $_.PathName -and $_.PathName -like "*$($ctx.ServerExeName)*" })
|
||||
if ($svcs.Count -gt 0) { Fail "another service ($(($svcs.Name) -join ', ')) also launches $($ctx.ServerExeName); two gateways on one box is forbidden" }
|
||||
} else { Info 'Win32_Service cross-check unavailable on this host (skipped)' }
|
||||
|
||||
if ($Rollback) { return }
|
||||
|
||||
# Sources
|
||||
if (-not $ServerSource -or -not $WorkerSource) { Fail '-ServerSource and -WorkerSource are required for an upgrade' }
|
||||
foreach ($s in @(@($ServerSource, $ctx.ServerExeName), @($WorkerSource, "$($script:WorkerProc).exe"))) {
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $s[0] $s[1]))) { Fail "staged source '$($s[0])' does not contain $($s[1])" }
|
||||
}
|
||||
if ((Test-Inside $ServerSource $ctx.ServerDir) -or (Test-Inside $WorkerSource $ctx.WorkerDir)) { Fail 'staging directory is inside a live directory' }
|
||||
$sSha = Get-ProductSha (Join-Path $ServerSource $ctx.ServerExeName)
|
||||
$wSha = Get-ProductSha (Join-Path $WorkerSource "$($script:WorkerProc).exe")
|
||||
Info "staged Server SHA: $sSha ; staged Worker SHA: $wSha"
|
||||
if ($sSha -and $wSha -and -not (Test-ShaMatch $sSha $wSha)) { Fail "Server ($sSha) and Worker ($wSha) were built from different commits (wire protocol risk)" }
|
||||
if ($ExpectedSha) {
|
||||
if (-not (Test-ShaMatch $sSha $ExpectedSha)) {
|
||||
if ($script:Dry) { Warn "staged Server SHA '$sSha' does not match -ExpectedSha '$ExpectedSha' (dry-run: continuing)" }
|
||||
else { Fail "staged Server SHA '$sSha' does not match -ExpectedSha '$ExpectedSha'" }
|
||||
}
|
||||
} else { Warn 'no -ExpectedSha given: the post-deploy SHA check will only record the SHA' }
|
||||
|
||||
# Disk space: 2x (server + worker + stores + payload), at least MinFreeGB
|
||||
if (-not $SkipFreeSpaceCheck) {
|
||||
$need = (Get-DirSize $ctx.ServerDir) + (Get-DirSize $ctx.WorkerDir) + (Get-DirSize $ServerSource) + (Get-DirSize $WorkerSource)
|
||||
foreach ($f in (Get-StoreFiles $ctx.AuthDb) + (Get-StoreFiles $ctx.SecretsDb)) { $need += (Get-Item -LiteralPath $f).Length }
|
||||
$need = [math]::Max($need * 2, $MinFreeGB * 1GB)
|
||||
$root = [IO.Path]::GetPathRoot((Resolve-Path -LiteralPath $ctx.ServerDir).Path)
|
||||
$free = (New-Object IO.DriveInfo $root).AvailableFreeSpace
|
||||
Info ("free on {0}: {1:N2} GB, required {2:N2} GB" -f $root, ($free / 1GB), ($need / 1GB))
|
||||
if ($free -lt $need) { Fail 'not enough free disk space for the backups' }
|
||||
}
|
||||
Say '== Preflight OK =='
|
||||
}
|
||||
|
||||
# --- stop / start ---------------------------------------------------------------------------------------------
|
||||
|
||||
function Wait-State([string]$Want, [int]$TimeoutSec) {
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
||||
do { $s = Get-ServiceState; if ($s -eq $Want) { return $true }; Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline)
|
||||
$false
|
||||
}
|
||||
|
||||
function Assert-NoGatewayProcess([string]$When) {
|
||||
$left = Get-GatewayProcesses
|
||||
if ($left.Count -gt 0) { Fail "${When}: gateway process(es) still alive: $(($left | ForEach-Object { "$($_.ProcessName)#$($_.Id)" }) -join ', ')" }
|
||||
}
|
||||
|
||||
function Stop-Gateway {
|
||||
Say '-- Stop --'
|
||||
$state = Get-ServiceState
|
||||
if ($state -eq 'SERVICE_STOPPED') { Info 'already stopped' }
|
||||
else {
|
||||
Step "nssm stop $ServiceName (wait up to $StopTimeoutSeconds s for SERVICE_STOPPED)" {
|
||||
[void](Invoke-Nssm -NssmArgs @('stop', $ServiceName) -AllowFail)
|
||||
if (-not (Wait-State 'SERVICE_STOPPED' $StopTimeoutSeconds)) {
|
||||
# STOP_PENDING trap: workers did not exit in time. Kill the Server (then Workers) by image name -
|
||||
# safe because preflight proved there is exactly one gateway on the box.
|
||||
Warn "service still $(Get-ServiceState) after $StopTimeoutSeconds s (STOP_PENDING trap) - killing gateway processes"
|
||||
foreach ($n in @($script:ServerProc, $script:WorkerProc)) {
|
||||
Get-Process -Name $n -ErrorAction SilentlyContinue | ForEach-Object { Say "[ACTION] Stop-Process -Id $($_.Id) -Force ($n)"; Stop-Process -Id $_.Id -Force }
|
||||
}
|
||||
if (-not (Wait-State 'SERVICE_STOPPED' 20)) {
|
||||
Warn 'still not STOPPED; killing the NSSM service host process'
|
||||
try {
|
||||
$svcPid = (Get-CimInstance Win32_Service -Filter "Name='$ServiceName'").ProcessId
|
||||
if ($svcPid) { Stop-Process -Id $svcPid -Force }
|
||||
} catch { Warn "could not kill the service host: $_" }
|
||||
if (-not (Wait-State 'SERVICE_STOPPED' 20)) { Fail "service '$ServiceName' will not stop; investigate manually (state $(Get-ServiceState))" }
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
Assert-NoGatewayProcess 'after stop'
|
||||
}
|
||||
}
|
||||
if (-not $script:Dry) { Assert-NoGatewayProcess 'after stop' }
|
||||
}
|
||||
|
||||
function Start-Gateway($ctx) {
|
||||
Say '-- Start --'
|
||||
if (-not $script:Dry) { Assert-NoGatewayProcess 'before start (OrphanWorkerTerminator guard)' }
|
||||
else { Say '[DRYRUN] assert no Server/Worker process exists (OrphanWorkerTerminator guard)' }
|
||||
Step "nssm start $ServiceName (once)" {
|
||||
[void](Invoke-Nssm -NssmArgs @('start', $ServiceName))
|
||||
}
|
||||
Step "wait for SERVICE_RUNNING and compare env entry count (expect $($ctx.EnvCount))" {
|
||||
if (-not (Wait-State 'SERVICE_RUNNING' 60)) { Fail "service did not reach RUNNING (state $(Get-ServiceState))" }
|
||||
$now = (Parse-EnvBlock (Get-NssmValue 'AppEnvironmentExtra')).Count
|
||||
if ($now -ne $ctx.EnvCount) { Fail "NSSM env block entry count changed: $($ctx.EnvCount) -> $now" }
|
||||
}
|
||||
}
|
||||
|
||||
# --- backup / swap -----------------------------------------------------------------------------------------------
|
||||
|
||||
function Get-Suffix { if ($Label) { "$($script:Stamp)-$Label" } else { $script:Stamp } }
|
||||
|
||||
function Backup-State($ctx) {
|
||||
Say '-- Backup (service stopped) --'
|
||||
$suffix = Get-Suffix
|
||||
$root = Split-Path -Parent $ctx.ServerDir
|
||||
$envFile = Join-Path $root "_env-AppEnvironmentExtra-$($script:Stamp).txt"
|
||||
Step "save the NSSM env block + Application/AppDirectory to $envFile (contains secrets: keep it ACL-restricted)" {
|
||||
$body = @("Application=$($ctx.Application)", "AppDirectory=$($ctx.AppDirectory)", '--- AppEnvironmentExtra ---', $ctx.EnvText)
|
||||
Set-Content -LiteralPath $envFile -Value $body -Encoding UTF8
|
||||
try { & icacls.exe $envFile /inheritance:r /grant:r 'SYSTEM:(F)' 'BUILTIN\Administrators:(F)' | Out-Null } catch { Warn "could not restrict ACL on ${envFile}: $_" }
|
||||
}
|
||||
foreach ($db in @($ctx.AuthDb, $ctx.SecretsDb)) {
|
||||
foreach ($f in (Get-StoreFiles $db)) {
|
||||
$dst = "$f.bak-$suffix"
|
||||
Step "copy store file $f -> $dst (live file is never moved)" { Copy-Item -LiteralPath $f -Destination $dst -Force }
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $db)) { Info "no store at $db to back up" }
|
||||
}
|
||||
}
|
||||
|
||||
function Swap-Directory([string]$Live, [string]$Source, [string]$Name, [bool]$KeepSettings) {
|
||||
$bak = "$Live.bak-$(Get-Suffix)"
|
||||
Step "rename $Name dir: $Live -> $bak" { Rename-Item -LiteralPath $Live -NewName (Split-Path -Leaf $bak) }
|
||||
Step "copy new ${Name}: $Source -> $Live" { Copy-Item -LiteralPath $Source -Destination $Live -Recurse -Force }
|
||||
if ($KeepSettings) {
|
||||
$from = if ($script:Dry) { $Live } else { $bak } # in a dry run nothing was renamed yet
|
||||
foreach ($f in 'appsettings.json', 'appsettings.Development.json') {
|
||||
if (Test-Path -LiteralPath (Join-Path $from $f)) {
|
||||
Step "restore live $f byte-identical from $bak" { Copy-Item -LiteralPath (Join-Path $bak $f) -Destination (Join-Path $Live $f) -Force }
|
||||
} else { Info "$f not present in the old $Name dir (nothing to carry over)" }
|
||||
}
|
||||
}
|
||||
$bak
|
||||
}
|
||||
|
||||
# --- verification ----------------------------------------------------------------------------------------------------
|
||||
|
||||
function Invoke-Verify($ctx) {
|
||||
Say '== Verify =='
|
||||
if ($script:Dry) {
|
||||
Say "[DRYRUN] GET $HealthBaseUrl/health/ready until 200 (<= $ReadyTimeoutSeconds s)"
|
||||
Say "[DRYRUN] GET $HealthBaseUrl/healthz ; read ProductVersion SHA of $(Join-Path $ctx.ServerDir $ctx.ServerExeName)$(if ($ExpectedSha) { " and require it to match $ExpectedSha" })"
|
||||
Say "[DRYRUN] service must stay SERVICE_RUNNING for $StableSeconds s with env entry count $($ctx.EnvCount)"
|
||||
return
|
||||
}
|
||||
$ready = $false; $deadline = (Get-Date).AddSeconds($ReadyTimeoutSeconds)
|
||||
while (-not $ready -and (Get-Date) -lt $deadline) {
|
||||
try { $r = Invoke-WebRequest -Uri "$HealthBaseUrl/health/ready" -UseBasicParsing -TimeoutSec 10; $ready = ($r.StatusCode -eq 200) }
|
||||
catch { Start-Sleep -Seconds 3 }
|
||||
}
|
||||
if (-not $ready) { Fail "$HealthBaseUrl/health/ready did not return 200 within $ReadyTimeoutSeconds s" }
|
||||
Info '/health/ready = 200'
|
||||
try { Info "/healthz = $((Invoke-WebRequest -Uri "$HealthBaseUrl/healthz" -UseBasicParsing -TimeoutSec 10).StatusCode)" } catch { Fail "/healthz failed: $_" }
|
||||
|
||||
$sha = Get-ProductSha (Join-Path $ctx.ServerDir $ctx.ServerExeName)
|
||||
Info "deployed Server SHA (ProductVersion): $sha"
|
||||
if ($ExpectedSha -and -not $Rollback -and -not (Test-ShaMatch $sha $ExpectedSha)) { Fail "deployed SHA '$sha' != expected '$ExpectedSha'" }
|
||||
|
||||
Info "watching the service for $StableSeconds s"
|
||||
$until = (Get-Date).AddSeconds($StableSeconds)
|
||||
while ((Get-Date) -lt $until) {
|
||||
if ((Get-ServiceState) -ne 'SERVICE_RUNNING') { Fail 'service left RUNNING during the stability window (restart loop?)' }
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
$n = (Parse-EnvBlock (Get-NssmValue 'AppEnvironmentExtra')).Count
|
||||
if ($n -ne $ctx.EnvCount) { Fail "env block entry count $n != preflight $($ctx.EnvCount)" }
|
||||
Info "env block still $n entries"
|
||||
Warn 'health is green with zero sessions; still prove the Worker path: open a session with a PRE-EXISTING API key and read a known tag (runbook 7.3)'
|
||||
}
|
||||
|
||||
# --- rollback ------------------------------------------------------------------------------------------------------------
|
||||
|
||||
function Find-Backup([string]$Live, [string]$Stamp) {
|
||||
$leaf = Split-Path -Leaf $Live; $parent = Split-Path -Parent $Live
|
||||
$cands = @(Get-ChildItem -LiteralPath $parent -Directory -Filter "$leaf.bak-*" -ErrorAction SilentlyContinue)
|
||||
if ($Stamp) { $cands = @($cands | Where-Object { $_.Name -like "$leaf.bak-$Stamp*" }) }
|
||||
$cands | Sort-Object Name -Descending | Select-Object -First 1
|
||||
}
|
||||
|
||||
function Invoke-RollbackSwap($ctx, [string]$Stamp, [bool]$Stores) {
|
||||
Say '== Rollback =='
|
||||
$sb = Find-Backup $ctx.ServerDir $Stamp
|
||||
$wb = Find-Backup $ctx.WorkerDir $Stamp
|
||||
if (-not $sb -or -not $wb) { Fail "no matching Server/Worker backup directories found (stamp '$Stamp')" }
|
||||
Info "rolling back to Server=$($sb.Name) Worker=$($wb.Name)"
|
||||
Stop-Gateway
|
||||
foreach ($pair in @(@($ctx.ServerDir, $sb), @($ctx.WorkerDir, $wb))) {
|
||||
$failed = "$($pair[0]).failed-$($script:Stamp)"
|
||||
if (Test-Path -LiteralPath $pair[0]) { Step "rename $($pair[0]) -> $failed" { Rename-Item -LiteralPath $pair[0] -NewName (Split-Path -Leaf $failed) } }
|
||||
Step "rename $($pair[1].FullName) -> $($pair[0])" { Rename-Item -LiteralPath $pair[1].FullName -NewName (Split-Path -Leaf $pair[0]) }
|
||||
}
|
||||
if ($Stores) {
|
||||
$bstamp = ($sb.Name -replace '^.*\.bak-', '')
|
||||
foreach ($db in @($ctx.AuthDb, $ctx.SecretsDb)) {
|
||||
foreach ($ext in '', '-wal', '-shm') {
|
||||
$live = "$db$ext"; $bak = "$db$ext.bak-$bstamp"
|
||||
if (Test-Path -LiteralPath $bak) { Step "restore store file $bak -> $live" { Copy-Item -LiteralPath $bak -Destination $live -Force } }
|
||||
elseif ($ext -and (Test-Path -LiteralPath $live) -and (Test-Path -LiteralPath "$db.bak-$bstamp")) {
|
||||
# a stale wal/shm must not pair with the restored db: move it aside, never delete
|
||||
Step "move stale $live aside -> $live.stale-$($script:Stamp)" { Rename-Item -LiteralPath $live -NewName ("$(Split-Path -Leaf $live).stale-$($script:Stamp)") }
|
||||
}
|
||||
elseif (-not $ext) { Warn "no backup $bak for $db; live store left untouched" }
|
||||
}
|
||||
}
|
||||
} else { Info 'stores left untouched (they live outside the swap); pass -RestoreStores after a schema bump' }
|
||||
Start-Gateway $ctx
|
||||
}
|
||||
|
||||
# --- main -------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
$exit = 0
|
||||
try {
|
||||
Say "Upgrade-MxGatewayService stamp=$($script:Stamp) mode=$(if ($Rollback) {'ROLLBACK'} else {'UPGRADE'}) dry=$($script:Dry)"
|
||||
$ctx = Get-Context
|
||||
Invoke-Preflight $ctx
|
||||
|
||||
if (-not $script:Dry) {
|
||||
$lockPath = Join-Path (Split-Path -Parent $ctx.ServerDir) '.mxgw-deploy.lock'
|
||||
try { $script:LockStream = [IO.File]::Open($lockPath, 'OpenOrCreate', 'ReadWrite', 'None') }
|
||||
catch { Fail "another deploy run holds $lockPath" }
|
||||
}
|
||||
|
||||
if ($Rollback) {
|
||||
Invoke-RollbackSwap $ctx $BackupStamp $RestoreStores.IsPresent
|
||||
Invoke-Verify $ctx
|
||||
} else {
|
||||
Stop-Gateway
|
||||
Backup-State $ctx
|
||||
$swapped = $false
|
||||
try {
|
||||
$null = Swap-Directory $ctx.ServerDir $ServerSource 'Server' (-not $TakeNewAppSettings)
|
||||
$null = Swap-Directory $ctx.WorkerDir $WorkerSource 'Worker' $false
|
||||
$swapped = $true
|
||||
Start-Gateway $ctx
|
||||
Invoke-Verify $ctx
|
||||
} catch {
|
||||
$err = $_
|
||||
Say "[FAIL] $($err.Exception.Message)"
|
||||
if ($script:Dry -or $NoAutoRollback) { throw }
|
||||
Warn 'upgrade failed - rolling back automatically'
|
||||
try {
|
||||
if (-not $swapped) { Warn 'swap did not complete; rollback swaps whatever .bak sets exist for this stamp' }
|
||||
Invoke-RollbackSwap $ctx $script:Stamp $false
|
||||
Invoke-Verify $ctx
|
||||
Warn 'ROLLED BACK to the previous build; original failure follows'
|
||||
} catch { Warn "AUTOMATIC ROLLBACK FAILED: $($_.Exception.Message). Manual intervention required (backups: *.bak-$(Get-Suffix))." }
|
||||
throw $err
|
||||
}
|
||||
}
|
||||
Say '== DONE =='
|
||||
} catch {
|
||||
$exit = 1
|
||||
Write-Host "ERROR: $($_.Exception.Message)" -ForegroundColor Red
|
||||
} finally {
|
||||
if ($script:LockStream) { $script:LockStream.Dispose() }
|
||||
try {
|
||||
if (-not $script:Dry -and $ctx) {
|
||||
$notes = Join-Path (Split-Path -Parent $ctx.ServerDir) "_DEPLOY-NOTES-$($script:Stamp).txt"
|
||||
Set-Content -LiteralPath $notes -Value $script:Notes -Encoding UTF8
|
||||
Write-Host "deploy notes: $notes"
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
exit $exit
|
||||
Reference in New Issue
Block a user