# ScadaBridge QA (vq03/vq04) 8e32f353 -> 061432c8 : ScadaBridge#143. Rendered per call: ACTION / TS. # prep | backup (vq03) | stop-site | stop-central | migrate (vq03) | swap | start-central | start-site | verify | rollback-swap $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' $Action = '__ACTION__' $ts = '__TS__' $sha = '061432c8' $fromSha = '8e32f353' $zipSha256 = 'B6E85ECC24186AF8B658FA6F1E38F080588EA5F78DD856A4356E696897FF4095' $sqlSha256 = 'A16A150DE1E1F6C4CB9153E1C53BED95A029D656F18C3DC3B61EC6575198F7A7' $stageDir = "E:\psrem-stage\sb-$sha" $stage = "$stageDir\unpacked" $root = 'E:\ApiInstall\ScadaBridge' $log = "$root\_DEPLOY-NOTES-$sha-$ts.txt" $settings = 'appsettings.json','appsettings.Central.json','appsettings.Site.json' $centralKey = 'HKLM:\SYSTEM\CurrentControlSet\Services\ScadaBridge-Central' $bak = "F:\MSSQL\BACKUP\ScadaBridge_pre-$($sha)_$ts.bak" function Say($m) { $line = "[{0:HH:mm:ss.fff}] {1} {2}" -f (Get-Date), $env:COMPUTERNAME, $m; $line; Add-Content -Path $log -Value $line } function Is-Retired($e) { $e -like 'AuditLog__HashChain__*' -or $e -like 'AuditLog__MaintenanceConnectionString=*' } function Digest($entries) { $h = [System.Security.Cryptography.SHA256]::Create(); $o = @() foreach ($e in $entries) { $i = $e.IndexOf('='); if ($i -lt 1) { $o += ''; continue } $v = $e.Substring($i+1); $x = -join ($h.ComputeHash([Text.Encoding]::UTF8.GetBytes($v)) | % { $_.ToString('x2') }) $o += ("{0} sha256:{1} len={2}" -f $e.Substring(0,$i), $x.Substring(0,12), $v.Length) } $h.Dispose(); $o | Sort-Object } function Wait-Ready($url, $seconds) { $deadline = (Get-Date).AddSeconds($seconds) while ((Get-Date) -lt $deadline) { try { if ((Invoke-WebRequest -UseBasicParsing $url -TimeoutSec 5).StatusCode -eq 200) { return $true } } catch {}; Start-Sleep 2 } return $false } function Stop-Svc($s) { if ((Get-Service $s).Status -ne 'Stopped') { Say "stopping $s"; Stop-Service $s -Force; (Get-Service $s).WaitForStatus('Stopped','00:02:00') } Say "$s stopped" } function Sql-Init { $envC = @((Get-ItemProperty $centralKey).Environment) $cs = ($envC | Where-Object { $_ -like 'ScadaBridge__Database__ConfigurationDb=*' }) -replace '^[^=]*=','' $script:sb = New-Object System.Data.SqlClient.SqlConnectionStringBuilder $cs $script:db = $sb['Database']; if (-not $db) { $script:db = $sb.InitialCatalog } if ($db -ne 'ScadaBridge') { throw "unexpected catalog" } $env:SQLCMDPASSWORD = $sb.Password } function Sql-Scalar($q) { $r = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -h -1 -W -b -Q "SET NOCOUNT ON; $q"; if ($LASTEXITCODE -ne 0) { throw "sql failed" }; ($r | Where-Object { $_ -and $_.Trim() } | Select-Object -First 1).Trim() } $chainQ = "SELECT CONCAT((SELECT COUNT(*) FROM __EFMigrationsHistory),'|',COL_LENGTH('dbo.AuditLog','ChainSeq'),'|',COL_LENGTH('dbo.AuditLog','RowHash'),'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT(*) FROM sys.indexes WHERE object_id=OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_ChainSeq','IX_AuditLog_Unchained')),'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_chainer') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_writer') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT_BIG(*) FROM dbo.AuditLog),'|',CONVERT(varchar(33),(SELECT MAX(OccurredAtUtc) FROM dbo.AuditLog),126))" switch ($Action) { 'prep' { Say "PREP $fromSha -> $sha" if ((Get-FileHash "$stageDir\scadabridge-$sha.zip" -Algorithm SHA256).Hash -ne $zipSha256) { throw "zip hash mismatch" } if (Test-Path $stage) { Remove-Item $stage -Recurse -Force } Expand-Archive "$stageDir\scadabridge-$sha.zip" $stage $sv = (Get-Item "$stage\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion $sc = (Get-Item "$stage\cli\scadabridge.exe").VersionInfo.ProductVersion if ($sv -notlike "*+$sha*" -or $sc -notlike "*+$sha*") { throw "staged exe $sv / $sc" } $lv = (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion if ($lv -notlike "*+$fromSha*") { throw "live exe $lv" } Say "staged $sv; live $lv; app files $((Get-ChildItem "$stage\app" -Recurse -File).Count), cli $((Get-ChildItem "$stage\cli" -Recurse -File).Count)" foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { & reg.exe export "HKLM\SYSTEM\CurrentControlSet\Services\$s" "$stageDir\_envbak-$s-$ts.reg" /y | Out-Null if ($LASTEXITCODE -ne 0) { throw "reg export $s failed" } Say "reg export $s" $e = @((Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Services\$s").Environment) Say ("$s env entries {0}" -f $e.Count); Digest $e | % { Say " env $s $_" } Say ("recovery ${s}: " + ((& sc.exe qfailure $s | Out-String) -replace '\s+',' ').Trim()) Say ("recovery ${s}: " + ((& sc.exe qfailureflag $s | Out-String) -replace '\s+',' ').Trim()) } $envC = @((Get-ItemProperty $centralKey).Environment) if (-not ($envC -contains 'AuditLog__HashChain__Enabled=true')) { throw "AuditLog__HashChain__Enabled=true not in Central env" } Say ("retired keys: " + ((@($envC | ? { Is-Retired $_ }) | % { $_.Substring(0,$_.IndexOf('=')) }) -join ',')) $keep = New-Item -ItemType Directory -Force -Path "$root\_appsettings-keep-$ts" foreach ($f in $settings) { Copy-Item "$root\app\$f" "$keep\$f"; Say ("keep {0} sha256 {1}" -f $f, (Get-FileHash "$keep\$f" -Algorithm SHA256).Hash) } } 'backup' { Sql-Init if ((Get-FileHash "$stageDir\migrate-64-to-65.idempotent.sql" -Algorithm SHA256).Hash -ne $sqlSha256) { throw "sql hash mismatch" } $pre = Sql-Scalar $chainQ; Say "schema before: $pre" if (-not $pre.StartsWith('64|')) { throw "expected 64 migrations" } & sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "BACKUP DATABASE [$db] TO DISK = N'$bak' WITH COPY_ONLY, CHECKSUM, COMPRESSION, INIT" | Out-Null if ($LASTEXITCODE -ne 0) { throw "backup failed" } & sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "RESTORE VERIFYONLY FROM DISK = N'$bak' WITH CHECKSUM" | Out-Null if ($LASTEXITCODE -ne 0) { throw "verifyonly failed" } Say "backup $bak VERIFYONLY ok" } 'stop-site' { Stop-Svc 'ScadaBridge-Site' } 'stop-central' { Stop-Svc 'ScadaBridge-Central' $deadline = (Get-Date).AddSeconds(30) while ((Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) -and (Get-Date) -lt $deadline) { Start-Sleep 1 } if (Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) { throw "Host.exe still running" } Say "no Host.exe remains" } 'migrate' { Sql-Init foreach ($h in 'wonder-app-vq03','wonder-app-vq04') { foreach ($p in 8085,8084) { try { Invoke-WebRequest -UseBasicParsing "http://$($h):$p/health/ready" -TimeoutSec 3 | Out-Null; throw "$h`:$p still answering - not all services stopped" } catch [System.Net.WebException] {} } } $migSql = "$stageDir\_migrate-xact-$ts.sql" $body = [IO.File]::ReadAllText("$stageDir\migrate-64-to-65.idempotent.sql", [Text.Encoding]::UTF8).TrimStart([char]0xFEFF) [IO.File]::WriteAllText($migSql, "SET XACT_ABORT ON;`r`nGO`r`n" + $body, (New-Object System.Text.UTF8Encoding $false)) Say "migrating 64 -> 65" $out = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -I -b -i $migSql 2>&1 $rc = $LASTEXITCODE $out | % { Say " sql: $_" } if ($rc -ne 0) { throw "migration failed (exit $rc)" } $post = Sql-Scalar $chainQ; Say "schema after: $post" if (-not $post.StartsWith('65|')) { throw "expected 65 migrations" } } 'swap' { if (Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) { throw "Host.exe running" } foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { & sc.exe failureflag $s 1 | Out-Null; if ($LASTEXITCODE -ne 0) { throw "failureflag $s" } } Rename-Item "$root\app" "app.bak.$ts"; Rename-Item "$root\cli" "cli.bak.$ts" Say "renamed app/cli -> *.bak.$ts" Copy-Item "$stage\app" "$root\app" -Recurse Copy-Item "$stage\cli" "$root\cli" -Recurse $keep = "$root\_appsettings-keep-$ts" foreach ($f in $settings) { Copy-Item "$keep\$f" "$root\app\$f" -Force if ((Get-FileHash "$keep\$f" -Algorithm SHA256).Hash -ne (Get-FileHash "$root\app\$f" -Algorithm SHA256).Hash) { throw "appsettings mismatch $f" } Say "restored $f (sha256 match)" } Say ("live Host " + (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion + " / CLI " + (Get-Item "$root\cli\scadabridge.exe").VersionInfo.ProductVersion) $before = @((Get-ItemProperty $centralKey).Environment) $keepEnv = @($before | Where-Object { -not (Is-Retired $_) }) Set-ItemProperty -Path $centralKey -Name Environment -Value ([string[]]$keepEnv) -Type MultiString $after = @((Get-ItemProperty $centralKey).Environment) foreach ($e in $keepEnv) { if ($after -notcontains $e) { throw "env entry changed" } } if (@($after | ? { Is-Retired $_ }).Count -ne 0) { throw "retired key still present" } Say ("central env entries {0} -> {1}" -f $before.Count, $after.Count); Digest $after | % { Say " env ScadaBridge-Central $_" } } 'start-central' { Say "starting ScadaBridge-Central"; Start-Service 'ScadaBridge-Central'; $ok = Wait-Ready 'http://localhost:8085/health/ready' 180; Say "central ready=$ok"; if (-not $ok) { throw "central not ready" } } 'start-site' { Say "starting ScadaBridge-Site"; Start-Service 'ScadaBridge-Site'; $ok = Wait-Ready 'http://localhost:8084/health/ready' 180; Say "site ready=$ok"; if (-not $ok) { throw "site not ready" } } 'verify' { foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { Say ("$s " + (Get-Service $s).Status) Say ("recovery ${s}: " + ((& sc.exe qfailure $s | Out-String) -replace '\s+',' ').Trim()) Say ("recovery ${s}: " + ((& sc.exe qfailureflag $s | Out-String) -replace '\s+',' ').Trim()) Digest @((Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Services\$s").Environment) | % { Say " env $s $_" } } $logDir = "$root\central\logs" $f = Get-ChildItem $logDir -File | Sort-Object LastWriteTime -Descending | Select-Object -First 1 Say ("central log {0}: retired-key warnings = {1}" -f $f.Name, @(Select-String -Path $f.FullName -Pattern 'no longer supported' -SimpleMatch).Count) if ($env:COMPUTERNAME -eq 'WONDER-APP-VQ03') { Sql-Init; Say ("schema now: " + (Sql-Scalar $chainQ)) } } 'rollback-swap' { foreach ($s in 'ScadaBridge-Site','ScadaBridge-Central') { Stop-Svc $s } if (Test-Path "$root\app") { Rename-Item "$root\app" "app.failed.$ts" } if (Test-Path "$root\cli") { Rename-Item "$root\cli" "cli.failed.$ts" } Rename-Item "$root\app.bak.$ts" 'app'; Rename-Item "$root\cli.bak.$ts" 'cli' & reg.exe import "$stageDir\_envbak-ScadaBridge-Central-$ts.reg" | Out-Null Say "rolled back binaries + Central env (services left STOPPED; restore DB next)" } default { throw "bad action $Action" } } Remove-Item Env:SQLCMDPASSWORD -ErrorAction SilentlyContinue Say "$Action OK" exit 0