50a5291b64
DEV: ~48 s site / ~29 s central outage; QA: pair stop/start, ~2m40s site / ~1m37s central. Chain schema + chainer role gone, env key removed, all gates pass on both tiers. #116 QA site drills (all reactors idle): kill → takeover +7.7 s; stop-and-hold → [leader-auto-down] DOWNING +27.2 s, takeover +27.4 s, recovery restored byte-identical. Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
156 lines
11 KiB
PowerShell
156 lines
11 KiB
PowerShell
# ScadaBridge QA (vq03/vq04) 8e32f353 -> 061432c8 : ScadaBridge#143. Rendered per call: ACTION / TS.
|
|
# prep | backup (vq03) | stop-site | stop-central | migrate (vq03) | swap | start-central | start-site | verify | rollback-swap
|
|
$ErrorActionPreference = 'Stop'
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$Action = '__ACTION__'
|
|
$ts = '__TS__'
|
|
$sha = '061432c8'
|
|
$fromSha = '8e32f353'
|
|
$zipSha256 = 'B6E85ECC24186AF8B658FA6F1E38F080588EA5F78DD856A4356E696897FF4095'
|
|
$sqlSha256 = 'A16A150DE1E1F6C4CB9153E1C53BED95A029D656F18C3DC3B61EC6575198F7A7'
|
|
$stageDir = "E:\psrem-stage\sb-$sha"
|
|
$stage = "$stageDir\unpacked"
|
|
$root = 'E:\ApiInstall\ScadaBridge'
|
|
$log = "$root\_DEPLOY-NOTES-$sha-$ts.txt"
|
|
$settings = 'appsettings.json','appsettings.Central.json','appsettings.Site.json'
|
|
$centralKey = 'HKLM:\SYSTEM\CurrentControlSet\Services\ScadaBridge-Central'
|
|
$bak = "F:\MSSQL\BACKUP\ScadaBridge_pre-$($sha)_$ts.bak"
|
|
function Say($m) { $line = "[{0:HH:mm:ss.fff}] {1} {2}" -f (Get-Date), $env:COMPUTERNAME, $m; $line; Add-Content -Path $log -Value $line }
|
|
function Is-Retired($e) { $e -like 'AuditLog__HashChain__*' -or $e -like 'AuditLog__MaintenanceConnectionString=*' }
|
|
function Digest($entries) {
|
|
$h = [System.Security.Cryptography.SHA256]::Create(); $o = @()
|
|
foreach ($e in $entries) { $i = $e.IndexOf('='); if ($i -lt 1) { $o += '<unparsable>'; continue }
|
|
$v = $e.Substring($i+1); $x = -join ($h.ComputeHash([Text.Encoding]::UTF8.GetBytes($v)) | % { $_.ToString('x2') })
|
|
$o += ("{0} sha256:{1} len={2}" -f $e.Substring(0,$i), $x.Substring(0,12), $v.Length) }
|
|
$h.Dispose(); $o | Sort-Object
|
|
}
|
|
function Wait-Ready($url, $seconds) {
|
|
$deadline = (Get-Date).AddSeconds($seconds)
|
|
while ((Get-Date) -lt $deadline) { try { if ((Invoke-WebRequest -UseBasicParsing $url -TimeoutSec 5).StatusCode -eq 200) { return $true } } catch {}; Start-Sleep 2 }
|
|
return $false
|
|
}
|
|
function Stop-Svc($s) {
|
|
if ((Get-Service $s).Status -ne 'Stopped') { Say "stopping $s"; Stop-Service $s -Force; (Get-Service $s).WaitForStatus('Stopped','00:02:00') }
|
|
Say "$s stopped"
|
|
}
|
|
function Sql-Init {
|
|
$envC = @((Get-ItemProperty $centralKey).Environment)
|
|
$cs = ($envC | Where-Object { $_ -like 'ScadaBridge__Database__ConfigurationDb=*' }) -replace '^[^=]*=',''
|
|
$script:sb = New-Object System.Data.SqlClient.SqlConnectionStringBuilder $cs
|
|
$script:db = $sb['Database']; if (-not $db) { $script:db = $sb.InitialCatalog }
|
|
if ($db -ne 'ScadaBridge') { throw "unexpected catalog" }
|
|
$env:SQLCMDPASSWORD = $sb.Password
|
|
}
|
|
function Sql-Scalar($q) { $r = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -h -1 -W -b -Q "SET NOCOUNT ON; $q"; if ($LASTEXITCODE -ne 0) { throw "sql failed" }; ($r | Where-Object { $_ -and $_.Trim() } | Select-Object -First 1).Trim() }
|
|
$chainQ = "SELECT CONCAT((SELECT COUNT(*) FROM __EFMigrationsHistory),'|',COL_LENGTH('dbo.AuditLog','ChainSeq'),'|',COL_LENGTH('dbo.AuditLog','RowHash'),'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT(*) FROM sys.indexes WHERE object_id=OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_ChainSeq','IX_AuditLog_Unchained')),'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_chainer') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_writer') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT_BIG(*) FROM dbo.AuditLog),'|',CONVERT(varchar(33),(SELECT MAX(OccurredAtUtc) FROM dbo.AuditLog),126))"
|
|
|
|
switch ($Action) {
|
|
'prep' {
|
|
Say "PREP $fromSha -> $sha"
|
|
if ((Get-FileHash "$stageDir\scadabridge-$sha.zip" -Algorithm SHA256).Hash -ne $zipSha256) { throw "zip hash mismatch" }
|
|
if (Test-Path $stage) { Remove-Item $stage -Recurse -Force }
|
|
Expand-Archive "$stageDir\scadabridge-$sha.zip" $stage
|
|
$sv = (Get-Item "$stage\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion
|
|
$sc = (Get-Item "$stage\cli\scadabridge.exe").VersionInfo.ProductVersion
|
|
if ($sv -notlike "*+$sha*" -or $sc -notlike "*+$sha*") { throw "staged exe $sv / $sc" }
|
|
$lv = (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion
|
|
if ($lv -notlike "*+$fromSha*") { throw "live exe $lv" }
|
|
Say "staged $sv; live $lv; app files $((Get-ChildItem "$stage\app" -Recurse -File).Count), cli $((Get-ChildItem "$stage\cli" -Recurse -File).Count)"
|
|
foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') {
|
|
& reg.exe export "HKLM\SYSTEM\CurrentControlSet\Services\$s" "$stageDir\_envbak-$s-$ts.reg" /y | Out-Null
|
|
if ($LASTEXITCODE -ne 0) { throw "reg export $s failed" }
|
|
Say "reg export $s"
|
|
$e = @((Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Services\$s").Environment)
|
|
Say ("$s env entries {0}" -f $e.Count); Digest $e | % { Say " env $s $_" }
|
|
Say ("recovery ${s}: " + ((& sc.exe qfailure $s | Out-String) -replace '\s+',' ').Trim())
|
|
Say ("recovery ${s}: " + ((& sc.exe qfailureflag $s | Out-String) -replace '\s+',' ').Trim())
|
|
}
|
|
$envC = @((Get-ItemProperty $centralKey).Environment)
|
|
if (-not ($envC -contains 'AuditLog__HashChain__Enabled=true')) { throw "AuditLog__HashChain__Enabled=true not in Central env" }
|
|
Say ("retired keys: " + ((@($envC | ? { Is-Retired $_ }) | % { $_.Substring(0,$_.IndexOf('=')) }) -join ','))
|
|
$keep = New-Item -ItemType Directory -Force -Path "$root\_appsettings-keep-$ts"
|
|
foreach ($f in $settings) { Copy-Item "$root\app\$f" "$keep\$f"; Say ("keep {0} sha256 {1}" -f $f, (Get-FileHash "$keep\$f" -Algorithm SHA256).Hash) }
|
|
}
|
|
'backup' {
|
|
Sql-Init
|
|
if ((Get-FileHash "$stageDir\migrate-64-to-65.idempotent.sql" -Algorithm SHA256).Hash -ne $sqlSha256) { throw "sql hash mismatch" }
|
|
$pre = Sql-Scalar $chainQ; Say "schema before: $pre"
|
|
if (-not $pre.StartsWith('64|')) { throw "expected 64 migrations" }
|
|
& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "BACKUP DATABASE [$db] TO DISK = N'$bak' WITH COPY_ONLY, CHECKSUM, COMPRESSION, INIT" | Out-Null
|
|
if ($LASTEXITCODE -ne 0) { throw "backup failed" }
|
|
& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "RESTORE VERIFYONLY FROM DISK = N'$bak' WITH CHECKSUM" | Out-Null
|
|
if ($LASTEXITCODE -ne 0) { throw "verifyonly failed" }
|
|
Say "backup $bak VERIFYONLY ok"
|
|
}
|
|
'stop-site' { Stop-Svc 'ScadaBridge-Site' }
|
|
'stop-central' {
|
|
Stop-Svc 'ScadaBridge-Central'
|
|
$deadline = (Get-Date).AddSeconds(30)
|
|
while ((Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) -and (Get-Date) -lt $deadline) { Start-Sleep 1 }
|
|
if (Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) { throw "Host.exe still running" }
|
|
Say "no Host.exe remains"
|
|
}
|
|
'migrate' {
|
|
Sql-Init
|
|
foreach ($h in 'wonder-app-vq03','wonder-app-vq04') { foreach ($p in 8085,8084) { try { Invoke-WebRequest -UseBasicParsing "http://$($h):$p/health/ready" -TimeoutSec 3 | Out-Null; throw "$h`:$p still answering - not all services stopped" } catch [System.Net.WebException] {} } }
|
|
$migSql = "$stageDir\_migrate-xact-$ts.sql"
|
|
$body = [IO.File]::ReadAllText("$stageDir\migrate-64-to-65.idempotent.sql", [Text.Encoding]::UTF8).TrimStart([char]0xFEFF)
|
|
[IO.File]::WriteAllText($migSql, "SET XACT_ABORT ON;`r`nGO`r`n" + $body, (New-Object System.Text.UTF8Encoding $false))
|
|
Say "migrating 64 -> 65"
|
|
$out = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -I -b -i $migSql 2>&1
|
|
$rc = $LASTEXITCODE
|
|
$out | % { Say " sql: $_" }
|
|
if ($rc -ne 0) { throw "migration failed (exit $rc)" }
|
|
$post = Sql-Scalar $chainQ; Say "schema after: $post"
|
|
if (-not $post.StartsWith('65|')) { throw "expected 65 migrations" }
|
|
}
|
|
'swap' {
|
|
if (Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) { throw "Host.exe running" }
|
|
foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { & sc.exe failureflag $s 1 | Out-Null; if ($LASTEXITCODE -ne 0) { throw "failureflag $s" } }
|
|
Rename-Item "$root\app" "app.bak.$ts"; Rename-Item "$root\cli" "cli.bak.$ts"
|
|
Say "renamed app/cli -> *.bak.$ts"
|
|
Copy-Item "$stage\app" "$root\app" -Recurse
|
|
Copy-Item "$stage\cli" "$root\cli" -Recurse
|
|
$keep = "$root\_appsettings-keep-$ts"
|
|
foreach ($f in $settings) {
|
|
Copy-Item "$keep\$f" "$root\app\$f" -Force
|
|
if ((Get-FileHash "$keep\$f" -Algorithm SHA256).Hash -ne (Get-FileHash "$root\app\$f" -Algorithm SHA256).Hash) { throw "appsettings mismatch $f" }
|
|
Say "restored $f (sha256 match)"
|
|
}
|
|
Say ("live Host " + (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion + " / CLI " + (Get-Item "$root\cli\scadabridge.exe").VersionInfo.ProductVersion)
|
|
$before = @((Get-ItemProperty $centralKey).Environment)
|
|
$keepEnv = @($before | Where-Object { -not (Is-Retired $_) })
|
|
Set-ItemProperty -Path $centralKey -Name Environment -Value ([string[]]$keepEnv) -Type MultiString
|
|
$after = @((Get-ItemProperty $centralKey).Environment)
|
|
foreach ($e in $keepEnv) { if ($after -notcontains $e) { throw "env entry changed" } }
|
|
if (@($after | ? { Is-Retired $_ }).Count -ne 0) { throw "retired key still present" }
|
|
Say ("central env entries {0} -> {1}" -f $before.Count, $after.Count); Digest $after | % { Say " env ScadaBridge-Central $_" }
|
|
}
|
|
'start-central' { Say "starting ScadaBridge-Central"; Start-Service 'ScadaBridge-Central'; $ok = Wait-Ready 'http://localhost:8085/health/ready' 180; Say "central ready=$ok"; if (-not $ok) { throw "central not ready" } }
|
|
'start-site' { Say "starting ScadaBridge-Site"; Start-Service 'ScadaBridge-Site'; $ok = Wait-Ready 'http://localhost:8084/health/ready' 180; Say "site ready=$ok"; if (-not $ok) { throw "site not ready" } }
|
|
'verify' {
|
|
foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') {
|
|
Say ("$s " + (Get-Service $s).Status)
|
|
Say ("recovery ${s}: " + ((& sc.exe qfailure $s | Out-String) -replace '\s+',' ').Trim())
|
|
Say ("recovery ${s}: " + ((& sc.exe qfailureflag $s | Out-String) -replace '\s+',' ').Trim())
|
|
Digest @((Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Services\$s").Environment) | % { Say " env $s $_" }
|
|
}
|
|
$logDir = "$root\central\logs"
|
|
$f = Get-ChildItem $logDir -File | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
|
Say ("central log {0}: retired-key warnings = {1}" -f $f.Name, @(Select-String -Path $f.FullName -Pattern 'no longer supported' -SimpleMatch).Count)
|
|
if ($env:COMPUTERNAME -eq 'WONDER-APP-VQ03') { Sql-Init; Say ("schema now: " + (Sql-Scalar $chainQ)) }
|
|
}
|
|
'rollback-swap' {
|
|
foreach ($s in 'ScadaBridge-Site','ScadaBridge-Central') { Stop-Svc $s }
|
|
if (Test-Path "$root\app") { Rename-Item "$root\app" "app.failed.$ts" }
|
|
if (Test-Path "$root\cli") { Rename-Item "$root\cli" "cli.failed.$ts" }
|
|
Rename-Item "$root\app.bak.$ts" 'app'; Rename-Item "$root\cli.bak.$ts" 'cli'
|
|
& reg.exe import "$stageDir\_envbak-ScadaBridge-Central-$ts.reg" | Out-Null
|
|
Say "rolled back binaries + Central env (services left STOPPED; restore DB next)"
|
|
}
|
|
default { throw "bad action $Action" }
|
|
}
|
|
Remove-Item Env:SQLCMDPASSWORD -ErrorAction SilentlyContinue
|
|
Say "$Action OK"
|
|
exit 0
|