deploy(#143): DEV vd03 + QA vq03/vq04 to 061432c8 — hash chain removed (migration 65), #116 QA site-pair drills

DEV: ~48 s site / ~29 s central outage; QA: pair stop/start, ~2m40s site / ~1m37s central.
Chain schema + chainer role gone, env key removed, all gates pass on both tiers.
#116 QA site drills (all reactors idle): kill → takeover +7.7 s; stop-and-hold →
[leader-auto-down] DOWNING +27.2 s, takeover +27.4 s, recovery restored byte-identical.

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
Joseph Doherty
2026-11-15 19:36:04 -05:00
parent 061432c8c4
commit 50a5291b64
7 changed files with 710 additions and 0 deletions
@@ -0,0 +1,182 @@
# ScadaBridge DEV (wonder-app-vd03) 8e32f353 -> 061432c8 : ScadaBridge#143 (hash-chain removal), migration 64 -> 65
# winrm --timeout 1200 ps vd03 -f dev-deploy.ps1
# Single-node box: ONE central + ONE site; DB on this host.
# Order: preflight gates -> reg export -> DB backup (COPY_ONLY,CHECKSUM,COMPRESSION + VERIFYONLY) -> keep appsettings
# -> stop Site, Central -> migrate 64->65 (XACT_ABORT, sqlcmd -I -b) -> failureflag -> swap app/cli -> restore appsettings
# -> remove retired AuditLog env keys from Central -> start Central, Site -> post checks.
# On failure after the stop: restore DB (if migrated) + old binaries + Central env (.reg), restart. Values are never printed.
$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
$sha = '061432c8'
$fromSha = '8e32f353'
$zipSha256 = 'B6E85ECC24186AF8B658FA6F1E38F080588EA5F78DD856A4356E696897FF4095'
$sqlSha256 = 'A16A150DE1E1F6C4CB9153E1C53BED95A029D656F18C3DC3B61EC6575198F7A7'
$stageDir = "E:\ApiInstall\_stage-sb-$sha"
$stage = "$stageDir\unpacked"
$root = 'E:\ApiInstall\ScadaBridge'
$bakDir = 'E:\ApiInstall\ScadaBridge-backups'
$ts = Get-Date -Format 'yyyyMMddTHHmmss'
$log = "$root\_DEPLOY-NOTES-$sha-$ts.txt"
$settings = 'appsettings.json','appsettings.Central.json','appsettings.Site.json'
$centralKey = 'HKLM:\SYSTEM\CurrentControlSet\Services\ScadaBridge-Central'
function Say($m) { $line = "[{0:HH:mm:ss.fff}] {1}" -f (Get-Date), $m; $line; Add-Content -Path $log -Value $line }
function Is-Retired($e) { $e -like 'AuditLog__HashChain__*' -or $e -like 'AuditLog__MaintenanceConnectionString=*' }
function Digest($entries) {
$h = [System.Security.Cryptography.SHA256]::Create(); $o = @()
foreach ($e in $entries) { $i = $e.IndexOf('='); if ($i -lt 1) { $o += '<unparsable>'; continue }
$v = $e.Substring($i+1); $x = -join ($h.ComputeHash([Text.Encoding]::UTF8.GetBytes($v)) | % { $_.ToString('x2') })
$o += ("{0} sha256:{1} len={2}" -f $e.Substring(0,$i), $x.Substring(0,12), $v.Length) }
$h.Dispose(); $o | Sort-Object
}
function Wait-Ready($url, $seconds) {
$deadline = (Get-Date).AddSeconds($seconds)
while ((Get-Date) -lt $deadline) { try { if ((Invoke-WebRequest -UseBasicParsing $url -TimeoutSec 5).StatusCode -eq 200) { return $true } } catch {}; Start-Sleep 2 }
return $false
}
function Stop-Both {
foreach ($s in 'ScadaBridge-Site','ScadaBridge-Central') {
if ((Get-Service $s).Status -ne 'Stopped') { Say "stopping $s"; Stop-Service $s -Force; (Get-Service $s).WaitForStatus('Stopped','00:02:00') }
Say "$s stopped"
}
$deadline = (Get-Date).AddSeconds(30)
while ((Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) -and (Get-Date) -lt $deadline) { Start-Sleep 1 }
if (Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) { throw "Host.exe still running after stop" }
Say "no Host.exe process remains"
}
function Start-Both {
Say "starting ScadaBridge-Central"; Start-Service 'ScadaBridge-Central'
$ok = Wait-Ready 'http://localhost:8085/health/ready' 180; Say "central ready=$ok"
if (-not $ok) { throw "central did not become ready" }
Say "starting ScadaBridge-Site"; Start-Service 'ScadaBridge-Site'
$ok = Wait-Ready 'http://localhost:8084/health/ready' 180; Say "site ready=$ok"
if (-not $ok) { throw "site did not become ready" }
}
Say "ScadaBridge#143 deploy $fromSha -> $sha on $env:COMPUTERNAME"
# ---- preflight gates (nothing changed yet) ----------------------------------------------------
if ((Get-FileHash "$stageDir\scadabridge-$sha.zip" -Algorithm SHA256).Hash -ne $zipSha256) { throw "zip hash mismatch" }
if ((Get-FileHash "$stageDir\migrate-64-to-65.idempotent.sql" -Algorithm SHA256).Hash -ne $sqlSha256) { throw "sql hash mismatch" }
if (Test-Path $stage) { Remove-Item $stage -Recurse -Force }
Expand-Archive "$stageDir\scadabridge-$sha.zip" $stage
$stagedVer = (Get-Item "$stage\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion
$stagedCli = (Get-Item "$stage\cli\scadabridge.exe").VersionInfo.ProductVersion
if ($stagedVer -notlike "*+$sha*" -or $stagedCli -notlike "*+$sha*") { throw "staged exe is $stagedVer / $stagedCli" }
$liveVer = (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion
if ($liveVer -notlike "*+$fromSha*") { throw "live exe is $liveVer, not $fromSha" }
Say "staged $stagedVer; live $liveVer"
Say ("staged app files {0}, cli files {1}" -f (Get-ChildItem "$stage\app" -Recurse -File).Count, (Get-ChildItem "$stage\cli" -Recurse -File).Count)
$envBefore = @((Get-ItemProperty $centralKey).Environment)
$retired = @($envBefore | Where-Object { Is-Retired $_ })
Say ("central env entries {0}; retired keys present: {1}" -f $envBefore.Count, (($retired | % { $_.Substring(0,$_.IndexOf('=')) }) -join ','))
if (-not ($envBefore -contains 'AuditLog__HashChain__Enabled=true')) { throw "AuditLog__HashChain__Enabled=true not found in the Central env" }
$cs = ($envBefore | Where-Object { $_ -like 'ScadaBridge__Database__ConfigurationDb=*' }) -replace '^[^=]*=',''
$sb = New-Object System.Data.SqlClient.SqlConnectionStringBuilder $cs
$db = $sb['Database']; if (-not $db) { $db = $sb.InitialCatalog }
if ($db -ne 'ScadaBridge') { throw "unexpected catalog" }
$env:SQLCMDPASSWORD = $sb.Password
function Sql-Scalar($q) { $r = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -h -1 -W -b -Q "SET NOCOUNT ON; $q"; if ($LASTEXITCODE -ne 0) { throw "sql failed: $q" }; ($r | Where-Object { $_ -and $_.Trim() } | Select-Object -First 1).Trim() }
$chainQ = "SELECT CONCAT((SELECT COUNT(*) FROM __EFMigrationsHistory),'|',COL_LENGTH('dbo.AuditLog','ChainSeq'),'|',COL_LENGTH('dbo.AuditLog','RowHash'),'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT(*) FROM sys.indexes WHERE object_id=OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_ChainSeq','IX_AuditLog_Unchained')),'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_chainer') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_writer') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT_BIG(*) FROM dbo.AuditLog))"
$pre = Sql-Scalar $chainQ
Say "schema before (migrations|ChainSeq|RowHash|Head|Tomb|chainIdx|chainer|purger|writer|rows): $pre"
if (-not $pre.StartsWith('64|')) { throw "expected 64 migrations: $pre" }
# ---- backups (services still running) ---------------------------------------------------------
foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') {
& reg.exe export "HKLM\SYSTEM\CurrentControlSet\Services\$s" "$stageDir\_envbak-$s-$ts.reg" /y | Out-Null
if ($LASTEXITCODE -ne 0) { throw "reg export $s failed" }
Say "reg export $s -> $stageDir\_envbak-$s-$ts.reg"
}
New-Item -ItemType Directory -Force -Path $bakDir | Out-Null
$bak = "$bakDir\ScadaBridge_pre-$($sha)_$ts.bak"
& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "BACKUP DATABASE [$db] TO DISK = N'$bak' WITH COPY_ONLY, CHECKSUM, COMPRESSION, INIT" | Out-Null
if ($LASTEXITCODE -ne 0) { throw "backup failed" }
& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "RESTORE VERIFYONLY FROM DISK = N'$bak' WITH CHECKSUM" | Out-Null
if ($LASTEXITCODE -ne 0) { throw "verifyonly failed" }
Say "db backup $bak ($((Get-Item $bak).Length) bytes, VERIFYONLY ok)"
$keep = New-Item -ItemType Directory -Force -Path "$root\_appsettings-keep-$ts"
foreach ($f in $settings) { Copy-Item "$root\app\$f" "$keep\$f"; Say ("keep {0} sha256 {1}" -f $f, (Get-FileHash "$keep\$f" -Algorithm SHA256).Hash) }
# ---- outage window -------------------------------------------------------------------------
$swapped = $false; $migrated = $false; $envChanged = $false
try {
Stop-Both
$migSql = "$stageDir\_migrate-xact-$ts.sql"
# one UTF-8 file WITHOUT any BOM (a BOM after a prepended header broke sqlcmd in #126)
$body = [IO.File]::ReadAllText("$stageDir\migrate-64-to-65.idempotent.sql", [Text.Encoding]::UTF8).TrimStart([char]0xFEFF)
[IO.File]::WriteAllText($migSql, "SET XACT_ABORT ON;`r`nGO`r`n" + $body, (New-Object System.Text.UTF8Encoding $false))
Say "migrating 64 -> 65"
$migrated = $true
$out = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -I -b -i $migSql 2>&1
$out | ForEach-Object { Say " sql: $_" }
if ($LASTEXITCODE -ne 0) { throw "migration failed (exit $LASTEXITCODE)" }
$post = Sql-Scalar $chainQ
Say "schema after: $post"
if (-not $post.StartsWith('65|')) { throw "expected 65 migrations after migrate: $post" }
foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') {
& sc.exe failureflag $s 1 | Out-Null; if ($LASTEXITCODE -ne 0) { throw "sc failureflag $s failed" }
}
Rename-Item "$root\app" "app.bak.$ts"; Rename-Item "$root\cli" "cli.bak.$ts"; $swapped = $true
Say "renamed app/cli -> *.bak.$ts"
Copy-Item "$stage\app" "$root\app" -Recurse
Copy-Item "$stage\cli" "$root\cli" -Recurse
foreach ($f in $settings) {
Copy-Item "$keep\$f" "$root\app\$f" -Force
if ((Get-FileHash "$keep\$f" -Algorithm SHA256).Hash -ne (Get-FileHash "$root\app\$f" -Algorithm SHA256).Hash) { throw "appsettings restore mismatch: $f" }
Say "restored $f (sha256 match)"
}
Say ("live Host " + (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion + " / CLI " + (Get-Item "$root\cli\scadabridge.exe").VersionInfo.ProductVersion)
$keepEnv = @($envBefore | Where-Object { -not (Is-Retired $_) })
Set-ItemProperty -Path $centralKey -Name Environment -Value ([string[]]$keepEnv) -Type MultiString; $envChanged = $true
$envAfter = @((Get-ItemProperty $centralKey).Environment)
foreach ($e in $keepEnv) { if ($envAfter -notcontains $e) { throw "an existing env entry changed" } }
if (@($envAfter | Where-Object { Is-Retired $_ }).Count -ne 0) { throw "retired env key still present" }
Say ("central env entries {0} -> {1}" -f $envBefore.Count, $envAfter.Count)
Digest $envAfter | ForEach-Object { Say " env $_" }
Start-Both
}
catch {
Say "ERROR: $($_.Exception.Message)"
Say "AUTO-ROLLBACK (migrated=$migrated swapped=$swapped envChanged=$envChanged)"
try {
Stop-Both
if ($migrated) {
& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "ALTER DATABASE [$db] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; RESTORE DATABASE [$db] FROM DISK = N'$bak' WITH REPLACE, CHECKSUM; ALTER DATABASE [$db] SET MULTI_USER;" | ForEach-Object { Say " restore: $_" }
if ($LASTEXITCODE -ne 0) { throw "DB restore failed" }
Say "DB restored from $bak"
}
if ($swapped) {
if (Test-Path "$root\app") { Rename-Item "$root\app" "app.failed.$ts" }
if (Test-Path "$root\cli") { Rename-Item "$root\cli" "cli.failed.$ts" }
Rename-Item "$root\app.bak.$ts" 'app'; Rename-Item "$root\cli.bak.$ts" 'cli'
Say "restored app/cli from *.bak.$ts"
}
if ($envChanged) { Set-ItemProperty -Path $centralKey -Name Environment -Value ([string[]]$envBefore) -Type MultiString; Say "central env restored" }
Start-Both
Say ("rolled back; live Host " + (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion)
} catch { Say "AUTO-ROLLBACK FAILED: $($_.Exception.Message) - manual action needed" }
Remove-Item Env:SQLCMDPASSWORD -ErrorAction SilentlyContinue
exit 1
}
# ---- post checks (verify of this deploy) ------------------------------------------------------
Start-Sleep 20
$q2 = Sql-Scalar "SELECT CONCAT(COUNT_BIG(*),'|',CONVERT(varchar(33),MAX(OccurredAtUtc),126)) FROM dbo.AuditLog"
Say "auditlog rows|newest: $q2"
$logDir = "$root\central\logs"
if (Test-Path $logDir) {
$f = Get-ChildItem $logDir -File | Sort-Object LastWriteTime -Descending | Select-Object -First 1
$hits = @(Select-String -Path $f.FullName -Pattern 'no longer supported' -SimpleMatch)
Say ("central log {0}: retired-key warnings since start = {1}" -f $f.Name, $hits.Count)
}
foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { Say ("$s " + (Get-Service $s).Status + " / " + ((& sc.exe qfailureflag $s | Out-String) -replace '\s+',' ').Trim()) }
Remove-Item Env:SQLCMDPASSWORD -ErrorAction SilentlyContinue
Say "DEPLOY DONE ts=$ts backup=$bak"
exit 0