From 98a8df1f9d7b881d5dd80e02ac79a8a03dd5ac96 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 19 Nov 2026 15:17:16 -0500 Subject: [PATCH] deploy(vd03): 2e0ce81e (#144/#145) + NAM AD login on DEV --- .../2026-11-19-144-145/dev-deploy.ps1 | 159 ++++++++++++++++++ .../2026-11-19-wonder-app-vd03-2e0ce81e.md | 57 +++++++ 2 files changed, 216 insertions(+) create mode 100644 docs/deployment/deploy-prep/2026-11-19-144-145/dev-deploy.ps1 create mode 100644 docs/deployment/deployment-records/2026-11-19-wonder-app-vd03-2e0ce81e.md diff --git a/docs/deployment/deploy-prep/2026-11-19-144-145/dev-deploy.ps1 b/docs/deployment/deploy-prep/2026-11-19-144-145/dev-deploy.ps1 new file mode 100644 index 00000000..f6fa98b5 --- /dev/null +++ b/docs/deployment/deploy-prep/2026-11-19-144-145/dev-deploy.ps1 @@ -0,0 +1,159 @@ +# ScadaBridge DEV (wonder-app-vd03) 061432c8 -> 2e0ce81e : ScadaBridge#144 + #145 (no role, no login). Binary-only: NO migration, NO env change. +# winrm --timeout 1200 ps vd03 -f dev-deploy.ps1 +# Single-node box: ONE central + ONE site; DB on this host. +# Order: preflight gates -> reg export -> DB backup (safety, COPY_ONLY) -> keep appsettings +# -> stop Site, Central -> failureflag -> swap app/cli -> restore appsettings -> start Central, Site -> post checks. +# On failure after the stop: old binaries back, restart. Values are never printed. +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +$sha = '2e0ce81e' +$fromSha = '061432c8' +$zipSha256 = '6C706B0A76B7E9EC7CA00E12CEA19C5D86814BEAF66B541F527CBFC3D6572E23' +$stageDir = "E:\ApiInstall\_stage-sb-$sha" +$stage = "$stageDir\unpacked" +$root = 'E:\ApiInstall\ScadaBridge' +$bakDir = 'E:\ApiInstall\ScadaBridge-backups' +$ts = Get-Date -Format 'yyyyMMddTHHmmss' +$log = "$root\_DEPLOY-NOTES-$sha-$ts.txt" +$settings = 'appsettings.json','appsettings.Central.json','appsettings.Site.json' +$centralKey = 'HKLM:\SYSTEM\CurrentControlSet\Services\ScadaBridge-Central' +function Say($m) { $line = "[{0:HH:mm:ss.fff}] {1}" -f (Get-Date), $m; $line; Add-Content -Path $log -Value $line } +function Is-Retired($e) { $e -like 'AuditLog__HashChain__*' -or $e -like 'AuditLog__MaintenanceConnectionString=*' } +function Digest($entries) { + $h = [System.Security.Cryptography.SHA256]::Create(); $o = @() + foreach ($e in $entries) { $i = $e.IndexOf('='); if ($i -lt 1) { $o += ''; continue } + $v = $e.Substring($i+1); $x = -join ($h.ComputeHash([Text.Encoding]::UTF8.GetBytes($v)) | % { $_.ToString('x2') }) + $o += ("{0} sha256:{1} len={2}" -f $e.Substring(0,$i), $x.Substring(0,12), $v.Length) } + $h.Dispose(); $o | Sort-Object +} +function Wait-Ready($url, $seconds) { + $deadline = (Get-Date).AddSeconds($seconds) + while ((Get-Date) -lt $deadline) { try { if ((Invoke-WebRequest -UseBasicParsing $url -TimeoutSec 5).StatusCode -eq 200) { return $true } } catch {}; Start-Sleep 2 } + return $false +} +function Stop-Both { + foreach ($s in 'ScadaBridge-Site','ScadaBridge-Central') { + if ((Get-Service $s).Status -ne 'Stopped') { Say "stopping $s"; Stop-Service $s -Force; (Get-Service $s).WaitForStatus('Stopped','00:02:00') } + Say "$s stopped" + } + $deadline = (Get-Date).AddSeconds(30) + while ((Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) -and (Get-Date) -lt $deadline) { Start-Sleep 1 } + if (Get-Process -Name 'ZB.MOM.WW.ScadaBridge.Host' -ErrorAction SilentlyContinue) { throw "Host.exe still running after stop" } + Say "no Host.exe process remains" +} +function Start-Both { + Say "starting ScadaBridge-Central"; Start-Service 'ScadaBridge-Central' + $ok = Wait-Ready 'http://localhost:8085/health/ready' 180; Say "central ready=$ok" + if (-not $ok) { throw "central did not become ready" } + Say "starting ScadaBridge-Site"; Start-Service 'ScadaBridge-Site' + $ok = Wait-Ready 'http://localhost:8084/health/ready' 180; Say "site ready=$ok" + if (-not $ok) { throw "site did not become ready" } +} + +Say "ScadaBridge#144/#145 deploy $fromSha -> $sha on $env:COMPUTERNAME" + +# ---- preflight gates (nothing changed yet) ---------------------------------------------------- +if ((Get-FileHash "$stageDir\scadabridge-$sha.zip" -Algorithm SHA256).Hash -ne $zipSha256) { throw "zip hash mismatch" } +if (Test-Path $stage) { Remove-Item $stage -Recurse -Force } +Expand-Archive "$stageDir\scadabridge-$sha.zip" $stage +$stagedVer = (Get-Item "$stage\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion +$stagedCli = (Get-Item "$stage\cli\scadabridge.exe").VersionInfo.ProductVersion +if ($stagedVer -notlike "*+$sha*" -or $stagedCli -notlike "*+$sha*") { throw "staged exe is $stagedVer / $stagedCli" } +$liveVer = (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion +if ($liveVer -notlike "*+$fromSha*") { throw "live exe is $liveVer, not $fromSha" } +Say "staged $stagedVer; live $liveVer" +Say ("staged app files {0}, cli files {1}" -f (Get-ChildItem "$stage\app" -Recurse -File).Count, (Get-ChildItem "$stage\cli" -Recurse -File).Count) + +$envBefore = @((Get-ItemProperty $centralKey).Environment) +$retired = @($envBefore | Where-Object { Is-Retired $_ }) +Say ("central env entries {0}; retired keys present: {1}" -f $envBefore.Count, $retired.Count) + +$cs = ($envBefore | Where-Object { $_ -like 'ScadaBridge__Database__ConfigurationDb=*' }) -replace '^[^=]*=','' +$sb = New-Object System.Data.SqlClient.SqlConnectionStringBuilder $cs +$db = $sb['Database']; if (-not $db) { $db = $sb.InitialCatalog } +if ($db -ne 'ScadaBridge') { throw "unexpected catalog" } +$env:SQLCMDPASSWORD = $sb.Password +function Sql-Scalar($q) { $r = & sqlcmd -S $sb.DataSource -d $db -U $sb.UserID -C -h -1 -W -b -Q "SET NOCOUNT ON; $q"; if ($LASTEXITCODE -ne 0) { throw "sql failed: $q" }; ($r | Where-Object { $_ -and $_.Trim() } | Select-Object -First 1).Trim() } +$chainQ = "SELECT CONCAT((SELECT COUNT(*) FROM __EFMigrationsHistory),'|',COL_LENGTH('dbo.AuditLog','ChainSeq'),'|',COL_LENGTH('dbo.AuditLog','RowHash'),'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT(*) FROM sys.indexes WHERE object_id=OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_ChainSeq','IX_AuditLog_Unchained')),'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_chainer') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL THEN 0 ELSE 1 END,'|',CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_writer') IS NULL THEN 0 ELSE 1 END,'|',(SELECT COUNT_BIG(*) FROM dbo.AuditLog))" +$pre = Sql-Scalar $chainQ +Say "schema before (migrations|ChainSeq|RowHash|Head|Tomb|chainIdx|chainer|purger|writer|rows): $pre" +if (-not $pre.StartsWith('65|')) { throw "expected 65 migrations: $pre" } + +# ---- backups (services still running) --------------------------------------------------------- +foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { + & reg.exe export "HKLM\SYSTEM\CurrentControlSet\Services\$s" "$stageDir\_envbak-$s-$ts.reg" /y | Out-Null + if ($LASTEXITCODE -ne 0) { throw "reg export $s failed" } + Say "reg export $s -> $stageDir\_envbak-$s-$ts.reg" +} +New-Item -ItemType Directory -Force -Path $bakDir | Out-Null +$bak = "$bakDir\ScadaBridge_pre-$($sha)_$ts.bak" +& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "BACKUP DATABASE [$db] TO DISK = N'$bak' WITH COPY_ONLY, CHECKSUM, COMPRESSION, INIT" | Out-Null +if ($LASTEXITCODE -ne 0) { throw "backup failed" } +& sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "RESTORE VERIFYONLY FROM DISK = N'$bak' WITH CHECKSUM" | Out-Null +if ($LASTEXITCODE -ne 0) { throw "verifyonly failed" } +Say "db backup $bak ($((Get-Item $bak).Length) bytes, VERIFYONLY ok)" +$keep = New-Item -ItemType Directory -Force -Path "$root\_appsettings-keep-$ts" +foreach ($f in $settings) { Copy-Item "$root\app\$f" "$keep\$f"; Say ("keep {0} sha256 {1}" -f $f, (Get-FileHash "$keep\$f" -Algorithm SHA256).Hash) } + +# ---- outage window ------------------------------------------------------------------------- +$swapped = $false; $migrated = $false; $envChanged = $false +try { + Stop-Both + + + foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { + & sc.exe failureflag $s 1 | Out-Null; if ($LASTEXITCODE -ne 0) { throw "sc failureflag $s failed" } + } + Rename-Item "$root\app" "app.bak.$ts"; Rename-Item "$root\cli" "cli.bak.$ts"; $swapped = $true + Say "renamed app/cli -> *.bak.$ts" + Copy-Item "$stage\app" "$root\app" -Recurse + Copy-Item "$stage\cli" "$root\cli" -Recurse + foreach ($f in $settings) { + Copy-Item "$keep\$f" "$root\app\$f" -Force + if ((Get-FileHash "$keep\$f" -Algorithm SHA256).Hash -ne (Get-FileHash "$root\app\$f" -Algorithm SHA256).Hash) { throw "appsettings restore mismatch: $f" } + Say "restored $f (sha256 match)" + } + Say ("live Host " + (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion + " / CLI " + (Get-Item "$root\cli\scadabridge.exe").VersionInfo.ProductVersion) + + + Start-Both +} +catch { + Say "ERROR: $($_.Exception.Message)" + Say "AUTO-ROLLBACK (migrated=$migrated swapped=$swapped envChanged=$envChanged)" + try { + Stop-Both + if ($migrated) { + & sqlcmd -S $sb.DataSource -d master -U $sb.UserID -C -b -Q "ALTER DATABASE [$db] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; RESTORE DATABASE [$db] FROM DISK = N'$bak' WITH REPLACE, CHECKSUM; ALTER DATABASE [$db] SET MULTI_USER;" | ForEach-Object { Say " restore: $_" } + if ($LASTEXITCODE -ne 0) { throw "DB restore failed" } + Say "DB restored from $bak" + } + if ($swapped) { + if (Test-Path "$root\app") { Rename-Item "$root\app" "app.failed.$ts" } + if (Test-Path "$root\cli") { Rename-Item "$root\cli" "cli.failed.$ts" } + Rename-Item "$root\app.bak.$ts" 'app'; Rename-Item "$root\cli.bak.$ts" 'cli' + Say "restored app/cli from *.bak.$ts" + } + if ($envChanged) { Set-ItemProperty -Path $centralKey -Name Environment -Value ([string[]]$envBefore) -Type MultiString; Say "central env restored" } + Start-Both + Say ("rolled back; live Host " + (Get-Item "$root\app\ZB.MOM.WW.ScadaBridge.Host.exe").VersionInfo.ProductVersion) + } catch { Say "AUTO-ROLLBACK FAILED: $($_.Exception.Message) - manual action needed" } + Remove-Item Env:SQLCMDPASSWORD -ErrorAction SilentlyContinue + exit 1 +} + +# ---- post checks (verify of this deploy) ------------------------------------------------------ +Start-Sleep 20 +$q2 = Sql-Scalar "SELECT CONCAT(COUNT_BIG(*),'|',CONVERT(varchar(33),MAX(OccurredAtUtc),126)) FROM dbo.AuditLog" +Say "auditlog rows|newest: $q2" +$logDir = "$root\central\logs" +if (Test-Path $logDir) { + $f = Get-ChildItem $logDir -File | Sort-Object LastWriteTime -Descending | Select-Object -First 1 + $hits = @(Select-String -Path $f.FullName -Pattern 'no longer supported' -SimpleMatch) + Say ("central log {0}: retired-key warnings since start = {1}" -f $f.Name, $hits.Count) +} +foreach ($s in 'ScadaBridge-Central','ScadaBridge-Site') { Say ("$s " + (Get-Service $s).Status + " / " + ((& sc.exe qfailureflag $s | Out-String) -replace '\s+',' ').Trim()) } +Remove-Item Env:SQLCMDPASSWORD -ErrorAction SilentlyContinue +Say "DEPLOY DONE ts=$ts backup=$bak" +exit 0 diff --git a/docs/deployment/deployment-records/2026-11-19-wonder-app-vd03-2e0ce81e.md b/docs/deployment/deployment-records/2026-11-19-wonder-app-vd03-2e0ce81e.md new file mode 100644 index 00000000..9250b9ff --- /dev/null +++ b/docs/deployment/deployment-records/2026-11-19-wonder-app-vd03-2e0ce81e.md @@ -0,0 +1,57 @@ +# 2026-11-19: DEV (wonder-app-vd03) `061432c8` → `2e0ce81e` (#144 + #145, no role no login), then NAM AD login on + +**Owner go (2026-11-19):** "ok, do it. winrm usage approved (for the vd03 deploy)", as the first step of the DEV NAM +LDAP cutover (scadaproj plan `docs/plans/2026-11-15-nam-ldap-cutover.md` §0; record +`scadaproj/docs/deploy/2026-11-19-vd03-nam-ldap/`). + +**Build:** `main` @ `2e0ce81e`, ProductVersion `1.0.0+2e0ce81ea04ac5e85ddb654cf33068594ba5c3c4`. Host and CLI +self-contained win-x64 from a clean detached worktree: 1056 app files, 224 cli files. The zip is 130 508 202 B, SHA-256 +`6c706b0a76b7e9ec7ca00e12cea19c5d86814beaf66b541f527cbfc3d6572e23`. + +**Delta:** 11 commits: #144 (`5a642d1c`, `2d8d5a3d`, `764e75c3`), #145 (`1d2170a1`, `a5243b1e`) and the #116 +`leader-auto-down` default (`5bf6d1fa`, already drilled on vd03 2026-11-15). **No migration** (65 stays 65), no env change. + +**Script:** `docs/deployment/deploy-prep/2026-11-19-144-145/dev-deploy.ps1`, derived from the 11-15 pack with the +migration and env steps removed. On-box notes: `E:\ApiInstall\ScadaBridge\_DEPLOY-NOTES-2e0ce81e-20261119T145229.txt`. + +## Window + +| Step | Local (EST) | +|---|---| +| Preflight: live `061432c8`, staged `2e0ce81e` Host+CLI, 65 migrations, AuditLog 2 856 rows | 14:52:30 → 14:52:42 | +| `reg export` both service keys; DB backup `E:\ApiInstall\ScadaBridge-backups\ScadaBridge_pre-2e0ce81e_20261119T145229.bak` (132 222 976 B, VERIFYONLY ok); appsettings kept | → 14:52:45 | +| Stop Site, then Central | 14:52:45 → 14:53:06 | +| Swap `app`/`cli` → `*.bak.20261119T145229`; appsettings restored (hash match) | → 14:53:12 | +| Central ready, Site ready | 14:53:24 / 14:53:33 | + +**Downtime:** site data path ~47 s, central API ~30 s. + +## Gates + +- Health ready 200 (central `:8085`, site `:8084`); both services `FAILURE_ACTIONS_ON_NONCRASH_FAILURES: TRUE`. +- No retired-key warning in the central log. +- Bogus inbound `X-API-Key` → 401. +- CLI (login still off) `instance list` → 6 instances. + +## Then: NAM AD login on (same day, 15:11) + +- `appsettings.Central.json`: `ScadaBridge:Security:Ldap` → NAM (LDAPS nam1 + nam2 fallback, `svc_igntadmin`, + `${secret:nam-ldap-svc-password}`, `sAMAccountName`); the `Auth` block (`DisableLogin` + `AllowOutsideDevelopment`) removed. +- Central env: `ScadaBridge__Security__Ldap__ServiceAccountPassword` (GLAuth) removed, 12 → 11 entries. +- Central restarted; Site not restarted. +- **Zero-role proven live** before any `UG_WW_*` row existed: `dohertj2` was refused at `/auth/login`, at `/auth/token` + and on the CLI. Log: "the LDAP bind succeeded but none of the user's 176 group(s) maps to a ScadaBridge role". +- `LdapGroupMappings` rows 6–9 added by SQL: `UG_WW_APP_ADMIN_DEV`→Administrator, `UG_WW_APP_IDE_DEV`→Designer, + `UG_WW_APP_IDE_RO_DEV`→Viewer, `UG_WW_HIST_ADMIN_DEV`→Deployer (DEV interim until `UG_WW_APP_DEPLOY_DEV` exists). +- Then `dohertj2`: cookie login ✓, `/auth/token` 200 with Administrator+Designer+Deployer, CLI `instance list` and + `security role-mapping list` ✓. +- MxGateway connection Connected, 810/810, good 399 / bad 411 (baseline). + +**Automation identity on DEV is now `dohertj2`** (HTTP Basic / `--password-file`); the login-off dev user is gone. + +## Rollback + +- **LDAP only:** restore `E:\ApiInstall\_ldapcut-20261119T145640\sb-appsettings.Central.json.pre-cutover` as + `app\appsettings.Central.json`, `reg import …\reg-ScadaBridge-Central.reg`, restart Central. +- **Binaries:** stop Site then Central; `app`/`cli` ← `*.bak.20261119T145229`; start Central then Site. No DB restore is + needed (schema unchanged); the backup above is there for safety.