Files
scadaproj/infra/otel-collector/wonder/Switch-AppsToOtlp.ps1
T
Joseph Doherty 71b8595742 feat(infra): per-box OTel collector on the wonder tiers; apps switched to OTLP (#54)
ZbOtelCollector (otelcol-contrib 0.162.0, NSSM, NT SERVICE\ZbOtelCollector, every
listener on 127.0.0.1, file + Prometheus exporters, size-capped rotation) on DEV
vd03 and QA vq03/vq04. OtOpcUa, ScadaBridge Central + Site, MxAccessGateway and
HistorianGateway export OTLP to http://127.0.0.1:4317 via two service-env entries
each; rolling restarts, standby-first on the QA pairs. Overview has no exporter
binding and is unchanged. Config, install/switch/verify scripts, runbook and the
dated rollout record under infra/otel-collector/wonder/; evidence doc, GAPS,
index row and deployment.md updated.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-12 22:10:15 -05:00

109 lines
6.4 KiB
PowerShell

<#
.SYNOPSIS
Adds the OTLP exporter settings to app services' environments and restarts them one at a time (scadaproj#54).
.DESCRIPTION
For each -Apps entry, in order:
1. reads the service environment (service-key 'Environment' for the .NET-hosted services, NSSM
Parameters\AppEnvironmentExtra for the NSSM ones) and exports the service key (.reg) to -BackupDir;
2. writes <Prefix>Telemetry__Exporter=Otlp and <Prefix>Telemetry__OtlpEndpoint=http://127.0.0.1:4317
(replacing any existing entries of those two names), as REG_MULTI_SZ, never via nssm/install scripts;
3. reads it back: every pre-existing entry must be hash-identical and the count must be +2 (or +0 when the
entries were already present);
4. stops the service (waits for Stopped and for its process to exit), starts it, waits for /health/ready 200.
If 3 or 4 fails, the original environment (held in memory) is written back, the service is restarted and the
script stops with exit 1. Only entry NAMES and SHA-256 prefixes are printed, never values.
-Rollback writes back the environment from the .reg backups instead (removes the two entries) and restarts.
.PARAMETER Apps Entries 'Service|Kind|Prefix|Port', Kind = Env or Nssm, Prefix e.g. 'OtOpcUa__' or '' (HistorianGateway).
#>
param(
[Parameter(Mandatory)] [string[]]$Apps,
[Parameter(Mandatory)] [string]$BackupDir,
[string]$Endpoint = 'http://127.0.0.1:4317',
[switch]$Rollback
)
$ErrorActionPreference = 'Stop'
function Say($m) { "[{0:HH:mm:ss}] {1}" -f (Get-Date), $m }
function Sha([string]$s) { $b = [Text.Encoding]::UTF8.GetBytes($s); -join ([Security.Cryptography.SHA256]::Create().ComputeHash($b)[0..5] | ForEach-Object { $_.ToString('x2') }) }
if (-not (Test-Path $BackupDir)) {
New-Item -ItemType Directory -Path $BackupDir | Out-Null
$acl = New-Object Security.AccessControl.DirectorySecurity
$acl.SetAccessRuleProtection($true, $false)
foreach ($id in 'NT AUTHORITY\SYSTEM','BUILTIN\Administrators') {
$acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule($id, 'FullControl', 'ContainerInherit,ObjectInherit', 'None', 'Allow')))
}
Set-Acl $BackupDir $acl
}
function Get-EnvLoc($svc, $kind) {
if ($kind -eq 'Nssm') { @{ Path = "HKLM:\SYSTEM\CurrentControlSet\Services\$svc\Parameters"; Name = 'AppEnvironmentExtra' } }
else { @{ Path = "HKLM:\SYSTEM\CurrentControlSet\Services\$svc"; Name = 'Environment' } }
}
function Read-Env($loc) { @((Get-ItemProperty $loc.Path -Name $loc.Name -ErrorAction SilentlyContinue).($loc.Name) | Where-Object { $_ }) }
function Write-Env($loc, [string[]]$lines) { Set-ItemProperty $loc.Path -Name $loc.Name -Type MultiString -Value $lines }
function Test-Ready($port) {
foreach ($h in 'localhost', '127.0.0.1', $env:COMPUTERNAME) {
try { $r = Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 "http://${h}:$port/health/ready"; if ($r.StatusCode -eq 200) { return $true } } catch {}
}
$false
}
function Restart-App($svc, $port) {
$t0 = Get-Date
$procId = (Get-CimInstance Win32_Service -Filter "Name='$svc'").ProcessId
Stop-Service $svc -NoWait -Force
$deadline = (Get-Date).AddSeconds(180)
while ((Get-Service $svc).Status -ne 'Stopped') { if ((Get-Date) -gt $deadline) { throw "$svc did not stop" }; Start-Sleep 1 }
while ($procId -and (Get-Process -Id $procId -ErrorAction SilentlyContinue)) { if ((Get-Date) -gt $deadline) { throw "$svc pid $procId still alive" }; Start-Sleep 1 }
$tStop = Get-Date
Start-Service $svc
$deadline = (Get-Date).AddSeconds(240)
while (-not (Test-Ready $port)) { if ((Get-Date) -gt $deadline) { throw "$svc not ready on :$port" }; Start-Sleep 2 }
$newPid = (Get-CimInstance Win32_Service -Filter "Name='$svc'").ProcessId
Say ("{0} restarted: stop {1:N0} s, stop->ready {2:N0} s, pid {3} -> {4}" -f $svc, ($tStop - $t0).TotalSeconds, ((Get-Date) - $tStop).TotalSeconds, $procId, $newPid)
}
$stamp = Get-Date -Format yyyyMMddTHHmmss
foreach ($entry in $Apps) {
$svc, $kind, $prefix, $port = $entry -split '\|'
$loc = Get-EnvLoc $svc $kind
$kExp = "${prefix}Telemetry__Exporter"; $kEnd = "${prefix}Telemetry__OtlpEndpoint"
$old = Read-Env $loc
if ($old.Count -eq 0) { throw "$svc has no environment at $($loc.Path)\$($loc.Name)" }
$oldHashes = $old | ForEach-Object { Sha $_ }
$pep = $old | Where-Object { $_ -match '^[^=]*Pepper=' } | ForEach-Object { ($_ -split '=',2)[0] + ':' + (Sha (($_ -split '=',2)[1])) }
Say "$svc env before: $($old.Count) entries, block $(Sha ($old -join "`n")), pepper [$($pep -join ',')]"
if ($Rollback) {
$new = @($old | Where-Object { -not ($_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=")) })
} else {
$reg = Join-Path $BackupDir "$svc-pre-otlp-$stamp.reg"
& reg.exe export ($loc.Path -replace '^HKLM:\\','HKLM\' -replace '\\Parameters$','') $reg /y | Out-Null
if ($LASTEXITCODE -ne 0) { throw "reg export failed for $svc" }
Set-Content (Join-Path $BackupDir "$svc-env-names-pre-$stamp.txt") ($old | ForEach-Object { ($_ -split '=',2)[0] })
$new = @($old | Where-Object { -not ($_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=")) }) + @("$kExp=Otlp", "$kEnd=$Endpoint")
}
$kept = @($old | Where-Object { -not ($_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=")) })
try {
Write-Env $loc $new
$back = Read-Env $loc
$backHashes = @($back | ForEach-Object { Sha $_ })
foreach ($k in $kept) { if ($backHashes -notcontains (Sha $k)) { throw "$svc readback lost entry $(($k -split '=',2)[0])" } }
if ($back.Count -ne $new.Count) { throw "$svc readback count $($back.Count) != $($new.Count)" }
$pep2 = $back | Where-Object { $_ -match '^[^=]*Pepper=' } | ForEach-Object { ($_ -split '=',2)[0] + ':' + (Sha (($_ -split '=',2)[1])) }
if (($pep -join ',') -ne ($pep2 -join ',')) { throw "$svc pepper changed" }
$added = $back | Where-Object { $_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=") }
Say "$svc env after: $($back.Count) entries ($($kept.Count) pre-existing hash-identical), pepper [$($pep2 -join ',')], telemetry: $($added -join ' ')"
Restart-App $svc $port
} catch {
Say "FAIL $svc : $($_.Exception.Message) -- restoring the original environment"
Write-Env $loc $old
try { Restart-App $svc $port } catch { Say "ROLLBACK RESTART FAILED $svc : $($_.Exception.Message)" }
exit 1
}
}
Say 'all done'