71b8595742
ZbOtelCollector (otelcol-contrib 0.162.0, NSSM, NT SERVICE\ZbOtelCollector, every listener on 127.0.0.1, file + Prometheus exporters, size-capped rotation) on DEV vd03 and QA vq03/vq04. OtOpcUa, ScadaBridge Central + Site, MxAccessGateway and HistorianGateway export OTLP to http://127.0.0.1:4317 via two service-env entries each; rolling restarts, standby-first on the QA pairs. Overview has no exporter binding and is unchanged. Config, install/switch/verify scripts, runbook and the dated rollout record under infra/otel-collector/wonder/; evidence doc, GAPS, index row and deployment.md updated. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
109 lines
6.4 KiB
PowerShell
109 lines
6.4 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Adds the OTLP exporter settings to app services' environments and restarts them one at a time (scadaproj#54).
|
|
|
|
.DESCRIPTION
|
|
For each -Apps entry, in order:
|
|
1. reads the service environment (service-key 'Environment' for the .NET-hosted services, NSSM
|
|
Parameters\AppEnvironmentExtra for the NSSM ones) and exports the service key (.reg) to -BackupDir;
|
|
2. writes <Prefix>Telemetry__Exporter=Otlp and <Prefix>Telemetry__OtlpEndpoint=http://127.0.0.1:4317
|
|
(replacing any existing entries of those two names), as REG_MULTI_SZ, never via nssm/install scripts;
|
|
3. reads it back: every pre-existing entry must be hash-identical and the count must be +2 (or +0 when the
|
|
entries were already present);
|
|
4. stops the service (waits for Stopped and for its process to exit), starts it, waits for /health/ready 200.
|
|
If 3 or 4 fails, the original environment (held in memory) is written back, the service is restarted and the
|
|
script stops with exit 1. Only entry NAMES and SHA-256 prefixes are printed, never values.
|
|
|
|
-Rollback writes back the environment from the .reg backups instead (removes the two entries) and restarts.
|
|
|
|
.PARAMETER Apps Entries 'Service|Kind|Prefix|Port', Kind = Env or Nssm, Prefix e.g. 'OtOpcUa__' or '' (HistorianGateway).
|
|
#>
|
|
param(
|
|
[Parameter(Mandatory)] [string[]]$Apps,
|
|
[Parameter(Mandatory)] [string]$BackupDir,
|
|
[string]$Endpoint = 'http://127.0.0.1:4317',
|
|
[switch]$Rollback
|
|
)
|
|
$ErrorActionPreference = 'Stop'
|
|
function Say($m) { "[{0:HH:mm:ss}] {1}" -f (Get-Date), $m }
|
|
function Sha([string]$s) { $b = [Text.Encoding]::UTF8.GetBytes($s); -join ([Security.Cryptography.SHA256]::Create().ComputeHash($b)[0..5] | ForEach-Object { $_.ToString('x2') }) }
|
|
|
|
if (-not (Test-Path $BackupDir)) {
|
|
New-Item -ItemType Directory -Path $BackupDir | Out-Null
|
|
$acl = New-Object Security.AccessControl.DirectorySecurity
|
|
$acl.SetAccessRuleProtection($true, $false)
|
|
foreach ($id in 'NT AUTHORITY\SYSTEM','BUILTIN\Administrators') {
|
|
$acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule($id, 'FullControl', 'ContainerInherit,ObjectInherit', 'None', 'Allow')))
|
|
}
|
|
Set-Acl $BackupDir $acl
|
|
}
|
|
|
|
function Get-EnvLoc($svc, $kind) {
|
|
if ($kind -eq 'Nssm') { @{ Path = "HKLM:\SYSTEM\CurrentControlSet\Services\$svc\Parameters"; Name = 'AppEnvironmentExtra' } }
|
|
else { @{ Path = "HKLM:\SYSTEM\CurrentControlSet\Services\$svc"; Name = 'Environment' } }
|
|
}
|
|
function Read-Env($loc) { @((Get-ItemProperty $loc.Path -Name $loc.Name -ErrorAction SilentlyContinue).($loc.Name) | Where-Object { $_ }) }
|
|
function Write-Env($loc, [string[]]$lines) { Set-ItemProperty $loc.Path -Name $loc.Name -Type MultiString -Value $lines }
|
|
|
|
function Test-Ready($port) {
|
|
foreach ($h in 'localhost', '127.0.0.1', $env:COMPUTERNAME) {
|
|
try { $r = Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 "http://${h}:$port/health/ready"; if ($r.StatusCode -eq 200) { return $true } } catch {}
|
|
}
|
|
$false
|
|
}
|
|
function Restart-App($svc, $port) {
|
|
$t0 = Get-Date
|
|
$procId = (Get-CimInstance Win32_Service -Filter "Name='$svc'").ProcessId
|
|
Stop-Service $svc -NoWait -Force
|
|
$deadline = (Get-Date).AddSeconds(180)
|
|
while ((Get-Service $svc).Status -ne 'Stopped') { if ((Get-Date) -gt $deadline) { throw "$svc did not stop" }; Start-Sleep 1 }
|
|
while ($procId -and (Get-Process -Id $procId -ErrorAction SilentlyContinue)) { if ((Get-Date) -gt $deadline) { throw "$svc pid $procId still alive" }; Start-Sleep 1 }
|
|
$tStop = Get-Date
|
|
Start-Service $svc
|
|
$deadline = (Get-Date).AddSeconds(240)
|
|
while (-not (Test-Ready $port)) { if ((Get-Date) -gt $deadline) { throw "$svc not ready on :$port" }; Start-Sleep 2 }
|
|
$newPid = (Get-CimInstance Win32_Service -Filter "Name='$svc'").ProcessId
|
|
Say ("{0} restarted: stop {1:N0} s, stop->ready {2:N0} s, pid {3} -> {4}" -f $svc, ($tStop - $t0).TotalSeconds, ((Get-Date) - $tStop).TotalSeconds, $procId, $newPid)
|
|
}
|
|
|
|
$stamp = Get-Date -Format yyyyMMddTHHmmss
|
|
foreach ($entry in $Apps) {
|
|
$svc, $kind, $prefix, $port = $entry -split '\|'
|
|
$loc = Get-EnvLoc $svc $kind
|
|
$kExp = "${prefix}Telemetry__Exporter"; $kEnd = "${prefix}Telemetry__OtlpEndpoint"
|
|
$old = Read-Env $loc
|
|
if ($old.Count -eq 0) { throw "$svc has no environment at $($loc.Path)\$($loc.Name)" }
|
|
$oldHashes = $old | ForEach-Object { Sha $_ }
|
|
$pep = $old | Where-Object { $_ -match '^[^=]*Pepper=' } | ForEach-Object { ($_ -split '=',2)[0] + ':' + (Sha (($_ -split '=',2)[1])) }
|
|
Say "$svc env before: $($old.Count) entries, block $(Sha ($old -join "`n")), pepper [$($pep -join ',')]"
|
|
|
|
if ($Rollback) {
|
|
$new = @($old | Where-Object { -not ($_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=")) })
|
|
} else {
|
|
$reg = Join-Path $BackupDir "$svc-pre-otlp-$stamp.reg"
|
|
& reg.exe export ($loc.Path -replace '^HKLM:\\','HKLM\' -replace '\\Parameters$','') $reg /y | Out-Null
|
|
if ($LASTEXITCODE -ne 0) { throw "reg export failed for $svc" }
|
|
Set-Content (Join-Path $BackupDir "$svc-env-names-pre-$stamp.txt") ($old | ForEach-Object { ($_ -split '=',2)[0] })
|
|
$new = @($old | Where-Object { -not ($_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=")) }) + @("$kExp=Otlp", "$kEnd=$Endpoint")
|
|
}
|
|
$kept = @($old | Where-Object { -not ($_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=")) })
|
|
try {
|
|
Write-Env $loc $new
|
|
$back = Read-Env $loc
|
|
$backHashes = @($back | ForEach-Object { Sha $_ })
|
|
foreach ($k in $kept) { if ($backHashes -notcontains (Sha $k)) { throw "$svc readback lost entry $(($k -split '=',2)[0])" } }
|
|
if ($back.Count -ne $new.Count) { throw "$svc readback count $($back.Count) != $($new.Count)" }
|
|
$pep2 = $back | Where-Object { $_ -match '^[^=]*Pepper=' } | ForEach-Object { ($_ -split '=',2)[0] + ':' + (Sha (($_ -split '=',2)[1])) }
|
|
if (($pep -join ',') -ne ($pep2 -join ',')) { throw "$svc pepper changed" }
|
|
$added = $back | Where-Object { $_.StartsWith("$kExp=") -or $_.StartsWith("$kEnd=") }
|
|
Say "$svc env after: $($back.Count) entries ($($kept.Count) pre-existing hash-identical), pepper [$($pep2 -join ',')], telemetry: $($added -join ' ')"
|
|
Restart-App $svc $port
|
|
} catch {
|
|
Say "FAIL $svc : $($_.Exception.Message) -- restoring the original environment"
|
|
Write-Env $loc $old
|
|
try { Restart-App $svc $port } catch { Say "ROLLBACK RESTART FAILED $svc : $($_.Exception.Message)" }
|
|
exit 1
|
|
}
|
|
}
|
|
Say 'all done'
|