71b8595742
ZbOtelCollector (otelcol-contrib 0.162.0, NSSM, NT SERVICE\ZbOtelCollector, every listener on 127.0.0.1, file + Prometheus exporters, size-capped rotation) on DEV vd03 and QA vq03/vq04. OtOpcUa, ScadaBridge Central + Site, MxAccessGateway and HistorianGateway export OTLP to http://127.0.0.1:4317 via two service-env entries each; rolling restarts, standby-first on the QA pairs. Overview has no exporter binding and is unchanged. Config, install/switch/verify scripts, runbook and the dated rollout record under infra/otel-collector/wonder/; evidence doc, GAPS, index row and deployment.md updated. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
66 lines
4.8 KiB
PowerShell
66 lines
4.8 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Read-only verification of ZbOtelCollector on this box (scadaproj#54): health, listeners, what each app sent.
|
|
.PARAMETER SinceMinutes Only count export requests whose newest timestamp is within this window.
|
|
.PARAMETER AppPorts App HTTP ports whose /metrics must still answer 200 (Prometheus stays served under OTLP).
|
|
#>
|
|
param([int]$SinceMinutes = 15, [int[]]$AppPorts = @(), [int]$Tail = 20)
|
|
$ErrorActionPreference = 'Continue'
|
|
$root = 'E:\ApiInstall\OtelCollector'
|
|
function Say($m) { "[{0:HH:mm:ss}] {1}" -f (Get-Date), $m }
|
|
$s = Get-CimInstance Win32_Service -Filter "Name='ZbOtelCollector'"
|
|
Say "service $($s.State) $($s.StartMode) $($s.StartName)"
|
|
try { Say ("health_check " + (Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 'http://127.0.0.1:13133/').StatusCode) } catch { Say "health_check FAIL $($_.Exception.Message)" }
|
|
$child = Get-CimInstance Win32_Process -Filter "ParentProcessId=$($s.ProcessId)" | Where-Object Name -eq 'otelcol-contrib.exe'
|
|
if ($child) { Get-NetTCPConnection -State Listen -OwningProcess $child.ProcessId | Sort-Object LocalPort | ForEach-Object { " listen $($_.LocalAddress):$($_.LocalPort)" }
|
|
" collector WS MB " + [int]((Get-Process -Id $child.ProcessId).WorkingSet64/1MB) }
|
|
Get-ChildItem "$root\data" | ForEach-Object { " file {0} {1:N1} MB {2:HH:mm:ss}" -f $_.Name, ($_.Length/1MB), $_.LastWriteTime }
|
|
" E: free GB " + [int]((Get-PSDrive E).Free/1GB)
|
|
|
|
$cut = [DateTimeOffset]::UtcNow.AddMinutes(-$SinceMinutes).ToUnixTimeMilliseconds() * 1000000
|
|
function Attr($res, $k) { ($res.attributes | Where-Object key -eq $k | Select-Object -First 1).value.stringValue }
|
|
foreach ($sig in 'metrics','traces','logs') {
|
|
$f = "$root\data\$sig.jsonl"
|
|
if (-not (Test-Path $f)) { Say "${sig}: no file"; continue }
|
|
$agg = @{}
|
|
foreach ($line in (Get-Content $f -Tail $Tail)) {
|
|
try { $o = $line | ConvertFrom-Json } catch { continue }
|
|
$rs = if ($sig -eq 'metrics') { $o.resourceMetrics } elseif ($sig -eq 'traces') { $o.resourceSpans } else { $o.resourceLogs }
|
|
foreach ($r in $rs) {
|
|
$key = "{0} | {1} | inst={2} | env={3} | site={4} role={5}" -f (Attr $r.resource 'service.name'), (Attr $r.resource 'host.name'), (Attr $r.resource 'service.instance.id'), (Attr $r.resource 'deployment.environment.name'), (Attr $r.resource 'site.id'), (Attr $r.resource 'node.role')
|
|
if (-not $agg[$key]) { $agg[$key] = @{ n = 0; names = @{}; newest = 0 } }
|
|
$a = $agg[$key]
|
|
if ($sig -eq 'metrics') {
|
|
foreach ($sm in $r.scopeMetrics) { foreach ($m in $sm.metrics) { $a.n++; $a.names[$m.name] = 1
|
|
foreach ($dp in @($m.sum.dataPoints) + @($m.gauge.dataPoints) + @($m.histogram.dataPoints)) { if ($dp -and [long]$dp.timeUnixNano -gt $a.newest) { $a.newest = [long]$dp.timeUnixNano } } } }
|
|
} elseif ($sig -eq 'traces') {
|
|
foreach ($ss in $r.scopeSpans) { foreach ($sp in $ss.spans) { $a.n++; $a.names[$sp.name] = 1; if ([long]$sp.endTimeUnixNano -gt $a.newest) { $a.newest = [long]$sp.endTimeUnixNano } } }
|
|
} else {
|
|
foreach ($sl in $r.scopeLogs) { foreach ($lr in $sl.logRecords) { $a.n++; if ([long]$lr.timeUnixNano -gt $a.newest) { $a.newest = [long]$lr.timeUnixNano } } }
|
|
}
|
|
}
|
|
}
|
|
Say "$sig (last $Tail export requests):"
|
|
foreach ($k in ($agg.Keys | Sort-Object)) {
|
|
$a = $agg[$k]
|
|
$newest = if ($a.newest) { [DateTimeOffset]::FromUnixTimeMilliseconds([long]($a.newest / 1000000)).UtcDateTime.ToString('HH:mm:ssZ') } else { '-' }
|
|
$fresh = if ($a.newest -ge $cut) { 'FRESH' } else { 'old' }
|
|
" $k | items=$($a.n) distinct=$($a.names.Count) newest=$newest $fresh"
|
|
if ($sig -eq 'traces') { " spans: " + (($a.names.Keys | Sort-Object | Select-Object -First 8) -join '; ') }
|
|
}
|
|
}
|
|
try {
|
|
$p = (Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 'http://127.0.0.1:8889/metrics').Content -split "`n"
|
|
$jobs = $p | Where-Object { $_ -notmatch '^#' -and $_ -match 'job="([^"]+)"' } | ForEach-Object { if ($_ -match 'job="([^"]+)"') { $Matches[1] } } | Group-Object | Sort-Object Name
|
|
Say "prometheus :8889 series per job:"; $jobs | ForEach-Object { " $($_.Name) $($_.Count)" }
|
|
} catch { Say "prometheus :8889 FAIL $($_.Exception.Message)" }
|
|
try {
|
|
$q = (Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 'http://127.0.0.1:8888/metrics').Content -split "`n"
|
|
Say "collector self-metrics:"
|
|
$q | Where-Object { $_ -match '^otelcol_(receiver_accepted|receiver_refused|exporter_sent|exporter_send_failed)_\w+' } | ForEach-Object { " " + ($_ -replace 'service_instance_id="[^"]*",?','' -replace 'service_name="[^"]*",?','' -replace 'service_version="[^"]*",?','') }
|
|
} catch { Say "self-metrics :8888 FAIL $($_.Exception.Message)" }
|
|
foreach ($port in $AppPorts) {
|
|
$c = 'FAIL'; foreach ($h in 'localhost', $env:COMPUTERNAME) { try { $c = (Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 "http://${h}:$port/metrics").StatusCode; break } catch {} }
|
|
" app :$port /metrics $c"
|
|
}
|