From 103f2f1931fc3868ecbe1ba91003c1d17cfb9708 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:53:26 -0400 Subject: [PATCH] feat(host+rig): give central a dedicated Prometheus scrape port (register row 56) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The row read 'central nodes expose no Prometheus endpoint'. That is not quite what was wrong, and the difference matters. Central already maps /metrics (Program.cs central branch) and telemetry is registered for both roles through the shared BindSharedOptions, so the endpoint rides endpoint routing and has always been served on central's web port (:5000 on the rig, anonymous — there is no FallbackPolicy). What central lacked was the site branch's dedicated MetricsPort, which is the port the established live-scrape recipe probes: docker exec bash -c 'exec 3<>/dev/tcp/localhost/8084 ...' So a probe of central on 8084 failed and central-only series (KPI/outbox counters, the bundle-import session gauge + its new rejection counter, alarm-cache gauges) read as unscrapeable while they were in fact being exported on a different port. Central now binds a dedicated HTTP/1.1 listener on ScadaBridge:Node:MetricsPort, so one probe recipe covers all eight rig nodes. HTTP/1.1-capable because a standard Prometheus scraper cannot negotiate h2c, which is why this cannot share the gRPC port. Read as a PRESENCE check, not GetValue's defaulted read. NodeOptions defaults MetricsPort to 8084, so a defaulted read would make every existing central node start binding a new port on upgrade — and a port already in use is a hard boot failure, not a degraded one. Central gains the listener only where an operator asked for it; site nodes are unchanged and still bind theirs unconditionally. The bind is also skipped when the port would collide with the gRPC port or an ASPNETCORE_URLS port, since Kestrel throws on a duplicate ListenAnyIP and /metrics is reachable on those ports anyway. Config only — no host port mapping added, because the probe runs inside the container. Applied to both docker/ centrals and both docker-env2/ centrals. FILE EDITS ONLY: no docker command was run, and deploy.sh was not invoked. --- .../central-node-a/appsettings.Central.json | 4 +- .../central-node-b/appsettings.Central.json | 4 +- docker/README.md | 39 +++++++++++++--- .../central-node-a/appsettings.Central.json | 4 +- .../central-node-b/appsettings.Central.json | 4 +- src/ZB.MOM.WW.ScadaBridge.Host/Program.cs | 44 +++++++++++++++++-- 6 files changed, 86 insertions(+), 13 deletions(-) diff --git a/docker-env2/central-node-a/appsettings.Central.json b/docker-env2/central-node-a/appsettings.Central.json index 02398abf..f4f6ba5e 100644 --- a/docker-env2/central-node-a/appsettings.Central.json +++ b/docker-env2/central-node-a/appsettings.Central.json @@ -4,7 +4,9 @@ "Role": "Central", "NodeName": "central-a", "NodeHostname": "scadabridge-env2-central-a", - "RemotingPort": 8081 + "RemotingPort": 8081, + "_metricsPort": "Register row 56 — mirrors docker/central-node-a. Dedicated HTTP/1.1 Prometheus scrape listener; central binds it only when this key is present. In-container port only, no host mapping.", + "MetricsPort": 8084 }, "Cluster": { "SeedNodes": [ diff --git a/docker-env2/central-node-b/appsettings.Central.json b/docker-env2/central-node-b/appsettings.Central.json index 41b93443..8850c3ca 100644 --- a/docker-env2/central-node-b/appsettings.Central.json +++ b/docker-env2/central-node-b/appsettings.Central.json @@ -4,7 +4,9 @@ "Role": "Central", "NodeName": "central-b", "NodeHostname": "scadabridge-env2-central-b", - "RemotingPort": 8081 + "RemotingPort": 8081, + "_metricsPort": "Register row 56 — mirrors docker/central-node-a. Dedicated HTTP/1.1 Prometheus scrape listener; central binds it only when this key is present. In-container port only, no host mapping.", + "MetricsPort": 8084 }, "Cluster": { "SeedNodes": [ diff --git a/docker/README.md b/docker/README.md index c59d0d8f..d2831de4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -181,12 +181,28 @@ Two traps it exists to avoid: Binding to them gives a `Connected` data connection with `Good` quality and *zero* event traffic, which looks healthy and probes nothing. Use the built-in simulated nodes (`SpikeData`, `RandomSignedInt32`, `StepUp`, `FastUInt1..5`, `SlowUInt1..5`). -- **`scadabridge_site_connection_up` does not mean what its name suggests.** It counts - open central→site gRPC *subscriber streams*, not OPC UA data connections, so it sits - at 0 on a perfectly healthy site with nobody watching. For data-connection health read - `dataConnectionEndpoints` / `dataConnectionTagQuality` off `scadabridge health site`. +- **`scadabridge_site_stream_subscribers_active` is not data-connection health.** It counts + open central→site gRPC *subscriber streams*, so it sits at 0 on a perfectly healthy site + with nobody watching. (Register row 54 renamed it from `scadabridge_site_connection_up`, + whose name invited exactly that misreading — it is what made row 48's "0 on all six site + nodes" look like a fault when it was healthy steady state.) -Verify (metrics are on container port 8084; the containers ship no curl/wget): +For real data-connection health, the same row added two honest gauges: + +| Series | Meaning | +| --- | --- | +| `scadabridge_site_data_connections_connected` | Connections currently in the `Connected` state | +| `scadabridge_site_data_connections_total` | Connections known to this node, in any state | + +`connected < total` is the alert condition — neither number alone separates "a device is +down" from "nothing is configured". Both read **0 on the standby node**, which is healthy: +connections are opened by the deployment path on the active node only, so alert on the +active node or on `max()` across the pair. The health report's `dataConnectionEndpoints` / +`dataConnectionTagQuality` (via `scadabridge health site`) remain the per-connection detail. + +Verify (metrics are on container port 8084 — **all eight nodes** as of register row 56, which +gave the two central nodes the same dedicated scrape listener the six site nodes already had; +the containers ship no curl/wget): ```bash scadabridge debug snapshot --id @@ -195,9 +211,20 @@ scadabridge debug stream --id # open a stream, then in another s docker exec scadabridge-site-a-a bash -c \ 'exec 3<>/dev/tcp/localhost/8084 && printf "GET /metrics HTTP/1.0\r\nHost: localhost\r\n\r\n" >&3 && cat <&3' \ - | grep -E 'site_connection_up|stream_batch_size' + | grep -E 'stream_subscribers_active|stream_batch_size|data_connections' + +# Central-only series (KPI/outbox counters, bundle-import session gauge + rejection +# counter, alarm-cache gauges) — same recipe, central container: +docker exec scadabridge-central-a bash -c \ + 'exec 3<>/dev/tcp/localhost/8084 && printf "GET /metrics HTTP/1.0\r\nHost: localhost\r\n\r\n" >&3 && cat <&3' \ + | grep -E 'import_sessions|alarm_cache' ``` +> Central binds 8084 only because `docker/central-node-*/appsettings.Central.json` now sets +> `ScadaBridge:Node:MetricsPort`. Unlike a site node, central reads that key **only when +> present** — a central deployment without it is unchanged and still serves `/metrics` on its +> web port, since the endpoint rides endpoint routing and is served on every bound listener. + ### After Code Changes Rebuild and redeploy. The Docker build cache skips NuGet restore when only source files change: diff --git a/docker/central-node-a/appsettings.Central.json b/docker/central-node-a/appsettings.Central.json index dea71fc3..0184c93e 100644 --- a/docker/central-node-a/appsettings.Central.json +++ b/docker/central-node-a/appsettings.Central.json @@ -4,7 +4,9 @@ "Role": "Central", "NodeName": "central-a", "NodeHostname": "scadabridge-central-a", - "RemotingPort": 8081 + "RemotingPort": 8081, + "_metricsPort": "Register row 56. Central maps /metrics on every listener it binds, so it was always reachable on the web port (:5000) — but not on 8084, which is the port the live-scrape recipe probes (docker exec ... bash -c ' { // The HTTP/1.1 (+ HTTP/2) surface from the configured URLs, re-declared so it @@ -217,6 +238,21 @@ try { listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2; }); + + // The Prometheus scrape listener (register row 56), when configured. Skipped + // when it would double-bind a port already declared above — Kestrel throws on a + // duplicate ListenAnyIP, and /metrics is reachable on that port either way since + // it rides endpoint routing. HTTP/1.1-capable on purpose: a standard Prometheus + // scraper cannot negotiate h2c, which is why this cannot share the gRPC port. + if (centralMetricsPort is { } cmp + && cmp != centralGrpcPort + && !httpPorts.Contains(cmp)) + { + options.ListenAnyIP(cmp, listenOptions => + { + listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http1AndHttp2; + }); + } }); // Shared components @@ -698,9 +734,11 @@ try builder.Configuration.AddConfiguration(configuration); // Captive-dependency validation (deferred register row 50) — the SITE half of the - // pair. This branch is the one that actually gains coverage: the docker rig sets - // ASPNETCORE_ENVIRONMENT=Development on the central containers but NOT on the site - // containers, so a site node has always built its provider unvalidated. + // pair. This branch is the one that actually gains coverage: the docker rig long + // set ASPNETCORE_ENVIRONMENT=Development on the central containers but NOT on the + // site containers, so a site node built its provider unvalidated. Register row 63 + // closed that — all six rig site containers (and docker-env2's two) now set it, so + // this validation actually runs there. builder.Host.UseDefaultServiceProvider(ServiceProviderValidation.Configure); // Serilog