Merge feat/site-node-health: serve health on site nodes + shared active-node check (Health 0.3.0)

This commit is contained in:
Joseph Doherty
2026-07-24 13:35:27 -04:00
14 changed files with 613 additions and 76 deletions
+27 -9
View File
@@ -344,9 +344,10 @@ try
// Ready tier (/health/ready): database + akka-cluster + required-singletons.
// Active tier (/health/active): active-node = oldest Up member (singleton host) AND
// database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
// rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
// NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
// placement after a restart and both sides claim leadership during a partition).
// rotation. The active-node check is the shared ActiveNodeHealthCheck, which selects by age
// as of Health 0.3.0; through 0.2.1 it selected by leadership, which diverges from singleton
// placement after a restart and names both sides during a partition, so this host kept a
// private copy until the shared one was fixed.
// The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
// the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
builder.Services.AddHealthChecks()
@@ -371,9 +372,12 @@ try
"required-singletons",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
// Resolves ActorSystem from DI, like the akka-cluster check above.
.AddTypeActivatedCheck<OldestNodeActiveHealthCheck>(
// Oldest-Up-member check (review 01 [High]), unscoped: every central member competes for
// the one active slot. Since ZB.MOM.WW.Health 0.3.0 this IS the shared check — it selects
// by age, so the host-local OldestNodeActiveHealthCheck that existed only to avoid the old
// leader-based selection is gone. Resolves ActorSystem from DI, like the akka-cluster
// check above.
.AddTypeActivatedCheck<ActiveNodeHealthCheck>(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
@@ -413,9 +417,9 @@ try
// consults IActiveNodeGate and defaults to "allow" when none is registered,
// which leaves the design's "central cluster only (active node)" guarantee
// unenforced in deployed binaries). The gate is backed by the same oldest-member
// evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
// inbound API and the /health/active endpoint Traefik routes against agree on
// which node is active.
// evaluator (ClusterActivityEvaluator, which delegates to the shared ClusterActiveNode) as
// the active-node check above, so the inbound API and the /health/active endpoint Traefik
// routes against agree on which node is active.
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.InboundAPI.IActiveNodeGate, ActiveNodeGate>();
// Admin-triggered manual failover of the central pair (Health page control,
@@ -648,6 +652,20 @@ try
// must be present before the Map* calls on the site role.
app.UseRouting();
// Map the canonical three-tier health endpoints on the site node:
// /health/ready — akka-cluster (site-pair membership, and the cluster-view data the
// family overview dashboard reads) + localdb (the consolidated site
// store — the only database a site node has).
// /health/active — active-node = this site pair's primary; 200 on the primary, 503 on
// the standby, the same contract central publishes.
// /healthz — bare process liveness.
// Checks are registered in SiteServiceRegistration.Configure. All three endpoints are
// anonymous and use the canonical ZbHealthWriter JSON; the site pipeline runs no
// authentication middleware and has no FallbackPolicy, so nothing extra is needed to keep
// them reachable. They are served on the HTTP/1.1 listener (metricsPort, default :8084)
// alongside /metrics — the gRPC listener is HTTP/2-only and unusable by a plain HTTP client.
app.MapZbHealth();
// Observability — mount the always-on Prometheus /metrics scrape endpoint.
// AddZbTelemetry (in SiteServiceRegistration.Configure → BindSharedOptions)
// wires the OTel Resource + standard instrumentation + Prometheus exporter;