dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(host): /health/active = oldest member + DB reachable — partition-safe Traefik routing

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-08 15:51:56 -04:00
parent da6d5ae12c
commit ff89e3145c
3 changed files with 105 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Host/Program.cs
+20 -13
View File
@@ -210,15 +210,19 @@ try
builder.Services.AddConfigurationDatabase(configDbConnectionString);
// Health checks for readiness gating — shared ZB.MOM.WW.Health probes.
// Check names and the ready/active tier split are preserved: database + akka-cluster
// carry the Ready tag (/health/ready), active-node carries the Active tag (/health/active).
// The Akka checks resolve ActorSystem from DI via the transient bridge registered below;
// Ready tier (/health/ready): database + akka-cluster + required-singletons.
// Active tier (/health/active): active-node = oldest Up member (singleton host) AND
// database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
// rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
// NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
// placement after a restart and both sides claim leadership during a partition).
// The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
// the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
builder.Services.AddHealthChecks()
.AddTypeActivatedCheck<DatabaseHealthCheck<ScadaBridgeDbContext>>(
"database",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })
.AddTypeActivatedCheck<AkkaClusterHealthCheck>(
"akka-cluster",
failureStatus: null,
@@ -236,7 +240,9 @@ try
"required-singletons",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
.AddTypeActivatedCheck<ActiveNodeHealthCheck>(
// Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
// Resolves ActorSystem from DI, like the akka-cluster check above.
.AddTypeActivatedCheck<OldestNodeActiveHealthCheck>(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
@@ -261,9 +267,9 @@ try
// standby-node gating is actually enforced (the InboundApiEndpointFilter
// consults IActiveNodeGate and defaults to "allow" when none is registered,
// which leaves the design's "central cluster only (active node)" guarantee
// unenforced in deployed binaries). The gate is backed by the same Akka
// cluster-leadership check as ActiveNodeHealthCheck above, so the inbound
// API and the /health/active endpoint Traefik routes against agree on
// unenforced in deployed binaries). The gate is backed by the same oldest-member
// evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
// inbound API and the /health/active endpoint Traefik routes against agree on
// which node is active.
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.InboundAPI.IActiveNodeGate, ActiveNodeGate>();
@@ -352,12 +358,13 @@ try
branch => branch.UseAuditWriteMiddleware());
// Map the canonical three-tier health endpoints in one call:
// /health/ready — Ready-tagged checks (database + akka-cluster). REQ-HOST-4a defines
// readiness as cluster membership + DB connectivity, explicitly NOT
// cluster leadership, so the leader-only active-node check is excluded
// /health/ready — Ready-tagged checks (database + akka-cluster + required-singletons).
// REQ-HOST-4a defines readiness as cluster membership + DB connectivity,
// explicitly NOT active-node status, so the active-node check is excluded
// (a fully operational standby central node still reports ready).
// /health/active — Active-tagged check (active-node); returns 200 only on the cluster
// leader; used by Traefik for routing.
// /health/active — Active-tagged checks (active-node = oldest Up member + database
// reachable); returns 200 only on the singleton-host node with a live
// DB; used by Traefik for routing.
// /healthz — bare process liveness; runs no checks (always 200 while the process
// is up). New tier added by adopting the shared library.
// All three are anonymous and use the canonical ZbHealthWriter JSON output.