dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge feat/site-node-health: serve health on site nodes + shared active-node check (Health 0.3.0)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-24 13:35:27 -04:00
parent c1216044a4 1c9159a7ce
commit b3dc17a5fe
14 changed files with 613 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Host/Health/ActiveNodeGate.cs
+2 -1
View File
@@ -14,7 +14,8 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <see cref="ClusterActivityEvaluator.SelfIsOldest(Cluster, string?)"/> — NOT
/// the cluster leader, whose address-ordered definition diverges from singleton
/// placement after a restart (review 01 [High]). This is the same evaluator
/// backing <see cref="OldestNodeActiveHealthCheck"/>, so
/// backing the shared <c>ActiveNodeHealthCheck</c> registered on the active tier
/// — both route to <c>ClusterActiveNode</c> — so
/// <see cref="InboundApiEndpointFilter"/> can return HTTP 503 on a standby.
///
/// Registered only in the Central-role branch of <c>Program.cs</c>. The gate
src/ZB.MOM.WW.ScadaBridge.Host/Health/ClusterActivityEvaluator.cs
+7 -10
View File
@@ -1,4 +1,5 @@
using Akka.Cluster;
using ZB.MOM.WW.Health.Akka;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
@@ -27,14 +28,10 @@ public static class ClusterActivityEvaluator
/// <param name="cluster">The Akka cluster to evaluate.</param>
/// <param name="role">Optional role scope; when set, only members with this role are considered.</param>
/// <returns>The oldest Up member in the role scope, or null when none is Up.</returns>
public static Member? OldestUpMember(Cluster cluster, string? role = null)
{
Member? oldest = null;
foreach (var m in cluster.State.Members.Where(m => m.Status == MemberStatus.Up))
{
if (role != null && !m.HasRole(role)) continue;
if (oldest == null || m.IsOlderThan(oldest)) oldest = m;
}
return oldest;
}
/// <remarks>
/// Delegates to the shared <see cref="ClusterActiveNode.OldestUpMember(Cluster, string?)"/>, so
/// the label a node shows and the answer <see cref="SelfIsOldest"/> gates on come from one rule.
/// </remarks>
public static Member? OldestUpMember(Cluster cluster, string? role = null) =>
ClusterActiveNode.OldestUpMember(cluster, role);
}
src/ZB.MOM.WW.ScadaBridge.Host/Health/OldestNodeActiveHealthCheck.cs
-34
View File
@@ -1,34 +0,0 @@
using Akka.Actor;
using Microsoft.Extensions.Diagnostics.HealthChecks;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <summary>
/// /health/active check: Healthy only on the oldest Up member — the node that
/// hosts the cluster singletons. Replaces the shared package's leader-based
/// ActiveNodeHealthCheck (review 01 [High]): leadership is address-ordered and
/// diverges from singleton placement after a restart, and during a partition
/// BOTH nodes compute themselves leader, so Traefik served both. Oldest-member
/// semantics keep exactly one active node through a partition (the non-oldest
/// side still sees the old oldest member until SBR resolves membership).
/// </summary>
public sealed class OldestNodeActiveHealthCheck : IHealthCheck
{
private readonly ActorSystem _system;
/// <summary>Initializes a new <see cref="OldestNodeActiveHealthCheck"/> bound to the running actor system.</summary>
/// <param name="system">The live cluster actor system.</param>
public OldestNodeActiveHealthCheck(ActorSystem system) => _system = system;
/// <summary>Reports Healthy only when this node is the oldest Up cluster member (the singleton host); otherwise reports Unhealthy.</summary>
/// <param name="context">The health check context (unused; the result depends only on cluster membership).</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the health check result.</returns>
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken ct = default)
{
var cluster = Akka.Cluster.Cluster.Get(_system);
return Task.FromResult(ClusterActivityEvaluator.SelfIsOldest(cluster)
? HealthCheckResult.Healthy("oldest Up member (singleton host)")
: HealthCheckResult.Unhealthy("not the oldest Up member (standby)"));
}
}
src/ZB.MOM.WW.ScadaBridge.Host/Health/SiteLocalDbHealthCheck.cs
+77
View File
@@ -0,0 +1,77 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <summary>
/// /health/ready check for the consolidated site database (LocalDb). Site nodes have no EF
/// context — central's <c>DatabaseHealthCheck&lt;ScadaBridgeDbContext&gt;</c> is central-only — so
/// the site store is probed directly through the registered <see cref="ILocalDb"/> with a trivial
/// <c>SELECT 1</c>. That is the honest readiness question for a site node: everything it persists
/// (OperationTracking, site events, store-and-forward, the replicated config tables) lives here.
/// </summary>
/// <remarks>
/// Replication state is reported as <c>data</c> ENRICHMENT ONLY and never affects the status.
/// Replication is default-OFF by design, so a node with no peer reports <c>connected=false</c> with
/// a real 0 backlog — the healthy steady state of an unreplicated node — and failing the check on
/// it would mark every correctly-configured single node unready.
/// </remarks>
public sealed class SiteLocalDbHealthCheck : IHealthCheck
{
private readonly ILocalDb _localDb;
private readonly ISyncStatus? _syncStatus;
/// <summary>Initializes a new <see cref="SiteLocalDbHealthCheck"/>.</summary>
/// <param name="localDb">The consolidated site database registered by <c>AddZbLocalDb</c>.</param>
/// <param name="syncStatus">
/// The replication status view, when the replication engine is registered. Optional so the check
/// still works in a composition that wires storage without replication.
/// </param>
public SiteLocalDbHealthCheck(ILocalDb localDb, ISyncStatus? syncStatus = null)
{
_localDb = localDb ?? throw new ArgumentNullException(nameof(localDb));
_syncStatus = syncStatus;
}
/// <summary>Probes the site database and reports its reachability.</summary>
/// <param name="context">The health check context (unused; the verdict depends only on the probe).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Healthy when the probe query succeeds; Unhealthy with the fault otherwise.</returns>
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
try
{
await _localDb.QueryAsync("SELECT 1;", static r => r.GetInt32(0), ct: cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
return HealthCheckResult.Unhealthy("Site LocalDb is not reachable.", ex);
}
return HealthCheckResult.Healthy("Site LocalDb reachable.", BuildData());
}
private IReadOnlyDictionary<string, object> BuildData()
{
var data = new Dictionary<string, object>(StringComparer.Ordinal);
if (_syncStatus is null)
return data;
data["replicationConnected"] = _syncStatus.Connected;
// Nullable on purpose upstream: null means "backlog unknown" (no provider wired or the poll
// failed), which must not be flattened to 0 — that would render a pair which cannot read its
// own oplog as perfectly caught up. Omit the key instead.
if (_syncStatus.OplogBacklog is { } backlog)
data["replicationOplogBacklog"] = backlog;
if (_syncStatus.PeerNodeId is { } peer)
data["replicationPeerNodeId"] = peer;
return data;
}
}
src/ZB.MOM.WW.ScadaBridge.Host/Health/SitePairActiveNodeHealthCheck.cs
+39
View File
@@ -0,0 +1,39 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <summary>
/// /health/active check for a SITE pair: Healthy only on the pair's primary — the oldest Up member
/// carrying the <c>site-{SiteId}</c> role. Same active/standby semantics central publishes, so a
/// consumer reads one contract fleet-wide (200 = Active, 503 = Standby).
/// </summary>
/// <remarks>
/// Deliberately NOT central's unscoped shared <c>ActiveNodeHealthCheck</c>: that one calls
/// <c>ClusterActivityEvaluator.SelfIsOldest(cluster)</c> with no role argument, so on a mesh that
/// carries more than one site's members it computes "oldest" across the wrong member set and a site
/// node could report Active because it happens to be the oldest member overall. Delegating to
/// <see cref="IClusterNodeProvider.SelfIsPrimary"/> reuses the role-scoped evaluation the site
/// already trusts for singleton placement and event-log purge — one primary definition per site,
/// not two that can disagree.
/// </remarks>
public sealed class SitePairActiveNodeHealthCheck : IHealthCheck
{
private readonly IClusterNodeProvider _nodeProvider;
/// <summary>Initializes a new <see cref="SitePairActiveNodeHealthCheck"/>.</summary>
/// <param name="nodeProvider">The role-scoped site cluster node provider.</param>
public SitePairActiveNodeHealthCheck(IClusterNodeProvider nodeProvider) =>
_nodeProvider = nodeProvider ?? throw new ArgumentNullException(nameof(nodeProvider));
/// <summary>Reports Healthy on the site pair's primary and Unhealthy on the standby.</summary>
/// <param name="context">The health check context (unused; the verdict depends only on cluster membership).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Healthy on the primary; Unhealthy (503 = Standby) otherwise.</returns>
public Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default) =>
Task.FromResult(_nodeProvider.SelfIsPrimary
? HealthCheckResult.Healthy("site pair primary (oldest Up member of the site role)")
: HealthCheckResult.Unhealthy("not the site pair primary (standby)"));
}
src/ZB.MOM.WW.ScadaBridge.Host/Program.cs
+27 -9
View File
@@ -344,9 +344,10 @@ try
// Ready tier (/health/ready): database + akka-cluster + required-singletons.
// Active tier (/health/active): active-node = oldest Up member (singleton host) AND
// database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
// rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
// NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
// placement after a restart and both sides claim leadership during a partition).
// rotation. The active-node check is the shared ActiveNodeHealthCheck, which selects by age
// as of Health 0.3.0; through 0.2.1 it selected by leadership, which diverges from singleton
// placement after a restart and names both sides during a partition, so this host kept a
// private copy until the shared one was fixed.
// The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
// the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
builder.Services.AddHealthChecks()
@@ -371,9 +372,12 @@ try
"required-singletons",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
// Resolves ActorSystem from DI, like the akka-cluster check above.
.AddTypeActivatedCheck<OldestNodeActiveHealthCheck>(
// Oldest-Up-member check (review 01 [High]), unscoped: every central member competes for
// the one active slot. Since ZB.MOM.WW.Health 0.3.0 this IS the shared check — it selects
// by age, so the host-local OldestNodeActiveHealthCheck that existed only to avoid the old
// leader-based selection is gone. Resolves ActorSystem from DI, like the akka-cluster
// check above.
.AddTypeActivatedCheck<ActiveNodeHealthCheck>(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
@@ -413,9 +417,9 @@ try
// consults IActiveNodeGate and defaults to "allow" when none is registered,
// which leaves the design's "central cluster only (active node)" guarantee
// unenforced in deployed binaries). The gate is backed by the same oldest-member
// evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
// inbound API and the /health/active endpoint Traefik routes against agree on
// which node is active.
// evaluator (ClusterActivityEvaluator, which delegates to the shared ClusterActiveNode) as
// the active-node check above, so the inbound API and the /health/active endpoint Traefik
// routes against agree on which node is active.
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.InboundAPI.IActiveNodeGate, ActiveNodeGate>();
// Admin-triggered manual failover of the central pair (Health page control,
@@ -648,6 +652,20 @@ try
// must be present before the Map* calls on the site role.
app.UseRouting();
// Map the canonical three-tier health endpoints on the site node:
// /health/ready — akka-cluster (site-pair membership, and the cluster-view data the
// family overview dashboard reads) + localdb (the consolidated site
// store — the only database a site node has).
// /health/active — active-node = this site pair's primary; 200 on the primary, 503 on
// the standby, the same contract central publishes.
// /healthz — bare process liveness.
// Checks are registered in SiteServiceRegistration.Configure. All three endpoints are
// anonymous and use the canonical ZbHealthWriter JSON; the site pipeline runs no
// authentication middleware and has no FallbackPolicy, so nothing extra is needed to keep
// them reachable. They are served on the HTTP/1.1 listener (metricsPort, default :8084)
// alongside /metrics — the gRPC listener is HTTP/2-only and unusable by a plain HTTP client.
app.MapZbHealth();
// Observability — mount the always-on Prometheus /metrics scrape endpoint.
// AddZbTelemetry (in SiteServiceRegistration.Configure → BindSharedOptions)
// wires the OTel Resource + standard instrumentation + Prometheus exporter;
src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs
+35
View File
@@ -1,5 +1,7 @@
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.ScadaBridge.AuditLog;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication;
@@ -203,6 +205,39 @@ public static class SiteServiceRegistration
return () => nodeProvider.SelfIsPrimary;
});
// Health checks — the shared ZB.MOM.WW.Health probes, mapped by MapZbHealth on the site's
// HTTP/1.1 listener (Program.cs). Site nodes served no health endpoints before this; the
// family overview dashboard probes every node the same way, so a site node has to answer
// the same two questions central does.
//
// Ready (/health/ready) akka-cluster — this node's site-pair membership. It also carries
// the cluster-view data (leader/memberCount/...) that the
// dashboard reads, free with ZB.MOM.WW.Health 0.2.0.
// localdb — the consolidated site store, which is the only
// database a site node has (central's EF DatabaseHealthCheck is
// central-only and does not apply here).
// Active (/health/active) active-node — site-pair primary vs standby.
//
// Registered HERE rather than in Program.cs, like the LocalDb registrations above, so the
// composition-root tests that build this graph actually cover them. The Akka check resolves
// ActorSystem from DI per probe via the singleton bridge registered above.
services.AddHealthChecks()
.AddTypeActivatedCheck<AkkaClusterHealthCheck>(
"akka-cluster",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready },
args: AkkaClusterStatusPolicy.Default)
.AddTypeActivatedCheck<SiteLocalDbHealthCheck>(
"localdb",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// NOT central's unscoped shared ActiveNodeHealthCheck — that one is role-less; see
// SitePairActiveNodeHealthCheck for why a site pair needs the role-scoped answer.
.AddTypeActivatedCheck<SitePairActiveNodeHealthCheck>(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
// Options binding
BindSharedOptions(services, config);