diff --git a/Directory.Packages.props b/Directory.Packages.props
index 71e1ff93..9ef663e2 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -90,9 +90,9 @@
to mark tests as Skipped (not silently Passed) when MSSQL is unreachable.
-->
-
-
-
+
+
+
diff --git a/src/ZB.MOM.WW.ScadaBridge.Communication/ClusterState/ActiveNodeEvaluator.cs b/src/ZB.MOM.WW.ScadaBridge.Communication/ClusterState/ActiveNodeEvaluator.cs
index 43ba93a7..58058ba8 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Communication/ClusterState/ActiveNodeEvaluator.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.Communication/ClusterState/ActiveNodeEvaluator.cs
@@ -1,4 +1,5 @@
using Akka.Cluster;
+using ZB.MOM.WW.Health.Akka;
namespace ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
@@ -32,17 +33,13 @@ public static class ActiveNodeEvaluator
/// The Akka cluster to evaluate.
/// Optional role scope; when set, only members with this role are considered.
/// true when self is Up and the oldest Up member in the role scope.
- public static bool SelfIsOldestUp(Cluster cluster, string? role = null)
- {
- var self = cluster.SelfMember;
- if (self.Status != MemberStatus.Up)
- return false;
- if (role != null && !self.HasRole(role))
- return false;
-
- return cluster.State.Members
- .Where(m => m.Status == MemberStatus.Up)
- .Where(m => role == null || m.HasRole(role))
- .All(m => m.UniqueAddress.Equals(self.UniqueAddress) || self.IsOlderThan(m));
- }
+ ///
+ /// Delegates to the shared
+ /// (ZB.MOM.WW.Health.Akka 0.3.0), which is this evaluator promoted into the shared library
+ /// after OtOpcUa independently needed the same rule and wrote its own third copy. The rule now has
+ /// one implementation family-wide; this method survives as Communication's entry point into it, so
+ /// the delivery gate and heartbeat keep their existing call sites and layering.
+ ///
+ public static bool SelfIsOldestUp(Cluster cluster, string? role = null) =>
+ ClusterActiveNode.SelfIsActive(cluster, role);
}
diff --git a/src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj b/src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj
index ccc4bf18..329f237f 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj
+++ b/src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj
@@ -28,6 +28,13 @@
+
+
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Health/ActiveNodeGate.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Health/ActiveNodeGate.cs
index db95aedf..efe6e911 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Host/Health/ActiveNodeGate.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.Host/Health/ActiveNodeGate.cs
@@ -14,7 +14,8 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// — NOT
/// the cluster leader, whose address-ordered definition diverges from singleton
/// placement after a restart (review 01 [High]). This is the same evaluator
-/// backing , so
+/// backing the shared ActiveNodeHealthCheck registered on the active tier
+/// — both route to ClusterActiveNode — so
/// can return HTTP 503 on a standby.
///
/// Registered only in the Central-role branch of Program.cs. The gate
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Health/ClusterActivityEvaluator.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Health/ClusterActivityEvaluator.cs
index b4729cb5..926049d0 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Host/Health/ClusterActivityEvaluator.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.Host/Health/ClusterActivityEvaluator.cs
@@ -1,4 +1,5 @@
using Akka.Cluster;
+using ZB.MOM.WW.Health.Akka;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
@@ -27,14 +28,10 @@ public static class ClusterActivityEvaluator
/// The Akka cluster to evaluate.
/// Optional role scope; when set, only members with this role are considered.
/// The oldest Up member in the role scope, or null when none is Up.
- public static Member? OldestUpMember(Cluster cluster, string? role = null)
- {
- Member? oldest = null;
- foreach (var m in cluster.State.Members.Where(m => m.Status == MemberStatus.Up))
- {
- if (role != null && !m.HasRole(role)) continue;
- if (oldest == null || m.IsOlderThan(oldest)) oldest = m;
- }
- return oldest;
- }
+ ///
+ /// Delegates to the shared , so
+ /// the label a node shows and the answer gates on come from one rule.
+ ///
+ public static Member? OldestUpMember(Cluster cluster, string? role = null) =>
+ ClusterActiveNode.OldestUpMember(cluster, role);
}
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Health/OldestNodeActiveHealthCheck.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Health/OldestNodeActiveHealthCheck.cs
deleted file mode 100644
index c20a1f81..00000000
--- a/src/ZB.MOM.WW.ScadaBridge.Host/Health/OldestNodeActiveHealthCheck.cs
+++ /dev/null
@@ -1,34 +0,0 @@
-using Akka.Actor;
-using Microsoft.Extensions.Diagnostics.HealthChecks;
-
-namespace ZB.MOM.WW.ScadaBridge.Host.Health;
-
-///
-/// /health/active check: Healthy only on the oldest Up member — the node that
-/// hosts the cluster singletons. Replaces the shared package's leader-based
-/// ActiveNodeHealthCheck (review 01 [High]): leadership is address-ordered and
-/// diverges from singleton placement after a restart, and during a partition
-/// BOTH nodes compute themselves leader, so Traefik served both. Oldest-member
-/// semantics keep exactly one active node through a partition (the non-oldest
-/// side still sees the old oldest member until SBR resolves membership).
-///
-public sealed class OldestNodeActiveHealthCheck : IHealthCheck
-{
- private readonly ActorSystem _system;
-
- /// Initializes a new bound to the running actor system.
- /// The live cluster actor system.
- public OldestNodeActiveHealthCheck(ActorSystem system) => _system = system;
-
- /// Reports Healthy only when this node is the oldest Up cluster member (the singleton host); otherwise reports Unhealthy.
- /// The health check context (unused; the result depends only on cluster membership).
- /// Cancellation token.
- /// A task that resolves to the health check result.
- public Task CheckHealthAsync(HealthCheckContext context, CancellationToken ct = default)
- {
- var cluster = Akka.Cluster.Cluster.Get(_system);
- return Task.FromResult(ClusterActivityEvaluator.SelfIsOldest(cluster)
- ? HealthCheckResult.Healthy("oldest Up member (singleton host)")
- : HealthCheckResult.Unhealthy("not the oldest Up member (standby)"));
- }
-}
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Health/SiteLocalDbHealthCheck.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Health/SiteLocalDbHealthCheck.cs
new file mode 100644
index 00000000..fb472298
--- /dev/null
+++ b/src/ZB.MOM.WW.ScadaBridge.Host/Health/SiteLocalDbHealthCheck.cs
@@ -0,0 +1,77 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.LocalDb;
+using ZB.MOM.WW.LocalDb.Replication;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Health;
+
+///
+/// /health/ready check for the consolidated site database (LocalDb). Site nodes have no EF
+/// context — central's DatabaseHealthCheck<ScadaBridgeDbContext> is central-only — so
+/// the site store is probed directly through the registered with a trivial
+/// SELECT 1. That is the honest readiness question for a site node: everything it persists
+/// (OperationTracking, site events, store-and-forward, the replicated config tables) lives here.
+///
+///
+/// Replication state is reported as data ENRICHMENT ONLY and never affects the status.
+/// Replication is default-OFF by design, so a node with no peer reports connected=false with
+/// a real 0 backlog — the healthy steady state of an unreplicated node — and failing the check on
+/// it would mark every correctly-configured single node unready.
+///
+public sealed class SiteLocalDbHealthCheck : IHealthCheck
+{
+ private readonly ILocalDb _localDb;
+ private readonly ISyncStatus? _syncStatus;
+
+ /// Initializes a new .
+ /// The consolidated site database registered by AddZbLocalDb.
+ ///
+ /// The replication status view, when the replication engine is registered. Optional so the check
+ /// still works in a composition that wires storage without replication.
+ ///
+ public SiteLocalDbHealthCheck(ILocalDb localDb, ISyncStatus? syncStatus = null)
+ {
+ _localDb = localDb ?? throw new ArgumentNullException(nameof(localDb));
+ _syncStatus = syncStatus;
+ }
+
+ /// Probes the site database and reports its reachability.
+ /// The health check context (unused; the verdict depends only on the probe).
+ /// Cancellation token.
+ /// Healthy when the probe query succeeds; Unhealthy with the fault otherwise.
+ public async Task CheckHealthAsync(
+ HealthCheckContext context,
+ CancellationToken cancellationToken = default)
+ {
+ try
+ {
+ await _localDb.QueryAsync("SELECT 1;", static r => r.GetInt32(0), ct: cancellationToken)
+ .ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ return HealthCheckResult.Unhealthy("Site LocalDb is not reachable.", ex);
+ }
+
+ return HealthCheckResult.Healthy("Site LocalDb reachable.", BuildData());
+ }
+
+ private IReadOnlyDictionary BuildData()
+ {
+ var data = new Dictionary(StringComparer.Ordinal);
+ if (_syncStatus is null)
+ return data;
+
+ data["replicationConnected"] = _syncStatus.Connected;
+
+ // Nullable on purpose upstream: null means "backlog unknown" (no provider wired or the poll
+ // failed), which must not be flattened to 0 — that would render a pair which cannot read its
+ // own oplog as perfectly caught up. Omit the key instead.
+ if (_syncStatus.OplogBacklog is { } backlog)
+ data["replicationOplogBacklog"] = backlog;
+
+ if (_syncStatus.PeerNodeId is { } peer)
+ data["replicationPeerNodeId"] = peer;
+
+ return data;
+ }
+}
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Health/SitePairActiveNodeHealthCheck.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Health/SitePairActiveNodeHealthCheck.cs
new file mode 100644
index 00000000..1ed049ea
--- /dev/null
+++ b/src/ZB.MOM.WW.ScadaBridge.Host/Health/SitePairActiveNodeHealthCheck.cs
@@ -0,0 +1,39 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Health;
+
+///
+/// /health/active check for a SITE pair: Healthy only on the pair's primary — the oldest Up member
+/// carrying the site-{SiteId} role. Same active/standby semantics central publishes, so a
+/// consumer reads one contract fleet-wide (200 = Active, 503 = Standby).
+///
+///
+/// Deliberately NOT central's unscoped shared ActiveNodeHealthCheck: that one calls
+/// ClusterActivityEvaluator.SelfIsOldest(cluster) with no role argument, so on a mesh that
+/// carries more than one site's members it computes "oldest" across the wrong member set and a site
+/// node could report Active because it happens to be the oldest member overall. Delegating to
+/// reuses the role-scoped evaluation the site
+/// already trusts for singleton placement and event-log purge — one primary definition per site,
+/// not two that can disagree.
+///
+public sealed class SitePairActiveNodeHealthCheck : IHealthCheck
+{
+ private readonly IClusterNodeProvider _nodeProvider;
+
+ /// Initializes a new .
+ /// The role-scoped site cluster node provider.
+ public SitePairActiveNodeHealthCheck(IClusterNodeProvider nodeProvider) =>
+ _nodeProvider = nodeProvider ?? throw new ArgumentNullException(nameof(nodeProvider));
+
+ /// Reports Healthy on the site pair's primary and Unhealthy on the standby.
+ /// The health check context (unused; the verdict depends only on cluster membership).
+ /// Cancellation token.
+ /// Healthy on the primary; Unhealthy (503 = Standby) otherwise.
+ public Task CheckHealthAsync(
+ HealthCheckContext context,
+ CancellationToken cancellationToken = default) =>
+ Task.FromResult(_nodeProvider.SelfIsPrimary
+ ? HealthCheckResult.Healthy("site pair primary (oldest Up member of the site role)")
+ : HealthCheckResult.Unhealthy("not the site pair primary (standby)"));
+}
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs
index 02b30671..e6752c06 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs
@@ -344,9 +344,10 @@ try
// Ready tier (/health/ready): database + akka-cluster + required-singletons.
// Active tier (/health/active): active-node = oldest Up member (singleton host) AND
// database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
- // rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
- // NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
- // placement after a restart and both sides claim leadership during a partition).
+ // rotation. The active-node check is the shared ActiveNodeHealthCheck, which selects by age
+ // as of Health 0.3.0; through 0.2.1 it selected by leadership, which diverges from singleton
+ // placement after a restart and names both sides during a partition, so this host kept a
+ // private copy until the shared one was fixed.
// The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
// the DatabaseHealthCheck resolves a scoped ScadaBridgeDbContext (no factory).
builder.Services.AddHealthChecks()
@@ -371,9 +372,12 @@ try
"required-singletons",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
- // Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
- // Resolves ActorSystem from DI, like the akka-cluster check above.
- .AddTypeActivatedCheck(
+ // Oldest-Up-member check (review 01 [High]), unscoped: every central member competes for
+ // the one active slot. Since ZB.MOM.WW.Health 0.3.0 this IS the shared check — it selects
+ // by age, so the host-local OldestNodeActiveHealthCheck that existed only to avoid the old
+ // leader-based selection is gone. Resolves ActorSystem from DI, like the akka-cluster
+ // check above.
+ .AddTypeActivatedCheck(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
@@ -413,9 +417,9 @@ try
// consults IActiveNodeGate and defaults to "allow" when none is registered,
// which leaves the design's "central cluster only (active node)" guarantee
// unenforced in deployed binaries). The gate is backed by the same oldest-member
- // evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
- // inbound API and the /health/active endpoint Traefik routes against agree on
- // which node is active.
+ // evaluator (ClusterActivityEvaluator, which delegates to the shared ClusterActiveNode) as
+ // the active-node check above, so the inbound API and the /health/active endpoint Traefik
+ // routes against agree on which node is active.
builder.Services.AddSingleton();
// Admin-triggered manual failover of the central pair (Health page control,
@@ -648,6 +652,20 @@ try
// must be present before the Map* calls on the site role.
app.UseRouting();
+ // Map the canonical three-tier health endpoints on the site node:
+ // /health/ready — akka-cluster (site-pair membership, and the cluster-view data the
+ // family overview dashboard reads) + localdb (the consolidated site
+ // store — the only database a site node has).
+ // /health/active — active-node = this site pair's primary; 200 on the primary, 503 on
+ // the standby, the same contract central publishes.
+ // /healthz — bare process liveness.
+ // Checks are registered in SiteServiceRegistration.Configure. All three endpoints are
+ // anonymous and use the canonical ZbHealthWriter JSON; the site pipeline runs no
+ // authentication middleware and has no FallbackPolicy, so nothing extra is needed to keep
+ // them reachable. They are served on the HTTP/1.1 listener (metricsPort, default :8084)
+ // alongside /metrics — the gRPC listener is HTTP/2-only and unusable by a plain HTTP client.
+ app.MapZbHealth();
+
// Observability — mount the always-on Prometheus /metrics scrape endpoint.
// AddZbTelemetry (in SiteServiceRegistration.Configure → BindSharedOptions)
// wires the OTel Resource + standard instrumentation + Prometheus exporter;
diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs b/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs
index 9734962a..c4ff637e 100644
--- a/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs
@@ -1,5 +1,7 @@
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
+using ZB.MOM.WW.Health;
+using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.ScadaBridge.AuditLog;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication;
@@ -203,6 +205,39 @@ public static class SiteServiceRegistration
return () => nodeProvider.SelfIsPrimary;
});
+ // Health checks — the shared ZB.MOM.WW.Health probes, mapped by MapZbHealth on the site's
+ // HTTP/1.1 listener (Program.cs). Site nodes served no health endpoints before this; the
+ // family overview dashboard probes every node the same way, so a site node has to answer
+ // the same two questions central does.
+ //
+ // Ready (/health/ready) akka-cluster — this node's site-pair membership. It also carries
+ // the cluster-view data (leader/memberCount/...) that the
+ // dashboard reads, free with ZB.MOM.WW.Health 0.2.0.
+ // localdb — the consolidated site store, which is the only
+ // database a site node has (central's EF DatabaseHealthCheck is
+ // central-only and does not apply here).
+ // Active (/health/active) active-node — site-pair primary vs standby.
+ //
+ // Registered HERE rather than in Program.cs, like the LocalDb registrations above, so the
+ // composition-root tests that build this graph actually cover them. The Akka check resolves
+ // ActorSystem from DI per probe via the singleton bridge registered above.
+ services.AddHealthChecks()
+ .AddTypeActivatedCheck(
+ "akka-cluster",
+ failureStatus: null,
+ tags: new[] { ZbHealthTags.Ready },
+ args: AkkaClusterStatusPolicy.Default)
+ .AddTypeActivatedCheck(
+ "localdb",
+ failureStatus: null,
+ tags: new[] { ZbHealthTags.Ready })
+ // NOT central's unscoped shared ActiveNodeHealthCheck — that one is role-less; see
+ // SitePairActiveNodeHealthCheck for why a site pair needs the role-scoped answer.
+ .AddTypeActivatedCheck(
+ "active-node",
+ failureStatus: null,
+ tags: new[] { ZbHealthTags.Active });
+
// Options binding
BindSharedOptions(services, config);
diff --git a/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/MonitoringAuthorizationTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/MonitoringAuthorizationTests.cs
index 8f4356fe..ce42b247 100644
--- a/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/MonitoringAuthorizationTests.cs
+++ b/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/MonitoringAuthorizationTests.cs
@@ -42,7 +42,9 @@ public class MonitoringAuthorizationTests
public void HealthDashboard_IsIntentionallyAllAuthenticatedRoles()
{
// Health Dashboard stays all-roles (no policy) per the design doc.
- var attr = AuthorizeOf();
+ // Fully qualified: the Communication.Health namespace otherwise makes
+ // the bare `Health` page type ambiguous.
+ var attr = AuthorizeOf();
Assert.NotNull(attr);
Assert.Null(attr!.Policy);
diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HealthCheckTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HealthCheckTests.cs
index c730038e..b73843f7 100644
--- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HealthCheckTests.cs
+++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HealthCheckTests.cs
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
+using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.ScadaBridge.Host.Health;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
@@ -207,11 +208,13 @@ public class HealthCheckTests : IDisposable
}
[Fact]
- public void ActiveNodeCheck_IsHostLocalOldestNodeCheck()
+ public void ActiveNodeCheck_IsTheSharedOldestUpMemberCheck()
{
- // The shared package's ActiveNodeHealthCheck is LEADER-based (review 01 [High]):
- // during a partition both nodes think they lead and Traefik serves both. The
- // active-node check must be the host-local oldest-member check instead.
+ // The property under test is that the active tier selects by AGE, not by leadership: during a
+ // partition both sides think they lead and Traefik would serve both (review 01 [High]).
+ // Through ZB.MOM.WW.Health 0.2.1 the shared check was leader-based, so this host carried a
+ // private OldestNodeActiveHealthCheck; 0.3.0 made the shared check age-based, so the private
+ // copy is gone and the shared type is now the correct answer here.
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
try
{
@@ -222,7 +225,7 @@ public class HealthCheckTests : IDisposable
Assert.Contains(ZbHealthTags.Active, active.Tags);
var check = active.Factory(factory.Services);
- Assert.IsType(check);
+ Assert.IsType(check);
}
finally
{
diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthCheckTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthCheckTests.cs
new file mode 100644
index 00000000..aa6ac71b
--- /dev/null
+++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthCheckTests.cs
@@ -0,0 +1,239 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.Health;
+using ZB.MOM.WW.Health.Akka;
+using ZB.MOM.WW.LocalDb;
+using ZB.MOM.WW.LocalDb.Registration;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.Host.Health;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
+
+///
+/// Site-node health endpoints. Site nodes previously served no health surface at all — only gRPC on
+/// the HTTP/2 listener and /metrics on the HTTP/1.1 one — so the family overview dashboard
+/// had no uniform way to probe them. These tests pin the three checks the site now registers and
+/// the tier each one lands in.
+///
+/// The registration assertions build the REAL container from
+/// rather than a hand-assembled
+/// ServiceCollection, for the reason spelled out in :
+/// this family's wiring defects have consistently been ones that only a container-built graph
+/// catches. Each registration is additionally ACTIVATED through its factory, so a check that is
+/// registered but cannot resolve its dependencies fails here instead of at the first live probe.
+///
+///
+public class SiteHealthCheckTests : IDisposable
+{
+ private readonly WebApplication _host;
+ private readonly string _tempDbPath;
+ private readonly string _tempSiteDbPath;
+ private readonly string _tempTrackingDbPath;
+
+ public SiteHealthCheckTests()
+ {
+ var stamp = Guid.NewGuid();
+ _tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_localdb_{stamp}.db");
+ _tempSiteDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_site_{stamp}.db");
+ _tempTrackingDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_track_{stamp}.db");
+
+ var builder = WebApplication.CreateBuilder();
+ builder.Configuration.Sources.Clear();
+ builder.Configuration.AddInMemoryCollection(new Dictionary
+ {
+ ["ScadaBridge:Node:Role"] = "Site",
+ ["ScadaBridge:Node:NodeName"] = "node-a",
+ ["ScadaBridge:Node:NodeHostname"] = "test-site",
+ ["ScadaBridge:Node:SiteId"] = "TestSite",
+ ["ScadaBridge:Node:RemotingPort"] = "0",
+ ["ScadaBridge:Node:GrpcPort"] = "0",
+ ["ScadaBridge:Database:SiteDbPath"] = _tempSiteDbPath,
+ ["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={_tempTrackingDbPath}",
+ ["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
+ ["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
+ ["LocalDb:Path"] = _tempDbPath,
+ });
+
+ builder.Services.AddGrpc();
+ builder.Services.AddSingleton();
+
+ SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
+
+ AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);
+
+ _host = builder.Build();
+ }
+
+ public void Dispose()
+ {
+ (_host as IDisposable)?.Dispose();
+ foreach (var path in new[] { _tempDbPath, _tempSiteDbPath, _tempTrackingDbPath })
+ {
+ try { File.Delete(path); } catch { /* best effort */ }
+ }
+
+ GC.SuppressFinalize(this);
+ }
+
+ private IReadOnlyList Registrations =>
+ _host.Services.GetRequiredService>().Value.Registrations.ToList();
+
+ [Fact]
+ public void Site_RegistersExactlyTheThreeSiteChecks()
+ {
+ // Exact-set, not "contains": an extra check silently changes what /health/ready means for
+ // every consumer, and a site node inheriting a central-only probe is precisely the mistake
+ // this asserts against.
+ Assert.Equal(
+ new[] { "active-node", "akka-cluster", "localdb" },
+ Registrations.Select(r => r.Name).OrderBy(n => n, StringComparer.Ordinal).ToArray());
+ }
+
+ [Theory]
+ [InlineData("akka-cluster", ZbHealthTags.Ready)]
+ [InlineData("localdb", ZbHealthTags.Ready)]
+ [InlineData("active-node", ZbHealthTags.Active)]
+ public void Site_ChecksCarryTheirIntendedTier(string name, string expectedTag)
+ {
+ var registration = Assert.Single(Registrations, r => r.Name == name);
+
+ // Exactly one tier tag each: MapZbHealth selects by tag, so a check tagged both Ready and
+ // Active would put standby-ness into readiness and drop the standby out of readiness too.
+ Assert.Equal(new[] { expectedTag }, registration.Tags.ToArray());
+ }
+
+ [Theory]
+ [InlineData("akka-cluster", typeof(AkkaClusterHealthCheck))]
+ [InlineData("localdb", typeof(SiteLocalDbHealthCheck))]
+ [InlineData("active-node", typeof(SitePairActiveNodeHealthCheck))]
+ public void Site_ChecksActivateToTheIntendedType(string name, Type expected)
+ {
+ var registration = Assert.Single(Registrations, r => r.Name == name);
+
+ // Running the factory is the point: a type-activated check whose constructor dependencies
+ // are not registered on site would throw here rather than on the first probe in production.
+ var check = registration.Factory(_host.Services);
+
+ Assert.IsType(expected, check);
+ }
+
+ [Fact]
+ public void Site_ActiveNodeCheck_IsRoleScoped_NotTheUnscopedCentralCheck()
+ {
+ // Central registers the shared ActiveNodeHealthCheck UNSCOPED — oldest Up member of the whole
+ // cluster. On a mesh carrying more than one site's members that computes "oldest" across the
+ // wrong member set, so a site node could report Active purely for being the oldest member
+ // overall. A site must answer for its own site-{SiteId} role, which SitePairActiveNodeHealthCheck
+ // does by delegating to the site's IClusterNodeProvider (itself now backed by the shared
+ // ClusterActiveNode, so the rule is shared even though the scoping is local).
+ var check = Assert.Single(Registrations, r => r.Name == "active-node").Factory(_host.Services);
+
+ Assert.IsType(check);
+ Assert.IsNotType(check);
+ }
+
+ [Theory]
+ [InlineData("database")]
+ [InlineData("required-singletons")]
+ public void Site_DoesNotRegisterCentralOnlyChecks(string centralOnlyName)
+ {
+ // Positive control first: if the harness ever stopped registering health checks at all,
+ // every "is absent" assertion below would pass vacuously and this test would be worthless.
+ Assert.Contains(Registrations, r => r.Name == "akka-cluster");
+
+ Assert.DoesNotContain(Registrations, r => r.Name == centralOnlyName);
+ }
+
+ [Fact]
+ public async Task Site_LocalDbCheck_ReportsHealthyAgainstTheRealSiteDatabase()
+ {
+ // Behaviour, not wiring: the probe runs against the consolidated site database this
+ // composition actually built, so a check that resolves but cannot query fails here.
+ var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb")
+ .Factory(_host.Services);
+
+ var result = await check.CheckHealthAsync(NewContext("localdb", check));
+
+ Assert.Equal(HealthStatus.Healthy, result.Status);
+ }
+
+ [Fact]
+ public async Task Site_LocalDbCheck_EnrichesWithReplicationStateWithoutFailingOnDefaultOff()
+ {
+ // Replication ships default-OFF, so the unreplicated steady state must stay Healthy while
+ // still surfacing the state as data — enrichment must never become a verdict.
+ var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb")
+ .Factory(_host.Services);
+
+ var result = await check.CheckHealthAsync(NewContext("localdb", check));
+
+ Assert.Equal(HealthStatus.Healthy, result.Status);
+ Assert.False(Assert.IsType(result.Data["replicationConnected"]));
+ Assert.Equal(0L, Assert.IsType(result.Data["replicationOplogBacklog"]));
+ }
+
+ [Fact]
+ public async Task Site_LocalDbCheck_ReportsUnhealthyWhenTheStoreIsUnreachable()
+ {
+ var check = new SiteLocalDbHealthCheck(new ThrowingLocalDb());
+
+ var result = await check.CheckHealthAsync(NewContext("localdb", check));
+
+ Assert.Equal(HealthStatus.Unhealthy, result.Status);
+ }
+
+ [Theory]
+ [InlineData(true, HealthStatus.Healthy)]
+ [InlineData(false, HealthStatus.Unhealthy)]
+ public async Task Site_ActiveNodeCheck_SplitsPrimaryFromStandby(bool selfIsPrimary, HealthStatus expected)
+ {
+ // 200 on the primary / 503 on the standby is the same contract central publishes, so a
+ // consumer reads one active-node meaning fleet-wide.
+ var check = new SitePairActiveNodeHealthCheck(new StubNodeProvider(selfIsPrimary));
+
+ var result = await check.CheckHealthAsync(NewContext("active-node", check));
+
+ Assert.Equal(expected, result.Status);
+ }
+
+ private static HealthCheckContext NewContext(string name, IHealthCheck check) => new()
+ {
+ Registration = new HealthCheckRegistration(name, check, HealthStatus.Unhealthy, tags: null),
+ };
+
+ private sealed class StubNodeProvider(bool selfIsPrimary) : IClusterNodeProvider
+ {
+ public bool SelfIsPrimary { get; } = selfIsPrimary;
+
+ public IReadOnlyList GetClusterNodes() => [];
+ }
+
+ private sealed class ThrowingLocalDb : ILocalDb
+ {
+ public Task ExecuteAsync(string sql, object? parameters = null, CancellationToken ct = default) =>
+ throw new InvalidOperationException("store unreachable");
+
+ public Task> QueryAsync(
+ string sql,
+ Func map,
+ object? parameters = null,
+ CancellationToken ct = default) =>
+ throw new InvalidOperationException("store unreachable");
+
+ public Task BeginTransactionAsync(CancellationToken ct = default) =>
+ throw new InvalidOperationException("store unreachable");
+
+ public Microsoft.Data.Sqlite.SqliteConnection CreateConnection() =>
+ throw new InvalidOperationException("store unreachable");
+
+ public ReplicatedTable RegisterReplicated(string tableName) =>
+ throw new InvalidOperationException("store unreachable");
+
+ public IReadOnlyDictionary ReplicatedTables =>
+ throw new InvalidOperationException("store unreachable");
+ }
+}
diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthEndpointTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthEndpointTests.cs
new file mode 100644
index 00000000..5be47b9e
--- /dev/null
+++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthEndpointTests.cs
@@ -0,0 +1,156 @@
+using System.Net;
+using System.Text.Json;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
+
+///
+/// Proves the site pipeline actually MAPS the health endpoints — the half
+/// cannot cover, since that fixture builds the site container
+/// from and never runs Program's endpoint wiring.
+/// Without this, deleting app.MapZbHealth() from the site branch would leave every
+/// registration test green while site nodes served 404 to the overview dashboard.
+///
+/// Boots the real Program in the Site role. Program builds its own
+/// ConfigurationBuilder before the web host exists, so its role and settings come from
+/// PROCESS-WIDE environment variables (SCADABRIDGE_CONFIG selects appsettings.Site.json)
+/// rather than from WithWebHostBuilder — which is why this fixture joins
+/// and restores every var it sets.
+///
+///
+[Collection(HostBootCollection.Name)]
+public class SiteHealthEndpointTests : IDisposable
+{
+ private readonly List _disposables = new();
+ private readonly Dictionary _previousEnv = new(StringComparer.Ordinal);
+ private readonly string _tempDbPath;
+
+ public SiteHealthEndpointTests()
+ {
+ _tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_ep_{Guid.NewGuid()}.db");
+
+ // Whole-key env overrides, the sanctioned path: supplying GrpcPsk concretely makes the
+ // pre-host secrets expander skip appsettings.Site.json's ${secret:SB-GRPC-PSK-site-1}
+ // token, so this test needs no seeded secrets store.
+ //
+ // The remoting port is moved off the site default (8082) so this fixture cannot collide
+ // with a locally running node, but it must be a REAL port and seed-nodes[0] must be this
+ // node itself — StartupValidator enforces both before the host is built, and it runs
+ // whether or not the Akka hosted service is later removed. Nothing binds it: the hosted
+ // service is removed below.
+ SetEnv("SCADABRIDGE_CONFIG", "Site");
+ SetEnv("ScadaBridge__Node__Role", "Site");
+ SetEnv("ScadaBridge__Node__SiteId", "TestSite");
+ SetEnv("ScadaBridge__Node__NodeHostname", "localhost");
+ SetEnv("ScadaBridge__Node__RemotingPort", "18082");
+ SetEnv("ScadaBridge__Cluster__SeedNodes__0", "akka.tcp://scadabridge@localhost:18082");
+ SetEnv("ScadaBridge__Cluster__SeedNodes__1", "akka.tcp://scadabridge@localhost:18085");
+ SetEnv("ScadaBridge__Communication__GrpcPsk", "test-psk-0123456789");
+ SetEnv("LocalDb__Path", _tempDbPath);
+ }
+
+ private void SetEnv(string key, string? value)
+ {
+ _previousEnv[key] = Environment.GetEnvironmentVariable(key);
+ Environment.SetEnvironmentVariable(key, value);
+ }
+
+ public void Dispose()
+ {
+ foreach (var d in _disposables)
+ {
+ try { d.Dispose(); } catch { /* best effort */ }
+ }
+
+ foreach (var (key, value) in _previousEnv)
+ {
+ Environment.SetEnvironmentVariable(key, value);
+ }
+
+ try { File.Delete(_tempDbPath); } catch { /* best effort */ }
+
+ GC.SuppressFinalize(this);
+ }
+
+ private HttpClient CreateSiteClient()
+ {
+ var factory = new WebApplicationFactory()
+ .WithWebHostBuilder(builder =>
+ {
+ // WebApplicationFactory defaults the environment to Development, which turns on
+ // ValidateOnBuild/ValidateScopes. The site graph carries scoped registrations whose
+ // dependencies are central-only (ReconcileService → IDeploymentManagerRepository,
+ // AuditLogKpiSampleSource → IAuditLogRepository) and are never resolved on a site
+ // node, so eager validation fails on a composition that runs fine in production.
+ // That is a pre-existing property of the site container, not of the health wiring
+ // under test — run this fixture the way a site node actually runs.
+ builder.UseEnvironment("Production");
+
+ // ConfigureTestServices runs after the app's own registrations, so this drops the
+ // hosted service that would form a real cluster while leaving the AkkaHostedService
+ // singleton resolvable (the site pipeline resolves it for shutdown ordering).
+ builder.ConfigureTestServices(AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly);
+ });
+ _disposables.Add(factory);
+
+ var client = factory.CreateClient();
+ _disposables.Add(client);
+ return client;
+ }
+
+ [Fact]
+ public async Task Site_MapsHealthReady_WithTheCanonicalJsonBody()
+ {
+ var response = await CreateSiteClient().GetAsync("/health/ready");
+
+ // Never 404. 200 vs 503 depends on cluster state this fixture does not form, so both are
+ // accepted — what is asserted is that the tier is mapped and answers in canonical shape.
+ Assert.NotEqual(HttpStatusCode.NotFound, response.StatusCode);
+ Assert.True(
+ response.StatusCode is HttpStatusCode.OK or HttpStatusCode.ServiceUnavailable,
+ $"Expected 200 or 503, got {(int)response.StatusCode}");
+
+ using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
+ var entries = doc.RootElement.GetProperty("entries");
+
+ Assert.True(entries.TryGetProperty("akka-cluster", out _), "ready tier must carry akka-cluster");
+ Assert.True(entries.TryGetProperty("localdb", out _), "ready tier must carry localdb");
+ // Readiness must NOT depend on being the primary, or a healthy standby would read unready.
+ Assert.False(entries.TryGetProperty("active-node", out _), "active-node belongs to the Active tier");
+ }
+
+ [Fact]
+ public async Task Site_MapsHealthActive_CarryingOnlyTheActiveNodeCheck()
+ {
+ var response = await CreateSiteClient().GetAsync("/health/active");
+
+ Assert.NotEqual(HttpStatusCode.NotFound, response.StatusCode);
+
+ using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
+ var entries = doc.RootElement.GetProperty("entries");
+
+ Assert.Equal(
+ new[] { "active-node" },
+ entries.EnumerateObject().Select(p => p.Name).ToArray());
+ }
+
+ [Fact]
+ public async Task Site_HealthEndpointsAreAnonymous()
+ {
+ // The dashboard authenticates nowhere. The site pipeline runs no authentication middleware
+ // today, so this is a regression pin: adding one later must not silently close these.
+ var client = CreateSiteClient();
+
+ foreach (var path in new[] { "/health/ready", "/health/active", "/healthz" })
+ {
+ var response = await client.GetAsync(path);
+
+ Assert.NotEqual(HttpStatusCode.Unauthorized, response.StatusCode);
+ Assert.NotEqual(HttpStatusCode.Forbidden, response.StatusCode);
+ Assert.NotEqual(HttpStatusCode.NotFound, response.StatusCode);
+ }
+ }
+}