dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge feat/site-node-health: serve health on site nodes + shared active-node check (Health 0.3.0)

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-24 13:35:27 -04:00
parent c1216044a4 1c9159a7ce
commit b3dc17a5fe
14 changed files with 613 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Directory.Packages.props
+3 -3
View File
@@ -90,9 +90,9 @@
to mark tests as Skipped (not silently Passed) when MSSQL is unreachable.
-->
<PackageVersion Include="Xunit.SkippableFact" Version="1.5.61" />
<PackageVersion Include="ZB.MOM.WW.Health" Version="0.1.0" />
<PackageVersion Include="ZB.MOM.WW.Health.Akka" Version="0.1.0" />
<PackageVersion Include="ZB.MOM.WW.Health.EntityFrameworkCore" Version="0.1.0" />
<PackageVersion Include="ZB.MOM.WW.Health" Version="0.3.0" />
<PackageVersion Include="ZB.MOM.WW.Health.Akka" Version="0.3.0" />
<PackageVersion Include="ZB.MOM.WW.Health.EntityFrameworkCore" Version="0.3.0" />
<PackageVersion Include="ZB.MOM.WW.Telemetry" Version="0.1.0" />
<PackageVersion Include="ZB.MOM.WW.Telemetry.Serilog" Version="0.1.0" />
<PackageVersion Include="ZB.MOM.WW.MxGateway.Client" Version="0.1.1" />
src/ZB.MOM.WW.ScadaBridge.Communication/ClusterState/ActiveNodeEvaluator.cs
+10 -13
View File
@@ -1,4 +1,5 @@
using Akka.Cluster;
using ZB.MOM.WW.Health.Akka;
namespace ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
@@ -32,17 +33,13 @@ public static class ActiveNodeEvaluator
/// <param name="cluster">The Akka cluster to evaluate.</param>
/// <param name="role">Optional role scope; when set, only members with this role are considered.</param>
/// <returns><c>true</c> when self is Up and the oldest Up member in the role scope.</returns>
public static bool SelfIsOldestUp(Cluster cluster, string? role = null)
{
var self = cluster.SelfMember;
if (self.Status != MemberStatus.Up)
return false;
if (role != null && !self.HasRole(role))
return false;
return cluster.State.Members
.Where(m => m.Status == MemberStatus.Up)
.Where(m => role == null || m.HasRole(role))
.All(m => m.UniqueAddress.Equals(self.UniqueAddress) || self.IsOlderThan(m));
}
/// <remarks>
/// Delegates to the shared <see cref="ClusterActiveNode.SelfIsActive"/>
/// (<c>ZB.MOM.WW.Health.Akka</c> 0.3.0), which is this evaluator promoted into the shared library
/// after OtOpcUa independently needed the same rule and wrote its own third copy. The rule now has
/// one implementation family-wide; this method survives as Communication's entry point into it, so
/// the delivery gate and heartbeat keep their existing call sites and layering.
/// </remarks>
public static bool SelfIsOldestUp(Cluster cluster, string? role = null) =>
ClusterActiveNode.SelfIsActive(cluster, role);
}
src/ZB.MOM.WW.ScadaBridge.Communication/ZB.MOM.WW.ScadaBridge.Communication.csproj
+7
View File
@@ -28,6 +28,13 @@
<PackageReference Include="Grpc.Net.Client" />
<PackageReference Include="Grpc.Tools" PrivateAssets="All" />
<PackageReference Include="ZB.MOM.WW.Configuration" />
<!-- For ClusterActiveNode — the shared "oldest Up member" primitive that ActiveNodeEvaluator was
promoted into. The store-and-forward delivery gate, the heartbeat IsActive stamp and the
/health/active tier must all name the SAME active node; delegating to one implementation is
what makes that structural. It lives in Health.Akka because that is the family's Akka-cluster
utility package (it already hosts AkkaClusterStatusPolicy and an endpoint gate); a dedicated
cluster package would be a better home if a third such primitive appears. -->
<PackageReference Include="ZB.MOM.WW.Health.Akka" />
</ItemGroup>
<ItemGroup>
src/ZB.MOM.WW.ScadaBridge.Host/Health/ActiveNodeGate.cs
+2 -1
View File
@@ -14,7 +14,8 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <see cref="ClusterActivityEvaluator.SelfIsOldest(Cluster, string?)"/> — NOT
/// the cluster leader, whose address-ordered definition diverges from singleton
/// placement after a restart (review 01 [High]). This is the same evaluator
/// backing <see cref="OldestNodeActiveHealthCheck"/>, so
/// backing the shared <c>ActiveNodeHealthCheck</c> registered on the active tier
/// — both route to <c>ClusterActiveNode</c> — so
/// <see cref="InboundApiEndpointFilter"/> can return HTTP 503 on a standby.
///
/// Registered only in the Central-role branch of <c>Program.cs</c>. The gate
src/ZB.MOM.WW.ScadaBridge.Host/Health/ClusterActivityEvaluator.cs
+7 -10
View File
@@ -1,4 +1,5 @@
using Akka.Cluster;
using ZB.MOM.WW.Health.Akka;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
@@ -27,14 +28,10 @@ public static class ClusterActivityEvaluator
/// <param name="cluster">The Akka cluster to evaluate.</param>
/// <param name="role">Optional role scope; when set, only members with this role are considered.</param>
/// <returns>The oldest Up member in the role scope, or null when none is Up.</returns>
public static Member? OldestUpMember(Cluster cluster, string? role = null)
{
Member? oldest = null;
foreach (var m in cluster.State.Members.Where(m => m.Status == MemberStatus.Up))
{
if (role != null && !m.HasRole(role)) continue;
if (oldest == null || m.IsOlderThan(oldest)) oldest = m;
}
return oldest;
}
/// <remarks>
/// Delegates to the shared <see cref="ClusterActiveNode.OldestUpMember(Cluster, string?)"/>, so
/// the label a node shows and the answer <see cref="SelfIsOldest"/> gates on come from one rule.
/// </remarks>
public static Member? OldestUpMember(Cluster cluster, string? role = null) =>
ClusterActiveNode.OldestUpMember(cluster, role);
}
src/ZB.MOM.WW.ScadaBridge.Host/Health/OldestNodeActiveHealthCheck.cs
-34
View File
@@ -1,34 +0,0 @@
using Akka.Actor;
using Microsoft.Extensions.Diagnostics.HealthChecks;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <summary>
/// /health/active check: Healthy only on the oldest Up member — the node that
/// hosts the cluster singletons. Replaces the shared package's leader-based
/// ActiveNodeHealthCheck (review 01 [High]): leadership is address-ordered and
/// diverges from singleton placement after a restart, and during a partition
/// BOTH nodes compute themselves leader, so Traefik served both. Oldest-member
/// semantics keep exactly one active node through a partition (the non-oldest
/// side still sees the old oldest member until SBR resolves membership).
/// </summary>
public sealed class OldestNodeActiveHealthCheck : IHealthCheck
{
private readonly ActorSystem _system;
/// <summary>Initializes a new <see cref="OldestNodeActiveHealthCheck"/> bound to the running actor system.</summary>
/// <param name="system">The live cluster actor system.</param>
public OldestNodeActiveHealthCheck(ActorSystem system) => _system = system;
/// <summary>Reports Healthy only when this node is the oldest Up cluster member (the singleton host); otherwise reports Unhealthy.</summary>
/// <param name="context">The health check context (unused; the result depends only on cluster membership).</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the health check result.</returns>
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken ct = default)
{
var cluster = Akka.Cluster.Cluster.Get(_system);
return Task.FromResult(ClusterActivityEvaluator.SelfIsOldest(cluster)
? HealthCheckResult.Healthy("oldest Up member (singleton host)")
: HealthCheckResult.Unhealthy("not the oldest Up member (standby)"));
}
}
src/ZB.MOM.WW.ScadaBridge.Host/Health/SiteLocalDbHealthCheck.cs
+77
View File
@@ -0,0 +1,77 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <summary>
/// /health/ready check for the consolidated site database (LocalDb). Site nodes have no EF
/// context — central's <c>DatabaseHealthCheck&lt;ScadaBridgeDbContext&gt;</c> is central-only — so
/// the site store is probed directly through the registered <see cref="ILocalDb"/> with a trivial
/// <c>SELECT 1</c>. That is the honest readiness question for a site node: everything it persists
/// (OperationTracking, site events, store-and-forward, the replicated config tables) lives here.
/// </summary>
/// <remarks>
/// Replication state is reported as <c>data</c> ENRICHMENT ONLY and never affects the status.
/// Replication is default-OFF by design, so a node with no peer reports <c>connected=false</c> with
/// a real 0 backlog — the healthy steady state of an unreplicated node — and failing the check on
/// it would mark every correctly-configured single node unready.
/// </remarks>
public sealed class SiteLocalDbHealthCheck : IHealthCheck
{
private readonly ILocalDb _localDb;
private readonly ISyncStatus? _syncStatus;
/// <summary>Initializes a new <see cref="SiteLocalDbHealthCheck"/>.</summary>
/// <param name="localDb">The consolidated site database registered by <c>AddZbLocalDb</c>.</param>
/// <param name="syncStatus">
/// The replication status view, when the replication engine is registered. Optional so the check
/// still works in a composition that wires storage without replication.
/// </param>
public SiteLocalDbHealthCheck(ILocalDb localDb, ISyncStatus? syncStatus = null)
{
_localDb = localDb ?? throw new ArgumentNullException(nameof(localDb));
_syncStatus = syncStatus;
}
/// <summary>Probes the site database and reports its reachability.</summary>
/// <param name="context">The health check context (unused; the verdict depends only on the probe).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Healthy when the probe query succeeds; Unhealthy with the fault otherwise.</returns>
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
try
{
await _localDb.QueryAsync("SELECT 1;", static r => r.GetInt32(0), ct: cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
return HealthCheckResult.Unhealthy("Site LocalDb is not reachable.", ex);
}
return HealthCheckResult.Healthy("Site LocalDb reachable.", BuildData());
}
private IReadOnlyDictionary<string, object> BuildData()
{
var data = new Dictionary<string, object>(StringComparer.Ordinal);
if (_syncStatus is null)
return data;
data["replicationConnected"] = _syncStatus.Connected;
// Nullable on purpose upstream: null means "backlog unknown" (no provider wired or the poll
// failed), which must not be flattened to 0 — that would render a pair which cannot read its
// own oplog as perfectly caught up. Omit the key instead.
if (_syncStatus.OplogBacklog is { } backlog)
data["replicationOplogBacklog"] = backlog;
if (_syncStatus.PeerNodeId is { } peer)
data["replicationPeerNodeId"] = peer;
return data;
}
}
src/ZB.MOM.WW.ScadaBridge.Host/Health/SitePairActiveNodeHealthCheck.cs
+39
View File
@@ -0,0 +1,39 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <summary>
/// /health/active check for a SITE pair: Healthy only on the pair's primary — the oldest Up member
/// carrying the <c>site-{SiteId}</c> role. Same active/standby semantics central publishes, so a
/// consumer reads one contract fleet-wide (200 = Active, 503 = Standby).
/// </summary>
/// <remarks>
/// Deliberately NOT central's unscoped shared <c>ActiveNodeHealthCheck</c>: that one calls
/// <c>ClusterActivityEvaluator.SelfIsOldest(cluster)</c> with no role argument, so on a mesh that
/// carries more than one site's members it computes "oldest" across the wrong member set and a site
/// node could report Active because it happens to be the oldest member overall. Delegating to
/// <see cref="IClusterNodeProvider.SelfIsPrimary"/> reuses the role-scoped evaluation the site
/// already trusts for singleton placement and event-log purge — one primary definition per site,
/// not two that can disagree.
/// </remarks>
public sealed class SitePairActiveNodeHealthCheck : IHealthCheck
{
private readonly IClusterNodeProvider _nodeProvider;
/// <summary>Initializes a new <see cref="SitePairActiveNodeHealthCheck"/>.</summary>
/// <param name="nodeProvider">The role-scoped site cluster node provider.</param>
public SitePairActiveNodeHealthCheck(IClusterNodeProvider nodeProvider) =>
_nodeProvider = nodeProvider ?? throw new ArgumentNullException(nameof(nodeProvider));
/// <summary>Reports Healthy on the site pair's primary and Unhealthy on the standby.</summary>
/// <param name="context">The health check context (unused; the verdict depends only on cluster membership).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Healthy on the primary; Unhealthy (503 = Standby) otherwise.</returns>
public Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default) =>
Task.FromResult(_nodeProvider.SelfIsPrimary
? HealthCheckResult.Healthy("site pair primary (oldest Up member of the site role)")
: HealthCheckResult.Unhealthy("not the site pair primary (standby)"));
}
src/ZB.MOM.WW.ScadaBridge.Host/Program.cs
+27 -9
View File
@@ -344,9 +344,10 @@ try
// Ready tier (/health/ready): database + akka-cluster + required-singletons.
// Active tier (/health/active): active-node = oldest Up member (singleton host) AND
// database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
// rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
// NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
// placement after a restart and both sides claim leadership during a partition).
// rotation. The active-node check is the shared ActiveNodeHealthCheck, which selects by age
// as of Health 0.3.0; through 0.2.1 it selected by leadership, which diverges from singleton
// placement after a restart and names both sides during a partition, so this host kept a
// private copy until the shared one was fixed.
// The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
// the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
builder.Services.AddHealthChecks()
@@ -371,9 +372,12 @@ try
"required-singletons",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
// Resolves ActorSystem from DI, like the akka-cluster check above.
.AddTypeActivatedCheck<OldestNodeActiveHealthCheck>(
// Oldest-Up-member check (review 01 [High]), unscoped: every central member competes for
// the one active slot. Since ZB.MOM.WW.Health 0.3.0 this IS the shared check — it selects
// by age, so the host-local OldestNodeActiveHealthCheck that existed only to avoid the old
// leader-based selection is gone. Resolves ActorSystem from DI, like the akka-cluster
// check above.
.AddTypeActivatedCheck<ActiveNodeHealthCheck>(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
@@ -413,9 +417,9 @@ try
// consults IActiveNodeGate and defaults to "allow" when none is registered,
// which leaves the design's "central cluster only (active node)" guarantee
// unenforced in deployed binaries). The gate is backed by the same oldest-member
// evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
// inbound API and the /health/active endpoint Traefik routes against agree on
// which node is active.
// evaluator (ClusterActivityEvaluator, which delegates to the shared ClusterActiveNode) as
// the active-node check above, so the inbound API and the /health/active endpoint Traefik
// routes against agree on which node is active.
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.InboundAPI.IActiveNodeGate, ActiveNodeGate>();
// Admin-triggered manual failover of the central pair (Health page control,
@@ -648,6 +652,20 @@ try
// must be present before the Map* calls on the site role.
app.UseRouting();
// Map the canonical three-tier health endpoints on the site node:
// /health/ready — akka-cluster (site-pair membership, and the cluster-view data the
// family overview dashboard reads) + localdb (the consolidated site
// store — the only database a site node has).
// /health/active — active-node = this site pair's primary; 200 on the primary, 503 on
// the standby, the same contract central publishes.
// /healthz — bare process liveness.
// Checks are registered in SiteServiceRegistration.Configure. All three endpoints are
// anonymous and use the canonical ZbHealthWriter JSON; the site pipeline runs no
// authentication middleware and has no FallbackPolicy, so nothing extra is needed to keep
// them reachable. They are served on the HTTP/1.1 listener (metricsPort, default :8084)
// alongside /metrics — the gRPC listener is HTTP/2-only and unusable by a plain HTTP client.
app.MapZbHealth();
// Observability — mount the always-on Prometheus /metrics scrape endpoint.
// AddZbTelemetry (in SiteServiceRegistration.Configure → BindSharedOptions)
// wires the OTel Resource + standard instrumentation + Prometheus exporter;
src/ZB.MOM.WW.ScadaBridge.Host/SiteServiceRegistration.cs
+35
View File
@@ -1,5 +1,7 @@
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.ScadaBridge.AuditLog;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication;
@@ -203,6 +205,39 @@ public static class SiteServiceRegistration
return () => nodeProvider.SelfIsPrimary;
});
// Health checks — the shared ZB.MOM.WW.Health probes, mapped by MapZbHealth on the site's
// HTTP/1.1 listener (Program.cs). Site nodes served no health endpoints before this; the
// family overview dashboard probes every node the same way, so a site node has to answer
// the same two questions central does.
//
// Ready (/health/ready) akka-cluster — this node's site-pair membership. It also carries
// the cluster-view data (leader/memberCount/...) that the
// dashboard reads, free with ZB.MOM.WW.Health 0.2.0.
// localdb — the consolidated site store, which is the only
// database a site node has (central's EF DatabaseHealthCheck is
// central-only and does not apply here).
// Active (/health/active) active-node — site-pair primary vs standby.
//
// Registered HERE rather than in Program.cs, like the LocalDb registrations above, so the
// composition-root tests that build this graph actually cover them. The Akka check resolves
// ActorSystem from DI per probe via the singleton bridge registered above.
services.AddHealthChecks()
.AddTypeActivatedCheck<AkkaClusterHealthCheck>(
"akka-cluster",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready },
args: AkkaClusterStatusPolicy.Default)
.AddTypeActivatedCheck<SiteLocalDbHealthCheck>(
"localdb",
failureStatus: null,
tags: new[] { ZbHealthTags.Ready })
// NOT central's unscoped shared ActiveNodeHealthCheck — that one is role-less; see
// SitePairActiveNodeHealthCheck for why a site pair needs the role-scoped answer.
.AddTypeActivatedCheck<SitePairActiveNodeHealthCheck>(
"active-node",
failureStatus: null,
tags: new[] { ZbHealthTags.Active });
// Options binding
BindSharedOptions(services, config);
tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Monitoring/MonitoringAuthorizationTests.cs
+3 -1
View File
@@ -42,7 +42,9 @@ public class MonitoringAuthorizationTests
public void HealthDashboard_IsIntentionallyAllAuthenticatedRoles()
{
// Health Dashboard stays all-roles (no policy) per the design doc.
var attr = AuthorizeOf<Health>();
// Fully qualified: the Communication.Health namespace otherwise makes
// the bare `Health` page type ambiguous.
var attr = AuthorizeOf<CentralUI.Components.Pages.Monitoring.Health>();
Assert.NotNull(attr);
Assert.Null(attr!.Policy);
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HealthCheckTests.cs
+8 -5
View File
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.ScadaBridge.Host.Health;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
@@ -207,11 +208,13 @@ public class HealthCheckTests : IDisposable
}
[Fact]
public void ActiveNodeCheck_IsHostLocalOldestNodeCheck()
public void ActiveNodeCheck_IsTheSharedOldestUpMemberCheck()
{
// The shared package's ActiveNodeHealthCheck is LEADER-based (review 01 [High]):
// during a partition both nodes think they lead and Traefik serves both. The
// active-node check must be the host-local oldest-member check instead.
// The property under test is that the active tier selects by AGE, not by leadership: during a
// partition both sides think they lead and Traefik would serve both (review 01 [High]).
// Through ZB.MOM.WW.Health 0.2.1 the shared check was leader-based, so this host carried a
// private OldestNodeActiveHealthCheck; 0.3.0 made the shared check age-based, so the private
// copy is gone and the shared type is now the correct answer here.
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
try
{
@@ -222,7 +225,7 @@ public class HealthCheckTests : IDisposable
Assert.Contains(ZbHealthTags.Active, active.Tags);
var check = active.Factory(factory.Services);
Assert.IsType<OldestNodeActiveHealthCheck>(check);
Assert.IsType<ActiveNodeHealthCheck>(check);
}
finally
{
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthCheckTests.cs
+239
View File
@@ -0,0 +1,239 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Registration;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.Host.Health;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <summary>
/// Site-node health endpoints. Site nodes previously served no health surface at all — only gRPC on
/// the HTTP/2 listener and <c>/metrics</c> on the HTTP/1.1 one — so the family overview dashboard
/// had no uniform way to probe them. These tests pin the three checks the site now registers and
/// the tier each one lands in.
/// <para>
/// The registration assertions build the REAL container from
/// <see cref="SiteServiceRegistration.Configure"/> rather than a hand-assembled
/// <c>ServiceCollection</c>, for the reason spelled out in <see cref="SiteLocalDbWiringTests"/>:
/// this family's wiring defects have consistently been ones that only a container-built graph
/// catches. Each registration is additionally ACTIVATED through its factory, so a check that is
/// registered but cannot resolve its dependencies fails here instead of at the first live probe.
/// </para>
/// </summary>
public class SiteHealthCheckTests : IDisposable
{
private readonly WebApplication _host;
private readonly string _tempDbPath;
private readonly string _tempSiteDbPath;
private readonly string _tempTrackingDbPath;
public SiteHealthCheckTests()
{
var stamp = Guid.NewGuid();
_tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_localdb_{stamp}.db");
_tempSiteDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_site_{stamp}.db");
_tempTrackingDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_track_{stamp}.db");
var builder = WebApplication.CreateBuilder();
builder.Configuration.Sources.Clear();
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["ScadaBridge:Node:Role"] = "Site",
["ScadaBridge:Node:NodeName"] = "node-a",
["ScadaBridge:Node:NodeHostname"] = "test-site",
["ScadaBridge:Node:SiteId"] = "TestSite",
["ScadaBridge:Node:RemotingPort"] = "0",
["ScadaBridge:Node:GrpcPort"] = "0",
["ScadaBridge:Database:SiteDbPath"] = _tempSiteDbPath,
["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={_tempTrackingDbPath}",
["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
["LocalDb:Path"] = _tempDbPath,
});
builder.Services.AddGrpc();
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);
_host = builder.Build();
}
public void Dispose()
{
(_host as IDisposable)?.Dispose();
foreach (var path in new[] { _tempDbPath, _tempSiteDbPath, _tempTrackingDbPath })
{
try { File.Delete(path); } catch { /* best effort */ }
}
GC.SuppressFinalize(this);
}
private IReadOnlyList<HealthCheckRegistration> Registrations =>
_host.Services.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations.ToList();
[Fact]
public void Site_RegistersExactlyTheThreeSiteChecks()
{
// Exact-set, not "contains": an extra check silently changes what /health/ready means for
// every consumer, and a site node inheriting a central-only probe is precisely the mistake
// this asserts against.
Assert.Equal(
new[] { "active-node", "akka-cluster", "localdb" },
Registrations.Select(r => r.Name).OrderBy(n => n, StringComparer.Ordinal).ToArray());
}
[Theory]
[InlineData("akka-cluster", ZbHealthTags.Ready)]
[InlineData("localdb", ZbHealthTags.Ready)]
[InlineData("active-node", ZbHealthTags.Active)]
public void Site_ChecksCarryTheirIntendedTier(string name, string expectedTag)
{
var registration = Assert.Single(Registrations, r => r.Name == name);
// Exactly one tier tag each: MapZbHealth selects by tag, so a check tagged both Ready and
// Active would put standby-ness into readiness and drop the standby out of readiness too.
Assert.Equal(new[] { expectedTag }, registration.Tags.ToArray());
}
[Theory]
[InlineData("akka-cluster", typeof(AkkaClusterHealthCheck))]
[InlineData("localdb", typeof(SiteLocalDbHealthCheck))]
[InlineData("active-node", typeof(SitePairActiveNodeHealthCheck))]
public void Site_ChecksActivateToTheIntendedType(string name, Type expected)
{
var registration = Assert.Single(Registrations, r => r.Name == name);
// Running the factory is the point: a type-activated check whose constructor dependencies
// are not registered on site would throw here rather than on the first probe in production.
var check = registration.Factory(_host.Services);
Assert.IsType(expected, check);
}
[Fact]
public void Site_ActiveNodeCheck_IsRoleScoped_NotTheUnscopedCentralCheck()
{
// Central registers the shared ActiveNodeHealthCheck UNSCOPED — oldest Up member of the whole
// cluster. On a mesh carrying more than one site's members that computes "oldest" across the
// wrong member set, so a site node could report Active purely for being the oldest member
// overall. A site must answer for its own site-{SiteId} role, which SitePairActiveNodeHealthCheck
// does by delegating to the site's IClusterNodeProvider (itself now backed by the shared
// ClusterActiveNode, so the rule is shared even though the scoping is local).
var check = Assert.Single(Registrations, r => r.Name == "active-node").Factory(_host.Services);
Assert.IsType<SitePairActiveNodeHealthCheck>(check);
Assert.IsNotType<ActiveNodeHealthCheck>(check);
}
[Theory]
[InlineData("database")]
[InlineData("required-singletons")]
public void Site_DoesNotRegisterCentralOnlyChecks(string centralOnlyName)
{
// Positive control first: if the harness ever stopped registering health checks at all,
// every "is absent" assertion below would pass vacuously and this test would be worthless.
Assert.Contains(Registrations, r => r.Name == "akka-cluster");
Assert.DoesNotContain(Registrations, r => r.Name == centralOnlyName);
}
[Fact]
public async Task Site_LocalDbCheck_ReportsHealthyAgainstTheRealSiteDatabase()
{
// Behaviour, not wiring: the probe runs against the consolidated site database this
// composition actually built, so a check that resolves but cannot query fails here.
var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb")
.Factory(_host.Services);
var result = await check.CheckHealthAsync(NewContext("localdb", check));
Assert.Equal(HealthStatus.Healthy, result.Status);
}
[Fact]
public async Task Site_LocalDbCheck_EnrichesWithReplicationStateWithoutFailingOnDefaultOff()
{
// Replication ships default-OFF, so the unreplicated steady state must stay Healthy while
// still surfacing the state as data — enrichment must never become a verdict.
var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb")
.Factory(_host.Services);
var result = await check.CheckHealthAsync(NewContext("localdb", check));
Assert.Equal(HealthStatus.Healthy, result.Status);
Assert.False(Assert.IsType<bool>(result.Data["replicationConnected"]));
Assert.Equal(0L, Assert.IsType<long>(result.Data["replicationOplogBacklog"]));
}
[Fact]
public async Task Site_LocalDbCheck_ReportsUnhealthyWhenTheStoreIsUnreachable()
{
var check = new SiteLocalDbHealthCheck(new ThrowingLocalDb());
var result = await check.CheckHealthAsync(NewContext("localdb", check));
Assert.Equal(HealthStatus.Unhealthy, result.Status);
}
[Theory]
[InlineData(true, HealthStatus.Healthy)]
[InlineData(false, HealthStatus.Unhealthy)]
public async Task Site_ActiveNodeCheck_SplitsPrimaryFromStandby(bool selfIsPrimary, HealthStatus expected)
{
// 200 on the primary / 503 on the standby is the same contract central publishes, so a
// consumer reads one active-node meaning fleet-wide.
var check = new SitePairActiveNodeHealthCheck(new StubNodeProvider(selfIsPrimary));
var result = await check.CheckHealthAsync(NewContext("active-node", check));
Assert.Equal(expected, result.Status);
}
private static HealthCheckContext NewContext(string name, IHealthCheck check) => new()
{
Registration = new HealthCheckRegistration(name, check, HealthStatus.Unhealthy, tags: null),
};
private sealed class StubNodeProvider(bool selfIsPrimary) : IClusterNodeProvider
{
public bool SelfIsPrimary { get; } = selfIsPrimary;
public IReadOnlyList<NodeStatus> GetClusterNodes() => [];
}
private sealed class ThrowingLocalDb : ILocalDb
{
public Task<int> ExecuteAsync(string sql, object? parameters = null, CancellationToken ct = default) =>
throw new InvalidOperationException("store unreachable");
public Task<IReadOnlyList<T>> QueryAsync<T>(
string sql,
Func<Microsoft.Data.Sqlite.SqliteDataReader, T> map,
object? parameters = null,
CancellationToken ct = default) =>
throw new InvalidOperationException("store unreachable");
public Task<ILocalDbTransaction> BeginTransactionAsync(CancellationToken ct = default) =>
throw new InvalidOperationException("store unreachable");
public Microsoft.Data.Sqlite.SqliteConnection CreateConnection() =>
throw new InvalidOperationException("store unreachable");
public ReplicatedTable RegisterReplicated(string tableName) =>
throw new InvalidOperationException("store unreachable");
public IReadOnlyDictionary<string, ReplicatedTable> ReplicatedTables =>
throw new InvalidOperationException("store unreachable");
}
}
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthEndpointTests.cs
+156
View File
@@ -0,0 +1,156 @@
using System.Net;
using System.Text.Json;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <summary>
/// Proves the site pipeline actually MAPS the health endpoints — the half
/// <see cref="SiteHealthCheckTests"/> cannot cover, since that fixture builds the site container
/// from <see cref="SiteServiceRegistration"/> and never runs <c>Program</c>'s endpoint wiring.
/// Without this, deleting <c>app.MapZbHealth()</c> from the site branch would leave every
/// registration test green while site nodes served 404 to the overview dashboard.
/// <para>
/// Boots the real <c>Program</c> in the Site role. <c>Program</c> builds its own
/// <c>ConfigurationBuilder</c> before the web host exists, so its role and settings come from
/// PROCESS-WIDE environment variables (<c>SCADABRIDGE_CONFIG</c> selects appsettings.Site.json)
/// rather than from <c>WithWebHostBuilder</c> — which is why this fixture joins
/// <see cref="HostBootCollection"/> and restores every var it sets.
/// </para>
/// </summary>
[Collection(HostBootCollection.Name)]
public class SiteHealthEndpointTests : IDisposable
{
private readonly List<IDisposable> _disposables = new();
private readonly Dictionary<string, string?> _previousEnv = new(StringComparer.Ordinal);
private readonly string _tempDbPath;
public SiteHealthEndpointTests()
{
_tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_ep_{Guid.NewGuid()}.db");
// Whole-key env overrides, the sanctioned path: supplying GrpcPsk concretely makes the
// pre-host secrets expander skip appsettings.Site.json's ${secret:SB-GRPC-PSK-site-1}
// token, so this test needs no seeded secrets store.
//
// The remoting port is moved off the site default (8082) so this fixture cannot collide
// with a locally running node, but it must be a REAL port and seed-nodes[0] must be this
// node itself — StartupValidator enforces both before the host is built, and it runs
// whether or not the Akka hosted service is later removed. Nothing binds it: the hosted
// service is removed below.
SetEnv("SCADABRIDGE_CONFIG", "Site");
SetEnv("ScadaBridge__Node__Role", "Site");
SetEnv("ScadaBridge__Node__SiteId", "TestSite");
SetEnv("ScadaBridge__Node__NodeHostname", "localhost");
SetEnv("ScadaBridge__Node__RemotingPort", "18082");
SetEnv("ScadaBridge__Cluster__SeedNodes__0", "akka.tcp://scadabridge@localhost:18082");
SetEnv("ScadaBridge__Cluster__SeedNodes__1", "akka.tcp://scadabridge@localhost:18085");
SetEnv("ScadaBridge__Communication__GrpcPsk", "test-psk-0123456789");
SetEnv("LocalDb__Path", _tempDbPath);
}
private void SetEnv(string key, string? value)
{
_previousEnv[key] = Environment.GetEnvironmentVariable(key);
Environment.SetEnvironmentVariable(key, value);
}
public void Dispose()
{
foreach (var d in _disposables)
{
try { d.Dispose(); } catch { /* best effort */ }
}
foreach (var (key, value) in _previousEnv)
{
Environment.SetEnvironmentVariable(key, value);
}
try { File.Delete(_tempDbPath); } catch { /* best effort */ }
GC.SuppressFinalize(this);
}
private HttpClient CreateSiteClient()
{
var factory = new WebApplicationFactory<Program>()
.WithWebHostBuilder(builder =>
{
// WebApplicationFactory defaults the environment to Development, which turns on
// ValidateOnBuild/ValidateScopes. The site graph carries scoped registrations whose
// dependencies are central-only (ReconcileService → IDeploymentManagerRepository,
// AuditLogKpiSampleSource → IAuditLogRepository) and are never resolved on a site
// node, so eager validation fails on a composition that runs fine in production.
// That is a pre-existing property of the site container, not of the health wiring
// under test — run this fixture the way a site node actually runs.
builder.UseEnvironment("Production");
// ConfigureTestServices runs after the app's own registrations, so this drops the
// hosted service that would form a real cluster while leaving the AkkaHostedService
// singleton resolvable (the site pipeline resolves it for shutdown ordering).
builder.ConfigureTestServices(AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly);
});
_disposables.Add(factory);
var client = factory.CreateClient();
_disposables.Add(client);
return client;
}
[Fact]
public async Task Site_MapsHealthReady_WithTheCanonicalJsonBody()
{
var response = await CreateSiteClient().GetAsync("/health/ready");
// Never 404. 200 vs 503 depends on cluster state this fixture does not form, so both are
// accepted — what is asserted is that the tier is mapped and answers in canonical shape.
Assert.NotEqual(HttpStatusCode.NotFound, response.StatusCode);
Assert.True(
response.StatusCode is HttpStatusCode.OK or HttpStatusCode.ServiceUnavailable,
$"Expected 200 or 503, got {(int)response.StatusCode}");
using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
var entries = doc.RootElement.GetProperty("entries");
Assert.True(entries.TryGetProperty("akka-cluster", out _), "ready tier must carry akka-cluster");
Assert.True(entries.TryGetProperty("localdb", out _), "ready tier must carry localdb");
// Readiness must NOT depend on being the primary, or a healthy standby would read unready.
Assert.False(entries.TryGetProperty("active-node", out _), "active-node belongs to the Active tier");
}
[Fact]
public async Task Site_MapsHealthActive_CarryingOnlyTheActiveNodeCheck()
{
var response = await CreateSiteClient().GetAsync("/health/active");
Assert.NotEqual(HttpStatusCode.NotFound, response.StatusCode);
using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
var entries = doc.RootElement.GetProperty("entries");
Assert.Equal(
new[] { "active-node" },
entries.EnumerateObject().Select(p => p.Name).ToArray());
}
[Fact]
public async Task Site_HealthEndpointsAreAnonymous()
{
// The dashboard authenticates nowhere. The site pipeline runs no authentication middleware
// today, so this is a regression pin: adding one later must not silently close these.
var client = CreateSiteClient();
foreach (var path in new[] { "/health/ready", "/health/active", "/healthz" })
{
var response = await client.GetAsync(path);
Assert.NotEqual(HttpStatusCode.Unauthorized, response.StatusCode);
Assert.NotEqual(HttpStatusCode.Forbidden, response.StatusCode);
Assert.NotEqual(HttpStatusCode.NotFound, response.StatusCode);
}
}
}