Joseph Doherty
255 lines
11 KiB
C#
255 lines
11 KiB
C#
using System.Linq;
|
|
using Microsoft.AspNetCore.Mvc.Testing;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.Health;
|
|
using ZB.MOM.WW.ScadaBridge.Host.Health;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
|
|
|
/// <summary>
|
|
/// WP-12: Tests for the three-tier health endpoints after adopting the shared
|
|
/// ZB.MOM.WW.Health probes. Verifies that /health/ready, /health/active and the new
|
|
/// /healthz tier are mapped, and that the readiness/active tier split is now carried by
|
|
/// the canonical <see cref="ZbHealthTags"/> (Ready for database + akka-cluster, Active for
|
|
/// active-node) rather than by check-name predicates. These are pure route/tag assertions
|
|
/// — they require no database, LDAP, or formed Akka cluster.
|
|
/// </summary>
|
|
public class HealthCheckTests : IDisposable
|
|
{
|
|
private readonly List<IDisposable> _disposables = new();
|
|
|
|
public HealthCheckTests()
|
|
{
|
|
// Host-003: connection strings are externalised; supply them via env vars.
|
|
_disposables.Add(new CentralDbTestEnvironment());
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
foreach (var d in _disposables)
|
|
{
|
|
try { d.Dispose(); } catch { /* best effort */ }
|
|
}
|
|
}
|
|
|
|
private WebApplicationFactory<Program> CreateCentralFactory()
|
|
{
|
|
var factory = new WebApplicationFactory<Program>()
|
|
.WithWebHostBuilder(builder =>
|
|
{
|
|
builder.ConfigureAppConfiguration((context, config) =>
|
|
{
|
|
config.AddInMemoryCollection(new Dictionary<string, string?>
|
|
{
|
|
["ScadaBridge:Node:NodeName"] = "central-a",
|
|
["ScadaBridge:Node:NodeHostname"] = "localhost",
|
|
["ScadaBridge:Node:RemotingPort"] = "0",
|
|
["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
|
|
["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
|
|
["ScadaBridge:Database:SkipMigrations"] = "true",
|
|
});
|
|
});
|
|
builder.UseSetting("ScadaBridge:Node:Role", "Central");
|
|
builder.UseSetting("ScadaBridge:Database:SkipMigrations", "true");
|
|
});
|
|
_disposables.Add(factory);
|
|
return factory;
|
|
}
|
|
|
|
private static IEnumerable<HealthCheckRegistration> Registrations(WebApplicationFactory<Program> factory) =>
|
|
factory.Services.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations;
|
|
|
|
[Fact]
|
|
public async Task HealthReady_Endpoint_IsMapped()
|
|
{
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
var client = factory.CreateClient();
|
|
_disposables.Add(client);
|
|
|
|
var response = await client.GetAsync("/health/ready");
|
|
|
|
// The endpoint exists and returns a status code. With test infrastructure
|
|
// (no real DB / cluster) the readiness checks may report Unhealthy, so we
|
|
// accept either 200 (Healthy/Degraded) or 503 (Unhealthy) — never 404.
|
|
Assert.NotEqual(System.Net.HttpStatusCode.NotFound, response.StatusCode);
|
|
Assert.True(
|
|
response.StatusCode == System.Net.HttpStatusCode.OK ||
|
|
response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable,
|
|
$"Expected 200 or 503, got {(int)response.StatusCode}");
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HealthActive_Endpoint_IsMapped()
|
|
{
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
var client = factory.CreateClient();
|
|
_disposables.Add(client);
|
|
|
|
var response = await client.GetAsync("/health/active");
|
|
|
|
Assert.NotEqual(System.Net.HttpStatusCode.NotFound, response.StatusCode);
|
|
Assert.True(
|
|
response.StatusCode == System.Net.HttpStatusCode.OK ||
|
|
response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable,
|
|
$"Expected 200 or 503, got {(int)response.StatusCode}");
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Healthz_LivenessEndpoint_IsMappedAndReturns200()
|
|
{
|
|
// New tier added by adopting the shared library: /healthz runs no checks, so it
|
|
// returns 200 as long as the process is up — independent of DB / cluster state.
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
var client = factory.CreateClient();
|
|
_disposables.Add(client);
|
|
|
|
var response = await client.GetAsync("/healthz");
|
|
|
|
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ReadyTier_Carries_Database_And_AkkaCluster()
|
|
{
|
|
// Host-001 regression guard: readiness reflects cluster membership + DB connectivity
|
|
// only (REQ-HOST-4a), NOT cluster leadership. The split is now carried by the Ready tag
|
|
// rather than a check-name predicate: database + akka-cluster are Ready-tagged, and the
|
|
// leader-only active-node check is NOT — so a fully operational standby central node
|
|
// still reports ready on /health/ready.
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
|
|
var registrations = Registrations(factory).ToDictionary(r => r.Name);
|
|
|
|
Assert.True(registrations.ContainsKey("database"), "Expected a 'database' health check.");
|
|
Assert.True(registrations.ContainsKey("akka-cluster"), "Expected an 'akka-cluster' health check.");
|
|
|
|
Assert.Contains(ZbHealthTags.Ready, registrations["database"].Tags);
|
|
Assert.Contains(ZbHealthTags.Ready, registrations["akka-cluster"].Tags);
|
|
|
|
// M2.14 (#28): readiness ALSO reflects "required cluster singletons running"
|
|
// (REQ-HOST-4a). The Central-only required-singletons check is Ready-tagged so
|
|
// it gates /health/ready alongside database + akka-cluster, but is leadership-
|
|
// agnostic (it does NOT carry the Active tag), so a ready standby stays ready.
|
|
Assert.True(registrations.ContainsKey("required-singletons"),
|
|
"Expected a 'required-singletons' health check.");
|
|
Assert.Contains(ZbHealthTags.Ready, registrations["required-singletons"].Tags);
|
|
Assert.DoesNotContain(ZbHealthTags.Active, registrations["required-singletons"].Tags);
|
|
|
|
// The leader-only active-node check must NOT be on the readiness tier.
|
|
Assert.DoesNotContain(ZbHealthTags.Ready, registrations["active-node"].Tags);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ActiveTier_Carries_ActiveNode_And_Database()
|
|
{
|
|
// Review 01 [High]: /health/active = oldest Up member AND database reachable, so a
|
|
// DB-dead active node is pulled from Traefik rotation. active-node + database carry
|
|
// the Active tag; akka-cluster does not (readiness-only).
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
|
|
var registrations = Registrations(factory).ToDictionary(r => r.Name);
|
|
|
|
Assert.True(registrations.ContainsKey("active-node"), "Expected an 'active-node' health check.");
|
|
Assert.Contains(ZbHealthTags.Active, registrations["active-node"].Tags);
|
|
|
|
// Database gates active too (DB-dead active node must drop out of rotation).
|
|
Assert.Contains(ZbHealthTags.Active, registrations["database"].Tags);
|
|
Assert.DoesNotContain(ZbHealthTags.Active, registrations["akka-cluster"].Tags);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ActiveNodeCheck_IsHostLocalOldestNodeCheck()
|
|
{
|
|
// The shared package's ActiveNodeHealthCheck is LEADER-based (review 01 [High]):
|
|
// during a partition both nodes think they lead and Traefik serves both. The
|
|
// active-node check must be the host-local oldest-member check instead.
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
|
|
var active = Registrations(factory).Single(r => r.Name == "active-node");
|
|
Assert.Contains(ZbHealthTags.Active, active.Tags);
|
|
|
|
var check = active.Factory(factory.Services);
|
|
Assert.IsType<OldestNodeActiveHealthCheck>(check);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void DatabaseCheck_AlsoGatesActive()
|
|
{
|
|
// Review 01 [High] recommendation: /health/active should require Ready
|
|
// (DB reachable) in addition to singleton-host status, so a DB-dead
|
|
// active node is pulled from Traefik rotation.
|
|
var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
|
|
var factory = CreateCentralFactory();
|
|
|
|
var registrations = Registrations(factory).ToDictionary(r => r.Name);
|
|
var db = registrations["database"];
|
|
Assert.Contains(ZbHealthTags.Ready, db.Tags);
|
|
Assert.Contains(ZbHealthTags.Active, db.Tags);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
|
|
}
|
|
}
|
|
}
|