using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; using ZB.MOM.WW.Health; using ZB.MOM.WW.Health.Akka; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.LocalDb.Registration; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; using ZB.MOM.WW.ScadaBridge.Host.Health; namespace ZB.MOM.WW.ScadaBridge.Host.Tests; /// /// Site-node health endpoints. Site nodes previously served no health surface at all — only gRPC on /// the HTTP/2 listener and /metrics on the HTTP/1.1 one — so the family overview dashboard /// had no uniform way to probe them. These tests pin the three checks the site now registers and /// the tier each one lands in. /// /// The registration assertions build the REAL container from /// rather than a hand-assembled /// ServiceCollection, for the reason spelled out in : /// this family's wiring defects have consistently been ones that only a container-built graph /// catches. Each registration is additionally ACTIVATED through its factory, so a check that is /// registered but cannot resolve its dependencies fails here instead of at the first live probe. /// /// public class SiteHealthCheckTests : IDisposable { private readonly WebApplication _host; private readonly string _tempDbPath; private readonly string _tempSiteDbPath; private readonly string _tempTrackingDbPath; /// Initializes the test host with a temporary site database and the site health checks. public SiteHealthCheckTests() { var stamp = Guid.NewGuid(); _tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_localdb_{stamp}.db"); _tempSiteDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_site_{stamp}.db"); _tempTrackingDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_track_{stamp}.db"); var builder = WebApplication.CreateBuilder(); builder.Configuration.Sources.Clear(); builder.Configuration.AddInMemoryCollection(new Dictionary { ["ScadaBridge:Node:Role"] = "Site", ["ScadaBridge:Node:NodeName"] = "node-a", ["ScadaBridge:Node:NodeHostname"] = "test-site", ["ScadaBridge:Node:SiteId"] = "TestSite", ["ScadaBridge:Node:RemotingPort"] = "0", ["ScadaBridge:Node:GrpcPort"] = "0", ["ScadaBridge:Database:SiteDbPath"] = _tempSiteDbPath, ["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={_tempTrackingDbPath}", ["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551", ["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552", ["LocalDb:Path"] = _tempDbPath, }); builder.Services.AddGrpc(); builder.Services.AddSingleton(); SiteServiceRegistration.Configure(builder.Services, builder.Configuration); AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services); _host = builder.Build(); } /// Releases the resources held by this instance. public void Dispose() { (_host as IDisposable)?.Dispose(); foreach (var path in new[] { _tempDbPath, _tempSiteDbPath, _tempTrackingDbPath }) { try { File.Delete(path); } catch { /* best effort */ } } GC.SuppressFinalize(this); } private IReadOnlyList Registrations => _host.Services.GetRequiredService>().Value.Registrations.ToList(); /// Verifies that a site registers exactly the three site health checks. [Fact] public void Site_RegistersExactlyTheThreeSiteChecks() { // Exact-set, not "contains": an extra check silently changes what /health/ready means for // every consumer, and a site node inheriting a central-only probe is precisely the mistake // this asserts against. Assert.Equal( new[] { "active-node", "akka-cluster", "localdb" }, Registrations.Select(r => r.Name).OrderBy(n => n, StringComparer.Ordinal).ToArray()); } /// Verifies that each site check carries its intended tier tag. /// The health check registration name. /// The tier tag the check is expected to carry. [Theory] [InlineData("akka-cluster", ZbHealthTags.Ready)] [InlineData("localdb", ZbHealthTags.Ready)] [InlineData("active-node", ZbHealthTags.Active)] public void Site_ChecksCarryTheirIntendedTier(string name, string expectedTag) { var registration = Assert.Single(Registrations, r => r.Name == name); // Exactly one tier tag each: MapZbHealth selects by tag, so a check tagged both Ready and // Active would put standby-ness into readiness and drop the standby out of readiness too. Assert.Equal(new[] { expectedTag }, registration.Tags.ToArray()); } /// Verifies that each site check resolves to its intended implementation type. /// The health check registration name. /// The expected check implementation type. [Theory] [InlineData("akka-cluster", typeof(AkkaClusterHealthCheck))] [InlineData("localdb", typeof(SiteLocalDbHealthCheck))] [InlineData("active-node", typeof(ActiveNodeHealthCheck))] public void Site_ChecksActivateToTheIntendedType(string name, Type expected) { var registration = Assert.Single(Registrations, r => r.Name == name); // Running the factory is the point: a type-activated check whose constructor dependencies // are not registered on site would throw here rather than on the first probe in production. var check = registration.Factory(_host.Services); Assert.IsType(expected, check); } /// Verifies the site active-node check answers Degraded (HTTP 200), not Unhealthy, while the ActorSystem is not yet available (ScadaBridge#97). /// A task that represents the asynchronous operation. [Fact] public async Task Site_ActiveNodeCheck_IsDegradedNotUnhealthy_DuringStartup() { // The Health spec says a node still booting is Degraded, not a failed standby. The old // private site check returned Unhealthy (503) whenever no ActorSystem was up yet. var registration = Assert.Single(Registrations, r => r.Name == "active-node"); using var empty = new ServiceCollection().BuildServiceProvider(); var check = registration.Factory(empty); var result = await check.CheckHealthAsync(NewContext("active-node", check)); Assert.Equal(HealthStatus.Degraded, result.Status); } /// Verifies the site active-node check reports the active node in its data and is scoped to the site's own role (ScadaBridge#97). /// A task that represents the asynchronous operation. [Fact] public async Task Site_ActiveNodeCheck_ReportsActiveNode_AndIsScopedToTheSiteRole() { var registration = Assert.Single(Registrations, r => r.Name == "active-node"); // A real single-node cluster that carries the site's role: it is the oldest Up member, so // it is Active and must name itself as activeNode with activeRole = site-TestSite. using var inRole = await StartSelfJoinedClusterAsync("site-TestSite"); var check = registration.Factory(inRole.Provider); var result = await check.CheckHealthAsync(NewContext("active-node", check)); Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("site-TestSite", result.Data["activeRole"]); Assert.Equal(inRole.SelfAddress, result.Data["activeNode"]); Assert.Equal(inRole.SelfAddress, result.Data["selfAddress"]); // A node that is the oldest member overall but carries ANOTHER site's role is not this // site's active node: the check is scoped, so it must not answer Healthy. using var otherRole = await StartSelfJoinedClusterAsync("site-Other"); var other = registration.Factory(otherRole.Provider); var otherResult = await other.CheckHealthAsync(NewContext("active-node", other)); Assert.NotEqual(HealthStatus.Healthy, otherResult.Status); Assert.False(otherResult.Data.ContainsKey("activeNode")); } private static async Task StartSelfJoinedClusterAsync(string role) { var config = Akka.Configuration.ConfigurationFactory.ParseString($$""" akka.actor.provider = cluster akka.remote.dot-netty.tcp.hostname = "127.0.0.1" akka.remote.dot-netty.tcp.port = 0 akka.cluster.roles = ["{{role}}"] akka.loglevel = OFF """); var system = Akka.Actor.ActorSystem.Create("hc" + Guid.NewGuid().ToString("N")[..8], config); var cluster = Akka.Cluster.Cluster.Get(system); cluster.Join(cluster.SelfAddress); var deadline = DateTime.UtcNow.AddSeconds(20); while (cluster.SelfMember.Status != Akka.Cluster.MemberStatus.Up && DateTime.UtcNow < deadline) await Task.Delay(50); Assert.Equal(Akka.Cluster.MemberStatus.Up, cluster.SelfMember.Status); var services = new ServiceCollection(); services.AddSingleton(system); return new ClusterHandle(system, services.BuildServiceProvider(), cluster.SelfAddress.ToString()); } private sealed class ClusterHandle(Akka.Actor.ActorSystem system, ServiceProvider provider, string selfAddress) : IDisposable { public IServiceProvider Provider { get; } = provider; public string SelfAddress { get; } = selfAddress; public void Dispose() { system.Terminate().Wait(TimeSpan.FromSeconds(10)); } } /// Verifies that a site does not register central-only health checks. /// The name of a check that only central should register. [Theory] [InlineData("database")] [InlineData("required-singletons")] public void Site_DoesNotRegisterCentralOnlyChecks(string centralOnlyName) { // Positive control first: if the harness ever stopped registering health checks at all, // every "is absent" assertion below would pass vacuously and this test would be worthless. Assert.Contains(Registrations, r => r.Name == "akka-cluster"); Assert.DoesNotContain(Registrations, r => r.Name == centralOnlyName); } /// Verifies that the LocalDb check reports healthy against the real site database. /// A task that represents the asynchronous operation. [Fact] public async Task Site_LocalDbCheck_ReportsHealthyAgainstTheRealSiteDatabase() { // Behaviour, not wiring: the probe runs against the consolidated site database this // composition actually built, so a check that resolves but cannot query fails here. var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb") .Factory(_host.Services); var result = await check.CheckHealthAsync(NewContext("localdb", check)); Assert.Equal(HealthStatus.Healthy, result.Status); } /// Verifies that the LocalDb check adds replication state without failing when replication is off by default. /// A task that represents the asynchronous operation. [Fact] public async Task Site_LocalDbCheck_EnrichesWithReplicationStateWithoutFailingOnDefaultOff() { // Replication ships default-OFF, so the unreplicated steady state must stay Healthy while // still surfacing the state as data — enrichment must never become a verdict. var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb") .Factory(_host.Services); var result = await check.CheckHealthAsync(NewContext("localdb", check)); Assert.Equal(HealthStatus.Healthy, result.Status); Assert.False(Assert.IsType(result.Data["replicationConnected"])); Assert.Equal(0L, Assert.IsType(result.Data["replicationOplogBacklog"])); } /// Verifies that the LocalDb check reports unhealthy when the store is unreachable. /// A task that represents the asynchronous operation. [Fact] public async Task Site_LocalDbCheck_ReportsUnhealthyWhenTheStoreIsUnreachable() { var check = new SiteLocalDbHealthCheck(new ThrowingLocalDb()); var result = await check.CheckHealthAsync(NewContext("localdb", check)); Assert.Equal(HealthStatus.Unhealthy, result.Status); } private static HealthCheckContext NewContext(string name, IHealthCheck check) => new() { Registration = new HealthCheckRegistration(name, check, HealthStatus.Unhealthy, tags: null), }; private sealed class ThrowingLocalDb : ILocalDb { /// Always throws to simulate an unreachable store. /// The SQL text (ignored). /// The statement parameters (ignored). /// A token to cancel the operation (ignored). /// Never returns; always throws . public Task ExecuteAsync(string sql, object? parameters = null, CancellationToken ct = default) => throw new InvalidOperationException("store unreachable"); /// Always throws to simulate an unreachable store. /// The row type. /// The SQL text (ignored). /// The row mapper (ignored). /// The query parameters (ignored). /// A token to cancel the operation (ignored). /// Never returns; always throws . public Task> QueryAsync( string sql, Func map, object? parameters = null, CancellationToken ct = default) => throw new InvalidOperationException("store unreachable"); /// Always throws to simulate an unreachable store. /// A token to cancel the operation (ignored). /// Never returns; always throws . public Task BeginTransactionAsync(CancellationToken ct = default) => throw new InvalidOperationException("store unreachable"); /// Always throws to simulate an unreachable store. /// Never returns; always throws . public Microsoft.Data.Sqlite.SqliteConnection CreateConnection() => throw new InvalidOperationException("store unreachable"); /// Always throws to simulate an unreachable store. /// The table name (ignored). /// Never returns; always throws . public ReplicatedTable RegisterReplicated(string tableName) => throw new InvalidOperationException("store unreachable"); /// Always throws to simulate an unreachable store. /// The table name (ignored). /// Whether to baseline existing rows (ignored). /// Never returns; always throws . public ReplicatedTable RegisterReplicated(string tableName, bool baselineExistingRows) => throw new InvalidOperationException("store unreachable"); /// Always throws to simulate an unreachable store. /// The table name (ignored). /// Never returns; always throws . public bool DeregisterReplicated(string tableName) => throw new InvalidOperationException("store unreachable"); /// Gets the replicated tables; always throws to simulate an unreachable store. public IReadOnlyDictionary ReplicatedTables => throw new InvalidOperationException("store unreachable"); } }