fix(host): /health/active = oldest member + DB reachable — partition-safe Traefik routing

2026-07-08 15:51:56 -04:00
parent da6d5ae12c
commit ff89e3145c
3 changed files with 105 additions and 17 deletions
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.Health;
+using ZB.MOM.WW.ScadaBridge.Host.Health;

 namespace ZB.MOM.WW.ScadaBridge.Host.Tests;

@@ -177,10 +178,11 @@ public class HealthCheckTests : IDisposable
    }

    [Fact]
-    public void ActiveTier_Carries_Only_ActiveNode()
+    public void ActiveTier_Carries_ActiveNode_And_Database()
    {
-        // The active-node leader check carries the Active tag (→ /health/active); the readiness
-        // checks do not, so /health/active reports leadership alone.
+        // Review 01 [High]: /health/active = oldest Up member AND database reachable, so a
+        // DB-dead active node is pulled from Traefik rotation. active-node + database carry
+        // the Active tag; akka-cluster does not (readiness-only).
        var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
        try
        {
@@ -192,7 +194,8 @@ public class HealthCheckTests : IDisposable
            Assert.True(registrations.ContainsKey("active-node"), "Expected an 'active-node' health check.");
            Assert.Contains(ZbHealthTags.Active, registrations["active-node"].Tags);

-            Assert.DoesNotContain(ZbHealthTags.Active, registrations["database"].Tags);
+            // Database gates active too (DB-dead active node must drop out of rotation).
+            Assert.Contains(ZbHealthTags.Active, registrations["database"].Tags);
            Assert.DoesNotContain(ZbHealthTags.Active, registrations["akka-cluster"].Tags);
        }
        finally
@@ -200,4 +203,51 @@ public class HealthCheckTests : IDisposable
            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
        }
    }
+
+    [Fact]
+    public void ActiveNodeCheck_IsHostLocalOldestNodeCheck()
+    {
+        // The shared package's ActiveNodeHealthCheck is LEADER-based (review 01 [High]):
+        // during a partition both nodes think they lead and Traefik serves both. The
+        // active-node check must be the host-local oldest-member check instead.
+        var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+        try
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
+            var factory = CreateCentralFactory();
+
+            var active = Registrations(factory).Single(r => r.Name == "active-node");
+            Assert.Contains(ZbHealthTags.Active, active.Tags);
+
+            var check = active.Factory(factory.Services);
+            Assert.IsType<OldestNodeActiveHealthCheck>(check);
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
+        }
+    }
+
+    [Fact]
+    public void DatabaseCheck_AlsoGatesActive()
+    {
+        // Review 01 [High] recommendation: /health/active should require Ready
+        // (DB reachable) in addition to singleton-host status, so a DB-dead
+        // active node is pulled from Traefik rotation.
+        var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+        try
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
+            var factory = CreateCentralFactory();
+
+            var registrations = Registrations(factory).ToDictionary(r => r.Name);
+            var db = registrations["database"];
+            Assert.Contains(ZbHealthTags.Ready, db.Tags);
+            Assert.Contains(ZbHealthTags.Active, db.Tags);
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
+        }
+    }
 }