fix(host): /health/active = oldest member + DB reachable — partition-safe Traefik routing

src/ZB.MOM.WW.ScadaBridge.Host/Health/OldestNodeActiveHealthCheck.cs

@@ -0,0 +1,31 @@
+using Akka.Actor;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace ZB.MOM.WW.ScadaBridge.Host.Health;
+
+/// <summary>
+/// /health/active check: Healthy only on the oldest Up member — the node that
+/// hosts the cluster singletons. Replaces the shared package's leader-based
+/// ActiveNodeHealthCheck (review 01 [High]): leadership is address-ordered and
+/// diverges from singleton placement after a restart, and during a partition
+/// BOTH nodes compute themselves leader, so Traefik served both. Oldest-member
+/// semantics keep exactly one active node through a partition (the non-oldest
+/// side still sees the old oldest member until SBR resolves membership).
+/// </summary>
+public sealed class OldestNodeActiveHealthCheck : IHealthCheck
+{
+    private readonly ActorSystem _system;
+
+    /// <summary>Initializes a new <see cref="OldestNodeActiveHealthCheck"/> bound to the running actor system.</summary>
+    /// <param name="system">The live cluster actor system.</param>
+    public OldestNodeActiveHealthCheck(ActorSystem system) => _system = system;
+
+    /// <inheritdoc />
+    public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken ct = default)
+    {
+        var cluster = Akka.Cluster.Cluster.Get(_system);
+        return Task.FromResult(ClusterActivityEvaluator.SelfIsOldest(cluster)
+            ? HealthCheckResult.Healthy("oldest Up member (singleton host)")
+            : HealthCheckResult.Unhealthy("not the oldest Up member (standby)"));
+    }
+}

src/ZB.MOM.WW.ScadaBridge.Host/Program.cs

@@ -210,15 +210,19 @@ try
         builder.Services.AddConfigurationDatabase(configDbConnectionString);
 
         // Health checks for readiness gating — shared ZB.MOM.WW.Health probes.
-        // Check names and the ready/active tier split are preserved: database + akka-cluster
+        // Ready tier (/health/ready): database + akka-cluster + required-singletons.
-        // carry the Ready tag (/health/ready), active-node carries the Active tag (/health/active).
+        // Active tier (/health/active): active-node = oldest Up member (singleton host) AND
-        // The Akka checks resolve ActorSystem from DI via the transient bridge registered below;
+        // database reachable (review 01 [High]) — a DB-dead active node drops from Traefik
+        // rotation, and the active-node check is the host-local OldestNodeActiveHealthCheck,
+        // NOT the shared leader-based ActiveNodeHealthCheck (leadership diverges from singleton
+        // placement after a restart and both sides claim leadership during a partition).
+        // The Akka checks resolve ActorSystem from DI via the singleton bridge registered below;
         // the DatabaseHealthCheck<TContext> resolves a scoped ScadaBridgeDbContext (no factory).
         builder.Services.AddHealthChecks()
             .AddTypeActivatedCheck<DatabaseHealthCheck<ScadaBridgeDbContext>>(
                 "database",
                 failureStatus: null,
-                tags: new[] { ZbHealthTags.Ready })
+                tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })
             .AddTypeActivatedCheck<AkkaClusterHealthCheck>(
                 "akka-cluster",
                 failureStatus: null,
@@ -236,7 +240,9 @@ try
                 "required-singletons",
                 failureStatus: null,
                 tags: new[] { ZbHealthTags.Ready })
-            .AddTypeActivatedCheck<ActiveNodeHealthCheck>(
+            // Host-local oldest-member check (review 01 [High]) — see OldestNodeActiveHealthCheck.
+            // Resolves ActorSystem from DI, like the akka-cluster check above.
+            .AddTypeActivatedCheck<OldestNodeActiveHealthCheck>(
                 "active-node",
                 failureStatus: null,
                 tags: new[] { ZbHealthTags.Active });
@@ -261,9 +267,9 @@ try
         // standby-node gating is actually enforced (the InboundApiEndpointFilter
         // consults IActiveNodeGate and defaults to "allow" when none is registered,
         // which leaves the design's "central cluster only (active node)" guarantee
-        // unenforced in deployed binaries). The gate is backed by the same Akka
+        // unenforced in deployed binaries). The gate is backed by the same oldest-member
-        // cluster-leadership check as ActiveNodeHealthCheck above, so the inbound
+        // evaluator (ClusterActivityEvaluator) as OldestNodeActiveHealthCheck above, so the
-        // API and the /health/active endpoint Traefik routes against agree on
+        // inbound API and the /health/active endpoint Traefik routes against agree on
         // which node is active.
         builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.InboundAPI.IActiveNodeGate, ActiveNodeGate>();
 
@@ -352,12 +358,13 @@ try
             branch => branch.UseAuditWriteMiddleware());
 
         // Map the canonical three-tier health endpoints in one call:
-        //   /health/ready  — Ready-tagged checks (database + akka-cluster). REQ-HOST-4a defines
+        //   /health/ready  — Ready-tagged checks (database + akka-cluster + required-singletons).
-        //                    readiness as cluster membership + DB connectivity, explicitly NOT
+        //                    REQ-HOST-4a defines readiness as cluster membership + DB connectivity,
-        //                    cluster leadership, so the leader-only active-node check is excluded
+        //                    explicitly NOT active-node status, so the active-node check is excluded
         //                    (a fully operational standby central node still reports ready).
-        //   /health/active — Active-tagged check (active-node); returns 200 only on the cluster
+        //   /health/active — Active-tagged checks (active-node = oldest Up member + database
-        //                    leader; used by Traefik for routing.
+        //                    reachable); returns 200 only on the singleton-host node with a live
+        //                    DB; used by Traefik for routing.
         //   /healthz       — bare process liveness; runs no checks (always 200 while the process
         //                    is up). New tier added by adopting the shared library.
         // All three are anonymous and use the canonical ZbHealthWriter JSON output.

tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HealthCheckTests.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.Health;
+using ZB.MOM.WW.ScadaBridge.Host.Health;
 
 namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
 
@@ -177,10 +178,11 @@ public class HealthCheckTests : IDisposable
     }
 
     [Fact]
-    public void ActiveTier_Carries_Only_ActiveNode()
+    public void ActiveTier_Carries_ActiveNode_And_Database()
     {
-        // The active-node leader check carries the Active tag (→ /health/active); the readiness
+        // Review 01 [High]: /health/active = oldest Up member AND database reachable, so a
-        // checks do not, so /health/active reports leadership alone.
+        // DB-dead active node is pulled from Traefik rotation. active-node + database carry
+        // the Active tag; akka-cluster does not (readiness-only).
         var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
         try
         {
@@ -192,7 +194,8 @@ public class HealthCheckTests : IDisposable
             Assert.True(registrations.ContainsKey("active-node"), "Expected an 'active-node' health check.");
             Assert.Contains(ZbHealthTags.Active, registrations["active-node"].Tags);
 
-            Assert.DoesNotContain(ZbHealthTags.Active, registrations["database"].Tags);
+            // Database gates active too (DB-dead active node must drop out of rotation).
+            Assert.Contains(ZbHealthTags.Active, registrations["database"].Tags);
             Assert.DoesNotContain(ZbHealthTags.Active, registrations["akka-cluster"].Tags);
         }
         finally
@@ -200,4 +203,51 @@ public class HealthCheckTests : IDisposable
             Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
         }
     }
+
+    [Fact]
+    public void ActiveNodeCheck_IsHostLocalOldestNodeCheck()
+    {
+        // The shared package's ActiveNodeHealthCheck is LEADER-based (review 01 [High]):
+        // during a partition both nodes think they lead and Traefik serves both. The
+        // active-node check must be the host-local oldest-member check instead.
+        var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+        try
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
+            var factory = CreateCentralFactory();
+
+            var active = Registrations(factory).Single(r => r.Name == "active-node");
+            Assert.Contains(ZbHealthTags.Active, active.Tags);
+
+            var check = active.Factory(factory.Services);
+            Assert.IsType<OldestNodeActiveHealthCheck>(check);
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
+        }
+    }
+
+    [Fact]
+    public void DatabaseCheck_AlsoGatesActive()
+    {
+        // Review 01 [High] recommendation: /health/active should require Ready
+        // (DB reachable) in addition to singleton-host status, so a DB-dead
+        // active node is pulled from Traefik rotation.
+        var previousEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+        try
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Central");
+            var factory = CreateCentralFactory();
+
+            var registrations = Registrations(factory).ToDictionary(r => r.Name);
+            var db = registrations["database"];
+            Assert.Contains(ZbHealthTags.Ready, db.Tags);
+            Assert.Contains(ZbHealthTags.Active, db.Tags);
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", previousEnv);
+        }
+    }
 }