Files
scadaproj/ZB.MOM.WW.Health/tests/ZB.MOM.WW.Health.Akka.Tests/ActiveNodeDecisionTests.cs
T
Joseph Doherty af1e0a86b2 fix(health): active tier returns 503 on a role-member-but-not-leader node (0.2.1)
The shared spec states the active tier "Fails (503) on a standby or
role-member-but-not-leader node", and the same spec maps Degraded to HTTP 200.
ActiveNodeDecision.Evaluate returned Degraded for exactly that case, so the tier
answered 200 on every node and could not distinguish an active node from a
standby over HTTP at all - only the response body differed.

The consequence was live, not theoretical: OtOpcUa's Traefik routes the admin UI
by this probe (docker-dev/traefik-dynamic.yml, scripts/install/traefik-dynamic.yml),
so both central nodes always passed and the leader-pinning the design called for
has silently never worked. Found while running the overview dashboard's live
acceptance, which reads the same tier and showed all six rig nodes as Active.
Filed as lmxopcua#494.

Role-member-but-not-leader now returns Unhealthy. The other two branches are
unchanged: a node that lacks the role stays Healthy (the probe is irrelevant to
it), and the startup-safety path still returns Degraded when the ActorSystem or
cluster is not yet available.

Blast radius is one consumer. OtOpcUa is the only user of the role-filtered mode;
ScadaBridge deliberately uses its own OldestNodeActiveHealthCheck (leadership is
address-ordered and diverges from the singleton host), and MxGateway and
HistorianGateway have no Akka cluster.

Version 0.2.1, three packages published to the Gitea feed and restore-verified.
70 tests pass.
2026-07-24 10:46:04 -04:00

140 lines
5.7 KiB
C#

using Akka.Actor;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.Health.Akka;
namespace ZB.MOM.WW.Health.Akka.Tests;
/// <summary>
/// Table-driven tests for the pure <see cref="ActiveNodeDecision.Evaluate"/> helper covering both
/// the role-less (ScadaBridge ActiveNode) and role-filtered (OtOpcUa AdminRoleLeader) matrices,
/// plus the startup-safety null-guards on <see cref="ActiveNodeHealthCheck"/> and
/// <see cref="AkkaActiveNodeGate"/> when no <see cref="ActorSystem"/> is registered.
/// </summary>
public sealed class ActiveNodeDecisionTests
{
// Role-less: requiredRole == null. hasRole is irrelevant. Healthy iff (selfUp && isLeader), else Unhealthy.
public static IEnumerable<object?[]> RoleLessCases() => new[]
{
new object?[] { true, true, false, (string?)null, HealthStatus.Healthy },
new object?[] { true, false, false, (string?)null, HealthStatus.Unhealthy },
new object?[] { false, true, false, (string?)null, HealthStatus.Unhealthy },
new object?[] { false, false, false, (string?)null, HealthStatus.Unhealthy },
};
[Theory]
[MemberData(nameof(RoleLessCases))]
public void Evaluate_RoleLess(bool selfUp, bool isLeader, bool hasRole, string? requiredRole, HealthStatus expected)
{
Assert.Equal(expected, ActiveNodeDecision.Evaluate(selfUp, isLeader, hasRole, requiredRole));
}
// Role-filtered: requiredRole != null.
// lacks role -> Healthy (probe irrelevant for this node)
// has role & is leader -> Healthy (selfUp is ignored — role-filtered mode only cares about leadership)
// has role & not leader -> Unhealthy (503) — the tier exists to be a 503 an LB can act on
public static IEnumerable<object[]> RoleFilteredCases() => new[]
{
// node lacks the role -> Healthy regardless of selfUp / isLeader
new object[] { true, true, false, "admin", HealthStatus.Healthy },
new object[] { true, false, false, "admin", HealthStatus.Healthy },
new object[] { false, false, false, "admin", HealthStatus.Healthy },
// node carries the role and is leader -> Healthy (selfUp=true)
new object[] { true, true, true, "admin", HealthStatus.Healthy },
// node carries the role and is leader -> Healthy (selfUp=false: role-filtered mode ignores selfUp)
new object[] { false, true, true, "admin", HealthStatus.Healthy },
// Node carries the role but is not leader -> Unhealthy, i.e. HTTP 503.
// Degraded would be a 200 (see the shared health SPEC), which made the active tier answer
// 200 on every node and silently broke OtOpcUa's Traefik leader-pinning. lmxopcua#494.
new object[] { true, false, true, "admin", HealthStatus.Unhealthy },
new object[] { false, false, true, "admin", HealthStatus.Unhealthy },
};
[Theory]
[MemberData(nameof(RoleFilteredCases))]
public void Evaluate_RoleFiltered(bool selfUp, bool isLeader, bool hasRole, string? requiredRole, HealthStatus expected)
{
Assert.Equal(expected, ActiveNodeDecision.Evaluate(selfUp, isLeader, hasRole, requiredRole));
}
[Fact]
public async Task HealthCheck_RoleLess_NoActorSystem_ReturnsDegraded()
{
var provider = new ServiceCollection().BuildServiceProvider();
var check = new ActiveNodeHealthCheck(provider);
var result = await check.CheckHealthAsync(NewContext(check));
Assert.Equal(HealthStatus.Degraded, result.Status);
}
[Fact]
public async Task HealthCheck_RoleFiltered_NoActorSystem_ReturnsDegraded()
{
var provider = new ServiceCollection().BuildServiceProvider();
var check = new ActiveNodeHealthCheck(provider, "admin");
var result = await check.CheckHealthAsync(NewContext(check));
Assert.Equal(HealthStatus.Degraded, result.Status);
}
[Fact]
public void Gate_NoActorSystem_IsActiveNodeFalse()
{
var provider = new ServiceCollection().BuildServiceProvider();
var gate = new AkkaActiveNodeGate(provider);
Assert.False(gate.IsActiveNode);
}
[Fact]
public async Task HealthCheck_RoleLess_ClusterInaccessible_ReturnsDegraded()
{
// ActorSystem present but Akka.Cluster not configured → Cluster.Get throws. The check must
// return Degraded (startup-safety rule), not let the exception escape (→ Unhealthy).
using var system = ActorSystem.Create("plain-no-cluster-roleless");
try
{
var provider = new ServiceCollection()
.AddSingleton(system)
.BuildServiceProvider();
var check = new ActiveNodeHealthCheck(provider);
var result = await check.CheckHealthAsync(NewContext(check));
Assert.Equal(HealthStatus.Degraded, result.Status);
}
finally
{
await system.Terminate();
}
}
[Fact]
public async Task HealthCheck_RoleFiltered_ClusterInaccessible_ReturnsDegraded()
{
using var system = ActorSystem.Create("plain-no-cluster-rolefiltered");
try
{
var provider = new ServiceCollection()
.AddSingleton(system)
.BuildServiceProvider();
var check = new ActiveNodeHealthCheck(provider, "admin");
var result = await check.CheckHealthAsync(NewContext(check));
Assert.Equal(HealthStatus.Degraded, result.Status);
}
finally
{
await system.Terminate();
}
}
private static HealthCheckContext NewContext(IHealthCheck check) => new()
{
Registration = new HealthCheckRegistration("active-node", check, HealthStatus.Unhealthy, tags: null),
};
}