fix(overview): locate the cluster view by data key, not by check name

Found by the live rig, and invisible to every unit test that existed.

LeaderResolver and InstanceCard.ClusterDataLine both looked the cluster view up
under the check name "akka-cluster". ScadaBridge registers the shared
AkkaClusterHealthCheck under that name; OtOpcUa registers the SAME check as
"akka". So once the OtOpcUa rig was rebuilt on Health 0.2.0 and started
publishing data.leader, its groups still rendered with no leader chip and no
evidence line - and a group with no votes is a legitimate state, so nothing
looked wrong. ScadaBridge's four groups were fully populated the whole time,
which is what made it look like the feature worked.

Both now find the entry by the DATA KEY ("leader"), which is the part of the
contract the shared check actually owns rather than the part each consumer names
for itself.

Regression tests: the leader is found under akka-cluster, akka, and an arbitrary
future name; and an entry that publishes data WITHOUT a leader (ScadaBridge's
localdb check, whose data carries replication counters) does not get picked in
preference to the one that answers the question. Every prior fixture used
ScadaBridge's name, which is precisely why this got through.

After the fix all seven cluster groups render a leader chip with zero
split-brain flags - design section 9 check 2 now passes for BOTH products.

174 tests, 0 warnings.
This commit is contained in:
Joseph Doherty
2026-07-24 10:38:39 -04:00
parent 41013dcab3
commit d95292c95d
5 changed files with 88 additions and 15 deletions
@@ -211,13 +211,21 @@
}
/// <summary>
/// Renders the akka-cluster check's data as the mockup's monospace evidence line.
/// Renders the cluster check's data as the mockup's monospace evidence line.
/// </summary>
/// <remarks>
/// Located by the data it carries rather than by check name: the two apps register the same
/// shared cluster check under different names (<c>akka-cluster</c> vs <c>akka</c>), so keying on
/// either one silently drops the evidence line for the other.
/// </remarks>
private string? ClusterDataLine
{
get
{
if (Instance.Report?.Entries.GetValueOrDefault(LeaderResolver.ClusterCheckName) is not { Data: not null } entry)
var entry = Instance.Report?.Entries.Values
.FirstOrDefault(e => e.DataString(LeaderResolver.LeaderDataKey) is { Length: > 0 });
if (entry is null)
return null;
var parts = new List<string>();
@@ -1,8 +1,8 @@
namespace ZB.MOM.WW.Overview.Polling;
/// <summary>
/// Derives per-group leader state from what the members of a cluster group report in their
/// <c>akka-cluster</c> health data.
/// Derives per-group leader state from whatever health check a member publishes its cluster view
/// under.
/// </summary>
/// <remarks>
/// Pure and static: given the same instance snapshots it always produces the same answer, which is
@@ -11,9 +11,6 @@ namespace ZB.MOM.WW.Overview.Polling;
/// </remarks>
public static class LeaderResolver
{
/// <summary>The health check whose data carries the cluster view.</summary>
public const string ClusterCheckName = "akka-cluster";
/// <summary>The data key holding the Akka leader address.</summary>
public const string LeaderDataKey = "leader";
@@ -55,7 +52,7 @@ public static class LeaderResolver
// every time a node goes away mid-failover — the exact moment the flag must stay honest.
var votes = members
.Where(m => m.Status is InstanceStatus.Up or InstanceStatus.Degraded)
.Select(m => m.Report?.Entries.GetValueOrDefault(ClusterCheckName)?.DataString(LeaderDataKey))
.Select(LeaderReportedBy)
.Where(address => !string.IsNullOrWhiteSpace(address))
.Select(address => address!)
.ToList();
@@ -82,6 +79,31 @@ public static class LeaderResolver
distinct);
}
/// <summary>Finds the leader address one instance reports, whatever check carries it.</summary>
/// <param name="instance">The instance whose report to search.</param>
/// <returns>The reported leader address, or null when this instance publishes none.</returns>
/// <remarks>
/// Matched on the DATA KEY, not on the check name. The two apps register the same shared
/// <c>AkkaClusterHealthCheck</c> under different names — ScadaBridge calls it
/// <c>akka-cluster</c>, OtOpcUa calls it <c>akka</c> — and a resolver keyed on either name
/// silently renders no leader chip for the other, which is exactly how this was missed until a
/// live rig showed OtOpcUa's groups bare while ScadaBridge's were fully populated. The data key
/// is the part of the contract the shared check actually owns.
/// </remarks>
internal static string? LeaderReportedBy(InstanceSnapshot instance)
{
if (instance.Report?.Entries is not { } entries)
return null;
foreach (var entry in entries.Values)
{
if (entry.DataString(LeaderDataKey) is { Length: > 0 } leader)
return leader;
}
return null;
}
/// <summary>
/// Maps an Akka address to a registry instance name by host.
/// </summary>
@@ -182,4 +182,43 @@ public class LeaderResolverTests
{
Assert.Equal(expected, LeaderResolver.HostOf(address));
}
[Theory]
[InlineData("akka-cluster")] // ScadaBridge
[InlineData("akka")] // OtOpcUa
[InlineData("cluster-view")] // any future app
public void Leader_IsFoundWhateverTheCheckIsCalled(string checkName)
{
const string NodeALeader = "akka.tcp://otopcua@node-a:4053";
// The two apps register the SAME shared AkkaClusterHealthCheck under different names, so a
// resolver keyed on one name renders no leader chip at all for the other — silently, since
// a group with no votes is a legitimate state. That is exactly how this shipped past
// review and unit tests and was only caught on a live rig.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("node-a", "pair", "http://node-a:8080", leader: NodeALeader, clusterCheckName: checkName),
Snapshots.Instance("node-b", "pair", "http://node-b:8080", leader: NodeALeader, clusterCheckName: checkName),
]);
var group = Assert.Single(groups);
Assert.Equal("node-a", group.LeaderInstance);
Assert.False(group.Disagreement);
}
[Fact]
public void Leader_IgnoresChecksThatPublishDataWithoutALeader()
{
// ScadaBridge's site nodes publish a localdb check whose data carries replication counters
// and no leader. Scanning entries must pick the one that actually answers the question,
// not merely the first one carrying a data object.
var body = HealthBody.Ready(
"Healthy",
("localdb", "Healthy", null),
("akka", "Healthy", "akka.tcp://otopcua@node-a:4053"));
var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
var instance = Snapshots.Instance("node-a", "pair", "http://node-a:8080") with { Report = report };
Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance));
}
}
@@ -150,7 +150,8 @@ internal static class Snapshots
string baseUrl = "http://host:8080",
bool hasActiveRole = true,
InstanceStatus? status = InstanceStatus.Up,
string? leader = null) =>
string? leader = null,
string clusterCheckName = "akka-cluster") =>
new()
{
Application = "App",
@@ -159,13 +160,13 @@ internal static class Snapshots
BaseUrl = baseUrl,
HasActiveRole = hasActiveRole,
Status = status,
Report = leader is null ? null : ReportWithLeader(leader),
Report = leader is null ? null : ReportWithLeader(leader, clusterCheckName),
StaleAfter = TimeSpan.FromSeconds(45),
};
private static ZbHealthReport ReportWithLeader(string leader)
private static ZbHealthReport ReportWithLeader(string leader, string clusterCheckName)
{
var body = HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader));
var body = HealthBody.Ready("Healthy", (clusterCheckName, "Healthy", leader));
return JsonSerializer.Deserialize<ZbHealthReport>(body)!;
}
}