fix(overview): rank activeNode over leader for the Active chip

The dashboard derived every leader chip from AkkaClusterHealthCheck's `leader`
data — ClusterState.Leader, the LOWEST-ADDRESSED Up member. That is not the node
in charge. The node holding the singletons, gated for writes, and answering 200
on /health/active is the OLDEST Up member. The two agree only until some node
restarts, after which the restarted node rejoins youngest but keeps its address.

Observed live on the rebuilt OtOpcUa rig during the Health 0.3.0 work: leader
central-1, active node central-2. The dashboard would have put the Active chip on
central-1 while /health/active said central-2 — the chip contradicting the tier
it exists to visualise. The Phase 4 acceptance run missed it because the rig had
just been formed, the one state in which the two orderings agree.

Health 0.3.0's active tier now publishes `activeNode` on every member, including
standbys, so the fix is to prefer it and keep `leader` as the fallback for an
instance on an older Health or whose active tier was not probed.

The instance card now shows `active` and `leader` side by side rather than
swapping one for the other: they are legitimately different nodes, and showing
both turns a silent contradiction into something readable at 3am.

Tests: 177 (+3) — precedence when the two disagree, a standby naming the active
node rather than itself, and the fallback still working with no activeNode.
This commit is contained in:
Joseph Doherty
2026-07-24 13:27:07 -04:00
parent 6a4fa220de
commit 344bdc4da5
4 changed files with 153 additions and 22 deletions
@@ -214,18 +214,32 @@
/// Renders the cluster check's data as the mockup's monospace evidence line. /// Renders the cluster check's data as the mockup's monospace evidence line.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// Located by the data it carries rather than by check name: the two apps register the same /// <para>
/// shared cluster check under different names (<c>akka-cluster</c> vs <c>akka</c>), so keying on /// Located by the data it carries rather than by check name: the two apps register the
/// either one silently drops the evidence line for the other. /// same shared checks under different names (<c>akka-cluster</c> vs <c>akka</c>,
/// <c>active-node</c> vs <c>cluster-primary</c>), so keying on either one silently drops
/// the evidence line for the other.
/// </para>
/// <para>
/// <c>active</c> and <c>leader</c> are shown side by side <b>on purpose</b>. They come
/// from different checks and are different nodes whenever any node has restarted — the
/// leader is the lowest-addressed Up member, the active node is the oldest. Showing both
/// turns the confusion that produced lmxopcua#494 into something an operator can read
/// straight off the card, instead of a silent disagreement between the chip and the tier.
/// </para>
/// </remarks> /// </remarks>
private string? ClusterDataLine private string? ClusterDataLine
{ {
get get
{ {
var entry = Instance.Report?.Entries.Values var entries = Instance.Report?.Entries.Values;
.FirstOrDefault(e => e.DataString(LeaderResolver.LeaderDataKey) is { Length: > 0 }); if (entries is null)
return null;
if (entry is null) var cluster = entries.FirstOrDefault(e => e.DataString(LeaderResolver.LeaderDataKey) is { Length: > 0 });
var active = entries.FirstOrDefault(e => e.DataString(LeaderResolver.ActiveNodeDataKey) is { Length: > 0 });
if (cluster is null && active is null)
return null; return null;
var parts = new List<string>(); var parts = new List<string>();
@@ -236,10 +250,11 @@
parts.Add($"<b>{Escape(label)}</b> {Escape(value)}"); parts.Add($"<b>{Escape(label)}</b> {Escape(value)}");
} }
Add("leader", entry.DataString(LeaderResolver.LeaderDataKey)); Add("active", active?.DataString(LeaderResolver.ActiveNodeDataKey));
Add("members", entry.DataInt("memberCount")?.ToString()); Add("leader", cluster?.DataString(LeaderResolver.LeaderDataKey));
Add("unreachable", entry.DataInt("unreachableCount")?.ToString()); Add("members", cluster?.DataInt("memberCount")?.ToString());
Add("self", entry.DataString("selfAddress")); Add("unreachable", cluster?.DataInt("unreachableCount")?.ToString());
Add("self", (cluster ?? active)?.DataString("selfAddress"));
return parts.Count == 0 ? null : string.Join(" · ", parts); return parts.Count == 0 ? null : string.Join(" · ", parts);
} }
@@ -11,7 +11,16 @@ namespace ZB.MOM.WW.Overview.Polling;
/// </remarks> /// </remarks>
public static class LeaderResolver public static class LeaderResolver
{ {
/// <summary>The data key holding the Akka leader address.</summary> /// <summary>
/// The data key holding the address of the node actually in charge — the oldest Up member,
/// published by the active tier's <c>ActiveNodeHealthCheck</c> from ZB.MOM.WW.Health 0.3.0.
/// </summary>
public const string ActiveNodeDataKey = "activeNode";
/// <summary>
/// The data key holding the Akka cluster leader address, published by
/// <c>AkkaClusterHealthCheck</c>. Fallback only — see <see cref="LeaderReportedBy"/>.
/// </summary>
public const string LeaderDataKey = "leader"; public const string LeaderDataKey = "leader";
/// <summary>Resolves leader state for every cluster group present in <paramref name="instances"/>.</summary> /// <summary>Resolves leader state for every cluster group present in <paramref name="instances"/>.</summary>
@@ -79,29 +88,52 @@ public static class LeaderResolver
distinct); distinct);
} }
/// <summary>Finds the leader address one instance reports, whatever check carries it.</summary> /// <summary>Finds the address of the in-charge node one instance reports.</summary>
/// <param name="instance">The instance whose report to search.</param> /// <param name="instance">The instance whose report to search.</param>
/// <returns>The reported leader address, or null when this instance publishes none.</returns> /// <returns>The reported address, or null when this instance publishes none.</returns>
/// <remarks> /// <remarks>
/// Matched on the DATA KEY, not on the check name. The two apps register the same shared /// <para>
/// <c>AkkaClusterHealthCheck</c> under different names — ScadaBridge calls it /// <b><c>activeNode</c> first, <c>leader</c> only as a fallback.</b> They are different
/// <c>akka-cluster</c>, OtOpcUa calls it <c>akka</c> — and a resolver keyed on either name /// nodes and the chip must show the first. The Akka cluster <i>leader</i> is the
/// silently renders no leader chip for the other, which is exactly how this was missed until a /// lowest-addressed Up member — address order, no relationship to time — while the node
/// live rig showed OtOpcUa's groups bare while ScadaBridge's were fully populated. The data key /// that actually holds the singletons and answers 200 on <c>/health/active</c> is the
/// is the part of the contract the shared check actually owns. /// <i>oldest</i> Up member. They agree only until some node restarts, after which the
/// restarted node rejoins youngest but keeps its address; observed live on the rebuilt
/// OtOpcUa rig, where the leader was <c>central-1</c> while <c>central-2</c> was the
/// active node. Ranking <c>leader</c> first would put the Active chip on the standby.
/// </para>
/// <para>
/// The fallback still matters: it is what an instance running Health &lt; 0.3.0, or one
/// whose active tier was not probed, publishes. Showing the leader there is better than
/// showing nothing, and it is the pre-0.3.0 behaviour unchanged.
/// </para>
/// <para>
/// Matched on the DATA KEY, not on the check name. The two apps register the same shared
/// checks under different names — ScadaBridge calls its cluster check
/// <c>akka-cluster</c>, OtOpcUa calls it <c>akka</c>; the active check is
/// <c>active-node</c> in one and <c>cluster-primary</c> in the other — and a resolver
/// keyed on any of those names silently renders no chip for the other app, which is
/// exactly how this was missed until a live rig showed OtOpcUa's groups bare while
/// ScadaBridge's were fully populated. The data key is the part of the contract the
/// shared checks actually own.
/// </para>
/// </remarks> /// </remarks>
internal static string? LeaderReportedBy(InstanceSnapshot instance) internal static string? LeaderReportedBy(InstanceSnapshot instance)
{ {
if (instance.Report?.Entries is not { } entries) if (instance.Report?.Entries is not { } entries)
return null; return null;
string? leaderFallback = null;
foreach (var entry in entries.Values) foreach (var entry in entries.Values)
{ {
if (entry.DataString(LeaderDataKey) is { Length: > 0 } leader) if (entry.DataString(ActiveNodeDataKey) is { Length: > 0 } activeNode)
return leader; return activeNode;
leaderFallback ??= entry.DataString(LeaderDataKey) is { Length: > 0 } leader ? leader : null;
} }
return null; return leaderFallback;
} }
/// <summary> /// <summary>
@@ -221,4 +221,63 @@ public class LeaderResolverTests
Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance)); Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance));
} }
[Fact]
public void ActiveNode_WinsOverLeader_WhenTheyDisagree()
{
// The two are DIFFERENT NODES after any restart: leader is the lowest-addressed Up member,
// the active node is the oldest. Observed live on the rebuilt OtOpcUa rig — leader central-1,
// active central-2 — so ranking leader first would put the Active chip on the standby, which
// is the same class of error as lmxopcua#494 rendered in the UI instead of in routing.
var body = HealthBody.WithData(
"Healthy",
("akka", "Healthy", new Dictionary<string, object>
{
["leader"] = "akka.tcp://otopcua@central-1:4053",
}),
("cluster-primary", "Healthy", new Dictionary<string, object>
{
["activeNode"] = "akka.tcp://otopcua@central-2:4053",
}));
var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
var instance = Snapshots.Instance("central-1", "MAIN", "http://central-1:8080") with { Report = report };
Assert.Equal("akka.tcp://otopcua@central-2:4053", LeaderResolver.LeaderReportedBy(instance));
}
[Fact]
public void ActiveNode_ReportedByAStandby_NamesTheActiveNode_NotItself()
{
// Every member publishes who IS active, so a standby votes for the same address the active
// node does. That is what lets the group resolve with one unanimous chip instead of a
// manufactured disagreement, and it is why the standby's payload alone is enough.
var body = HealthBody.WithData(
"Unhealthy",
("cluster-primary", "Unhealthy", new Dictionary<string, object>
{
["activeNode"] = "akka.tcp://otopcua@site-a-1:4053",
["selfAddress"] = "akka.tcp://otopcua@site-a-2:4053",
}));
var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
var instance = Snapshots.Instance("site-a-2", "SITE-A", "http://site-a-2:8080") with { Report = report };
Assert.Equal("akka.tcp://otopcua@site-a-1:4053", LeaderResolver.LeaderReportedBy(instance));
}
[Fact]
public void Leader_IsStillUsed_WhenNoActiveNodeIsPublished()
{
// An instance on Health < 0.3.0, or one whose active tier was not probed, publishes only
// `leader`. Showing it beats showing nothing, and it keeps the pre-0.3.0 behaviour intact.
var body = HealthBody.WithData(
"Healthy",
("akka-cluster", "Healthy", new Dictionary<string, object>
{
["leader"] = "akka.tcp://scadabridge@central-a:4053",
}));
var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
var instance = Snapshots.Instance("central-a", "central", "http://central-a:8080") with { Report = report };
Assert.Equal("akka.tcp://scadabridge@central-a:4053", LeaderResolver.LeaderReportedBy(instance));
}
} }
@@ -139,6 +139,31 @@ internal static class HealthBody
return JsonSerializer.Serialize(payload); return JsonSerializer.Serialize(payload);
} }
/// <summary>
/// Builds a body whose entries carry arbitrary <c>data</c>, for the cases where the key under
/// test is not <c>leader</c> — notably the active tier's <c>activeNode</c>.
/// </summary>
public static string WithData(string status, params (string Name, string Status, Dictionary<string, object>? Data)[] entries)
{
var payload = new
{
status,
totalDurationMs = 1.5,
entries = entries.ToDictionary(
e => e.Name,
e => (object)new
{
status = e.Status,
description = $"{e.Name} says {e.Status}",
durationMs = 0.5,
data = e.Data,
},
StringComparer.Ordinal),
};
return JsonSerializer.Serialize(payload);
}
} }
/// <summary>Builds <see cref="InstanceSnapshot"/> values for the pure-logic tests.</summary> /// <summary>Builds <see cref="InstanceSnapshot"/> values for the pure-logic tests.</summary>