fix(overview): locate the cluster view by data key, not by check name
Found by the live rig, and invisible to every unit test that existed.
LeaderResolver and InstanceCard.ClusterDataLine both looked the cluster view up
under the check name "akka-cluster". ScadaBridge registers the shared
AkkaClusterHealthCheck under that name; OtOpcUa registers the SAME check as
"akka". So once the OtOpcUa rig was rebuilt on Health 0.2.0 and started
publishing data.leader, its groups still rendered with no leader chip and no
evidence line - and a group with no votes is a legitimate state, so nothing
looked wrong. ScadaBridge's four groups were fully populated the whole time,
which is what made it look like the feature worked.
Both now find the entry by the DATA KEY ("leader"), which is the part of the
contract the shared check actually owns rather than the part each consumer names
for itself.
Regression tests: the leader is found under akka-cluster, akka, and an arbitrary
future name; and an entry that publishes data WITHOUT a leader (ScadaBridge's
localdb check, whose data carries replication counters) does not get picked in
preference to the one that answers the question. Every prior fixture used
ScadaBridge's name, which is precisely why this got through.
After the fix all seven cluster groups render a leader chip with zero
split-brain flags - design section 9 check 2 now passes for BOTH products.
174 tests, 0 warnings.
This commit is contained in:
@@ -211,13 +211,21 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Renders the akka-cluster check's data as the mockup's monospace evidence line.
|
/// Renders the cluster check's data as the mockup's monospace evidence line.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Located by the data it carries rather than by check name: the two apps register the same
|
||||||
|
/// shared cluster check under different names (<c>akka-cluster</c> vs <c>akka</c>), so keying on
|
||||||
|
/// either one silently drops the evidence line for the other.
|
||||||
|
/// </remarks>
|
||||||
private string? ClusterDataLine
|
private string? ClusterDataLine
|
||||||
{
|
{
|
||||||
get
|
get
|
||||||
{
|
{
|
||||||
if (Instance.Report?.Entries.GetValueOrDefault(LeaderResolver.ClusterCheckName) is not { Data: not null } entry)
|
var entry = Instance.Report?.Entries.Values
|
||||||
|
.FirstOrDefault(e => e.DataString(LeaderResolver.LeaderDataKey) is { Length: > 0 });
|
||||||
|
|
||||||
|
if (entry is null)
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
var parts = new List<string>();
|
var parts = new List<string>();
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
namespace ZB.MOM.WW.Overview.Polling;
|
namespace ZB.MOM.WW.Overview.Polling;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Derives per-group leader state from what the members of a cluster group report in their
|
/// Derives per-group leader state from whatever health check a member publishes its cluster view
|
||||||
/// <c>akka-cluster</c> health data.
|
/// under.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// Pure and static: given the same instance snapshots it always produces the same answer, which is
|
/// Pure and static: given the same instance snapshots it always produces the same answer, which is
|
||||||
@@ -11,9 +11,6 @@ namespace ZB.MOM.WW.Overview.Polling;
|
|||||||
/// </remarks>
|
/// </remarks>
|
||||||
public static class LeaderResolver
|
public static class LeaderResolver
|
||||||
{
|
{
|
||||||
/// <summary>The health check whose data carries the cluster view.</summary>
|
|
||||||
public const string ClusterCheckName = "akka-cluster";
|
|
||||||
|
|
||||||
/// <summary>The data key holding the Akka leader address.</summary>
|
/// <summary>The data key holding the Akka leader address.</summary>
|
||||||
public const string LeaderDataKey = "leader";
|
public const string LeaderDataKey = "leader";
|
||||||
|
|
||||||
@@ -55,7 +52,7 @@ public static class LeaderResolver
|
|||||||
// every time a node goes away mid-failover — the exact moment the flag must stay honest.
|
// every time a node goes away mid-failover — the exact moment the flag must stay honest.
|
||||||
var votes = members
|
var votes = members
|
||||||
.Where(m => m.Status is InstanceStatus.Up or InstanceStatus.Degraded)
|
.Where(m => m.Status is InstanceStatus.Up or InstanceStatus.Degraded)
|
||||||
.Select(m => m.Report?.Entries.GetValueOrDefault(ClusterCheckName)?.DataString(LeaderDataKey))
|
.Select(LeaderReportedBy)
|
||||||
.Where(address => !string.IsNullOrWhiteSpace(address))
|
.Where(address => !string.IsNullOrWhiteSpace(address))
|
||||||
.Select(address => address!)
|
.Select(address => address!)
|
||||||
.ToList();
|
.ToList();
|
||||||
@@ -82,6 +79,31 @@ public static class LeaderResolver
|
|||||||
distinct);
|
distinct);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Finds the leader address one instance reports, whatever check carries it.</summary>
|
||||||
|
/// <param name="instance">The instance whose report to search.</param>
|
||||||
|
/// <returns>The reported leader address, or null when this instance publishes none.</returns>
|
||||||
|
/// <remarks>
|
||||||
|
/// Matched on the DATA KEY, not on the check name. The two apps register the same shared
|
||||||
|
/// <c>AkkaClusterHealthCheck</c> under different names — ScadaBridge calls it
|
||||||
|
/// <c>akka-cluster</c>, OtOpcUa calls it <c>akka</c> — and a resolver keyed on either name
|
||||||
|
/// silently renders no leader chip for the other, which is exactly how this was missed until a
|
||||||
|
/// live rig showed OtOpcUa's groups bare while ScadaBridge's were fully populated. The data key
|
||||||
|
/// is the part of the contract the shared check actually owns.
|
||||||
|
/// </remarks>
|
||||||
|
internal static string? LeaderReportedBy(InstanceSnapshot instance)
|
||||||
|
{
|
||||||
|
if (instance.Report?.Entries is not { } entries)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
foreach (var entry in entries.Values)
|
||||||
|
{
|
||||||
|
if (entry.DataString(LeaderDataKey) is { Length: > 0 } leader)
|
||||||
|
return leader;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Maps an Akka address to a registry instance name by host.
|
/// Maps an Akka address to a registry instance name by host.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -182,4 +182,43 @@ public class LeaderResolverTests
|
|||||||
{
|
{
|
||||||
Assert.Equal(expected, LeaderResolver.HostOf(address));
|
Assert.Equal(expected, LeaderResolver.HostOf(address));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("akka-cluster")] // ScadaBridge
|
||||||
|
[InlineData("akka")] // OtOpcUa
|
||||||
|
[InlineData("cluster-view")] // any future app
|
||||||
|
public void Leader_IsFoundWhateverTheCheckIsCalled(string checkName)
|
||||||
|
{
|
||||||
|
const string NodeALeader = "akka.tcp://otopcua@node-a:4053";
|
||||||
|
|
||||||
|
// The two apps register the SAME shared AkkaClusterHealthCheck under different names, so a
|
||||||
|
// resolver keyed on one name renders no leader chip at all for the other — silently, since
|
||||||
|
// a group with no votes is a legitimate state. That is exactly how this shipped past
|
||||||
|
// review and unit tests and was only caught on a live rig.
|
||||||
|
var groups = LeaderResolver.Resolve(
|
||||||
|
[
|
||||||
|
Snapshots.Instance("node-a", "pair", "http://node-a:8080", leader: NodeALeader, clusterCheckName: checkName),
|
||||||
|
Snapshots.Instance("node-b", "pair", "http://node-b:8080", leader: NodeALeader, clusterCheckName: checkName),
|
||||||
|
]);
|
||||||
|
|
||||||
|
var group = Assert.Single(groups);
|
||||||
|
Assert.Equal("node-a", group.LeaderInstance);
|
||||||
|
Assert.False(group.Disagreement);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Leader_IgnoresChecksThatPublishDataWithoutALeader()
|
||||||
|
{
|
||||||
|
// ScadaBridge's site nodes publish a localdb check whose data carries replication counters
|
||||||
|
// and no leader. Scanning entries must pick the one that actually answers the question,
|
||||||
|
// not merely the first one carrying a data object.
|
||||||
|
var body = HealthBody.Ready(
|
||||||
|
"Healthy",
|
||||||
|
("localdb", "Healthy", null),
|
||||||
|
("akka", "Healthy", "akka.tcp://otopcua@node-a:4053"));
|
||||||
|
var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
|
||||||
|
var instance = Snapshots.Instance("node-a", "pair", "http://node-a:8080") with { Report = report };
|
||||||
|
|
||||||
|
Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -150,7 +150,8 @@ internal static class Snapshots
|
|||||||
string baseUrl = "http://host:8080",
|
string baseUrl = "http://host:8080",
|
||||||
bool hasActiveRole = true,
|
bool hasActiveRole = true,
|
||||||
InstanceStatus? status = InstanceStatus.Up,
|
InstanceStatus? status = InstanceStatus.Up,
|
||||||
string? leader = null) =>
|
string? leader = null,
|
||||||
|
string clusterCheckName = "akka-cluster") =>
|
||||||
new()
|
new()
|
||||||
{
|
{
|
||||||
Application = "App",
|
Application = "App",
|
||||||
@@ -159,13 +160,13 @@ internal static class Snapshots
|
|||||||
BaseUrl = baseUrl,
|
BaseUrl = baseUrl,
|
||||||
HasActiveRole = hasActiveRole,
|
HasActiveRole = hasActiveRole,
|
||||||
Status = status,
|
Status = status,
|
||||||
Report = leader is null ? null : ReportWithLeader(leader),
|
Report = leader is null ? null : ReportWithLeader(leader, clusterCheckName),
|
||||||
StaleAfter = TimeSpan.FromSeconds(45),
|
StaleAfter = TimeSpan.FromSeconds(45),
|
||||||
};
|
};
|
||||||
|
|
||||||
private static ZbHealthReport ReportWithLeader(string leader)
|
private static ZbHealthReport ReportWithLeader(string leader, string clusterCheckName)
|
||||||
{
|
{
|
||||||
var body = HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader));
|
var body = HealthBody.Ready("Healthy", (clusterCheckName, "Healthy", leader));
|
||||||
return JsonSerializer.Deserialize<ZbHealthReport>(body)!;
|
return JsonSerializer.Deserialize<ZbHealthReport>(body)!;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -217,11 +217,14 @@
|
|||||||
"PRODUCT FIX during acceptance: Program.cs now calls builder.WebHost.UseStaticWebAssets() unconditionally. The host only calls it when the environment is literally 'Development', so running the build output under any other name (checking a staging registry locally) left the Theme RCL's _content assets 404 and the page rendered as a blank white screen with its markup fully present. No-op once published, so the container was never affected.",
|
"PRODUCT FIX during acceptance: Program.cs now calls builder.WebHost.UseStaticWebAssets() unconditionally. The host only calls it when the environment is literally 'Development', so running the build output under any other name (checking a staging registry locally) left the Theme RCL's _content assets 404 and the page rendered as a blank white screen with its markup fully present. No-op once published, so the container was never affected.",
|
||||||
"RIG DEFECT (pre-existing, both products): self-first Akka seed lists mean a pair booted cold self-forms two 1-member clusters that never join. Observed fleet-wide on ScadaBridge after a --force-recreate: every node named ITSELF leader and every node answered /health/active 200. Restarting ONE node of a pair makes it join the survivor and the pair converges to one Leader + one Active + one Standby. This is the user's own open backlog item (docs/plans/2026-07-22-initjoin-selfform-fallback.md, unexecuted). The dashboard surfaced it fleet-wide in one screen, which is what it is for.",
|
"RIG DEFECT (pre-existing, both products): self-first Akka seed lists mean a pair booted cold self-forms two 1-member clusters that never join. Observed fleet-wide on ScadaBridge after a --force-recreate: every node named ITSELF leader and every node answered /health/active 200. Restarting ONE node of a pair makes it join the survivor and the pair converges to one Leader + one Active + one Standby. This is the user's own open backlog item (docs/plans/2026-07-22-initjoin-selfform-fallback.md, unexecuted). The dashboard surfaced it fleet-wide in one screen, which is what it is for.",
|
||||||
"Leader and Active legitimately diverge after a restart: Akka leader is lowest-address, ScadaBridge's active-node check is oldest Up member. site-c-a shows LEADER + STANDBY while site-c-b shows ACTIVE. Same divergence CLAUDE.md documents for OtOpcUa's Primary vs RoleLeader.",
|
"Leader and Active legitimately diverge after a restart: Akka leader is lowest-address, ScadaBridge's active-node check is oldest Up member. site-c-a shows LEADER + STANDBY while site-c-b shows ACTIVE. Same divergence CLAUDE.md documents for OtOpcUa's Primary vs RoleLeader.",
|
||||||
"OUTSTANDING: the OtOpcUa docker-dev rig still runs a pre-0.2.0 build. Redeploying it needs a decision because the repo is checked out on feat/mesh-phase6, which carries 4 commits and an untracked Phase 7 plan beyond the base of feat/health-0.2.0-bump."
|
"OUTSTANDING: the OtOpcUa docker-dev rig still runs a pre-0.2.0 build. Redeploying it needs a decision because the repo is checked out on feat/mesh-phase6, which carries 4 commits and an untracked Phase 7 plan beyond the base of feat/health-0.2.0-bump.",
|
||||||
|
"PRODUCT BUG found by the live rig: LeaderResolver and InstanceCard.ClusterDataLine both keyed the cluster view on the check NAME 'akka-cluster'. ScadaBridge registers the shared AkkaClusterHealthCheck under that name; OtOpcUa registers the SAME check as 'akka'. The result was a silent miss \u2014 OtOpcUa groups rendered with no leader chip and no evidence line, which is indistinguishable from a group that simply has not reported yet. Both now locate the entry by the DATA KEY ('leader'), which is the part of the contract the shared check owns. Regression tests cover akka-cluster / akka / an arbitrary future name, plus an entry that publishes data WITHOUT a leader (ScadaBridge's localdb check) so scanning cannot pick the wrong entry. Unit tests and review had both missed it because every fixture used ScadaBridge's name.",
|
||||||
|
"OTOPCUA CONTRACT GAP (not a dashboard bug, worth filing): /health/active does not answer 'am I the active node'. On the standby admin (central-2) the admin-leader check returns Degraded ('Role admin member but not leader'), and ASP.NET maps Degraded to 200 \u2014 so a standby looks Active. On driver-only nodes it returns Healthy with the description 'Active for role admin (or not a role member)', i.e. it passes vacuously for every node that is not an admin at all. Meanwhile CLAUDE.md documents the real election as the oldest Up driver member per Cluster (IsDriverPrimary, ServiceLevel 250 vs 240), which this tier never consults. For the tier to mean what MapZbHealth's status mapping implies, a non-active node must return Unhealthy (503) \u2014 which is exactly what ScadaBridge's SitePairActiveNodeHealthCheck does, and why ScadaBridge reads correctly.",
|
||||||
|
"OtOpcUa rig rebuilt from a new branch feat/mesh-phase6-health-0.2.0 = feat/mesh-phase6 + a cherry-pick of the 3-line Health 0.2.0 bump (2e515c34). Checking out feat/health-0.2.0-bump directly would have regressed the rig past four commits of in-progress Phase 6/7 work. All three meshes formed cleanly (2 members, 0 unreachable, both members agreeing on one leader) \u2014 OtOpcUa's per-pair self-first seeding did NOT hit the cold-boot self-form race that ScadaBridge did."
|
||||||
],
|
],
|
||||||
"phase4_results": {
|
"phase4_results": {
|
||||||
"1_all_render_single_active_per_pair": "PASS (ScadaBridge, all 4 pairs: one Active + one Standby, verified live) / BLOCKED (OtOpcUa rig runs pre-0.2.0 build with self-formed 1-member meshes, so all 6 nodes report Active)",
|
"1_all_render_single_active_per_pair": "PARTIAL. ScadaBridge PASS: all four pairs render one Active + one Standby (active 200 / 503). OtOpcUa CANNOT PASS as built: all six nodes answer /health/active 200 because that tier does not gate on the redundancy Primary \u2014 see the OtOpcUa active-tier finding.",
|
||||||
"2_leader_chip_and_split_brain": "PASS (ScadaBridge). Leader chip moved from site-c-a to site-c-b when site-c-a was stopped, and the split-brain chip appeared while the pair was self-formed then cleared once it genuinely joined. BLOCKED for OtOpcUa (deployed build emits no data object, so no leader chip \u2014 the designed graceful degradation, confirmed)",
|
"2_leader_chip_and_split_brain": "PASS for BOTH products after the rig redeploy and the LeaderResolver fix. All seven cluster groups (ScadaBridge central/site-a/b/c, OtOpcUa MAIN/SITE-A/SITE-B) render a Leader chip with zero split-brain flags. Verified earlier that the chip MOVES when the leader is stopped and that the split-brain flag appears while a pair is self-formed and clears once it genuinely joins.",
|
||||||
"3_kill_and_restart": "PASS. Stopped scadabridge-site-c-a 12:01:27; Unreachable by 12:02:33 (two-strike damping = 2 failing sweeps). Restarted 12:04:01; Up again by 12:04:16, i.e. on the first good poll (recovery is deliberately asymmetric)",
|
"3_kill_and_restart": "PASS. Stopped scadabridge-site-c-a 12:01:27; Unreachable by 12:02:33 (two-strike damping = 2 failing sweeps). Restarted 12:04:01; Up again by 12:04:16, i.e. on the first good poll (recovery is deliberately asymmetric)",
|
||||||
"4_degraded_with_failing_entry": "PASS. Canned Degraded and Unhealthy endpoints render as Degraded/Down with the failing entry and its description visible, and check-dot ok/warn/bad classes per check",
|
"4_degraded_with_failing_entry": "PASS. Canned Degraded and Unhealthy endpoints render as Degraded/Down with the failing entry and its description visible, and check-dot ok/warn/bad classes per check",
|
||||||
"5_cache_first": "PASS. Five consecutive page loads in 0.00-0.02s, all showing the identical 'last sweep 12:17:44' header \u2014 the render never triggers a poll (an inline sweep could not finish in 10ms with two unreachable targets on 3s/10s timeouts)",
|
"5_cache_first": "PASS. Five consecutive page loads in 0.00-0.02s, all showing the identical 'last sweep 12:17:44' header \u2014 the render never triggers a poll (an inline sweep could not finish in 10ms with two unreachable targets on 3s/10s timeouts)",
|
||||||
|
|||||||
Reference in New Issue
Block a user