feat(overview): poller, leader aggregation and the dashboard UI
Tasks 3.4-3.6 of the overview-dashboard plan. Polling (3.4/3.5): - OverviewPollerService: one timer at the fastest configured cadence, each target polled when its own interval is due, fan-out per sweep, no retries (the next tick is the retry, so damping stays the only place that decides whether a failure is worth showing). - OverviewSnapshotStore: atomic swap of a complete immutable graph plus a change event. Pages read it and never probe — the cache-first requirement. - LeaderResolver: per-group leader from akka-cluster data, with the split-brain flag scoped to members that are actually answering, so an ordinary failover is not misread as a disagreement. UI (3.6): ThemeShell + StatusPill/TechCard composition against the mockup; instance cards carry the status stripe (dashed for Unreachable), the check detail list and the raw ready/active signal line. Two defects found by verification rather than by review: - The active tier answers with a status code and an EMPTY body, but the client demanded parseable JSON on every probe — so a healthy pair rendered as "role unknown" instead of Active/Standby. Split into ProbeAsync (ready tier, body is the payload) and ProbeStatusAsync (active tier, code is the answer). Caught by a live smoke run against two fake health endpoints; the poller tests now serve an empty body so they hold the fix. - SweepAsync published even when cancellation was already requested, leaving a half-probed registry as the store's final state on shutdown. It now checks the token itself rather than relying on the transport to throw. Also: app.UseAntiforgery() is required despite the anonymous-by-design pipeline — AddRazorComponents stamps antiforgery metadata unconditionally and the endpoint middleware hard-fails without it (every page was 500). Chosen over DisableAntiforgery() so a future form is protected by default. 147 tests, 0 warnings. Live-verified end to end against fake endpoints: Up/Unreachable/Active/Standby, leader chip, cluster-data line, KPI counts.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,185 @@
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Leader aggregation and the split-brain flag. The disagreement rule is the one output an operator
|
||||
/// would be woken by, so both directions are pinned: it must fire when two members genuinely claim
|
||||
/// different leaders, and it must NOT fire for the ordinary cases that superficially resemble that
|
||||
/// — a node that stopped answering, a node running a pre-0.2.0 Health package.
|
||||
/// </summary>
|
||||
public class LeaderResolverTests
|
||||
{
|
||||
private const string LeaderA = "akka.tcp://scadabridge@site-a-a:8082";
|
||||
private const string LeaderB = "akka.tcp://scadabridge@site-a-b:8082";
|
||||
|
||||
[Fact]
|
||||
public void Resolve_AgreeingPair_NamesTheLeaderInstance()
|
||||
{
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderA),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Equal("site-a", group.Name);
|
||||
Assert.Equal("site-a-a", group.LeaderInstance);
|
||||
Assert.Equal(LeaderA, group.LeaderAddress);
|
||||
Assert.False(group.Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_MembersClaimDifferentLeaders_FlagsDisagreement()
|
||||
{
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.True(group.Disagreement);
|
||||
Assert.Equal([LeaderA, LeaderB], group.ReportedLeaders);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(InstanceStatus.Down)]
|
||||
[InlineData(InstanceStatus.Unreachable)]
|
||||
public void Resolve_FailedMemberDoesNotVote_SoAFailoverIsNotMisreadAsSplitBrain(InstanceStatus failed)
|
||||
{
|
||||
// Mid-failover the departed node's last body still names the old leader. Counting it would
|
||||
// raise a split-brain warning during the most normal event in a redundant pair's life.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: failed, leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.False(group.Disagreement);
|
||||
Assert.Equal("site-a-b", group.LeaderInstance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_DegradedMemberStillVotes()
|
||||
{
|
||||
// Degraded means answering with a full cluster view — its opinion on the leader is current.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: InstanceStatus.Degraded, leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
|
||||
]);
|
||||
|
||||
Assert.True(Assert.Single(groups).Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_NoMemberPublishesLeaderData_YieldsNoLeaderAndNoWarning()
|
||||
{
|
||||
// The whole-fleet-on-Health-0.1.0 case: no data key anywhere. It must degrade to "leader
|
||||
// unknown", never to a false split-brain warning.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084"),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Null(group.LeaderAddress);
|
||||
Assert.Null(group.LeaderInstance);
|
||||
Assert.False(group.Disagreement);
|
||||
Assert.Empty(group.ReportedLeaders);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_OneMemberOnOldHealthPackage_UsesTheOneThatDoesReport()
|
||||
{
|
||||
// A partly-bumped fleet is the expected state during rollout, and one silent member is not
|
||||
// a disagreement.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Equal("site-a-a", group.LeaderInstance);
|
||||
Assert.False(group.Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_UnresolvableAddress_KeepsTheRawAddress()
|
||||
{
|
||||
// A leader on a host that is not in the registry (a node nobody added) still tells the
|
||||
// operator something useful — showing nothing would hide the discovery.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: "akka.tcp://scadabridge@ghost:8082"),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Null(group.LeaderInstance);
|
||||
Assert.Equal("akka.tcp://scadabridge@ghost:8082", group.LeaderAddress);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_MatchesOnHostOnly_BecauseAkkaAndHttpPortsDiffer()
|
||||
{
|
||||
// The registry knows the HTTP port (8084); the leader address carries the Akka remoting
|
||||
// port (8082). Matching on authority instead of host would never resolve anything.
|
||||
var name = LeaderResolver.ResolveToInstanceName(
|
||||
LeaderA,
|
||||
[Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084")]);
|
||||
|
||||
Assert.Equal("site-a-a", name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_GroupWithoutActiveRole_IsNotAGroupAtAll()
|
||||
{
|
||||
// Instances grouped only for layout have no leader; a chip there would invent a concept.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("gw-1", "gateways", "http://gw-1:5120", hasActiveRole: false),
|
||||
Snapshots.Instance("gw-2", "gateways", "http://gw-2:5120", hasActiveRole: false),
|
||||
]);
|
||||
|
||||
Assert.Empty(groups);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_UngroupedInstances_ProduceNoGroups()
|
||||
{
|
||||
Assert.Empty(LeaderResolver.Resolve([Snapshots.Instance("solo")]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_SeparateGroups_AreScopedIndependently()
|
||||
{
|
||||
// Two site pairs, each with its own leader: per-Cluster election is the family's actual
|
||||
// topology, so a leader from one group must never be attributed to another.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-b-a", "site-b", "http://site-b-a:8084", leader: "akka.tcp://scadabridge@site-b-a:8082"),
|
||||
]);
|
||||
|
||||
Assert.Equal(2, groups.Count);
|
||||
Assert.Equal("site-a-a", groups[0].LeaderInstance);
|
||||
Assert.Equal("site-b-a", groups[1].LeaderInstance);
|
||||
Assert.All(groups, g => Assert.False(g.Disagreement));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("akka.tcp://scadabridge@site-a-a:8082", "site-a-a")]
|
||||
[InlineData("akka.tcp://otopcua@10.100.0.35:4053", "10.100.0.35")]
|
||||
[InlineData("akka.tcp://sys@host", "host")]
|
||||
[InlineData("not-an-address", null)]
|
||||
[InlineData("", null)]
|
||||
[InlineData(null, null)]
|
||||
public void HostOf_ParsesAkkaAddresses(string? address, string? expected)
|
||||
{
|
||||
Assert.Equal(expected, LeaderResolver.HostOf(address));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user