feat(overview): poller, leader aggregation and the dashboard UI
Tasks 3.4-3.6 of the overview-dashboard plan. Polling (3.4/3.5): - OverviewPollerService: one timer at the fastest configured cadence, each target polled when its own interval is due, fan-out per sweep, no retries (the next tick is the retry, so damping stays the only place that decides whether a failure is worth showing). - OverviewSnapshotStore: atomic swap of a complete immutable graph plus a change event. Pages read it and never probe — the cache-first requirement. - LeaderResolver: per-group leader from akka-cluster data, with the split-brain flag scoped to members that are actually answering, so an ordinary failover is not misread as a disagreement. UI (3.6): ThemeShell + StatusPill/TechCard composition against the mockup; instance cards carry the status stripe (dashed for Unreachable), the check detail list and the raw ready/active signal line. Two defects found by verification rather than by review: - The active tier answers with a status code and an EMPTY body, but the client demanded parseable JSON on every probe — so a healthy pair rendered as "role unknown" instead of Active/Standby. Split into ProbeAsync (ready tier, body is the payload) and ProbeStatusAsync (active tier, code is the answer). Caught by a live smoke run against two fake health endpoints; the poller tests now serve an empty body so they hold the fix. - SweepAsync published even when cancellation was already requested, leaving a half-probed registry as the store's final state on shutdown. It now checks the token itself rather than relying on the transport to throw. Also: app.UseAntiforgery() is required despite the anonymous-by-design pipeline — AddRazorComponents stamps antiforgery metadata unconditionally and the endpoint middleware hard-fails without it (every page was 500). Chosen over DisableAntiforgery() so a future form is protected by default. 147 tests, 0 warnings. Live-verified end to end against fake endpoints: Up/Unreachable/Active/Standby, leader chip, cluster-data line, KPI counts.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,249 @@
|
||||
using Bunit;
|
||||
using ZB.MOM.WW.Overview.Components.Widgets;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The instance card is where a reading becomes a decision, so the tests pin what an operator can
|
||||
/// actually distinguish at a glance: which of the four states is shown, whether it is still
|
||||
/// current, and whether the raw evidence behind it is visible.
|
||||
/// </summary>
|
||||
public class InstanceCardTests : TestContext
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static InstanceSnapshot Card(
|
||||
InstanceStatus? status = InstanceStatus.Up,
|
||||
ActiveState active = ActiveState.NotApplicable,
|
||||
DateTimeOffset? lastPolled = null,
|
||||
DateTimeOffset? lastReachable = null,
|
||||
TimeSpan? staleAfter = null,
|
||||
string? error = null,
|
||||
ZbHealthReport? report = null,
|
||||
string? managementUrl = null) =>
|
||||
new()
|
||||
{
|
||||
Application = "ScadaBridge",
|
||||
Instance = "site-a-a",
|
||||
Group = "site-a",
|
||||
BaseUrl = "http://site-a-a:8084",
|
||||
ManagementUrl = managementUrl,
|
||||
ManagementLabel = "AdminUI",
|
||||
HasActiveRole = active != ActiveState.NotApplicable,
|
||||
Status = status,
|
||||
Active = active,
|
||||
LastPolledUtc = lastPolled ?? Now,
|
||||
LastReachableUtc = lastReachable ?? Now,
|
||||
Latency = TimeSpan.FromMilliseconds(18),
|
||||
Error = error,
|
||||
Report = report,
|
||||
StaleAfter = staleAfter ?? TimeSpan.FromSeconds(45),
|
||||
};
|
||||
|
||||
private IRenderedComponent<InstanceCard> Render(InstanceSnapshot instance, bool isLeader = false) =>
|
||||
RenderComponent<InstanceCard>(p => p
|
||||
.Add(c => c.Instance, instance)
|
||||
.Add(c => c.Now, Now)
|
||||
.Add(c => c.IsLeader, isLeader));
|
||||
|
||||
[Theory]
|
||||
[InlineData(InstanceStatus.Up, "up", "Up")]
|
||||
[InlineData(InstanceStatus.Degraded, "degraded", "Degraded")]
|
||||
[InlineData(InstanceStatus.Down, "down", "Down")]
|
||||
[InlineData(InstanceStatus.Unreachable, "unreachable", "Unreachable")]
|
||||
public void Card_CarriesItsStatusAsBothClassAndLabel(InstanceStatus status, string cssClass, string label)
|
||||
{
|
||||
var card = Render(Card(status));
|
||||
|
||||
Assert.Contains(cssClass, card.Find("article").ClassList, StringComparer.Ordinal);
|
||||
Assert.Contains(label, card.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnreachableAndDown_AreVisuallyDistinct_NotJustDifferentWords()
|
||||
{
|
||||
// The CSS gives Unreachable a dashed border precisely because both are red. If the class
|
||||
// did not differ, the two would be indistinguishable on a wall display.
|
||||
var down = Render(Card(InstanceStatus.Down)).Find("article").ClassList;
|
||||
var unreachable = Render(Card(InstanceStatus.Unreachable)).Find("article").ClassList;
|
||||
|
||||
Assert.NotEqual(down, unreachable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NeverPolled_RendersPending_NotHealthy()
|
||||
{
|
||||
var card = Render(Card(status: null, lastPolled: null));
|
||||
|
||||
Assert.Contains("pending", card.Find("article").ClassList, StringComparer.Ordinal);
|
||||
Assert.Contains("Pending", card.Markup, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(">Up<", card.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleCard_IsMarkedStaleButKeepsItsLastKnownStatusClass()
|
||||
{
|
||||
// Both facts matter: the reading was Up, and it is no longer current. Dropping either one
|
||||
// loses information the operator needs.
|
||||
var card = Render(Card(lastPolled: Now.AddMinutes(-5)));
|
||||
|
||||
var classes = card.Find("article").ClassList;
|
||||
Assert.Contains("stale", classes, StringComparer.Ordinal);
|
||||
Assert.Contains("up", classes, StringComparer.Ordinal);
|
||||
Assert.Contains("Stale", card.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleCard_CountsFromLastContact_NotFromTheLastPollAttempt()
|
||||
{
|
||||
// The poller kept trying for ten minutes and kept failing. "Stale since 5 s ago" would
|
||||
// suggest the instance was fine moments ago; it has actually been gone for ten minutes.
|
||||
var card = Render(Card(
|
||||
status: InstanceStatus.Unreachable,
|
||||
lastPolled: Now.AddSeconds(-5),
|
||||
lastReachable: Now.AddMinutes(-10),
|
||||
staleAfter: TimeSpan.FromSeconds(1)));
|
||||
|
||||
Assert.Contains("Stale since", card.Markup, StringComparison.Ordinal);
|
||||
Assert.Contains("10 min ago", card.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ActiveState.Active, "Active")]
|
||||
[InlineData(ActiveState.Standby, "Standby")]
|
||||
[InlineData(ActiveState.Unknown, "Role ?")]
|
||||
public void ActiveState_IsBadged(ActiveState state, string expected)
|
||||
{
|
||||
Assert.Contains(expected, Render(Card(active: state)).Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NoActiveRole_ShowsNoActivityBadgeAtAll()
|
||||
{
|
||||
// A single-instance gateway has no active/standby concept; a badge would invent one.
|
||||
var markup = Render(Card(active: ActiveState.NotApplicable)).Markup;
|
||||
|
||||
Assert.DoesNotContain("Standby", markup, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("Role ?", markup, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("· active", markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LeaderBadge_IsShownOnlyWhenTheCardIsTheLeader()
|
||||
{
|
||||
Assert.Contains("Leader", Render(Card(), isLeader: true).Markup, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("Leader", Render(Card(), isLeader: false).Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Checks_AreListedWithAHealthyCount()
|
||||
{
|
||||
var card = Render(Card(report: Report(("localdb", "Healthy", null), ("akka-cluster", "Unhealthy", null))));
|
||||
|
||||
Assert.Contains("1 / 2 healthy", card.Markup, StringComparison.Ordinal);
|
||||
Assert.Equal(2, card.FindAll(".check-row").Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClusterData_IsRenderedAsRawEvidence()
|
||||
{
|
||||
var card = Render(Card(report: Report(("akka-cluster", "Healthy", "akka.tcp://scadabridge@site-a-a:8082"))));
|
||||
|
||||
var data = card.Find(".check-data");
|
||||
Assert.Contains("akka.tcp://scadabridge@site-a-a:8082", data.TextContent, StringComparison.Ordinal);
|
||||
Assert.Contains("leader", data.TextContent, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClusterData_FromAnInstanceWithoutIt_IsOmittedEntirely()
|
||||
{
|
||||
// Every pre-0.2.0 node, and every check that publishes no data. The card must simply not
|
||||
// show the line rather than showing an empty one.
|
||||
var card = Render(Card(report: Report(("localdb", "Healthy", null))));
|
||||
|
||||
Assert.Empty(card.FindAll(".check-data"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClusterData_IsHtmlEscaped()
|
||||
{
|
||||
// The value is remote input from another application. Operationally trusted, but a health
|
||||
// check that ever emits markup must not be able to inject it into this page.
|
||||
var card = Render(Card(report: Report(("akka-cluster", "Healthy", "<img src=x onerror=alert(1)>"))));
|
||||
|
||||
var data = card.Find(".check-data");
|
||||
Assert.Empty(data.QuerySelectorAll("img"));
|
||||
Assert.Contains("<img src=x onerror=alert(1)>", data.TextContent, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManagementLink_OpensInANewTabWithNoopener()
|
||||
{
|
||||
var link = Render(Card(managementUrl: "http://site-a-a:9000/")).Find(".panel-foot a");
|
||||
|
||||
Assert.Equal("http://site-a-a:9000/", link.GetAttribute("href"));
|
||||
Assert.Equal("_blank", link.GetAttribute("target"));
|
||||
Assert.Equal("noopener", link.GetAttribute("rel"));
|
||||
Assert.Contains("AdminUI", link.TextContent, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NoManagementUrl_RendersNoDeadLink()
|
||||
{
|
||||
var card = Render(Card(managementUrl: null));
|
||||
|
||||
Assert.Empty(card.FindAll(".panel-foot a"));
|
||||
Assert.Contains("no management link", card.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(InstanceStatus.Up, ActiveState.Active, "ready 200 · active 200")]
|
||||
[InlineData(InstanceStatus.Up, ActiveState.Standby, "ready 200 · active 503")]
|
||||
[InlineData(InstanceStatus.Down, ActiveState.NotApplicable, "ready 503")]
|
||||
[InlineData(InstanceStatus.Unreachable, ActiveState.NotApplicable, "ready fail")]
|
||||
public void RawSignalLine_ShowsWhatWasActuallyReceived(
|
||||
InstanceStatus status, ActiveState active, string expected)
|
||||
{
|
||||
// The footer is the escape hatch from every derived label above it: when the dashboard and
|
||||
// a curl disagree, this line is what reconciles them.
|
||||
Assert.Contains(
|
||||
expected,
|
||||
Render(Card(status, active)).Find("[data-testid=raw-signals]").TextContent,
|
||||
StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProbeError_IsSurfacedOnTheCard()
|
||||
{
|
||||
var card = Render(Card(InstanceStatus.Unreachable, error: "Connection refused"));
|
||||
|
||||
Assert.Contains("Connection refused", card.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Host_IsShownWithoutTheScheme()
|
||||
{
|
||||
Assert.Contains("site-a-a:8084", Render(Card()).Find(".inst-host").TextContent, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(4, "4 s ago")]
|
||||
[InlineData(90, "2 min ago")]
|
||||
[InlineData(7200, "2 h ago")]
|
||||
[InlineData(172800, "2 d ago")]
|
||||
public void Relative_ScalesTheUnit(int seconds, string expected)
|
||||
{
|
||||
Assert.Equal(expected, InstanceCard.Relative(TimeSpan.FromSeconds(seconds)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Relative_NegativeAge_ReadsAsNow_NotAsTheFuture()
|
||||
{
|
||||
// Clock skew between this host and a probed one is routine; "-3 s ago" is nonsense.
|
||||
Assert.Equal("0 s ago", InstanceCard.Relative(TimeSpan.FromSeconds(-3)));
|
||||
}
|
||||
|
||||
private static ZbHealthReport Report(params (string Name, string Status, string? Leader)[] entries) =>
|
||||
System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(HealthBody.Ready("Healthy", entries))!;
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Leader aggregation and the split-brain flag. The disagreement rule is the one output an operator
|
||||
/// would be woken by, so both directions are pinned: it must fire when two members genuinely claim
|
||||
/// different leaders, and it must NOT fire for the ordinary cases that superficially resemble that
|
||||
/// — a node that stopped answering, a node running a pre-0.2.0 Health package.
|
||||
/// </summary>
|
||||
public class LeaderResolverTests
|
||||
{
|
||||
private const string LeaderA = "akka.tcp://scadabridge@site-a-a:8082";
|
||||
private const string LeaderB = "akka.tcp://scadabridge@site-a-b:8082";
|
||||
|
||||
[Fact]
|
||||
public void Resolve_AgreeingPair_NamesTheLeaderInstance()
|
||||
{
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderA),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Equal("site-a", group.Name);
|
||||
Assert.Equal("site-a-a", group.LeaderInstance);
|
||||
Assert.Equal(LeaderA, group.LeaderAddress);
|
||||
Assert.False(group.Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_MembersClaimDifferentLeaders_FlagsDisagreement()
|
||||
{
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.True(group.Disagreement);
|
||||
Assert.Equal([LeaderA, LeaderB], group.ReportedLeaders);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(InstanceStatus.Down)]
|
||||
[InlineData(InstanceStatus.Unreachable)]
|
||||
public void Resolve_FailedMemberDoesNotVote_SoAFailoverIsNotMisreadAsSplitBrain(InstanceStatus failed)
|
||||
{
|
||||
// Mid-failover the departed node's last body still names the old leader. Counting it would
|
||||
// raise a split-brain warning during the most normal event in a redundant pair's life.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: failed, leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.False(group.Disagreement);
|
||||
Assert.Equal("site-a-b", group.LeaderInstance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_DegradedMemberStillVotes()
|
||||
{
|
||||
// Degraded means answering with a full cluster view — its opinion on the leader is current.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: InstanceStatus.Degraded, leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
|
||||
]);
|
||||
|
||||
Assert.True(Assert.Single(groups).Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_NoMemberPublishesLeaderData_YieldsNoLeaderAndNoWarning()
|
||||
{
|
||||
// The whole-fleet-on-Health-0.1.0 case: no data key anywhere. It must degrade to "leader
|
||||
// unknown", never to a false split-brain warning.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084"),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Null(group.LeaderAddress);
|
||||
Assert.Null(group.LeaderInstance);
|
||||
Assert.False(group.Disagreement);
|
||||
Assert.Empty(group.ReportedLeaders);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_OneMemberOnOldHealthPackage_UsesTheOneThatDoesReport()
|
||||
{
|
||||
// A partly-bumped fleet is the expected state during rollout, and one silent member is not
|
||||
// a disagreement.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Equal("site-a-a", group.LeaderInstance);
|
||||
Assert.False(group.Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_UnresolvableAddress_KeepsTheRawAddress()
|
||||
{
|
||||
// A leader on a host that is not in the registry (a node nobody added) still tells the
|
||||
// operator something useful — showing nothing would hide the discovery.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: "akka.tcp://scadabridge@ghost:8082"),
|
||||
]);
|
||||
|
||||
var group = Assert.Single(groups);
|
||||
Assert.Null(group.LeaderInstance);
|
||||
Assert.Equal("akka.tcp://scadabridge@ghost:8082", group.LeaderAddress);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_MatchesOnHostOnly_BecauseAkkaAndHttpPortsDiffer()
|
||||
{
|
||||
// The registry knows the HTTP port (8084); the leader address carries the Akka remoting
|
||||
// port (8082). Matching on authority instead of host would never resolve anything.
|
||||
var name = LeaderResolver.ResolveToInstanceName(
|
||||
LeaderA,
|
||||
[Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084")]);
|
||||
|
||||
Assert.Equal("site-a-a", name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_GroupWithoutActiveRole_IsNotAGroupAtAll()
|
||||
{
|
||||
// Instances grouped only for layout have no leader; a chip there would invent a concept.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("gw-1", "gateways", "http://gw-1:5120", hasActiveRole: false),
|
||||
Snapshots.Instance("gw-2", "gateways", "http://gw-2:5120", hasActiveRole: false),
|
||||
]);
|
||||
|
||||
Assert.Empty(groups);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_UngroupedInstances_ProduceNoGroups()
|
||||
{
|
||||
Assert.Empty(LeaderResolver.Resolve([Snapshots.Instance("solo")]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolve_SeparateGroups_AreScopedIndependently()
|
||||
{
|
||||
// Two site pairs, each with its own leader: per-Cluster election is the family's actual
|
||||
// topology, so a leader from one group must never be attributed to another.
|
||||
var groups = LeaderResolver.Resolve(
|
||||
[
|
||||
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
|
||||
Snapshots.Instance("site-b-a", "site-b", "http://site-b-a:8084", leader: "akka.tcp://scadabridge@site-b-a:8082"),
|
||||
]);
|
||||
|
||||
Assert.Equal(2, groups.Count);
|
||||
Assert.Equal("site-a-a", groups[0].LeaderInstance);
|
||||
Assert.Equal("site-b-a", groups[1].LeaderInstance);
|
||||
Assert.All(groups, g => Assert.False(g.Disagreement));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("akka.tcp://scadabridge@site-a-a:8082", "site-a-a")]
|
||||
[InlineData("akka.tcp://otopcua@10.100.0.35:4053", "10.100.0.35")]
|
||||
[InlineData("akka.tcp://sys@host", "host")]
|
||||
[InlineData("not-an-address", null)]
|
||||
[InlineData("", null)]
|
||||
[InlineData(null, null)]
|
||||
public void HostOf_ParsesAkkaAddresses(string? address, string? expected)
|
||||
{
|
||||
Assert.Equal(expected, LeaderResolver.HostOf(address));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
using Bunit;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The page as a whole: the KPI strip an operator scans first, the group headers that carry the
|
||||
/// split-brain warning, and the pause control — which has to freeze the view without pretending
|
||||
/// the view is still live.
|
||||
/// </summary>
|
||||
public class OverviewPageTests : TestContext
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private readonly OverviewSnapshotStore _store = new();
|
||||
private readonly FakeTimeProvider _time = new(Now);
|
||||
|
||||
public OverviewPageTests()
|
||||
{
|
||||
Services.AddSingleton(_store);
|
||||
Services.AddSingleton<TimeProvider>(_time);
|
||||
}
|
||||
|
||||
private static InstanceSnapshot Node(
|
||||
string name,
|
||||
InstanceStatus? status,
|
||||
string? group = null,
|
||||
string? leader = null,
|
||||
DateTimeOffset? lastPolled = null) =>
|
||||
new()
|
||||
{
|
||||
Application = "ScadaBridge",
|
||||
Instance = name,
|
||||
Group = group,
|
||||
BaseUrl = $"http://{name}:8084",
|
||||
HasActiveRole = group is not null,
|
||||
Status = status,
|
||||
LastPolledUtc = lastPolled ?? Now,
|
||||
LastReachableUtc = lastPolled ?? Now,
|
||||
StaleAfter = TimeSpan.FromSeconds(45),
|
||||
Report = leader is null
|
||||
? null
|
||||
: System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(
|
||||
HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader))),
|
||||
};
|
||||
|
||||
private static OverviewSnapshot Snapshot(params InstanceSnapshot[] instances)
|
||||
{
|
||||
var applications = instances
|
||||
.GroupBy(i => i.Application, StringComparer.Ordinal)
|
||||
.Select(g => new ApplicationSnapshot(g.Key, g.ToList(), LeaderResolver.Resolve(g.ToList())))
|
||||
.ToList();
|
||||
|
||||
return new OverviewSnapshot(Now, applications);
|
||||
}
|
||||
|
||||
private IRenderedComponent<Components.Pages.Overview> RenderPage() =>
|
||||
RenderComponent<Components.Pages.Overview>();
|
||||
|
||||
[Fact]
|
||||
public void EmptyStore_RendersAnExplicitEmptyState()
|
||||
{
|
||||
// Before the first sweep. It must say so rather than render a KPI strip full of zeroes,
|
||||
// which would read as "the whole fleet is registered and nothing is up".
|
||||
var page = RenderPage();
|
||||
|
||||
Assert.NotNull(page.Find("[data-testid=empty-registry]"));
|
||||
Assert.Empty(page.FindAll(".agg-grid"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void KpiStrip_CountsEachStatus()
|
||||
{
|
||||
_store.Publish(Snapshot(
|
||||
Node("a", InstanceStatus.Up),
|
||||
Node("b", InstanceStatus.Up),
|
||||
Node("c", InstanceStatus.Degraded),
|
||||
Node("d", InstanceStatus.Down),
|
||||
Node("e", InstanceStatus.Unreachable)));
|
||||
|
||||
var page = RenderPage();
|
||||
|
||||
Assert.Equal("2", page.Find("[data-testid=kpi-up]").TextContent);
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-degraded]").TextContent);
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-down]").TextContent);
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-unreachable]").TextContent);
|
||||
Assert.Equal("0", page.Find("[data-testid=kpi-stale]").TextContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleInstances_CountAsStaleAndNotAlsoAsUp()
|
||||
{
|
||||
// Double-counting would make the KPI totals exceed the instance count and would let a
|
||||
// frozen dashboard keep reporting a healthy fleet.
|
||||
_store.Publish(Snapshot(
|
||||
Node("a", InstanceStatus.Up),
|
||||
Node("b", InstanceStatus.Up, lastPolled: Now.AddMinutes(-5))));
|
||||
|
||||
var page = RenderPage();
|
||||
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-up]").TextContent);
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-stale]").TextContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupHeader_NamesTheLeader()
|
||||
{
|
||||
const string leader = "akka.tcp://scadabridge@a:8082";
|
||||
_store.Publish(Snapshot(
|
||||
Node("a", InstanceStatus.Up, "site-a", leader),
|
||||
Node("b", InstanceStatus.Up, "site-a", leader)));
|
||||
|
||||
Assert.Contains("Leader · a", RenderPage().Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupHeader_WarnsOnDisagreement()
|
||||
{
|
||||
// The one output on this page that means "go look right now".
|
||||
_store.Publish(Snapshot(
|
||||
Node("a", InstanceStatus.Up, "site-a", "akka.tcp://scadabridge@a:8082"),
|
||||
Node("b", InstanceStatus.Up, "site-a", "akka.tcp://scadabridge@b:8082")));
|
||||
|
||||
Assert.Contains("Split brain", RenderPage().Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AgreeingGroup_ShowsNoWarning()
|
||||
{
|
||||
const string leader = "akka.tcp://scadabridge@a:8082";
|
||||
_store.Publish(Snapshot(
|
||||
Node("a", InstanceStatus.Up, "site-a", leader),
|
||||
Node("b", InstanceStatus.Up, "site-a", leader)));
|
||||
|
||||
Assert.DoesNotContain("Split brain", RenderPage().Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PublishedSnapshot_ReRendersThePage()
|
||||
{
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Up)));
|
||||
var page = RenderPage();
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-up]").TextContent);
|
||||
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Down)));
|
||||
|
||||
page.WaitForAssertion(() => Assert.Equal("0", page.Find("[data-testid=kpi-up]").TextContent));
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-down]").TextContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Paused_StopsAdoptingNewSnapshots()
|
||||
{
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Up)));
|
||||
var page = RenderPage();
|
||||
|
||||
page.Find("[data-testid=pause-toggle]").Click();
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Down)));
|
||||
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-up]").TextContent);
|
||||
Assert.Contains("Resume auto-refresh", page.Markup, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resumed_AdoptsWhateverTheStoreHoldsNow()
|
||||
{
|
||||
// Resume must not replay the snapshots missed while paused — it must jump to current.
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Up)));
|
||||
var page = RenderPage();
|
||||
page.Find("[data-testid=pause-toggle]").Click();
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Down)));
|
||||
|
||||
page.Find("[data-testid=pause-toggle]").Click();
|
||||
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-down]").TextContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PausedView_AgesIntoStaleOnItsOwn()
|
||||
{
|
||||
// This is why pause freezes adoption rather than stopping the clock: a paused dashboard
|
||||
// greys out by itself, so it cannot be mistaken for a live one.
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Up)));
|
||||
var page = RenderPage();
|
||||
page.Find("[data-testid=pause-toggle]").Click();
|
||||
|
||||
_time.Advance(TimeSpan.FromMinutes(5));
|
||||
page.Render();
|
||||
|
||||
Assert.Equal("1", page.Find("[data-testid=kpi-stale]").TextContent);
|
||||
Assert.Equal("0", page.Find("[data-testid=kpi-up]").TextContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Disposal_UnsubscribesFromTheStore()
|
||||
{
|
||||
// A leaked handler would call StateHasChanged on a disposed renderer every sweep, for the
|
||||
// life of the process — one leak per page load.
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Up)));
|
||||
var page = RenderPage();
|
||||
|
||||
page.Instance.Dispose();
|
||||
|
||||
var exception = Record.Exception(() => _store.Publish(Snapshot(Node("a", InstanceStatus.Down))));
|
||||
Assert.Null(exception);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplicationSection_CarriesAnAnchorMatchingTheRailLink()
|
||||
{
|
||||
// The rail's application links are fragment anchors; a mismatch makes every one dead.
|
||||
_store.Publish(Snapshot(Node("a", InstanceStatus.Up)));
|
||||
|
||||
var section = RenderPage().Find("[data-testid=application]");
|
||||
|
||||
Assert.Equal(
|
||||
ZB.MOM.WW.Overview.Components.Layout.MainLayout.Slug("ScadaBridge"),
|
||||
section.GetAttribute("id"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OneCardPerRegisteredInstance()
|
||||
{
|
||||
_store.Publish(Snapshot(
|
||||
Node("a", InstanceStatus.Up, "site-a"),
|
||||
Node("b", InstanceStatus.Up, "site-a"),
|
||||
Node("gw", InstanceStatus.Up)));
|
||||
|
||||
Assert.Equal(3, RenderPage().FindAll("[data-testid=instance-card]").Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UngroupedInstances_RenderAfterTheClusterGroups()
|
||||
{
|
||||
_store.Publish(Snapshot(
|
||||
Node("gw", InstanceStatus.Up),
|
||||
Node("a", InstanceStatus.Up, "site-a")));
|
||||
|
||||
// Enumerated, not indexed: bunit 1.40 was built against AngleSharp 1.2 and its
|
||||
// RefreshableElementCollection indexer calls an IHtmlCollection member whose signature
|
||||
// changed by 1.5.2 — the version this project pins for GHSA reasons. LINQ goes through
|
||||
// IEnumerable and is unaffected.
|
||||
var cards = RenderPage()
|
||||
.FindAll("[data-testid=instance-card]")
|
||||
.Select(c => c.GetAttribute("data-instance"))
|
||||
.ToList();
|
||||
|
||||
Assert.Equal(["a", "gw"], cards);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Sweep behaviour, driven directly rather than through the timer: the rules worth pinning are what
|
||||
/// a sweep probes, what it does with the answers, and when it declines to probe at all.
|
||||
/// </summary>
|
||||
public class OverviewPollerServiceTests
|
||||
{
|
||||
private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static OverviewOptions Registry(params ApplicationEntry[] applications) =>
|
||||
new()
|
||||
{
|
||||
PollIntervalSeconds = 10,
|
||||
TimeoutSeconds = 3,
|
||||
StaleAfterSeconds = 45,
|
||||
Applications = applications,
|
||||
};
|
||||
|
||||
private static ApplicationEntry App(string name, params InstanceEntry[] instances) =>
|
||||
new() { Name = name, ManagementLabel = "AdminUI", Instances = instances };
|
||||
|
||||
private static InstanceEntry Node(
|
||||
string name,
|
||||
string baseUrl,
|
||||
string? group = null,
|
||||
bool hasActiveRole = false,
|
||||
int? pollIntervalSeconds = null) =>
|
||||
new()
|
||||
{
|
||||
Name = name,
|
||||
BaseUrl = baseUrl,
|
||||
Group = group,
|
||||
HasActiveRole = hasActiveRole,
|
||||
PollIntervalSeconds = pollIntervalSeconds,
|
||||
};
|
||||
|
||||
private static (OverviewPollerService Poller, OverviewSnapshotStore Store) Build(
|
||||
OverviewOptions options,
|
||||
HttpMessageHandler handler,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
var store = new OverviewSnapshotStore();
|
||||
var poller = new OverviewPollerService(
|
||||
Options.Create(options),
|
||||
new StubHttpClientFactory(handler),
|
||||
store,
|
||||
timeProvider,
|
||||
NullLogger<OverviewPollerService>.Instance);
|
||||
|
||||
return (poller, store);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_ProbesReadyForEveryInstance()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var snapshot = await poller.SweepAsync();
|
||||
|
||||
Assert.Equal(2, snapshot.AllInstances.Count());
|
||||
Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
|
||||
Assert.Equal(2, handler.Requests.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_TrailingSlashInBaseUrl_DoesNotDoubleTheSeparator()
|
||||
{
|
||||
// "//health/ready" would 404 on every instance — a registry typo class that must not matter.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080/"))), handler, new FakeTimeProvider(Start));
|
||||
|
||||
var snapshot = await poller.SweepAsync();
|
||||
|
||||
Assert.Equal(InstanceStatus.Up, Assert.Single(snapshot.AllInstances).Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_ActiveTierProbedOnlyForInstancesThatHaveTheRole()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Status("http://a:8080/health/active", HttpStatusCode.OK)
|
||||
.Json("http://gw:5120/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true), Node("gw", "http://gw:5120"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var snapshot = await poller.SweepAsync();
|
||||
|
||||
Assert.DoesNotContain("http://gw:5120/health/active", handler.Requests, StringComparer.Ordinal);
|
||||
var byName = snapshot.AllInstances.ToDictionary(i => i.Instance, StringComparer.Ordinal);
|
||||
Assert.Equal(ActiveState.Active, byName["a"].Active);
|
||||
Assert.Equal(ActiveState.NotApplicable, byName["gw"].Active);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_StandbyAnswers503OnActive_AndIsStillUp()
|
||||
{
|
||||
// 503 on the ACTIVE tier is the standby's correct answer and says nothing about health —
|
||||
// conflating the two tiers would paint every healthy standby in the fleet as Down.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Status("http://b:8080/health/active", HttpStatusCode.ServiceUnavailable);
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("b", "http://b:8080", "pair", hasActiveRole: true))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
|
||||
Assert.Equal(InstanceStatus.Up, instance.Status);
|
||||
Assert.Equal(ActiveState.Standby, instance.Active);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_UnreachableInstance_SkipsTheActiveProbe()
|
||||
{
|
||||
// Spending a second full timeout to confirm what the first one established would double
|
||||
// the worst-case sweep duration for every dead node in the registry.
|
||||
var handler = new RoutingHandler().Fails("http://a:8080/health/ready", "Connection refused");
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
await poller.SweepAsync();
|
||||
|
||||
Assert.DoesNotContain("http://a:8080/health/active", handler.Requests, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_SickInstanceStillGetsItsActiveProbe()
|
||||
{
|
||||
// A 503 on ready means the app is talking. Whether the sick node is the ACTIVE half is
|
||||
// precisely what decides whether this is an outage or a standby nobody is using.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy"))
|
||||
.Status("http://a:8080/health/active", HttpStatusCode.OK);
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
|
||||
Assert.Equal(InstanceStatus.Down, instance.Status);
|
||||
Assert.Equal(ActiveState.Active, instance.Active);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_FirstFailureAfterAGoodPoll_IsDampedThenAdopted()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var time = new FakeTimeProvider(Start);
|
||||
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
|
||||
|
||||
Assert.Equal(InstanceStatus.Up, Assert.Single((await poller.SweepAsync()).AllInstances).Status);
|
||||
|
||||
handler.Fails("http://a:8080/health/ready", "Connection refused");
|
||||
|
||||
time.Advance(TimeSpan.FromSeconds(10));
|
||||
var damped = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
Assert.Equal(InstanceStatus.Up, damped.Status);
|
||||
Assert.Equal(1, damped.ConsecutiveFailures);
|
||||
|
||||
time.Advance(TimeSpan.FromSeconds(10));
|
||||
var adopted = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
Assert.Equal(InstanceStatus.Unreachable, adopted.Status);
|
||||
Assert.Equal(2, adopted.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_LastReachableSurvivesAFailedPoll()
|
||||
{
|
||||
// "last seen" must keep pointing at the last real contact, not reset to null the moment
|
||||
// the instance goes away — that timestamp is how an operator judges how long it has been out.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var time = new FakeTimeProvider(Start);
|
||||
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
|
||||
|
||||
await poller.SweepAsync();
|
||||
handler.Fails("http://a:8080/health/ready", "Connection refused");
|
||||
time.Advance(TimeSpan.FromSeconds(10));
|
||||
|
||||
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
|
||||
Assert.Equal(Start, instance.LastReachableUtc);
|
||||
Assert.Equal(Start.AddSeconds(10), instance.LastPolledUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_BeforeAnInstanceIsDue_LeavesItAlone()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var time = new FakeTimeProvider(Start);
|
||||
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
|
||||
|
||||
await poller.SweepAsync();
|
||||
time.Advance(TimeSpan.FromSeconds(1));
|
||||
await poller.SweepAsync();
|
||||
|
||||
Assert.Single(handler.Requests);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_PerInstanceInterval_IsHonoured()
|
||||
{
|
||||
// The override is only real if a slow instance actually skips ticks while a fast one runs.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://fast:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Json("http://slow:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var time = new FakeTimeProvider(Start);
|
||||
var (poller, _) = Build(
|
||||
Registry(App(
|
||||
"App",
|
||||
Node("fast", "http://fast:8080", pollIntervalSeconds: 5),
|
||||
Node("slow", "http://slow:8080", pollIntervalSeconds: 30))),
|
||||
handler,
|
||||
time);
|
||||
|
||||
for (var i = 0; i < 4; i++)
|
||||
{
|
||||
await poller.SweepAsync();
|
||||
time.Advance(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
Assert.Equal(4, handler.Requests.Count(r => r.Contains("fast", StringComparison.Ordinal)));
|
||||
Assert.Equal(1, handler.Requests.Count(r => r.Contains("slow", StringComparison.Ordinal)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_DueCheckToleratesAnEarlyTick()
|
||||
{
|
||||
// A tick arriving a few ms early must not push the instance to half its configured rate.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var time = new FakeTimeProvider(Start);
|
||||
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
|
||||
|
||||
await poller.SweepAsync();
|
||||
time.Advance(TimeSpan.FromMilliseconds(9_900));
|
||||
await poller.SweepAsync();
|
||||
|
||||
Assert.Equal(2, handler.Requests.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_PublishesToTheStore()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
|
||||
OverviewSnapshot? notified = null;
|
||||
store.Changed += s => notified = s;
|
||||
|
||||
var returned = await poller.SweepAsync();
|
||||
|
||||
Assert.Same(returned, store.Current);
|
||||
Assert.Same(returned, notified);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_CarriesRegistryMetadataOntoEveryCard()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Status("http://a:8080/health/active", HttpStatusCode.OK);
|
||||
var application = App("ScadaBridge", Node("central-a", "http://a:8080", "central", hasActiveRole: true));
|
||||
application.Instances[0].ManagementUrl = "http://a:9000/";
|
||||
var (poller, _) = Build(Registry(application), handler, new FakeTimeProvider(Start));
|
||||
|
||||
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
|
||||
Assert.Equal("ScadaBridge", instance.Application);
|
||||
Assert.Equal("central", instance.Group);
|
||||
Assert.Equal("AdminUI", instance.ManagementLabel);
|
||||
Assert.Equal("http://a:9000/", instance.ManagementUrl);
|
||||
Assert.Equal(TimeSpan.FromSeconds(45), instance.StaleAfter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_ResolvesGroupLeadersIntoTheSnapshot()
|
||||
{
|
||||
const string leader = "akka.tcp://scadabridge@a:8082";
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader)))
|
||||
.Status("http://a:8084/health/active", HttpStatusCode.OK)
|
||||
.Json("http://b:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader)))
|
||||
.Status("http://b:8084/health/active", HttpStatusCode.ServiceUnavailable);
|
||||
var (poller, _) = Build(
|
||||
Registry(App(
|
||||
"ScadaBridge",
|
||||
Node("site-a-a", "http://a:8084", "site-a", hasActiveRole: true),
|
||||
Node("site-a-b", "http://b:8084", "site-a", hasActiveRole: true))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var application = Assert.Single((await poller.SweepAsync()).Applications);
|
||||
var group = Assert.Single(application.Groups);
|
||||
|
||||
Assert.Equal("site-a", group.Name);
|
||||
Assert.Equal("site-a-a", group.LeaderInstance);
|
||||
Assert.False(group.Disagreement);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_WorstStatusRollsUpAcrossTheApplication()
|
||||
{
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Json("http://b:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy"));
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var application = Assert.Single((await poller.SweepAsync()).Applications);
|
||||
|
||||
Assert.Equal(InstanceStatus.Down, application.Worst);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_OneSlowInstanceDoesNotDelayTheOthers()
|
||||
{
|
||||
// The fan-out must be genuinely parallel: a serial sweep over a fleet with several dead
|
||||
// nodes would take timeout × dead-node-count and starve the whole dashboard.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var options = Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080")));
|
||||
var gate = new SemaphoreSlim(0);
|
||||
var (poller, _) = Build(options, new GatedHandler(handler, gate), new FakeTimeProvider(Start));
|
||||
|
||||
var sweep = poller.SweepAsync();
|
||||
Assert.False(sweep.IsCompleted);
|
||||
|
||||
// Both probes must already be in flight; releasing one at a time proves neither waited.
|
||||
gate.Release(2);
|
||||
var snapshot = await sweep.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_UnregisteredInstances_RenderAsPendingNotAsHealthy()
|
||||
{
|
||||
// Between host start and the first probe the registry is already published so the page has
|
||||
// its layout. Those cards must read "pending", never an unmeasured "Up".
|
||||
var handler = new RoutingHandler();
|
||||
var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
|
||||
|
||||
Assert.Empty(store.Current.Applications);
|
||||
|
||||
handler.Fails("http://a:8080/health/ready", "Connection refused");
|
||||
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
|
||||
|
||||
Assert.Equal(InstanceStatus.Unreachable, instance.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PendingSnapshot_IsNeverStale()
|
||||
{
|
||||
// Staleness on a card that has never been polled would be meaningless and alarming.
|
||||
var pending = Snapshots.Instance("a", status: null);
|
||||
|
||||
Assert.False(pending.IsStale(Start.AddYears(1)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Snapshot_GoesStaleOnceItsWindowElapses()
|
||||
{
|
||||
var polled = Snapshots.Instance("a") with { LastPolledUtc = Start, StaleAfter = TimeSpan.FromSeconds(45) };
|
||||
|
||||
Assert.False(polled.IsStale(Start.AddSeconds(45)));
|
||||
Assert.True(polled.IsStale(Start.AddSeconds(46)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_HostShutdown_AbandonsTheSweepInsteadOfRecordingFailures()
|
||||
{
|
||||
// Writing "Unreachable" across the registry on the way out would leave a lie in the store.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
|
||||
using var shutdown = new CancellationTokenSource();
|
||||
await shutdown.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => poller.SweepAsync(shutdown.Token));
|
||||
|
||||
Assert.Empty(store.Current.Applications);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyRegistry_StartsAndStopsWithoutProbing()
|
||||
{
|
||||
var (poller, _) = Build(Registry(), new RoutingHandler(), new FakeTimeProvider(Start));
|
||||
|
||||
await poller.StartAsync(CancellationToken.None);
|
||||
await poller.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>Holds every request until the test releases it, proving the fan-out is concurrent.</summary>
|
||||
private sealed class GatedHandler(RoutingHandler inner, SemaphoreSlim gate) : HttpMessageHandler
|
||||
{
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
return await inner.HandleAsync(request).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The store is the cache-first seam: a page reads it and never probes. These tests pin that it
|
||||
/// always has something to serve and that subscribers hear about replacements.
|
||||
/// </summary>
|
||||
public class OverviewSnapshotStoreTests
|
||||
{
|
||||
private static OverviewSnapshot SnapshotAt(int second) =>
|
||||
new(new DateTimeOffset(2026, 7, 24, 12, 0, second, TimeSpan.Zero), []);
|
||||
|
||||
[Fact]
|
||||
public void Current_BeforeAnyPublish_IsEmptyNotNull()
|
||||
{
|
||||
// A page can render before the poller's first sweep finishes; a null here would be a
|
||||
// NullReferenceException on the landing page of a monitoring dashboard.
|
||||
var store = new OverviewSnapshotStore();
|
||||
|
||||
Assert.NotNull(store.Current);
|
||||
Assert.Empty(store.Current.Applications);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Publish_ReplacesCurrent()
|
||||
{
|
||||
var store = new OverviewSnapshotStore();
|
||||
var snapshot = SnapshotAt(1);
|
||||
|
||||
store.Publish(snapshot);
|
||||
|
||||
Assert.Same(snapshot, store.Current);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Publish_NotifiesSubscribersWithTheNewSnapshot()
|
||||
{
|
||||
var store = new OverviewSnapshotStore();
|
||||
OverviewSnapshot? received = null;
|
||||
store.Changed += s => received = s;
|
||||
|
||||
var snapshot = SnapshotAt(2);
|
||||
store.Publish(snapshot);
|
||||
|
||||
Assert.Same(snapshot, received);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Publish_AfterUnsubscribe_DoesNotNotify()
|
||||
{
|
||||
// Blazor circuits come and go; a handler that outlives its component would call
|
||||
// StateHasChanged on a disposed renderer on every sweep.
|
||||
var store = new OverviewSnapshotStore();
|
||||
var calls = 0;
|
||||
void Handler(OverviewSnapshot _) => calls++;
|
||||
|
||||
store.Changed += Handler;
|
||||
store.Publish(SnapshotAt(1));
|
||||
store.Changed -= Handler;
|
||||
store.Publish(SnapshotAt(2));
|
||||
|
||||
Assert.Equal(1, calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Publish_Null_Throws()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() => new OverviewSnapshotStore().Publish(null!));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Current_ReadDuringPublish_ObservesOneWholeSnapshot()
|
||||
{
|
||||
// The tearing guarantee: a reader always sees a complete graph, never a half-swapped one.
|
||||
var store = new OverviewSnapshotStore();
|
||||
var a = SnapshotAt(1);
|
||||
var b = SnapshotAt(2);
|
||||
store.Publish(a);
|
||||
|
||||
using var stop = new CancellationTokenSource();
|
||||
var reader = Task.Run(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
var seen = store.Current;
|
||||
Assert.True(ReferenceEquals(seen, a) || ReferenceEquals(seen, b));
|
||||
}
|
||||
});
|
||||
|
||||
for (var i = 0; i < 1_000; i++)
|
||||
store.Publish(i % 2 == 0 ? b : a);
|
||||
|
||||
await stop.CancelAsync();
|
||||
await reader;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>A clock the test drives by hand.</summary>
|
||||
/// <remarks>
|
||||
/// Only <see cref="GetUtcNow"/> is overridden: the poller's sweep — the part with rules worth
|
||||
/// pinning — is exercised directly, so no test here needs a fake timer.
|
||||
/// </remarks>
|
||||
internal sealed class FakeTimeProvider(DateTimeOffset start) : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _now = start;
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _now;
|
||||
|
||||
public void Advance(TimeSpan by) => _now += by;
|
||||
}
|
||||
|
||||
/// <summary>Serves canned responses per absolute URL and counts what was requested.</summary>
|
||||
internal sealed class RoutingHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Dictionary<string, Func<HttpResponseMessage>> _routes = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public List<string> Requests { get; } = [];
|
||||
|
||||
public RoutingHandler Json(string url, HttpStatusCode status, string body)
|
||||
{
|
||||
_routes[url] = () => new HttpResponseMessage(status)
|
||||
{
|
||||
Content = new StringContent(body, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A status-code-only response with an EMPTY body — what the active tier actually returns when
|
||||
/// it is a bare gate endpoint rather than a full health report.
|
||||
/// </summary>
|
||||
public RoutingHandler Status(string url, HttpStatusCode status)
|
||||
{
|
||||
_routes[url] = () => new HttpResponseMessage(status)
|
||||
{
|
||||
Content = new StringContent(string.Empty, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
return this;
|
||||
}
|
||||
|
||||
public RoutingHandler Fails(string url, string message)
|
||||
{
|
||||
_routes[url] = () => throw new HttpRequestException(message);
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Lets a wrapping handler delegate here without re-entering the protected member.</summary>
|
||||
public Task<HttpResponseMessage> HandleAsync(HttpRequestMessage request) =>
|
||||
SendAsync(request, CancellationToken.None);
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var url = request.RequestUri!.ToString();
|
||||
|
||||
lock (Requests)
|
||||
Requests.Add(url);
|
||||
|
||||
if (!_routes.TryGetValue(url, out var factory))
|
||||
return Task.FromException<HttpResponseMessage>(new HttpRequestException($"no route for {url}"));
|
||||
|
||||
try
|
||||
{
|
||||
return Task.FromResult(factory());
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
return Task.FromException<HttpResponseMessage>(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Hands every named client the same test handler.</summary>
|
||||
internal sealed class StubHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory
|
||||
{
|
||||
public HttpClient CreateClient(string name) =>
|
||||
new(handler, disposeHandler: false) { Timeout = Timeout.InfiniteTimeSpan };
|
||||
}
|
||||
|
||||
/// <summary>Builds canonical health bodies so tests state intent, not JSON.</summary>
|
||||
internal static class HealthBody
|
||||
{
|
||||
public static string Ready(string status, params (string Name, string Status, string? Leader)[] entries)
|
||||
{
|
||||
var payload = new
|
||||
{
|
||||
status,
|
||||
totalDurationMs = 1.5,
|
||||
entries = entries.ToDictionary(
|
||||
e => e.Name,
|
||||
e => (object)new
|
||||
{
|
||||
status = e.Status,
|
||||
description = $"{e.Name} says {e.Status}",
|
||||
durationMs = 0.5,
|
||||
data = e.Leader is null ? null : new Dictionary<string, object> { ["leader"] = e.Leader },
|
||||
},
|
||||
StringComparer.Ordinal),
|
||||
};
|
||||
|
||||
return JsonSerializer.Serialize(payload);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Builds <see cref="InstanceSnapshot"/> values for the pure-logic tests.</summary>
|
||||
internal static class Snapshots
|
||||
{
|
||||
public static InstanceSnapshot Instance(
|
||||
string name,
|
||||
string? group = null,
|
||||
string baseUrl = "http://host:8080",
|
||||
bool hasActiveRole = true,
|
||||
InstanceStatus? status = InstanceStatus.Up,
|
||||
string? leader = null) =>
|
||||
new()
|
||||
{
|
||||
Application = "App",
|
||||
Instance = name,
|
||||
Group = group,
|
||||
BaseUrl = baseUrl,
|
||||
HasActiveRole = hasActiveRole,
|
||||
Status = status,
|
||||
Report = leader is null ? null : ReportWithLeader(leader),
|
||||
StaleAfter = TimeSpan.FromSeconds(45),
|
||||
};
|
||||
|
||||
private static ZbHealthReport ReportWithLeader(string leader)
|
||||
{
|
||||
var body = HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader));
|
||||
return JsonSerializer.Deserialize<ZbHealthReport>(body)!;
|
||||
}
|
||||
}
|
||||
@@ -169,6 +169,59 @@ public class ZbHealthReportClientTests
|
||||
() => client.ProbeAsync("http://node/health/ready", ProbeTimeout, shutdown.Token));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.OK, 200)]
|
||||
[InlineData(HttpStatusCode.ServiceUnavailable, 503)]
|
||||
public async Task ProbeStatus_EmptyBody_IsStillAnAnswer(HttpStatusCode status, int expected)
|
||||
{
|
||||
// The active tier answers with a status code; the body is not part of that contract. An
|
||||
// empty 200/503 is the endpoint telling us the role plainly, and insisting on parseable
|
||||
// JSON here reports every such node as "role unknown".
|
||||
var client = ClientReturning(status, string.Empty);
|
||||
|
||||
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
||||
|
||||
Assert.True(result.Reachable);
|
||||
Assert.Equal(expected, result.StatusCode);
|
||||
Assert.Null(result.Report);
|
||||
Assert.Null(result.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProbeStatus_StillParsesABodyWhenOneIsThere()
|
||||
{
|
||||
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
||||
|
||||
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
||||
|
||||
Assert.Equal("Healthy", result.Report!.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProbeStatus_TransportFailure_IsStillUnreachable()
|
||||
{
|
||||
// Relaxing the body requirement must not relax what "reachable" means.
|
||||
var client = new ZbHealthReportClient(
|
||||
new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused"))));
|
||||
|
||||
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Null(result.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ReadyTierStillDemandsAParseableBody()
|
||||
{
|
||||
// The ready tier's body IS the payload — the per-check list and the leader data come from
|
||||
// it. An empty 200 there means something other than a family app is on that port.
|
||||
var client = ClientReturning(HttpStatusCode.OK, string.Empty);
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
}
|
||||
|
||||
private sealed class StubHandler((HttpStatusCode Status, string Body, string ContentType) response)
|
||||
: HttpMessageHandler
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user