5fe7135e2c
Tasks 3.4-3.6 of the overview-dashboard plan. Polling (3.4/3.5): - OverviewPollerService: one timer at the fastest configured cadence, each target polled when its own interval is due, fan-out per sweep, no retries (the next tick is the retry, so damping stays the only place that decides whether a failure is worth showing). - OverviewSnapshotStore: atomic swap of a complete immutable graph plus a change event. Pages read it and never probe — the cache-first requirement. - LeaderResolver: per-group leader from akka-cluster data, with the split-brain flag scoped to members that are actually answering, so an ordinary failover is not misread as a disagreement. UI (3.6): ThemeShell + StatusPill/TechCard composition against the mockup; instance cards carry the status stripe (dashed for Unreachable), the check detail list and the raw ready/active signal line. Two defects found by verification rather than by review: - The active tier answers with a status code and an EMPTY body, but the client demanded parseable JSON on every probe — so a healthy pair rendered as "role unknown" instead of Active/Standby. Split into ProbeAsync (ready tier, body is the payload) and ProbeStatusAsync (active tier, code is the answer). Caught by a live smoke run against two fake health endpoints; the poller tests now serve an empty body so they hold the fix. - SweepAsync published even when cancellation was already requested, leaving a half-probed registry as the store's final state on shutdown. It now checks the token itself rather than relying on the transport to throw. Also: app.UseAntiforgery() is required despite the anonymous-by-design pipeline — AddRazorComponents stamps antiforgery metadata unconditionally and the endpoint middleware hard-fails without it (every page was 500). Chosen over DisableAntiforgery() so a future form is protected by default. 147 tests, 0 warnings. Live-verified end to end against fake endpoints: Up/Unreachable/Active/Standby, leader chip, cluster-data line, KPI counts.
253 lines
9.6 KiB
C#
253 lines
9.6 KiB
C#
using System.Net;
|
|
using System.Text;
|
|
using ZB.MOM.WW.Overview.Polling;
|
|
|
|
namespace ZB.MOM.WW.Overview.Tests;
|
|
|
|
/// <summary>
|
|
/// Golden-payload tests for the health client. The fixtures are hand-written to the exact shape
|
|
/// <c>ZbHealthWriter</c> emits — including an entry WITH <c>data</c> (0.2.0+) and one WITHOUT
|
|
/// (every check that publishes none, and every pre-0.2.0 node), because tolerating both is what
|
|
/// lets the dashboard run against a fleet that is only partly bumped.
|
|
/// </summary>
|
|
public class ZbHealthReportClientTests
|
|
{
|
|
private const string ReadyHealthyWithData = """
|
|
{
|
|
"status": "Healthy",
|
|
"totalDurationMs": 12.34,
|
|
"entries": {
|
|
"akka-cluster": {
|
|
"status": "Healthy",
|
|
"description": "Akka cluster member status: Up",
|
|
"durationMs": 1.23,
|
|
"data": {
|
|
"selfAddress": "akka.tcp://scadabridge@site-a-a:8082",
|
|
"selfRoles": ["Site", "site-a"],
|
|
"memberCount": 2,
|
|
"unreachableCount": 0,
|
|
"leader": "akka.tcp://scadabridge@site-a-a:8082"
|
|
}
|
|
},
|
|
"localdb": {
|
|
"status": "Healthy",
|
|
"description": "Site LocalDb reachable.",
|
|
"durationMs": 0.45
|
|
}
|
|
}
|
|
}
|
|
""";
|
|
|
|
private const string ReadyUnhealthyNoData = """
|
|
{
|
|
"status": "Unhealthy",
|
|
"totalDurationMs": 3001.5,
|
|
"entries": {
|
|
"database": {
|
|
"status": "Unhealthy",
|
|
"description": null,
|
|
"durationMs": 3000.1
|
|
}
|
|
}
|
|
}
|
|
""";
|
|
|
|
private static ZbHealthReportClient ClientReturning(
|
|
HttpStatusCode statusCode,
|
|
string body,
|
|
string contentType = "application/json") =>
|
|
new(new HttpClient(new StubHandler((statusCode, body, contentType))));
|
|
|
|
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(3);
|
|
|
|
[Fact]
|
|
public async Task Probe_HealthyBody_ParsesEnvelopeAndEntries()
|
|
{
|
|
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
|
|
Assert.True(result.Reachable);
|
|
Assert.Equal(200, result.StatusCode);
|
|
Assert.Equal("Healthy", result.Report!.Status);
|
|
Assert.Equal(12.34, result.Report.TotalDurationMs);
|
|
Assert.Equal(new[] { "akka-cluster", "localdb" }, result.Report.Entries.Keys.OrderBy(k => k, StringComparer.Ordinal));
|
|
Assert.Equal("Site LocalDb reachable.", result.Report.Entries["localdb"].Description);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_EntryWithData_ExposesLeaderAndCounts()
|
|
{
|
|
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
var entry = result.Report!.Entries["akka-cluster"];
|
|
|
|
Assert.Equal("akka.tcp://scadabridge@site-a-a:8082", entry.DataString("leader"));
|
|
Assert.Equal(2, entry.DataInt("memberCount"));
|
|
Assert.Equal(0, entry.DataInt("unreachableCount"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_EntryWithoutData_ReadsAsNullNotAsAnError()
|
|
{
|
|
// The 0.1.0-era / no-data case. It must degrade to "no leader shown", never to a parse
|
|
// failure — that tolerance is what keeps the dashboard usable mid-rollout.
|
|
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
var entry = result.Report!.Entries["localdb"];
|
|
|
|
Assert.True(result.Reachable);
|
|
Assert.Null(entry.Data);
|
|
Assert.Null(entry.DataString("leader"));
|
|
Assert.Null(entry.DataInt("memberCount"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_503_IsAParseableAnswer_NotAFailedProbe()
|
|
{
|
|
// 503 carries a full body naming the failing check — the single most useful payload the
|
|
// dashboard receives. Treating it as a transport failure would throw that away.
|
|
var client = ClientReturning(HttpStatusCode.ServiceUnavailable, ReadyUnhealthyNoData);
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
|
|
Assert.True(result.Reachable);
|
|
Assert.Equal(503, result.StatusCode);
|
|
Assert.Equal("Unhealthy", result.Report!.Status);
|
|
Assert.Null(result.Report.Entries["database"].Description);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("<html><body>502 Bad Gateway</body></html>")]
|
|
[InlineData("not json at all")]
|
|
public async Task Probe_NonJsonBody_IsUnreachable(string body)
|
|
{
|
|
var client = ClientReturning(HttpStatusCode.OK, body, contentType: "text/html");
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
|
|
Assert.False(result.Reachable);
|
|
Assert.Null(result.Report);
|
|
Assert.NotNull(result.Error);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_ConnectionRefused_IsUnreachableAndDoesNotThrow()
|
|
{
|
|
var client = new ZbHealthReportClient(
|
|
new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused"))));
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
|
|
Assert.False(result.Reachable);
|
|
Assert.Contains("refused", result.Error, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_Timeout_IsUnreachableAndReportsTheTimeout()
|
|
{
|
|
var client = new ZbHealthReportClient(new HttpClient(new HangingHandler()));
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", TimeSpan.FromMilliseconds(50));
|
|
|
|
Assert.False(result.Reachable);
|
|
Assert.Contains("timed out", result.Error, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_HostShutdown_PropagatesCancellation()
|
|
{
|
|
// A shutdown must NOT be recorded as a timed-out instance — the sweep is being abandoned,
|
|
// and writing "Unreachable" for every node on the way out would be a lie in the snapshot.
|
|
using var shutdown = new CancellationTokenSource();
|
|
await shutdown.CancelAsync();
|
|
var client = new ZbHealthReportClient(new HttpClient(new HangingHandler()));
|
|
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
|
() => client.ProbeAsync("http://node/health/ready", ProbeTimeout, shutdown.Token));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(HttpStatusCode.OK, 200)]
|
|
[InlineData(HttpStatusCode.ServiceUnavailable, 503)]
|
|
public async Task ProbeStatus_EmptyBody_IsStillAnAnswer(HttpStatusCode status, int expected)
|
|
{
|
|
// The active tier answers with a status code; the body is not part of that contract. An
|
|
// empty 200/503 is the endpoint telling us the role plainly, and insisting on parseable
|
|
// JSON here reports every such node as "role unknown".
|
|
var client = ClientReturning(status, string.Empty);
|
|
|
|
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
|
|
|
Assert.True(result.Reachable);
|
|
Assert.Equal(expected, result.StatusCode);
|
|
Assert.Null(result.Report);
|
|
Assert.Null(result.Error);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ProbeStatus_StillParsesABodyWhenOneIsThere()
|
|
{
|
|
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
|
|
|
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
|
|
|
Assert.Equal("Healthy", result.Report!.Status);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ProbeStatus_TransportFailure_IsStillUnreachable()
|
|
{
|
|
// Relaxing the body requirement must not relax what "reachable" means.
|
|
var client = new ZbHealthReportClient(
|
|
new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused"))));
|
|
|
|
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
|
|
|
Assert.False(result.Reachable);
|
|
Assert.Null(result.StatusCode);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Probe_ReadyTierStillDemandsAParseableBody()
|
|
{
|
|
// The ready tier's body IS the payload — the per-check list and the leader data come from
|
|
// it. An empty 200 there means something other than a family app is on that port.
|
|
var client = ClientReturning(HttpStatusCode.OK, string.Empty);
|
|
|
|
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
|
|
|
Assert.False(result.Reachable);
|
|
}
|
|
|
|
private sealed class StubHandler((HttpStatusCode Status, string Body, string ContentType) response)
|
|
: HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request, CancellationToken cancellationToken) =>
|
|
Task.FromResult(new HttpResponseMessage(response.Status)
|
|
{
|
|
Content = new StringContent(response.Body, Encoding.UTF8, response.ContentType),
|
|
});
|
|
}
|
|
|
|
private sealed class ThrowingHandler(Exception exception) : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request, CancellationToken cancellationToken) =>
|
|
Task.FromException<HttpResponseMessage>(exception);
|
|
}
|
|
|
|
private sealed class HangingHandler : HttpMessageHandler
|
|
{
|
|
protected override async Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false);
|
|
return new HttpResponseMessage(HttpStatusCode.OK);
|
|
}
|
|
}
|
|
}
|