Files
scadaproj/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZbHealthReportClientTests.cs
T
Joseph Doherty c25acd6093 fix(overview): one bad health response no longer stops the dashboard host (scadaproj#29)
ZbHealthReportClient caught only HttpRequestException/OperationCanceledException,
so an unknown Content-Type charset (InvalidOperationException from
ReadAsStringAsync) escaped the probe, faulted Task.WhenAll and the poller
BackgroundService, and StopHost took the dashboard down.

- client: final catch-all returns an Unreachable result for that instance
- poller: per-target Guarded probe + catch-and-log per sweep in ExecuteAsync
- tests: unknown charset (both tiers), arbitrary handler exception, sweep with
  one bad instance, and the BackgroundService survival loop
- code-reviews: Overview-001 Resolved; README regenerated

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 14:42:23 -05:00

297 lines
12 KiB
C#

using System.Net;
using System.Text;
using ZB.MOM.WW.Overview.Polling;
namespace ZB.MOM.WW.Overview.Tests;
/// <summary>
/// Golden-payload tests for the health client. The fixtures are hand-written to the exact shape
/// <c>ZbHealthWriter</c> emits — including an entry WITH <c>data</c> (0.2.0+) and one WITHOUT
/// (every check that publishes none, and every pre-0.2.0 node), because tolerating both is what
/// lets the dashboard run against a fleet that is only partly bumped.
/// </summary>
public class ZbHealthReportClientTests
{
private const string ReadyHealthyWithData = """
{
"status": "Healthy",
"totalDurationMs": 12.34,
"entries": {
"akka-cluster": {
"status": "Healthy",
"description": "Akka cluster member status: Up",
"durationMs": 1.23,
"data": {
"selfAddress": "akka.tcp://scadabridge@site-a-a:8082",
"selfRoles": ["Site", "site-a"],
"memberCount": 2,
"unreachableCount": 0,
"leader": "akka.tcp://scadabridge@site-a-a:8082"
}
},
"localdb": {
"status": "Healthy",
"description": "Site LocalDb reachable.",
"durationMs": 0.45
}
}
}
""";
private const string ReadyUnhealthyNoData = """
{
"status": "Unhealthy",
"totalDurationMs": 3001.5,
"entries": {
"database": {
"status": "Unhealthy",
"description": null,
"durationMs": 3000.1
}
}
}
""";
private static ZbHealthReportClient ClientReturning(
HttpStatusCode statusCode,
string body,
string contentType = "application/json") =>
new(new HttpClient(new StubHandler((statusCode, body, contentType))));
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(3);
[Fact]
public async Task Probe_HealthyBody_ParsesEnvelopeAndEntries()
{
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
Assert.True(result.Reachable);
Assert.Equal(200, result.StatusCode);
Assert.Equal("Healthy", result.Report!.Status);
Assert.Equal(12.34, result.Report.TotalDurationMs);
Assert.Equal(new[] { "akka-cluster", "localdb" }, result.Report.Entries.Keys.OrderBy(k => k, StringComparer.Ordinal));
Assert.Equal("Site LocalDb reachable.", result.Report.Entries["localdb"].Description);
}
[Fact]
public async Task Probe_EntryWithData_ExposesLeaderAndCounts()
{
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
var entry = result.Report!.Entries["akka-cluster"];
Assert.Equal("akka.tcp://scadabridge@site-a-a:8082", entry.DataString("leader"));
Assert.Equal(2, entry.DataInt("memberCount"));
Assert.Equal(0, entry.DataInt("unreachableCount"));
}
[Fact]
public async Task Probe_EntryWithoutData_ReadsAsNullNotAsAnError()
{
// The 0.1.0-era / no-data case. It must degrade to "no leader shown", never to a parse
// failure — that tolerance is what keeps the dashboard usable mid-rollout.
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
var entry = result.Report!.Entries["localdb"];
Assert.True(result.Reachable);
Assert.Null(entry.Data);
Assert.Null(entry.DataString("leader"));
Assert.Null(entry.DataInt("memberCount"));
}
[Fact]
public async Task Probe_503_IsAParseableAnswer_NotAFailedProbe()
{
// 503 carries a full body naming the failing check — the single most useful payload the
// dashboard receives. Treating it as a transport failure would throw that away.
var client = ClientReturning(HttpStatusCode.ServiceUnavailable, ReadyUnhealthyNoData);
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
Assert.True(result.Reachable);
Assert.Equal(503, result.StatusCode);
Assert.Equal("Unhealthy", result.Report!.Status);
Assert.Null(result.Report.Entries["database"].Description);
}
[Theory]
[InlineData("<html><body>502 Bad Gateway</body></html>")]
[InlineData("not json at all")]
public async Task Probe_NonJsonBody_IsUnreachable(string body)
{
var client = ClientReturning(HttpStatusCode.OK, body, contentType: "text/html");
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
Assert.False(result.Reachable);
Assert.Null(result.Report);
Assert.NotNull(result.Error);
}
[Fact]
public async Task Probe_ConnectionRefused_IsUnreachableAndDoesNotThrow()
{
var client = new ZbHealthReportClient(
new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused"))));
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
Assert.False(result.Reachable);
Assert.Contains("refused", result.Error, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task Probe_Timeout_IsUnreachableAndReportsTheTimeout()
{
var client = new ZbHealthReportClient(new HttpClient(new HangingHandler()));
var result = await client.ProbeAsync("http://node/health/ready", TimeSpan.FromMilliseconds(50));
Assert.False(result.Reachable);
Assert.Contains("timed out", result.Error, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task Probe_HostShutdown_PropagatesCancellation()
{
// A shutdown must NOT be recorded as a timed-out instance — the sweep is being abandoned,
// and writing "Unreachable" for every node on the way out would be a lie in the snapshot.
using var shutdown = new CancellationTokenSource();
await shutdown.CancelAsync();
var client = new ZbHealthReportClient(new HttpClient(new HangingHandler()));
await Assert.ThrowsAnyAsync<OperationCanceledException>(
() => client.ProbeAsync("http://node/health/ready", ProbeTimeout, shutdown.Token));
}
[Theory]
[InlineData(HttpStatusCode.OK, 200)]
[InlineData(HttpStatusCode.ServiceUnavailable, 503)]
public async Task ProbeStatus_EmptyBody_IsStillAnAnswer(HttpStatusCode status, int expected)
{
// The active tier answers with a status code; the body is not part of that contract. An
// empty 200/503 is the endpoint telling us the role plainly, and insisting on parseable
// JSON here reports every such node as "role unknown".
var client = ClientReturning(status, string.Empty);
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
Assert.True(result.Reachable);
Assert.Equal(expected, result.StatusCode);
Assert.Null(result.Report);
Assert.Null(result.Error);
}
[Fact]
public async Task ProbeStatus_StillParsesABodyWhenOneIsThere()
{
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
Assert.Equal("Healthy", result.Report!.Status);
}
[Fact]
public async Task ProbeStatus_TransportFailure_IsStillUnreachable()
{
// Relaxing the body requirement must not relax what "reachable" means.
var client = new ZbHealthReportClient(
new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused"))));
var result = await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
Assert.False(result.Reachable);
Assert.Null(result.StatusCode);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Probe_UnknownCharset_IsUnreachableNotThrown(bool bodyRequired)
{
// Overview-001 (scadaproj#29): ReadAsStringAsync throws InvalidOperationException for a
// Content-Type charset it cannot map to an Encoding. That escaped ProbeAsync, faulted the
// poller BackgroundService and the default StopHost policy took the whole dashboard down.
var client = new ZbHealthReportClient(new HttpClient(new BogusCharsetHandler()));
var result = bodyRequired
? await client.ProbeAsync("http://node/health/ready", ProbeTimeout)
: await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
Assert.False(result.Reachable);
Assert.Equal(200, result.StatusCode);
Assert.NotNull(result.Error);
}
[Fact]
public async Task Probe_ArbitraryHandlerException_IsUnreachableNotThrown()
{
var client = new ZbHealthReportClient(
new HttpClient(new ThrowingHandler(new InvalidOperationException("boom"))));
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
Assert.False(result.Reachable);
Assert.Null(result.StatusCode);
Assert.Contains("boom", result.Error, StringComparison.Ordinal);
}
[Fact]
public async Task Probe_ReadyTierStillDemandsAParseableBody()
{
// The ready tier's body IS the payload — the per-check list and the leader data come from
// it. An empty 200 there means something other than a family app is on that port.
var client = ClientReturning(HttpStatusCode.OK, string.Empty);
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
Assert.False(result.Reachable);
}
private sealed class StubHandler((HttpStatusCode Status, string Body, string ContentType) response)
: HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken) =>
Task.FromResult(new HttpResponseMessage(response.Status)
{
Content = new StringContent(response.Body, Encoding.UTF8, response.ContentType),
});
}
/// <summary>A 200 whose Content-Type names a charset no Encoding maps to.</summary>
internal sealed class BogusCharsetHandler : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken)
{
var content = new ByteArrayContent(Encoding.UTF8.GetBytes("""{"status":"Healthy","entries":{}}"""));
content.Headers.TryAddWithoutValidation("Content-Type", "application/json; charset=bogus-xyz");
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content });
}
}
private sealed class ThrowingHandler(Exception exception) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken) =>
Task.FromException<HttpResponseMessage>(exception);
}
private sealed class HangingHandler : HttpMessageHandler
{
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken)
{
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false);
return new HttpResponseMessage(HttpStatusCode.OK);
}
}
}