fix(overview): one bad health response no longer stops the dashboard host (scadaproj#29)
ZbHealthReportClient caught only HttpRequestException/OperationCanceledException, so an unknown Content-Type charset (InvalidOperationException from ReadAsStringAsync) escaped the probe, faulted Task.WhenAll and the poller BackgroundService, and StopHost took the dashboard down. - client: final catch-all returns an Unreachable result for that instance - poller: per-target Guarded probe + catch-and-log per sweep in ExecuteAsync - tests: unknown charset (both tiers), arbitrary handler exception, sweep with one bad instance, and the BackgroundService survival loop - code-reviews: Overview-001 Resolved; README regenerated Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Observability;
|
||||
@@ -434,6 +435,76 @@ public class OverviewPollerServiceTests
|
||||
Assert.True(polled.IsStale(Start.AddSeconds(46)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_OneInstanceWithAnUnreadableBody_DoesNotFaultTheSweep()
|
||||
{
|
||||
// Overview-001 (scadaproj#29): one instance answering with an unknown charset faulted the
|
||||
// Task.WhenAll, the BackgroundService and then the host. It must cost that instance only.
|
||||
var routes = new RoutingHandler()
|
||||
.Json("http://good:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Status("http://good:8080/health/active", HttpStatusCode.OK);
|
||||
var handler = new BadCharsetForHost("bad", routes);
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("good", "http://good:8080"), Node("bad", "http://bad:8080"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var snapshot = await poller.SweepAsync();
|
||||
|
||||
var good = snapshot.AllInstances.Single(i => i.Instance == "good");
|
||||
var bad = snapshot.AllInstances.Single(i => i.Instance == "bad");
|
||||
Assert.Equal(InstanceStatus.Up, good.Status);
|
||||
Assert.Equal(InstanceStatus.Unreachable, bad.Status);
|
||||
Assert.NotNull(bad.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteLoop_SurvivesAnInstanceWithAnUnreadableBody()
|
||||
{
|
||||
// The end-to-end shape of the finding: the BackgroundService must keep running.
|
||||
var routes = new RoutingHandler();
|
||||
var handler = new BadCharsetForHost("bad", routes);
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("bad", "http://bad:8080"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
await poller.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
for (var i = 0; i < 100 && handler.Calls == 0; i++)
|
||||
await Task.Delay(20);
|
||||
await Task.Delay(100);
|
||||
|
||||
Assert.True(handler.Calls > 0);
|
||||
Assert.False(poller.ExecuteTask!.IsFaulted, poller.ExecuteTask.Exception?.ToString());
|
||||
}
|
||||
finally
|
||||
{
|
||||
await poller.StopAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Answers every request to <c>host</c> with a 200 carrying an unknown charset.</summary>
|
||||
private sealed class BadCharsetForHost(string host, RoutingHandler inner) : HttpMessageHandler
|
||||
{
|
||||
private int _calls;
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!string.Equals(request.RequestUri!.Host, host, StringComparison.OrdinalIgnoreCase))
|
||||
return inner.HandleAsync(request);
|
||||
|
||||
Interlocked.Increment(ref _calls);
|
||||
var content = new ByteArrayContent(Encoding.UTF8.GetBytes(HealthBody.Ready("Healthy")));
|
||||
content.Headers.TryAddWithoutValidation("Content-Type", "application/json; charset=bogus-xyz");
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content });
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_HostShutdown_AbandonsTheSweepInsteadOfRecordingFailures()
|
||||
{
|
||||
|
||||
@@ -210,6 +210,38 @@ public class ZbHealthReportClientTests
|
||||
Assert.Null(result.StatusCode);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public async Task Probe_UnknownCharset_IsUnreachableNotThrown(bool bodyRequired)
|
||||
{
|
||||
// Overview-001 (scadaproj#29): ReadAsStringAsync throws InvalidOperationException for a
|
||||
// Content-Type charset it cannot map to an Encoding. That escaped ProbeAsync, faulted the
|
||||
// poller BackgroundService and the default StopHost policy took the whole dashboard down.
|
||||
var client = new ZbHealthReportClient(new HttpClient(new BogusCharsetHandler()));
|
||||
|
||||
var result = bodyRequired
|
||||
? await client.ProbeAsync("http://node/health/ready", ProbeTimeout)
|
||||
: await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Equal(200, result.StatusCode);
|
||||
Assert.NotNull(result.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ArbitraryHandlerException_IsUnreachableNotThrown()
|
||||
{
|
||||
var client = new ZbHealthReportClient(
|
||||
new HttpClient(new ThrowingHandler(new InvalidOperationException("boom"))));
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Null(result.StatusCode);
|
||||
Assert.Contains("boom", result.Error, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ReadyTierStillDemandsAParseableBody()
|
||||
{
|
||||
@@ -233,6 +265,18 @@ public class ZbHealthReportClientTests
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>A 200 whose Content-Type names a charset no Encoding maps to.</summary>
|
||||
internal sealed class BogusCharsetHandler : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var content = new ByteArrayContent(Encoding.UTF8.GetBytes("""{"status":"Healthy","entries":{}}"""));
|
||||
content.Headers.TryAddWithoutValidation("Content-Type", "application/json; charset=bogus-xyz");
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content });
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingHandler(Exception exception) : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
|
||||
Reference in New Issue
Block a user