fix(overview): one bad health response no longer stops the dashboard host (scadaproj#29)
ZbHealthReportClient caught only HttpRequestException/OperationCanceledException, so an unknown Content-Type charset (InvalidOperationException from ReadAsStringAsync) escaped the probe, faulted Task.WhenAll and the poller BackgroundService, and StopHost took the dashboard down. - client: final catch-all returns an Unreachable result for that instance - poller: per-target Guarded probe + catch-and-log per sweep in ExecuteAsync - tests: unknown charset (both tiers), arbitrary handler exception, sweep with one bad instance, and the BackgroundService survival loop - code-reviews: Overview-001 Resolved; README regenerated Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -85,7 +85,17 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
{
|
||||
do
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// One bad sweep must not end the loop: a faulted BackgroundService stops the
|
||||
// whole host under the default StopHost policy (Overview-001). The heartbeat is
|
||||
// not stamped, so the dashboard's own health check still notices the gap.
|
||||
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
||||
}
|
||||
@@ -152,7 +162,7 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = target.Timings.Timeout;
|
||||
var ready = await probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken).ConfigureAwait(false);
|
||||
var ready = await Guarded(probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Skip the active probe when nothing answered the ready probe: it would spend a second full
|
||||
// timeout to learn what we already know, and ObserveActive maps a missing probe to Unknown
|
||||
@@ -168,7 +178,7 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
// not exist. So probe unconditionally and let ObserveActive stay role-gated below.
|
||||
HealthProbeResult? active = null;
|
||||
if (ready.Reachable)
|
||||
active = await probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken).ConfigureAwait(false);
|
||||
active = await Guarded(probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var observed = StatusDerivation.Observe(ready);
|
||||
var (status, failures) = StatusDerivation.Damp(
|
||||
@@ -190,6 +200,23 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Defence in depth behind <see cref="ZbHealthReportClient"/>'s own catch-all: a probe that
|
||||
/// throws anything but a shutdown cancellation becomes an unreachable result for THIS instance
|
||||
/// only, instead of faulting the sweep's <c>Task.WhenAll</c> (Overview-001).
|
||||
/// </summary>
|
||||
private static async Task<HealthProbeResult> Guarded(Task<HealthProbeResult> probe, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await probe.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested))
|
||||
{
|
||||
return new HealthProbeResult(false, null, null, TimeSpan.Zero, $"probe failed: {ex.GetType().Name}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private OverviewSnapshot BuildSnapshot(DateTimeOffset now)
|
||||
{
|
||||
var applications = _targets
|
||||
|
||||
@@ -48,7 +48,10 @@ public sealed class ZbHealthReportClient
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
/// <returns>
|
||||
/// The probe result. This never throws for anything the remote end does — only for a
|
||||
/// cancellation of <paramref name="cancellationToken"/> (host shutdown).
|
||||
/// </returns>
|
||||
public Task<HealthProbeResult> ProbeAsync(
|
||||
string url,
|
||||
TimeSpan timeout,
|
||||
@@ -62,7 +65,10 @@ public sealed class ZbHealthReportClient
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
/// <returns>
|
||||
/// The probe result. This never throws for anything the remote end does — only for a
|
||||
/// cancellation of <paramref name="cancellationToken"/> (host shutdown).
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// For the active tier the status code IS the answer — 200 means active, 503 means standby —
|
||||
/// and the body carries nothing the dashboard uses. Demanding a parseable body here would
|
||||
@@ -89,6 +95,7 @@ public sealed class ZbHealthReportClient
|
||||
// timed-out instance.
|
||||
using var timeoutSource = new CancellationTokenSource(timeout);
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
int? statusCode = null;
|
||||
|
||||
try
|
||||
{
|
||||
@@ -96,8 +103,8 @@ public sealed class ZbHealthReportClient
|
||||
.GetAsync(url, HttpCompletionOption.ResponseContentRead, linked.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
statusCode = (int)response.StatusCode;
|
||||
var body = await response.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false);
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
ZbHealthReport? report;
|
||||
try
|
||||
@@ -132,6 +139,14 @@ public sealed class ZbHealthReportClient
|
||||
{
|
||||
return Failed(startedAt, null, ex.Message);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Anything else a remote answer can provoke — e.g. InvalidOperationException from
|
||||
// ReadAsStringAsync when the Content-Type names an unknown charset — is still a bad
|
||||
// answer from ONE instance. Letting it escape faulted the poller BackgroundService and
|
||||
// the host's StopHost policy took the whole dashboard down (Overview-001).
|
||||
return Failed(startedAt, statusCode, $"unreadable response: {ex.GetType().Name}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static HealthProbeResult Failed(long startedAt, int? statusCode, string error) =>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Observability;
|
||||
@@ -434,6 +435,76 @@ public class OverviewPollerServiceTests
|
||||
Assert.True(polled.IsStale(Start.AddSeconds(46)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_OneInstanceWithAnUnreadableBody_DoesNotFaultTheSweep()
|
||||
{
|
||||
// Overview-001 (scadaproj#29): one instance answering with an unknown charset faulted the
|
||||
// Task.WhenAll, the BackgroundService and then the host. It must cost that instance only.
|
||||
var routes = new RoutingHandler()
|
||||
.Json("http://good:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Status("http://good:8080/health/active", HttpStatusCode.OK);
|
||||
var handler = new BadCharsetForHost("bad", routes);
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("good", "http://good:8080"), Node("bad", "http://bad:8080"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
var snapshot = await poller.SweepAsync();
|
||||
|
||||
var good = snapshot.AllInstances.Single(i => i.Instance == "good");
|
||||
var bad = snapshot.AllInstances.Single(i => i.Instance == "bad");
|
||||
Assert.Equal(InstanceStatus.Up, good.Status);
|
||||
Assert.Equal(InstanceStatus.Unreachable, bad.Status);
|
||||
Assert.NotNull(bad.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteLoop_SurvivesAnInstanceWithAnUnreadableBody()
|
||||
{
|
||||
// The end-to-end shape of the finding: the BackgroundService must keep running.
|
||||
var routes = new RoutingHandler();
|
||||
var handler = new BadCharsetForHost("bad", routes);
|
||||
var (poller, _) = Build(
|
||||
Registry(App("App", Node("bad", "http://bad:8080"))),
|
||||
handler,
|
||||
new FakeTimeProvider(Start));
|
||||
|
||||
await poller.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
for (var i = 0; i < 100 && handler.Calls == 0; i++)
|
||||
await Task.Delay(20);
|
||||
await Task.Delay(100);
|
||||
|
||||
Assert.True(handler.Calls > 0);
|
||||
Assert.False(poller.ExecuteTask!.IsFaulted, poller.ExecuteTask.Exception?.ToString());
|
||||
}
|
||||
finally
|
||||
{
|
||||
await poller.StopAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Answers every request to <c>host</c> with a 200 carrying an unknown charset.</summary>
|
||||
private sealed class BadCharsetForHost(string host, RoutingHandler inner) : HttpMessageHandler
|
||||
{
|
||||
private int _calls;
|
||||
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!string.Equals(request.RequestUri!.Host, host, StringComparison.OrdinalIgnoreCase))
|
||||
return inner.HandleAsync(request);
|
||||
|
||||
Interlocked.Increment(ref _calls);
|
||||
var content = new ByteArrayContent(Encoding.UTF8.GetBytes(HealthBody.Ready("Healthy")));
|
||||
content.Headers.TryAddWithoutValidation("Content-Type", "application/json; charset=bogus-xyz");
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content });
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_HostShutdown_AbandonsTheSweepInsteadOfRecordingFailures()
|
||||
{
|
||||
|
||||
@@ -210,6 +210,38 @@ public class ZbHealthReportClientTests
|
||||
Assert.Null(result.StatusCode);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public async Task Probe_UnknownCharset_IsUnreachableNotThrown(bool bodyRequired)
|
||||
{
|
||||
// Overview-001 (scadaproj#29): ReadAsStringAsync throws InvalidOperationException for a
|
||||
// Content-Type charset it cannot map to an Encoding. That escaped ProbeAsync, faulted the
|
||||
// poller BackgroundService and the default StopHost policy took the whole dashboard down.
|
||||
var client = new ZbHealthReportClient(new HttpClient(new BogusCharsetHandler()));
|
||||
|
||||
var result = bodyRequired
|
||||
? await client.ProbeAsync("http://node/health/ready", ProbeTimeout)
|
||||
: await client.ProbeStatusAsync("http://node/health/active", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Equal(200, result.StatusCode);
|
||||
Assert.NotNull(result.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ArbitraryHandlerException_IsUnreachableNotThrown()
|
||||
{
|
||||
var client = new ZbHealthReportClient(
|
||||
new HttpClient(new ThrowingHandler(new InvalidOperationException("boom"))));
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Null(result.StatusCode);
|
||||
Assert.Contains("boom", result.Error, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ReadyTierStillDemandsAParseableBody()
|
||||
{
|
||||
@@ -233,6 +265,18 @@ public class ZbHealthReportClientTests
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>A 200 whose Content-Type names a charset no Encoding maps to.</summary>
|
||||
internal sealed class BogusCharsetHandler : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var content = new ByteArrayContent(Encoding.UTF8.GetBytes("""{"status":"Healthy","entries":{}}"""));
|
||||
content.Headers.TryAddWithoutValidation("Content-Type", "application/json; charset=bogus-xyz");
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = content });
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingHandler(Exception exception) : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
|
||||
Reference in New Issue
Block a user