fix(overview): one bad health response no longer stops the dashboard host (scadaproj#29)

ZbHealthReportClient caught only HttpRequestException/OperationCanceledException,
so an unknown Content-Type charset (InvalidOperationException from
ReadAsStringAsync) escaped the probe, faulted Task.WhenAll and the poller
BackgroundService, and StopHost took the dashboard down.

- client: final catch-all returns an Unreachable result for that instance
- poller: per-target Guarded probe + catch-and-log per sweep in ExecuteAsync
- tests: unknown charset (both tiers), arbitrary handler exception, sweep with
  one bad instance, and the BackgroundService survival loop
- code-reviews: Overview-001 Resolved; README regenerated

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 14:42:23 -05:00
parent 477245f5db
commit c25acd6093
6 changed files with 172 additions and 17 deletions
@@ -85,7 +85,17 @@ public sealed class OverviewPollerService : BackgroundService
{
do
{
await SweepAsync(stoppingToken).ConfigureAwait(false);
try
{
await SweepAsync(stoppingToken).ConfigureAwait(false);
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
// One bad sweep must not end the loop: a faulted BackgroundService stops the
// whole host under the default StopHost policy (Overview-001). The heartbeat is
// not stamped, so the dashboard's own health check still notices the gap.
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
}
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
}
@@ -152,7 +162,7 @@ public sealed class OverviewPollerService : BackgroundService
CancellationToken cancellationToken)
{
var timeout = target.Timings.Timeout;
var ready = await probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken).ConfigureAwait(false);
var ready = await Guarded(probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken), cancellationToken).ConfigureAwait(false);
// Skip the active probe when nothing answered the ready probe: it would spend a second full
// timeout to learn what we already know, and ObserveActive maps a missing probe to Unknown
@@ -168,7 +178,7 @@ public sealed class OverviewPollerService : BackgroundService
// not exist. So probe unconditionally and let ObserveActive stay role-gated below.
HealthProbeResult? active = null;
if (ready.Reachable)
active = await probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken).ConfigureAwait(false);
active = await Guarded(probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken), cancellationToken).ConfigureAwait(false);
var observed = StatusDerivation.Observe(ready);
var (status, failures) = StatusDerivation.Damp(
@@ -190,6 +200,23 @@ public sealed class OverviewPollerService : BackgroundService
};
}
/// <summary>
/// Defence in depth behind <see cref="ZbHealthReportClient"/>'s own catch-all: a probe that
/// throws anything but a shutdown cancellation becomes an unreachable result for THIS instance
/// only, instead of faulting the sweep's <c>Task.WhenAll</c> (Overview-001).
/// </summary>
private static async Task<HealthProbeResult> Guarded(Task<HealthProbeResult> probe, CancellationToken cancellationToken)
{
try
{
return await probe.ConfigureAwait(false);
}
catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested))
{
return new HealthProbeResult(false, null, null, TimeSpan.Zero, $"probe failed: {ex.GetType().Name}: {ex.Message}");
}
}
private OverviewSnapshot BuildSnapshot(DateTimeOffset now)
{
var applications = _targets
@@ -48,7 +48,10 @@ public sealed class ZbHealthReportClient
/// <param name="url">Absolute URL of a health endpoint.</param>
/// <param name="timeout">Per-request timeout.</param>
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
/// <returns>
/// The probe result. This never throws for anything the remote end does — only for a
/// cancellation of <paramref name="cancellationToken"/> (host shutdown).
/// </returns>
public Task<HealthProbeResult> ProbeAsync(
string url,
TimeSpan timeout,
@@ -62,7 +65,10 @@ public sealed class ZbHealthReportClient
/// <param name="url">Absolute URL of a health endpoint.</param>
/// <param name="timeout">Per-request timeout.</param>
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
/// <returns>
/// The probe result. This never throws for anything the remote end does — only for a
/// cancellation of <paramref name="cancellationToken"/> (host shutdown).
/// </returns>
/// <remarks>
/// For the active tier the status code IS the answer — 200 means active, 503 means standby —
/// and the body carries nothing the dashboard uses. Demanding a parseable body here would
@@ -89,6 +95,7 @@ public sealed class ZbHealthReportClient
// timed-out instance.
using var timeoutSource = new CancellationTokenSource(timeout);
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
int? statusCode = null;
try
{
@@ -96,8 +103,8 @@ public sealed class ZbHealthReportClient
.GetAsync(url, HttpCompletionOption.ResponseContentRead, linked.Token)
.ConfigureAwait(false);
statusCode = (int)response.StatusCode;
var body = await response.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false);
var statusCode = (int)response.StatusCode;
ZbHealthReport? report;
try
@@ -132,6 +139,14 @@ public sealed class ZbHealthReportClient
{
return Failed(startedAt, null, ex.Message);
}
catch (Exception ex)
{
// Anything else a remote answer can provoke — e.g. InvalidOperationException from
// ReadAsStringAsync when the Content-Type names an unknown charset — is still a bad
// answer from ONE instance. Letting it escape faulted the poller BackgroundService and
// the host's StopHost policy took the whole dashboard down (Overview-001).
return Failed(startedAt, statusCode, $"unreadable response: {ex.GetType().Name}: {ex.Message}");
}
}
private static HealthProbeResult Failed(long startedAt, int? statusCode, string error) =>