fix(overview): one bad health response no longer stops the dashboard host (scadaproj#29)
ZbHealthReportClient caught only HttpRequestException/OperationCanceledException, so an unknown Content-Type charset (InvalidOperationException from ReadAsStringAsync) escaped the probe, faulted Task.WhenAll and the poller BackgroundService, and StopHost took the dashboard down. - client: final catch-all returns an Unreachable result for that instance - poller: per-target Guarded probe + catch-and-log per sweep in ExecuteAsync - tests: unknown charset (both tiers), arbitrary handler exception, sweep with one bad instance, and the BackgroundService survival loop - code-reviews: Overview-001 Resolved; README regenerated Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -85,7 +85,17 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
{
|
||||
do
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// One bad sweep must not end the loop: a faulted BackgroundService stops the
|
||||
// whole host under the default StopHost policy (Overview-001). The heartbeat is
|
||||
// not stamped, so the dashboard's own health check still notices the gap.
|
||||
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
||||
}
|
||||
@@ -152,7 +162,7 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = target.Timings.Timeout;
|
||||
var ready = await probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken).ConfigureAwait(false);
|
||||
var ready = await Guarded(probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Skip the active probe when nothing answered the ready probe: it would spend a second full
|
||||
// timeout to learn what we already know, and ObserveActive maps a missing probe to Unknown
|
||||
@@ -168,7 +178,7 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
// not exist. So probe unconditionally and let ObserveActive stay role-gated below.
|
||||
HealthProbeResult? active = null;
|
||||
if (ready.Reachable)
|
||||
active = await probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken).ConfigureAwait(false);
|
||||
active = await Guarded(probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var observed = StatusDerivation.Observe(ready);
|
||||
var (status, failures) = StatusDerivation.Damp(
|
||||
@@ -190,6 +200,23 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Defence in depth behind <see cref="ZbHealthReportClient"/>'s own catch-all: a probe that
|
||||
/// throws anything but a shutdown cancellation becomes an unreachable result for THIS instance
|
||||
/// only, instead of faulting the sweep's <c>Task.WhenAll</c> (Overview-001).
|
||||
/// </summary>
|
||||
private static async Task<HealthProbeResult> Guarded(Task<HealthProbeResult> probe, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await probe.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested))
|
||||
{
|
||||
return new HealthProbeResult(false, null, null, TimeSpan.Zero, $"probe failed: {ex.GetType().Name}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private OverviewSnapshot BuildSnapshot(DateTimeOffset now)
|
||||
{
|
||||
var applications = _targets
|
||||
|
||||
@@ -48,7 +48,10 @@ public sealed class ZbHealthReportClient
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
/// <returns>
|
||||
/// The probe result. This never throws for anything the remote end does — only for a
|
||||
/// cancellation of <paramref name="cancellationToken"/> (host shutdown).
|
||||
/// </returns>
|
||||
public Task<HealthProbeResult> ProbeAsync(
|
||||
string url,
|
||||
TimeSpan timeout,
|
||||
@@ -62,7 +65,10 @@ public sealed class ZbHealthReportClient
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
/// <returns>
|
||||
/// The probe result. This never throws for anything the remote end does — only for a
|
||||
/// cancellation of <paramref name="cancellationToken"/> (host shutdown).
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// For the active tier the status code IS the answer — 200 means active, 503 means standby —
|
||||
/// and the body carries nothing the dashboard uses. Demanding a parseable body here would
|
||||
@@ -89,6 +95,7 @@ public sealed class ZbHealthReportClient
|
||||
// timed-out instance.
|
||||
using var timeoutSource = new CancellationTokenSource(timeout);
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
int? statusCode = null;
|
||||
|
||||
try
|
||||
{
|
||||
@@ -96,8 +103,8 @@ public sealed class ZbHealthReportClient
|
||||
.GetAsync(url, HttpCompletionOption.ResponseContentRead, linked.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
statusCode = (int)response.StatusCode;
|
||||
var body = await response.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false);
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
ZbHealthReport? report;
|
||||
try
|
||||
@@ -132,6 +139,14 @@ public sealed class ZbHealthReportClient
|
||||
{
|
||||
return Failed(startedAt, null, ex.Message);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Anything else a remote answer can provoke — e.g. InvalidOperationException from
|
||||
// ReadAsStringAsync when the Content-Type names an unknown charset — is still a bad
|
||||
// answer from ONE instance. Letting it escape faulted the poller BackgroundService and
|
||||
// the host's StopHost policy took the whole dashboard down (Overview-001).
|
||||
return Failed(startedAt, statusCode, $"unreadable response: {ex.GetType().Name}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static HealthProbeResult Failed(long startedAt, int? statusCode, string error) =>
|
||||
|
||||
Reference in New Issue
Block a user