fix(overview): P3 review fixes 003-005 - no redirects on probes, per-target publish so a hung instance cannot stale the rest, probe docs (scadaproj#48)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 14:54:12 -05:00
parent 362ba8d75f
commit 3ef43ca926
9 changed files with 233 additions and 44 deletions
@@ -33,6 +33,7 @@ public sealed class OverviewPollerService : BackgroundService
private readonly OverviewMetrics _metrics;
private readonly PollCycleHeartbeat _heartbeat;
private readonly List<PollTarget> _targets;
private readonly object _publishLock = new();
/// <summary>Initializes a new <see cref="OverviewPollerService"/>.</summary>
/// <param name="options">The validated registry.</param>
@@ -81,21 +82,18 @@ public sealed class OverviewPollerService : BackgroundService
using var timer = new PeriodicTimer(tick, _timeProvider);
// Sweeps are NOT awaited by the tick loop (Overview-004): a hung instance with a long
// timeout override must not delay the next tick for every other instance. A target that is
// still being probed is skipped by later sweeps (see PollTarget.TryClaim), so slow probes
// never pile up, and each probe publishes its own result as soon as it lands.
var pending = new List<Task>();
try
{
do
{
try
{
await SweepAsync(stoppingToken).ConfigureAwait(false);
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
// One bad sweep must not end the loop: a faulted BackgroundService stops the
// whole host under the default StopHost policy (Overview-001). The heartbeat is
// not stamped, so the dashboard's own health check still notices the gap.
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
}
pending.RemoveAll(t => t.IsCompleted);
pending.Add(SweepSafeAsync(stoppingToken));
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
}
@@ -103,6 +101,37 @@ public sealed class OverviewPollerService : BackgroundService
{
// Ordinary shutdown.
}
finally
{
// Probes observe the cancelled token; let them unwind before the host tears down the client.
try
{
await Task.WhenAll(pending).ConfigureAwait(false);
}
catch
{
// SweepSafeAsync already logged; nothing to add at shutdown.
}
}
}
private async Task SweepSafeAsync(CancellationToken stoppingToken)
{
try
{
await SweepAsync(stoppingToken).ConfigureAwait(false);
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
// One bad sweep must not end the loop: a faulted BackgroundService stops the
// whole host under the default StopHost policy (Overview-001). The heartbeat is
// not stamped, so the dashboard's own health check still notices the gap.
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Ordinary shutdown.
}
}
/// <summary>Runs one sweep: probes every due instance and publishes the resulting snapshot.</summary>
@@ -118,7 +147,8 @@ public sealed class OverviewPollerService : BackgroundService
var startedAt = _timeProvider.GetTimestamp();
var now = _timeProvider.GetUtcNow();
var due = _targets.Where(t => IsDue(t, now)).ToList();
// TryClaim last: only a target that is actually going to be probed may be marked in flight.
var due = _targets.Where(t => IsDue(t, now) && t.TryClaim()).ToList();
if (due.Count > 0)
{
@@ -127,12 +157,9 @@ public sealed class OverviewPollerService : BackgroundService
var client = _httpClientFactory.CreateClient(HttpClientName);
var probe = new ZbHealthReportClient(client);
var results = await Task
.WhenAll(due.Select(target => ProbeAsync(probe, target, now, cancellationToken)))
await Task
.WhenAll(due.Select(target => ProbeAndPublishAsync(probe, target, now, cancellationToken)))
.ConfigureAwait(false);
for (var i = 0; i < due.Count; i++)
due[i].Last = results[i];
}
// Re-checked after the fan-out: a shutdown that lands mid-sweep leaves the probed targets
@@ -140,8 +167,12 @@ public sealed class OverviewPollerService : BackgroundService
// misleading picture into the store for as long as the process takes to exit.
cancellationToken.ThrowIfCancellationRequested();
var snapshot = BuildSnapshot(now);
_store.Publish(snapshot);
OverviewSnapshot snapshot;
lock (_publishLock)
{
snapshot = BuildSnapshot(now);
_store.Publish(snapshot);
}
_metrics.RecordSweep(_timeProvider.GetElapsedTime(startedAt));
@@ -152,6 +183,35 @@ public sealed class OverviewPollerService : BackgroundService
return snapshot;
}
/// <summary>
/// Probes one target, then records and publishes its result immediately — the sweep's remaining
/// laggards must not hold back a result that is already in hand (Overview-004).
/// </summary>
private async Task ProbeAndPublishAsync(
ZbHealthReportClient probe,
PollTarget target,
DateTimeOffset now,
CancellationToken cancellationToken)
{
try
{
var result = await ProbeAsync(probe, target, now, cancellationToken).ConfigureAwait(false);
lock (_publishLock)
{
target.Last = result;
// A shutdown mid-probe must not publish a half-probed mix (see SweepAsync).
if (!cancellationToken.IsCancellationRequested)
_store.Publish(BuildSnapshot(now));
}
}
finally
{
target.Release();
}
}
private bool IsDue(PollTarget target, DateTimeOffset now) =>
target.Last?.LastPolledUtc is not { } polled || now - polled + DueTolerance >= target.Timings.PollInterval;
@@ -283,9 +343,17 @@ public sealed class OverviewPollerService : BackgroundService
public required InstanceSnapshot Template { get; init; }
/// <summary>
/// The last published result, or null before the first probe. Written only by the sweep
/// loop after its <c>Task.WhenAll</c> has completed, so it is never touched concurrently.
/// The last published result, or null before the first probe. Written only under the
/// poller's publish lock, by the probe task that holds this target's claim.
/// </summary>
public InstanceSnapshot? Last { get; set; }
private int _inFlight;
/// <summary>Claims the target for one probe; false when a previous probe is still running.</summary>
public bool TryClaim() => Interlocked.CompareExchange(ref _inFlight, 1, 0) == 0;
/// <summary>Releases the claim taken by <see cref="TryClaim"/>.</summary>
public void Release() => Volatile.Write(ref _inFlight, 0);
}
}
@@ -31,7 +31,11 @@ public static class PollingServiceCollectionExtensions
// Health bodies are small and must not be served from anywhere but the instance itself.
client.DefaultRequestHeaders.CacheControl = new() { NoCache = true, NoStore = true };
});
})
// A followed redirect would turn an auth wall's 200 login page into "Active"
// (Overview-003). With redirects off, a 3xx surfaces as its own status code and the
// active tier maps it to Unknown; the ready tier already requires a parseable body.
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { AllowAutoRedirect = false });
services.AddHostedService<OverviewPollerService>();
@@ -5,9 +5,11 @@ namespace ZB.MOM.WW.Overview.Registry;
/// <c>Overview</c> configuration section and validated by <see cref="OverviewOptionsValidator"/>.
/// </summary>
/// <remarks>
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, plus
/// <c>{BaseUrl}/health/active</c> when <see cref="InstanceEntry.HasActiveRole"/>. There is
/// deliberately no per-instance probe-mode switch: one probe model, one status derivation.
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, then
/// <c>{BaseUrl}/health/active</c> whenever ready answered, regardless of
/// <see cref="InstanceEntry.HasActiveRole"/> (single-instance gateways publish their session counts
/// on the active tier). <see cref="InstanceEntry.HasActiveRole"/> only controls whether an
/// Active/Standby badge is derived from the active answer. There is deliberately no per-instance probe-mode switch: one probe model, one status derivation.
/// </remarks>
public sealed class OverviewOptions
{
@@ -75,8 +77,9 @@ public sealed class InstanceEntry
/// <summary>
/// Whether this instance participates in an active/standby pair. False for the single-instance
/// gateways, which have no such concept — their cards show status without an Active badge and
/// are never probed for <c>/health/active</c>.
/// gateways, which have no such concept — their cards show status without an Active badge. The
/// active tier is still probed for them (it carries their session counts); this flag only gates
/// the Active/Standby interpretation.
/// </summary>
public bool HasActiveRole { get; set; }