fix(overview): P3 review fixes 003-005 - no redirects on probes, per-target publish so a hung instance cannot stale the rest, probe docs (scadaproj#48)
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -33,6 +33,7 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
private readonly OverviewMetrics _metrics;
|
||||
private readonly PollCycleHeartbeat _heartbeat;
|
||||
private readonly List<PollTarget> _targets;
|
||||
private readonly object _publishLock = new();
|
||||
|
||||
/// <summary>Initializes a new <see cref="OverviewPollerService"/>.</summary>
|
||||
/// <param name="options">The validated registry.</param>
|
||||
@@ -81,21 +82,18 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
|
||||
using var timer = new PeriodicTimer(tick, _timeProvider);
|
||||
|
||||
// Sweeps are NOT awaited by the tick loop (Overview-004): a hung instance with a long
|
||||
// timeout override must not delay the next tick for every other instance. A target that is
|
||||
// still being probed is skipped by later sweeps (see PollTarget.TryClaim), so slow probes
|
||||
// never pile up, and each probe publishes its own result as soon as it lands.
|
||||
var pending = new List<Task>();
|
||||
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// One bad sweep must not end the loop: a faulted BackgroundService stops the
|
||||
// whole host under the default StopHost policy (Overview-001). The heartbeat is
|
||||
// not stamped, so the dashboard's own health check still notices the gap.
|
||||
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
|
||||
}
|
||||
pending.RemoveAll(t => t.IsCompleted);
|
||||
pending.Add(SweepSafeAsync(stoppingToken));
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
||||
}
|
||||
@@ -103,6 +101,37 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
{
|
||||
// Ordinary shutdown.
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Probes observe the cancelled token; let them unwind before the host tears down the client.
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(pending).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// SweepSafeAsync already logged; nothing to add at shutdown.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SweepSafeAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// One bad sweep must not end the loop: a faulted BackgroundService stops the
|
||||
// whole host under the default StopHost policy (Overview-001). The heartbeat is
|
||||
// not stamped, so the dashboard's own health check still notices the gap.
|
||||
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// Ordinary shutdown.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs one sweep: probes every due instance and publishes the resulting snapshot.</summary>
|
||||
@@ -118,7 +147,8 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
|
||||
var startedAt = _timeProvider.GetTimestamp();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var due = _targets.Where(t => IsDue(t, now)).ToList();
|
||||
// TryClaim last: only a target that is actually going to be probed may be marked in flight.
|
||||
var due = _targets.Where(t => IsDue(t, now) && t.TryClaim()).ToList();
|
||||
|
||||
if (due.Count > 0)
|
||||
{
|
||||
@@ -127,12 +157,9 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
var client = _httpClientFactory.CreateClient(HttpClientName);
|
||||
var probe = new ZbHealthReportClient(client);
|
||||
|
||||
var results = await Task
|
||||
.WhenAll(due.Select(target => ProbeAsync(probe, target, now, cancellationToken)))
|
||||
await Task
|
||||
.WhenAll(due.Select(target => ProbeAndPublishAsync(probe, target, now, cancellationToken)))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
for (var i = 0; i < due.Count; i++)
|
||||
due[i].Last = results[i];
|
||||
}
|
||||
|
||||
// Re-checked after the fan-out: a shutdown that lands mid-sweep leaves the probed targets
|
||||
@@ -140,8 +167,12 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
// misleading picture into the store for as long as the process takes to exit.
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var snapshot = BuildSnapshot(now);
|
||||
_store.Publish(snapshot);
|
||||
OverviewSnapshot snapshot;
|
||||
lock (_publishLock)
|
||||
{
|
||||
snapshot = BuildSnapshot(now);
|
||||
_store.Publish(snapshot);
|
||||
}
|
||||
|
||||
_metrics.RecordSweep(_timeProvider.GetElapsedTime(startedAt));
|
||||
|
||||
@@ -152,6 +183,35 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probes one target, then records and publishes its result immediately — the sweep's remaining
|
||||
/// laggards must not hold back a result that is already in hand (Overview-004).
|
||||
/// </summary>
|
||||
private async Task ProbeAndPublishAsync(
|
||||
ZbHealthReportClient probe,
|
||||
PollTarget target,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await ProbeAsync(probe, target, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
lock (_publishLock)
|
||||
{
|
||||
target.Last = result;
|
||||
|
||||
// A shutdown mid-probe must not publish a half-probed mix (see SweepAsync).
|
||||
if (!cancellationToken.IsCancellationRequested)
|
||||
_store.Publish(BuildSnapshot(now));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
target.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsDue(PollTarget target, DateTimeOffset now) =>
|
||||
target.Last?.LastPolledUtc is not { } polled || now - polled + DueTolerance >= target.Timings.PollInterval;
|
||||
|
||||
@@ -283,9 +343,17 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
public required InstanceSnapshot Template { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The last published result, or null before the first probe. Written only by the sweep
|
||||
/// loop after its <c>Task.WhenAll</c> has completed, so it is never touched concurrently.
|
||||
/// The last published result, or null before the first probe. Written only under the
|
||||
/// poller's publish lock, by the probe task that holds this target's claim.
|
||||
/// </summary>
|
||||
public InstanceSnapshot? Last { get; set; }
|
||||
|
||||
private int _inFlight;
|
||||
|
||||
/// <summary>Claims the target for one probe; false when a previous probe is still running.</summary>
|
||||
public bool TryClaim() => Interlocked.CompareExchange(ref _inFlight, 1, 0) == 0;
|
||||
|
||||
/// <summary>Releases the claim taken by <see cref="TryClaim"/>.</summary>
|
||||
public void Release() => Volatile.Write(ref _inFlight, 0);
|
||||
}
|
||||
}
|
||||
|
||||
+5
-1
@@ -31,7 +31,11 @@ public static class PollingServiceCollectionExtensions
|
||||
|
||||
// Health bodies are small and must not be served from anywhere but the instance itself.
|
||||
client.DefaultRequestHeaders.CacheControl = new() { NoCache = true, NoStore = true };
|
||||
});
|
||||
})
|
||||
// A followed redirect would turn an auth wall's 200 login page into "Active"
|
||||
// (Overview-003). With redirects off, a 3xx surfaces as its own status code and the
|
||||
// active tier maps it to Unknown; the ready tier already requires a parseable body.
|
||||
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { AllowAutoRedirect = false });
|
||||
|
||||
services.AddHostedService<OverviewPollerService>();
|
||||
|
||||
|
||||
@@ -5,9 +5,11 @@ namespace ZB.MOM.WW.Overview.Registry;
|
||||
/// <c>Overview</c> configuration section and validated by <see cref="OverviewOptionsValidator"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, plus
|
||||
/// <c>{BaseUrl}/health/active</c> when <see cref="InstanceEntry.HasActiveRole"/>. There is
|
||||
/// deliberately no per-instance probe-mode switch: one probe model, one status derivation.
|
||||
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, then
|
||||
/// <c>{BaseUrl}/health/active</c> whenever ready answered, regardless of
|
||||
/// <see cref="InstanceEntry.HasActiveRole"/> (single-instance gateways publish their session counts
|
||||
/// on the active tier). <see cref="InstanceEntry.HasActiveRole"/> only controls whether an
|
||||
/// Active/Standby badge is derived from the active answer. There is deliberately no per-instance probe-mode switch: one probe model, one status derivation.
|
||||
/// </remarks>
|
||||
public sealed class OverviewOptions
|
||||
{
|
||||
@@ -75,8 +77,9 @@ public sealed class InstanceEntry
|
||||
|
||||
/// <summary>
|
||||
/// Whether this instance participates in an active/standby pair. False for the single-instance
|
||||
/// gateways, which have no such concept — their cards show status without an Active badge and
|
||||
/// are never probed for <c>/health/active</c>.
|
||||
/// gateways, which have no such concept — their cards show status without an Active badge. The
|
||||
/// active tier is still probed for them (it carries their session counts); this flag only gates
|
||||
/// the Active/Standby interpretation.
|
||||
/// </summary>
|
||||
public bool HasActiveRole { get; set; }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user