fix(overview): P3 review fixes 003-005 - no redirects on probes, per-target publish so a hung instance cannot stale the rest, probe docs (scadaproj#48)
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -4,6 +4,13 @@ The hosted family dashboard (Blazor Server). Build/test from this directory: `do
|
||||
|
||||
**Backlog / open work:** lives in Gitea issues on `dohertj2/scadaproj` (`tea issues list --repo dohertj2/scadaproj`) — currently #14 (re-run the two OtOpcUa-blocked Phase 4 acceptance checks). The history below is a record, not a status tracker; file new follow-ups as issues.
|
||||
|
||||
## Probe behaviour (P3 review fixes, scadaproj#48)
|
||||
|
||||
- The probe client does not follow redirects (Overview-003): an auth-wall 302 on `/health/active` maps to Unknown, not Active.
|
||||
- Probes are not gated by the sweep (Overview-004): each target publishes its own result when it lands, and a target still
|
||||
being probed is skipped by later ticks, so a hung instance with a long `TimeoutSeconds` cannot stale the others.
|
||||
- `/health/active` is probed whenever ready answered; `HasActiveRole` only gates the Active/Standby interpretation (Overview-005).
|
||||
|
||||
## Index history (migrated from the scadaproj root CLAUDE.md)
|
||||
|
||||
> Moved verbatim out of the scadaproj root `CLAUDE.md` on 2026-09-03 so it loads only when needed. **This file is now the authoritative place for this history — append new entries here and keep the root index row short.**
|
||||
|
||||
@@ -33,6 +33,7 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
private readonly OverviewMetrics _metrics;
|
||||
private readonly PollCycleHeartbeat _heartbeat;
|
||||
private readonly List<PollTarget> _targets;
|
||||
private readonly object _publishLock = new();
|
||||
|
||||
/// <summary>Initializes a new <see cref="OverviewPollerService"/>.</summary>
|
||||
/// <param name="options">The validated registry.</param>
|
||||
@@ -81,21 +82,18 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
|
||||
using var timer = new PeriodicTimer(tick, _timeProvider);
|
||||
|
||||
// Sweeps are NOT awaited by the tick loop (Overview-004): a hung instance with a long
|
||||
// timeout override must not delay the next tick for every other instance. A target that is
|
||||
// still being probed is skipped by later sweeps (see PollTarget.TryClaim), so slow probes
|
||||
// never pile up, and each probe publishes its own result as soon as it lands.
|
||||
var pending = new List<Task>();
|
||||
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// One bad sweep must not end the loop: a faulted BackgroundService stops the
|
||||
// whole host under the default StopHost policy (Overview-001). The heartbeat is
|
||||
// not stamped, so the dashboard's own health check still notices the gap.
|
||||
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
|
||||
}
|
||||
pending.RemoveAll(t => t.IsCompleted);
|
||||
pending.Add(SweepSafeAsync(stoppingToken));
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
||||
}
|
||||
@@ -103,6 +101,37 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
{
|
||||
// Ordinary shutdown.
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Probes observe the cancelled token; let them unwind before the host tears down the client.
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(pending).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// SweepSafeAsync already logged; nothing to add at shutdown.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SweepSafeAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// One bad sweep must not end the loop: a faulted BackgroundService stops the
|
||||
// whole host under the default StopHost policy (Overview-001). The heartbeat is
|
||||
// not stamped, so the dashboard's own health check still notices the gap.
|
||||
_logger.LogError(ex, "Overview poll sweep failed; retrying on the next tick.");
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// Ordinary shutdown.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs one sweep: probes every due instance and publishes the resulting snapshot.</summary>
|
||||
@@ -118,7 +147,8 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
|
||||
var startedAt = _timeProvider.GetTimestamp();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var due = _targets.Where(t => IsDue(t, now)).ToList();
|
||||
// TryClaim last: only a target that is actually going to be probed may be marked in flight.
|
||||
var due = _targets.Where(t => IsDue(t, now) && t.TryClaim()).ToList();
|
||||
|
||||
if (due.Count > 0)
|
||||
{
|
||||
@@ -127,12 +157,9 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
var client = _httpClientFactory.CreateClient(HttpClientName);
|
||||
var probe = new ZbHealthReportClient(client);
|
||||
|
||||
var results = await Task
|
||||
.WhenAll(due.Select(target => ProbeAsync(probe, target, now, cancellationToken)))
|
||||
await Task
|
||||
.WhenAll(due.Select(target => ProbeAndPublishAsync(probe, target, now, cancellationToken)))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
for (var i = 0; i < due.Count; i++)
|
||||
due[i].Last = results[i];
|
||||
}
|
||||
|
||||
// Re-checked after the fan-out: a shutdown that lands mid-sweep leaves the probed targets
|
||||
@@ -140,8 +167,12 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
// misleading picture into the store for as long as the process takes to exit.
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var snapshot = BuildSnapshot(now);
|
||||
_store.Publish(snapshot);
|
||||
OverviewSnapshot snapshot;
|
||||
lock (_publishLock)
|
||||
{
|
||||
snapshot = BuildSnapshot(now);
|
||||
_store.Publish(snapshot);
|
||||
}
|
||||
|
||||
_metrics.RecordSweep(_timeProvider.GetElapsedTime(startedAt));
|
||||
|
||||
@@ -152,6 +183,35 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probes one target, then records and publishes its result immediately — the sweep's remaining
|
||||
/// laggards must not hold back a result that is already in hand (Overview-004).
|
||||
/// </summary>
|
||||
private async Task ProbeAndPublishAsync(
|
||||
ZbHealthReportClient probe,
|
||||
PollTarget target,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await ProbeAsync(probe, target, now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
lock (_publishLock)
|
||||
{
|
||||
target.Last = result;
|
||||
|
||||
// A shutdown mid-probe must not publish a half-probed mix (see SweepAsync).
|
||||
if (!cancellationToken.IsCancellationRequested)
|
||||
_store.Publish(BuildSnapshot(now));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
target.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsDue(PollTarget target, DateTimeOffset now) =>
|
||||
target.Last?.LastPolledUtc is not { } polled || now - polled + DueTolerance >= target.Timings.PollInterval;
|
||||
|
||||
@@ -283,9 +343,17 @@ public sealed class OverviewPollerService : BackgroundService
|
||||
public required InstanceSnapshot Template { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The last published result, or null before the first probe. Written only by the sweep
|
||||
/// loop after its <c>Task.WhenAll</c> has completed, so it is never touched concurrently.
|
||||
/// The last published result, or null before the first probe. Written only under the
|
||||
/// poller's publish lock, by the probe task that holds this target's claim.
|
||||
/// </summary>
|
||||
public InstanceSnapshot? Last { get; set; }
|
||||
|
||||
private int _inFlight;
|
||||
|
||||
/// <summary>Claims the target for one probe; false when a previous probe is still running.</summary>
|
||||
public bool TryClaim() => Interlocked.CompareExchange(ref _inFlight, 1, 0) == 0;
|
||||
|
||||
/// <summary>Releases the claim taken by <see cref="TryClaim"/>.</summary>
|
||||
public void Release() => Volatile.Write(ref _inFlight, 0);
|
||||
}
|
||||
}
|
||||
|
||||
+5
-1
@@ -31,7 +31,11 @@ public static class PollingServiceCollectionExtensions
|
||||
|
||||
// Health bodies are small and must not be served from anywhere but the instance itself.
|
||||
client.DefaultRequestHeaders.CacheControl = new() { NoCache = true, NoStore = true };
|
||||
});
|
||||
})
|
||||
// A followed redirect would turn an auth wall's 200 login page into "Active"
|
||||
// (Overview-003). With redirects off, a 3xx surfaces as its own status code and the
|
||||
// active tier maps it to Unknown; the ready tier already requires a parseable body.
|
||||
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { AllowAutoRedirect = false });
|
||||
|
||||
services.AddHostedService<OverviewPollerService>();
|
||||
|
||||
|
||||
@@ -5,9 +5,11 @@ namespace ZB.MOM.WW.Overview.Registry;
|
||||
/// <c>Overview</c> configuration section and validated by <see cref="OverviewOptionsValidator"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, plus
|
||||
/// <c>{BaseUrl}/health/active</c> when <see cref="InstanceEntry.HasActiveRole"/>. There is
|
||||
/// deliberately no per-instance probe-mode switch: one probe model, one status derivation.
|
||||
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, then
|
||||
/// <c>{BaseUrl}/health/active</c> whenever ready answered, regardless of
|
||||
/// <see cref="InstanceEntry.HasActiveRole"/> (single-instance gateways publish their session counts
|
||||
/// on the active tier). <see cref="InstanceEntry.HasActiveRole"/> only controls whether an
|
||||
/// Active/Standby badge is derived from the active answer. There is deliberately no per-instance probe-mode switch: one probe model, one status derivation.
|
||||
/// </remarks>
|
||||
public sealed class OverviewOptions
|
||||
{
|
||||
@@ -75,8 +77,9 @@ public sealed class InstanceEntry
|
||||
|
||||
/// <summary>
|
||||
/// Whether this instance participates in an active/standby pair. False for the single-instance
|
||||
/// gateways, which have no such concept — their cards show status without an Active badge and
|
||||
/// are never probed for <c>/health/active</c>.
|
||||
/// gateways, which have no such concept — their cards show status without an Active badge. The
|
||||
/// active tier is still probed for them (it carries their session counts); this flag only gates
|
||||
/// the Active/Standby interpretation.
|
||||
/// </summary>
|
||||
public bool HasActiveRole { get; set; }
|
||||
|
||||
|
||||
@@ -401,6 +401,55 @@ public class OverviewPollerServiceTests
|
||||
Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_HungInstanceDoesNotHoldUpResultsOrLaterSweepsForTheOthers()
|
||||
{
|
||||
// Overview-004: a hung instance (long timeout override) used to gate the whole sweep, so
|
||||
// every other instance went Stale between polls. The fast instance must publish as soon as
|
||||
// it answers, and a later sweep must not wait for the hung one.
|
||||
var handler = new RoutingHandler()
|
||||
.Json("http://fast:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
|
||||
.Json("http://slow:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
|
||||
var options = Registry(App("App", Node("fast", "http://fast:8080"), Node("slow", "http://slow:8080")));
|
||||
var time = new FakeTimeProvider(Start);
|
||||
var (poller, store) = Build(options, new HangingHandler(handler, "slow"), time);
|
||||
|
||||
var first = poller.SweepAsync();
|
||||
|
||||
// The fast result is published while the slow probe is still hanging.
|
||||
await WaitUntilAsync(() => store.Current.AllInstances.Any(i => i.Instance == "fast" && i.Status == InstanceStatus.Up));
|
||||
Assert.False(first.IsCompleted);
|
||||
|
||||
time.Advance(TimeSpan.FromSeconds(11));
|
||||
var second = await poller.SweepAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.False(first.IsCompleted);
|
||||
Assert.Equal(2, ReadyProbes(handler, "fast"));
|
||||
Assert.Equal(InstanceStatus.Up, second.AllInstances.Single(i => i.Instance == "fast").Status);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(10);
|
||||
while (!condition())
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < deadline, "condition not met in time");
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class HangingHandler(RoutingHandler inner, string hangingHost) : HttpMessageHandler
|
||||
{
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.RequestUri!.Host == hangingHost)
|
||||
await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return await inner.HandleAsync(request).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_UnregisteredInstances_RenderAsPendingNotAsHealthy()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>Overview-003: the pooled probe client must not follow redirects.</summary>
|
||||
public sealed class ProbeRedirectTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ActiveProbe_AuthRedirectToLoginPage_IsNotReportedAsActive()
|
||||
{
|
||||
var port = FreePort();
|
||||
var prefix = $"http://127.0.0.1:{port}/";
|
||||
using var listener = new HttpListener();
|
||||
listener.Prefixes.Add(prefix);
|
||||
listener.Start();
|
||||
var server = Task.Run(async () =>
|
||||
{
|
||||
while (listener.IsListening)
|
||||
{
|
||||
HttpListenerContext ctx;
|
||||
try { ctx = await listener.GetContextAsync(); } catch { return; }
|
||||
if (ctx.Request.Url!.AbsolutePath == "/login")
|
||||
{
|
||||
ctx.Response.StatusCode = 200;
|
||||
ctx.Response.ContentType = "text/html";
|
||||
await ctx.Response.OutputStream.WriteAsync("<html>login</html>"u8.ToArray());
|
||||
}
|
||||
else
|
||||
{
|
||||
ctx.Response.StatusCode = 302;
|
||||
ctx.Response.RedirectLocation = "/login";
|
||||
}
|
||||
ctx.Response.Close();
|
||||
}
|
||||
});
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddOverviewPolling();
|
||||
await using var provider = services.BuildServiceProvider();
|
||||
var client = provider.GetRequiredService<IHttpClientFactory>().CreateClient(OverviewPollerService.HttpClientName);
|
||||
|
||||
var probe = await new ZbHealthReportClient(client)
|
||||
.ProbeStatusAsync($"{prefix}health/active", TimeSpan.FromSeconds(5));
|
||||
|
||||
listener.Stop();
|
||||
Assert.Equal(302, probe.StatusCode);
|
||||
Assert.Equal(ActiveState.Unknown, StatusDerivation.ObserveActive(true, probe));
|
||||
}
|
||||
|
||||
private static int FreePort()
|
||||
{
|
||||
var l = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
|
||||
l.Start();
|
||||
var port = ((IPEndPoint)l.LocalEndpoint).Port;
|
||||
l.Stop();
|
||||
return port;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user