fix(health): flag metrics-stale for never-reported sites via FirstSeenAt anchor — null LastReportReceivedAt no longer skips the check (plan R2-01 T8)

This commit is contained in:
Joseph Doherty
2026-07-13 10:45:22 -04:00
parent e4765b2eed
commit 11efe4be05
4 changed files with 55 additions and 5 deletions
@@ -47,6 +47,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
LatestReport = report,
LastReportReceivedAt = now,
LastHeartbeatAt = now,
FirstSeenAt = now,
LastSequenceNumber = report.SequenceNumber,
IsOnline = true,
IsMetricsStale = false
@@ -117,6 +118,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
LatestReport = null,
LastReportReceivedAt = null,
LastHeartbeatAt = receivedAt,
FirstSeenAt = receivedAt,
LastSequenceNumber = 0,
IsOnline = true
};
@@ -297,16 +299,22 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
// when no report has arrived within MetricsStaleTimeout. Distinct from
// offline — a site whose HealthReportSender crashed keeps heartbeating
// and would otherwise show "online with frozen metrics forever".
if (state.LastReportReceivedAt is { } lastReport
&& now - lastReport > _options.MetricsStaleTimeout
// A site that has NEVER reported (heartbeat-only registration,
// LastReportReceivedAt == null) anchors on FirstSeenAt instead —
// otherwise a pipeline dead from first boot is never flagged (N3).
var reportAnchor = state.LastReportReceivedAt ?? state.FirstSeenAt;
if (reportAnchor is { } anchor
&& now - anchor > _options.MetricsStaleTimeout
&& !state.IsMetricsStale)
{
var stale = state with { IsMetricsStale = true };
if (_siteStates.TryUpdate(kvp.Key, stale, state))
{
_logger.LogWarning(
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s (timeout: {Timeout}s)",
state.SiteId, (now - lastReport).TotalSeconds, _options.MetricsStaleTimeout.TotalSeconds);
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s{NeverNote} (timeout: {Timeout}s)",
state.SiteId, (now - anchor).TotalSeconds,
state.LastReportReceivedAt is null ? " (never reported since first contact)" : string.Empty,
_options.MetricsStaleTimeout.TotalSeconds);
}
// CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip.
}
@@ -40,6 +40,17 @@ public sealed record SiteHealthState
/// </summary>
public DateTimeOffset LastHeartbeatAt { get; init; }
/// <summary>
/// Gets the instant this site was first observed by the aggregator (via a
/// report or a heartbeat); <c>null</c> only for states not built by the
/// aggregator (hand-constructed fixtures). Anchors metrics-staleness for a
/// site that has NEVER delivered a report (review 01 round-2 N3): with
/// <see cref="LastReportReceivedAt"/> null the staleness check previously
/// skipped the site forever. <see cref="LastHeartbeatAt"/> cannot anchor
/// this — every heartbeat advances it.
/// </summary>
public DateTimeOffset? FirstSeenAt { get; init; }
/// <summary>Gets the sequence number of the last accepted health report, used to reject out-of-order duplicates.</summary>
public long LastSequenceNumber { get; init; }
/// <summary>Gets a value indicating whether the site is currently considered online.</summary>