fix(health): flag metrics-stale for never-reported sites via FirstSeenAt anchor — null LastReportReceivedAt no longer skips the check (plan R2-01 T8)
This commit is contained in:
@@ -47,6 +47,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
|
||||
LatestReport = report,
|
||||
LastReportReceivedAt = now,
|
||||
LastHeartbeatAt = now,
|
||||
FirstSeenAt = now,
|
||||
LastSequenceNumber = report.SequenceNumber,
|
||||
IsOnline = true,
|
||||
IsMetricsStale = false
|
||||
@@ -117,6 +118,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
|
||||
LatestReport = null,
|
||||
LastReportReceivedAt = null,
|
||||
LastHeartbeatAt = receivedAt,
|
||||
FirstSeenAt = receivedAt,
|
||||
LastSequenceNumber = 0,
|
||||
IsOnline = true
|
||||
};
|
||||
@@ -297,16 +299,22 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
|
||||
// when no report has arrived within MetricsStaleTimeout. Distinct from
|
||||
// offline — a site whose HealthReportSender crashed keeps heartbeating
|
||||
// and would otherwise show "online with frozen metrics forever".
|
||||
if (state.LastReportReceivedAt is { } lastReport
|
||||
&& now - lastReport > _options.MetricsStaleTimeout
|
||||
// A site that has NEVER reported (heartbeat-only registration,
|
||||
// LastReportReceivedAt == null) anchors on FirstSeenAt instead —
|
||||
// otherwise a pipeline dead from first boot is never flagged (N3).
|
||||
var reportAnchor = state.LastReportReceivedAt ?? state.FirstSeenAt;
|
||||
if (reportAnchor is { } anchor
|
||||
&& now - anchor > _options.MetricsStaleTimeout
|
||||
&& !state.IsMetricsStale)
|
||||
{
|
||||
var stale = state with { IsMetricsStale = true };
|
||||
if (_siteStates.TryUpdate(kvp.Key, stale, state))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s (timeout: {Timeout}s)",
|
||||
state.SiteId, (now - lastReport).TotalSeconds, _options.MetricsStaleTimeout.TotalSeconds);
|
||||
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s{NeverNote} (timeout: {Timeout}s)",
|
||||
state.SiteId, (now - anchor).TotalSeconds,
|
||||
state.LastReportReceivedAt is null ? " (never reported since first contact)" : string.Empty,
|
||||
_options.MetricsStaleTimeout.TotalSeconds);
|
||||
}
|
||||
// CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip.
|
||||
}
|
||||
|
||||
@@ -40,6 +40,17 @@ public sealed record SiteHealthState
|
||||
/// </summary>
|
||||
public DateTimeOffset LastHeartbeatAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the instant this site was first observed by the aggregator (via a
|
||||
/// report or a heartbeat); <c>null</c> only for states not built by the
|
||||
/// aggregator (hand-constructed fixtures). Anchors metrics-staleness for a
|
||||
/// site that has NEVER delivered a report (review 01 round-2 N3): with
|
||||
/// <see cref="LastReportReceivedAt"/> null the staleness check previously
|
||||
/// skipped the site forever. <see cref="LastHeartbeatAt"/> cannot anchor
|
||||
/// this — every heartbeat advances it.
|
||||
/// </summary>
|
||||
public DateTimeOffset? FirstSeenAt { get; init; }
|
||||
|
||||
/// <summary>Gets the sequence number of the last accepted health report, used to reject out-of-order duplicates.</summary>
|
||||
public long LastSequenceNumber { get; init; }
|
||||
/// <summary>Gets a value indicating whether the site is currently considered online.</summary>
|
||||
|
||||
Reference in New Issue
Block a user