feat(health): metrics-stale signal + status-transition timestamps; spec now matches heartbeat-liveness code

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs

@@ -48,7 +48,9 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
                     LastReportReceivedAt = now,
                     LastHeartbeatAt = now,
                     LastSequenceNumber = report.SequenceNumber,
-                    IsOnline = true
+                    IsOnline = true,
+                    IsMetricsStale = false
+                    // LastStatusChangeAt stays null: first observation is not a flip.
                 };
 
                 if (_siteStates.TryAdd(report.SiteId, registered))
@@ -76,7 +78,12 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
                 LastReportReceivedAt = now,
                 LastHeartbeatAt = now,
                 LastSequenceNumber = report.SequenceNumber,
-                IsOnline = true
+                IsOnline = true,
+                // A fresh report is proof the metrics pipeline is alive again.
+                IsMetricsStale = false,
+                // Stamp a flip only when this report brings the site back online;
+                // a report from an already-online site leaves the timestamp be.
+                LastStatusChangeAt = existing.IsOnline ? existing.LastStatusChangeAt : now
             };
 
             if (_siteStates.TryUpdate(report.SiteId, updated, existing))
@@ -155,7 +162,14 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
             var updated = existing with
             {
                 LastHeartbeatAt = newHeartbeat,
-                IsOnline = true
+                IsOnline = true,
+                // Stamp the flip when this heartbeat promotes an offline site back
+                // online. IsMetricsStale is deliberately left untouched — a
+                // heartbeat says nothing about whether the metrics pipeline
+                // recovered; only a full report clears staleness.
+                LastStatusChangeAt = existing.IsOnline
+                    ? existing.LastStatusChangeAt
+                    : _timeProvider.GetUtcNow()
             };
 
             if (_siteStates.TryUpdate(siteId, updated, existing))
@@ -246,18 +260,41 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
                 : _options.OfflineTimeout;
 
             var elapsed = now - state.LastHeartbeatAt;
-            if (elapsed <= timeout)
-                continue;
-
-            // Atomically swap to an offline copy. If the CAS loses to a
-            // concurrent report/heartbeat the site was just heard from, so
-            // leaving it online is the correct outcome — no retry needed.
-            var offline = state with { IsOnline = false };
-            if (_siteStates.TryUpdate(kvp.Key, offline, state))
+            if (elapsed > timeout)
             {
-                _logger.LogWarning(
-                    "Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
-                    state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
+                // Atomically swap to an offline copy, stamping the flip. If the
+                // CAS loses to a concurrent report/heartbeat the site was just
+                // heard from, so leaving it online is correct — no retry needed.
+                var offline = state with { IsOnline = false, LastStatusChangeAt = now };
+                if (_siteStates.TryUpdate(kvp.Key, offline, state))
+                {
+                    _logger.LogWarning(
+                        "Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
+                        state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
+                }
+
+                // Whether the CAS won or lost, do not also evaluate staleness on
+                // this stale snapshot — offline supersedes metrics-stale.
+                continue;
+            }
+
+            // Second signal: the site is still live (heartbeats within timeout)
+            // but its full-report pipeline may have died. Flag it metrics-stale
+            // when no report has arrived within MetricsStaleTimeout. Distinct from
+            // offline — a site whose HealthReportSender crashed keeps heartbeating
+            // and would otherwise show "online with frozen metrics forever".
+            if (state.LastReportReceivedAt is { } lastReport
+                && now - lastReport > _options.MetricsStaleTimeout
+                && !state.IsMetricsStale)
+            {
+                var stale = state with { IsMetricsStale = true };
+                if (_siteStates.TryUpdate(kvp.Key, stale, state))
+                {
+                    _logger.LogWarning(
+                        "Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s (timeout: {Timeout}s)",
+                        state.SiteId, (now - lastReport).TotalSeconds, _options.MetricsStaleTimeout.TotalSeconds);
+                }
+                // CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip.
             }
         }
     }

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/HealthMonitoringOptions.cs

@@ -19,4 +19,17 @@ public class HealthMonitoringOptions
     /// the design doc grants real sites. Default: 3x the report interval.
     /// </summary>
     public TimeSpan CentralOfflineTimeout { get; set; } = TimeSpan.FromMinutes(3);
+
+    /// <summary>
+    /// Duration an online site may go without a full health <em>report</em>
+    /// (heartbeats notwithstanding) before it is flagged
+    /// <see cref="SiteHealthState.IsMetricsStale"/>. Distinct from
+    /// <see cref="OfflineTimeout"/>, which is heartbeat-based liveness: a site
+    /// whose HealthReportSender has died keeps heartbeating and stays "online",
+    /// but its metrics freeze — this window surfaces that. Must be positive and
+    /// no shorter than <see cref="ReportInterval"/> (a window shorter than one
+    /// report interval would flag every healthy site). Default: 2 minutes
+    /// (~4 missed 30s reports).
+    /// </summary>
+    public TimeSpan MetricsStaleTimeout { get; set; } = TimeSpan.FromMinutes(2);
 }

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/HealthMonitoringOptionsValidator.cs

@@ -45,5 +45,24 @@ public sealed class HealthMonitoringOptionsValidator : OptionsValidatorBase<Heal
             $"must be >= OfflineTimeout ({options.OfflineTimeout}): the synthetic 'central' site has " +
             "no heartbeat source and is fed only by the slower self-report loop, so it needs at " +
             "least as much offline grace as a real site.");
+
+        builder.RequireThat(options.MetricsStaleTimeout > TimeSpan.Zero,
+            $"ScadaBridge:HealthMonitoring:MetricsStaleTimeout must be a positive duration " +
+            $"(was {options.MetricsStaleTimeout}); it is the window an online site may go without " +
+            "a full report before being flagged metrics-stale.");
+
+        // A stale window shorter than one report interval flags every healthy
+        // site (reports simply can't arrive faster than they are emitted). Same
+        // De Morgan guard shape as the Central/Offline pair above: stay silent
+        // when either field is non-positive so the dedicated positive-duration
+        // checks own that failure rather than double-firing here.
+        builder.RequireThat(
+            !(options.ReportInterval > TimeSpan.Zero
+              && options.MetricsStaleTimeout > TimeSpan.Zero
+              && options.MetricsStaleTimeout < options.ReportInterval),
+            $"ScadaBridge:HealthMonitoring:MetricsStaleTimeout ({options.MetricsStaleTimeout}) " +
+            $"must be >= ReportInterval ({options.ReportInterval}): a stale window shorter than one " +
+            "report interval would flag every healthy site, since reports cannot arrive faster than " +
+            "they are emitted.");
     }
 }

src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthState.cs

@@ -44,4 +44,23 @@ public sealed record SiteHealthState
     public long LastSequenceNumber { get; init; }
     /// <summary>Gets a value indicating whether the site is currently considered online.</summary>
     public bool IsOnline { get; init; }
+
+    /// <summary>
+    /// Gets a value indicating whether the site is online (heartbeats still
+    /// arriving) but has sent no full <see cref="SiteHealthReport"/> within
+    /// <see cref="HealthMonitoringOptions.MetricsStaleTimeout"/>. This is a signal
+    /// <em>distinct</em> from liveness: a site whose HealthReportSender died keeps
+    /// heartbeating and would otherwise show "online with frozen metrics forever".
+    /// Cleared whenever a fresh report is processed. Heartbeats never set or clear
+    /// it — they say nothing about the metrics pipeline.
+    /// </summary>
+    public bool IsMetricsStale { get; init; }
+
+    /// <summary>
+    /// Gets the instant of the last online↔offline flip, or <c>null</c> if the
+    /// site has never changed status since it was first observed. Answers "when
+    /// did this site drop / come back". Not moved by report/heartbeat traffic that
+    /// leaves the online/offline status unchanged, nor by the metrics-stale flag.
+    /// </summary>
+    public DateTimeOffset? LastStatusChangeAt { get; init; }
 }