feat(health): metrics-stale signal + status-transition timestamps; spec now matches heartbeat-liveness code

tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/CentralHealthAggregatorTests.cs

@@ -40,6 +40,23 @@ public class CentralHealthAggregatorTests
             _timeProvider);
     }
 
+    /// <summary>
+    /// Builds a fresh aggregator + manual time provider with an explicit
+    /// <paramref name="metricsStaleTimeout"/> so the metrics-staleness tests can
+    /// pick a window independent of the shared 60s <c>OfflineTimeout</c>.
+    /// </summary>
+    private static (CentralHealthAggregator Aggregator, TestTimeProvider Time) NewAggregator(
+        TimeSpan? metricsStaleTimeout = null)
+    {
+        var time = new TestTimeProvider(DateTimeOffset.UtcNow);
+        var opts = new HealthMonitoringOptions { OfflineTimeout = TimeSpan.FromSeconds(60) };
+        if (metricsStaleTimeout is { } mst)
+            opts.MetricsStaleTimeout = mst;
+        var aggregator = new CentralHealthAggregator(
+            Options.Create(opts), NullLogger<CentralHealthAggregator>.Instance, time);
+        return (aggregator, time);
+    }
+
     private static SiteHealthReport MakeReport(string siteId, long seq) =>
         new(
             SiteId: siteId,
@@ -423,4 +440,56 @@ public class CentralHealthAggregatorTests
         _aggregator.ProcessReport(MakeReport("site-1", 11));
         Assert.Equal(11, _aggregator.GetSiteState("site-1")!.LastSequenceNumber);
     }
+
+    /// <summary>
+    /// Review 01 [Medium]: a site whose HealthReportSender died keeps
+    /// heartbeating and previously showed "online with frozen metrics forever".
+    /// Metrics staleness is a distinct signal from liveness: heartbeats keep the
+    /// site online, but the absence of a full report for
+    /// <see cref="HealthMonitoringOptions.MetricsStaleTimeout"/> flags it stale.
+    /// </summary>
+    [Fact]
+    public void HeartbeatingSiteWithStaleReports_IsFlaggedMetricsStale_ButStaysOnline()
+    {
+        var (aggregator, time) = NewAggregator(metricsStaleTimeout: TimeSpan.FromMinutes(2));
+        aggregator.ProcessReport(MakeReport("site-a", 1));
+        time.Advance(TimeSpan.FromMinutes(3));
+        aggregator.MarkHeartbeat("site-a", time.GetUtcNow()); // heartbeats keep flowing
+        aggregator.CheckForOfflineSites();
+        var state = aggregator.GetSiteState("site-a")!;
+        Assert.True(state.IsOnline);        // heartbeat-based liveness unchanged
+        Assert.True(state.IsMetricsStale);  // NEW distinct signal
+    }
+
+    /// <summary>A fresh full report clears the metrics-stale flag.</summary>
+    [Fact]
+    public void FreshReport_ClearsMetricsStale()
+    {
+        var (aggregator, time) = NewAggregator(metricsStaleTimeout: TimeSpan.FromMinutes(2));
+        aggregator.ProcessReport(MakeReport("site-a", 1));
+        time.Advance(TimeSpan.FromMinutes(3));
+        aggregator.CheckForOfflineSites();
+        aggregator.ProcessReport(MakeReport("site-a", 2));
+        Assert.False(aggregator.GetSiteState("site-a")!.IsMetricsStale);
+    }
+
+    /// <summary>
+    /// Review 01 underdeveloped #7: "when did the site drop" was unanswerable.
+    /// Every online↔offline flip stamps <see cref="SiteHealthState.LastStatusChangeAt"/>.
+    /// </summary>
+    [Fact]
+    public void OnlineOfflineTransitions_RecordLastStatusChangeAt()
+    {
+        var (aggregator, time) = NewAggregator();
+        aggregator.ProcessReport(MakeReport("site-a", 1));
+        time.Advance(TimeSpan.FromMinutes(5)); // > OfflineTimeout
+        aggregator.CheckForOfflineSites();
+        var offlineAt = aggregator.GetSiteState("site-a")!.LastStatusChangeAt;
+        Assert.Equal(time.GetUtcNow(), offlineAt);
+        // Advance so the recovery flip lands at a demonstrably later instant than
+        // the offline flip — proves LastStatusChangeAt is re-stamped on each flip.
+        time.Advance(TimeSpan.FromSeconds(10));
+        aggregator.MarkHeartbeat("site-a", time.GetUtcNow());
+        Assert.NotEqual(offlineAt, aggregator.GetSiteState("site-a")!.LastStatusChangeAt);
+    }
 }

tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/HealthMonitoringOptionsValidatorTests.cs

@@ -70,4 +70,27 @@ public class HealthMonitoringOptionsValidatorTests
         Assert.True(result.Failed);
         Assert.Contains("CentralOfflineTimeout", result.FailureMessage);
     }
+
+    [Fact]
+    public void ZeroMetricsStaleTimeout_IsRejected()
+    {
+        var result = Validate(new HealthMonitoringOptions { MetricsStaleTimeout = TimeSpan.Zero });
+
+        Assert.True(result.Failed);
+        Assert.Contains("MetricsStaleTimeout", result.FailureMessage);
+    }
+
+    [Fact]
+    public void MetricsStaleTimeout_ShorterThanReportInterval_IsRejected()
+    {
+        // A stale window shorter than one report interval flags every site.
+        var result = Validate(new HealthMonitoringOptions
+        {
+            ReportInterval = TimeSpan.FromSeconds(30),
+            MetricsStaleTimeout = TimeSpan.FromSeconds(10)
+        });
+
+        Assert.True(result.Failed);
+        Assert.Contains("MetricsStaleTimeout", result.FailureMessage);
+    }
 }