fix(health-monitoring): resolve HealthMonitoring-004,006,010,011,012 — heartbeat-doc accuracy, testable sequence seeding, logged failures, dead-code removal

src/ScadaLink.HealthMonitoring/CentralHealthAggregator.cs

@@ -210,10 +210,12 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
             var state = kvp.Value;
             if (!state.IsOnline) continue;
 
-            // Use LastHeartbeatAt — heartbeats arrive frequently from any
+            // Use LastHeartbeatAt — heartbeats arrive every ~5s from any
             // healthy site node (cadence owned by Cluster Infrastructure /
-            // SiteCommunicationActor), so OfflineTimeout only fires when no
-            // node can reach central, not during single-node failovers.
+            // SiteCommunicationActor — CommunicationOptions.TransportHeartbeatInterval),
+            // so the 60s OfflineTimeout tolerates several missed heartbeats and
+            // only fires when no node can reach central, not during single-node
+            // failovers.
             //
             // The synthetic "central" site has no heartbeat source — its only
             // signal is the 30s CentralHealthReportLoop self-report — so it gets

src/ScadaLink.HealthMonitoring/CentralHealthReportLoop.cs

@@ -29,22 +29,31 @@ public class CentralHealthReportLoop : BackgroundService
 
     // Seeded with Unix-ms so reports from a newly-elected central leader
     // always sort after reports from any prior leader for siteId="central".
-    private long _sequenceNumber = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+    // The clock is read through the injected TimeProvider so the seeding is
+    // deterministically testable.
+    private long _sequenceNumber;
 
     public CentralHealthReportLoop(
         ISiteHealthCollector collector,
         ICentralHealthAggregator aggregator,
         IClusterNodeProvider clusterNodeProvider,
         IOptions<HealthMonitoringOptions> options,
-        ILogger<CentralHealthReportLoop> logger)
+        ILogger<CentralHealthReportLoop> logger,
+        TimeProvider? timeProvider = null)
     {
         _collector = collector;
         _aggregator = aggregator;
         _clusterNodeProvider = clusterNodeProvider;
         _options = options.Value;
         _logger = logger;
+        _sequenceNumber = (timeProvider ?? TimeProvider.System).GetUtcNow().ToUnixTimeMilliseconds();
     }
 
+    /// <summary>
+    /// Current sequence number (for testing).
+    /// </summary>
+    public long CurrentSequenceNumber => Interlocked.Read(ref _sequenceNumber);
+
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         _logger.LogInformation(

src/ScadaLink.HealthMonitoring/HealthReportSender.cs

@@ -8,7 +8,12 @@ namespace ScadaLink.HealthMonitoring;
 
 /// <summary>
 /// Periodically collects a SiteHealthReport and sends it to central via Akka remoting.
-/// Sequence numbers are monotonic, starting at 1, and reset on service restart.
+/// Sequence numbers are monotonic and reset on service restart. They are <b>not</b>
+/// zero/one-based: the per-process counter is seeded with the current Unix epoch
+/// (milliseconds) at construction so that, after a failover, reports from a
+/// freshly-active node always sort after reports from any prior active node for the
+/// same site — otherwise the central aggregator's sequence-number guard would
+/// silently reject the new active's first reports as stale.
 /// </summary>
 public class HealthReportSender : BackgroundService
 {
@@ -24,8 +29,9 @@ public class HealthReportSender : BackgroundService
     // node always sort after reports from any prior active node for the same
     // site. Without this seeding, failover would silently drop the new
     // active's first reports because their per-process counter starts below
-    // the prior active's last sequence number.
-    private long _sequenceNumber = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+    // the prior active's last sequence number. The clock is read through the
+    // injected TimeProvider so the seeding is deterministically testable.
+    private long _sequenceNumber;
 
     public HealthReportSender(
         ISiteHealthCollector collector,
@@ -34,7 +40,8 @@ public class HealthReportSender : BackgroundService
         ILogger<HealthReportSender> logger,
         ISiteIdentityProvider siteIdentityProvider,
         StoreAndForwardStorage? sfStorage = null,
-        IClusterNodeProvider? clusterNodeProvider = null)
+        IClusterNodeProvider? clusterNodeProvider = null,
+        TimeProvider? timeProvider = null)
     {
         _collector = collector;
         _transport = transport;
@@ -43,6 +50,7 @@ public class HealthReportSender : BackgroundService
         _siteId = siteIdentityProvider.SiteId;
         _sfStorage = sfStorage;
         _clusterNodeProvider = clusterNodeProvider;
+        _sequenceNumber = (timeProvider ?? TimeProvider.System).GetUtcNow().ToUnixTimeMilliseconds();
     }
 
     /// <summary>
@@ -73,7 +81,14 @@ public class HealthReportSender : BackgroundService
                     {
                         _collector.SetClusterNodes(_clusterNodeProvider.GetClusterNodes());
                     }
-                    catch { /* Non-fatal */ }
+                    catch (Exception ex)
+                    {
+                        // Non-fatal — the report ships with the previous cluster
+                        // node list. Logged so a persistent failure is diagnosable.
+                        _logger.LogWarning(ex,
+                            "Failed to refresh cluster nodes for health report (site {SiteId}); using stale list",
+                            _siteId);
+                    }
                 }
 
                 if (_sfStorage != null)
@@ -83,7 +98,13 @@ public class HealthReportSender : BackgroundService
                         var parkedCount = await _sfStorage.GetParkedMessageCountAsync();
                         _collector.SetParkedMessageCount(parkedCount);
                     }
-                    catch { /* Non-fatal — parked count will be 0 */ }
+                    catch (Exception ex)
+                    {
+                        // Non-fatal — parked count will be 0 in this report.
+                        _logger.LogWarning(ex,
+                            "Failed to query parked message count for health report (site {SiteId})",
+                            _siteId);
+                    }
 
                     try
                     {
@@ -97,7 +118,13 @@ public class HealthReportSender : BackgroundService
                             kvp => kvp.Value);
                         _collector.SetStoreAndForwardDepths(depths);
                     }
-                    catch { /* Non-fatal — buffer depths will be empty */ }
+                    catch (Exception ex)
+                    {
+                        // Non-fatal — buffer depths will be empty in this report.
+                        _logger.LogWarning(ex,
+                            "Failed to query store-and-forward buffer depths for health report (site {SiteId})",
+                            _siteId);
+                    }
                 }
 
                 var seq = Interlocked.Increment(ref _sequenceNumber);

src/ScadaLink.HealthMonitoring/ICentralHealthAggregator.cs

@@ -13,9 +13,14 @@ public interface ICentralHealthAggregator
     /// <summary>
     /// Bumps the last-seen timestamp for a site, keeping it marked online
     /// between full 30s reports when heartbeats are arriving — protects against
-    /// the offline threshold firing on a transiently delayed report. A heartbeat
-    /// for a site with no aggregator state yet (e.g. just after a central
-    /// restart/failover) registers that site as online with no
+    /// the offline threshold firing on a transiently delayed report. Heartbeat
+    /// cadence is owned by the Cluster Infrastructure / <c>SiteCommunicationActor</c>
+    /// (the application-level heartbeat to central, sent every
+    /// <c>CommunicationOptions.TransportHeartbeatInterval</c> — 5s by default);
+    /// the 60s <see cref="HealthMonitoringOptions.OfflineTimeout"/> therefore
+    /// tolerates several missed heartbeats. A heartbeat for a site with no
+    /// aggregator state yet (e.g. just after a central restart/failover)
+    /// registers that site as online with no
     /// <see cref="SiteHealthState.LatestReport"/>, so reachable sites are not
     /// shown as "unknown" during the failover window.
     /// </summary>

src/ScadaLink.HealthMonitoring/ServiceCollectionExtensions.cs

@@ -38,10 +38,4 @@ public static class ServiceCollectionExtensions
         services.AddHostedService<CentralHealthReportLoop>();
         return services;
     }
-
-    public static IServiceCollection AddHealthMonitoringActors(this IServiceCollection services)
-    {
-        // Placeholder for Akka actor registration (Phase 4+)
-        return services;
-    }
 }

src/ScadaLink.HealthMonitoring/SiteHealthState.cs

@@ -30,8 +30,9 @@ public sealed record SiteHealthState
     /// Time the most recent signal of any kind (full report OR heartbeat) was
     /// received. Drives offline detection — heartbeats from the standby keep the
     /// site marked online even when the active node is unable to produce a report
-    /// (mid-failover, brief stalls). See the heartbeat scheduler owned by the
-    /// Cluster Infrastructure / SiteCommunicationActor for the actual cadence.
+    /// (mid-failover, brief stalls). Heartbeat cadence is owned by the Cluster
+    /// Infrastructure / SiteCommunicationActor (every
+    /// CommunicationOptions.TransportHeartbeatInterval — 5s by default).
     /// </summary>
     public DateTimeOffset LastHeartbeatAt { get; init; }