fix(health-monitoring): resolve HealthMonitoring-004,006,010,011,012 — heartbeat-doc accuracy, testable sequence seeding, logged failures, dead-code removal

src/ScadaLink.HealthMonitoring/HealthReportSender.cs

@@ -8,7 +8,12 @@ namespace ScadaLink.HealthMonitoring;
 
 /// <summary>
 /// Periodically collects a SiteHealthReport and sends it to central via Akka remoting.
-/// Sequence numbers are monotonic, starting at 1, and reset on service restart.
+/// Sequence numbers are monotonic and reset on service restart. They are <b>not</b>
+/// zero/one-based: the per-process counter is seeded with the current Unix epoch
+/// (milliseconds) at construction so that, after a failover, reports from a
+/// freshly-active node always sort after reports from any prior active node for the
+/// same site — otherwise the central aggregator's sequence-number guard would
+/// silently reject the new active's first reports as stale.
 /// </summary>
 public class HealthReportSender : BackgroundService
 {
@@ -24,8 +29,9 @@ public class HealthReportSender : BackgroundService
     // node always sort after reports from any prior active node for the same
     // site. Without this seeding, failover would silently drop the new
     // active's first reports because their per-process counter starts below
-    // the prior active's last sequence number.
-    private long _sequenceNumber = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+    // the prior active's last sequence number. The clock is read through the
+    // injected TimeProvider so the seeding is deterministically testable.
+    private long _sequenceNumber;
 
     public HealthReportSender(
         ISiteHealthCollector collector,
@@ -34,7 +40,8 @@ public class HealthReportSender : BackgroundService
         ILogger<HealthReportSender> logger,
         ISiteIdentityProvider siteIdentityProvider,
         StoreAndForwardStorage? sfStorage = null,
-        IClusterNodeProvider? clusterNodeProvider = null)
+        IClusterNodeProvider? clusterNodeProvider = null,
+        TimeProvider? timeProvider = null)
     {
         _collector = collector;
         _transport = transport;
@@ -43,6 +50,7 @@ public class HealthReportSender : BackgroundService
         _siteId = siteIdentityProvider.SiteId;
         _sfStorage = sfStorage;
         _clusterNodeProvider = clusterNodeProvider;
+        _sequenceNumber = (timeProvider ?? TimeProvider.System).GetUtcNow().ToUnixTimeMilliseconds();
     }
 
     /// <summary>
@@ -73,7 +81,14 @@ public class HealthReportSender : BackgroundService
                     {
                         _collector.SetClusterNodes(_clusterNodeProvider.GetClusterNodes());
                     }
-                    catch { /* Non-fatal */ }
+                    catch (Exception ex)
+                    {
+                        // Non-fatal — the report ships with the previous cluster
+                        // node list. Logged so a persistent failure is diagnosable.
+                        _logger.LogWarning(ex,
+                            "Failed to refresh cluster nodes for health report (site {SiteId}); using stale list",
+                            _siteId);
+                    }
                 }
 
                 if (_sfStorage != null)
@@ -83,7 +98,13 @@ public class HealthReportSender : BackgroundService
                         var parkedCount = await _sfStorage.GetParkedMessageCountAsync();
                         _collector.SetParkedMessageCount(parkedCount);
                     }
-                    catch { /* Non-fatal — parked count will be 0 */ }
+                    catch (Exception ex)
+                    {
+                        // Non-fatal — parked count will be 0 in this report.
+                        _logger.LogWarning(ex,
+                            "Failed to query parked message count for health report (site {SiteId})",
+                            _siteId);
+                    }
 
                     try
                     {
@@ -97,7 +118,13 @@ public class HealthReportSender : BackgroundService
                             kvp => kvp.Value);
                         _collector.SetStoreAndForwardDepths(depths);
                     }
-                    catch { /* Non-fatal — buffer depths will be empty */ }
+                    catch (Exception ex)
+                    {
+                        // Non-fatal — buffer depths will be empty in this report.
+                        _logger.LogWarning(ex,
+                            "Failed to query store-and-forward buffer depths for health report (site {SiteId})",
+                            _siteId);
+                    }
                 }
 
                 var seq = Interlocked.Increment(ref _sequenceNumber);