dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(health): metrics-stale signal + status-transition timestamps; spec now matches heartbeat-liveness code

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-08 16:21:30 -04:00
parent 15d91d760f
commit c73b7faa11
7 changed files with 199 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs
+51 -14
View File
@@ -48,7 +48,9 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
LastReportReceivedAt = now,
LastHeartbeatAt = now,
LastSequenceNumber = report.SequenceNumber,
IsOnline = true
IsOnline = true,
IsMetricsStale = false
// LastStatusChangeAt stays null: first observation is not a flip.
};
if (_siteStates.TryAdd(report.SiteId, registered))
@@ -76,7 +78,12 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
LastReportReceivedAt = now,
LastHeartbeatAt = now,
LastSequenceNumber = report.SequenceNumber,
IsOnline = true
IsOnline = true,
// A fresh report is proof the metrics pipeline is alive again.
IsMetricsStale = false,
// Stamp a flip only when this report brings the site back online;
// a report from an already-online site leaves the timestamp be.
LastStatusChangeAt = existing.IsOnline ? existing.LastStatusChangeAt : now
};
if (_siteStates.TryUpdate(report.SiteId, updated, existing))
@@ -155,7 +162,14 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
var updated = existing with
{
LastHeartbeatAt = newHeartbeat,
IsOnline = true
IsOnline = true,
// Stamp the flip when this heartbeat promotes an offline site back
// online. IsMetricsStale is deliberately left untouched — a
// heartbeat says nothing about whether the metrics pipeline
// recovered; only a full report clears staleness.
LastStatusChangeAt = existing.IsOnline
? existing.LastStatusChangeAt
: _timeProvider.GetUtcNow()
};
if (_siteStates.TryUpdate(siteId, updated, existing))
@@ -246,18 +260,41 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
: _options.OfflineTimeout;
var elapsed = now - state.LastHeartbeatAt;
if (elapsed <= timeout)
continue;
// Atomically swap to an offline copy. If the CAS loses to a
// concurrent report/heartbeat the site was just heard from, so
// leaving it online is the correct outcome — no retry needed.
var offline = state with { IsOnline = false };
if (_siteStates.TryUpdate(kvp.Key, offline, state))
if (elapsed > timeout)
{
_logger.LogWarning(
"Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
// Atomically swap to an offline copy, stamping the flip. If the
// CAS loses to a concurrent report/heartbeat the site was just
// heard from, so leaving it online is correct — no retry needed.
var offline = state with { IsOnline = false, LastStatusChangeAt = now };
if (_siteStates.TryUpdate(kvp.Key, offline, state))
{
_logger.LogWarning(
"Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
}
// Whether the CAS won or lost, do not also evaluate staleness on
// this stale snapshot — offline supersedes metrics-stale.
continue;
}
// Second signal: the site is still live (heartbeats within timeout)
// but its full-report pipeline may have died. Flag it metrics-stale
// when no report has arrived within MetricsStaleTimeout. Distinct from
// offline — a site whose HealthReportSender crashed keeps heartbeating
// and would otherwise show "online with frozen metrics forever".
if (state.LastReportReceivedAt is { } lastReport
&& now - lastReport > _options.MetricsStaleTimeout
&& !state.IsMetricsStale)
{
var stale = state with { IsMetricsStale = true };
if (_siteStates.TryUpdate(kvp.Key, stale, state))
{
_logger.LogWarning(
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s (timeout: {Timeout}s)",
state.SiteId, (now - lastReport).TotalSeconds, _options.MetricsStaleTimeout.TotalSeconds);
}
// CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip.
}
}
}