refactor(health): move ICentralHealthAggregator and SiteHealthState to Commons (ScadaBridge#71)
Interface goes to Commons.Interfaces.Services and the state record to Commons.Messages.Health, so CentralCommunicationActor no longer needs the HealthMonitoring namespace. Implementation stays in HealthMonitoring. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -1,3 +1,4 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for central-side health aggregation.
|
||||
/// Consumed by Central UI to display site health dashboards.
|
||||
/// </summary>
|
||||
public interface ICentralHealthAggregator
|
||||
{
|
||||
/// <summary>
|
||||
/// Processes an incoming health report from a site and updates the aggregated state.
|
||||
/// </summary>
|
||||
/// <param name="report">The health report received from the site.</param>
|
||||
void ProcessReport(SiteHealthReport report);
|
||||
|
||||
/// <summary>
|
||||
/// Bumps the last-seen timestamp for a site, keeping it marked online
|
||||
/// between full 30s reports when heartbeats are arriving — protects against
|
||||
/// the offline threshold firing on a transiently delayed report. Heartbeat
|
||||
/// cadence is owned by the Cluster Infrastructure / <c>SiteCommunicationActor</c>
|
||||
/// (the application-level heartbeat to central, sent every
|
||||
/// <c>CommunicationOptions.TransportHeartbeatInterval</c> — 5s by default);
|
||||
/// the 60s <c>HealthMonitoringOptions.OfflineTimeout</c> therefore
|
||||
/// tolerates several missed heartbeats. A heartbeat for a site with no
|
||||
/// aggregator state yet (e.g. just after a central restart/failover)
|
||||
/// registers that site as online with no
|
||||
/// <see cref="SiteHealthState.LatestReport"/>, so reachable sites are not
|
||||
/// shown as "unknown" during the failover window.
|
||||
/// </summary>
|
||||
/// <param name="siteId">The string identifier of the site that sent the heartbeat.</param>
|
||||
/// <param name="receivedAt">The UTC timestamp when the heartbeat was received.</param>
|
||||
void MarkHeartbeat(string siteId, DateTimeOffset receivedAt);
|
||||
|
||||
/// <summary>Returns a snapshot of all currently tracked site health states.</summary>
|
||||
/// <returns>A dictionary mapping site id to its current <see cref="SiteHealthState"/>.</returns>
|
||||
IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates();
|
||||
/// <summary>Returns the current health state for the specified site, or null if not tracked.</summary>
|
||||
/// <param name="siteId">The string identifier of the site to look up.</param>
|
||||
/// <returns>The <see cref="SiteHealthState"/> for the site, or null if not yet seen.</returns>
|
||||
SiteHealthState? GetSiteState(string siteId);
|
||||
|
||||
/// <summary>
|
||||
/// Removes tracked sites not in <paramref name="knownSiteIds"/> (i.e. deleted
|
||||
/// from configuration). The synthetic central id is always retained.
|
||||
/// Self-healing: called from the CentralCommunicationActor's periodic site
|
||||
/// refresh, so a deleted site disappears within one refresh interval without
|
||||
/// needing a dedicated deletion event. Without this the in-memory state only
|
||||
/// ever grew, leaving a deleted site as a permanently-offline dashboard tile
|
||||
/// (and a live KPI sample source) forever.
|
||||
/// </summary>
|
||||
/// <param name="knownSiteIds">The identifiers of all currently-configured sites.</param>
|
||||
void PruneUnknownSites(IReadOnlyCollection<string> knownSiteIds);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory state for a single site's health, stored by the central aggregator.
|
||||
/// Immutable: every state transition produces a new instance which the aggregator
|
||||
/// installs into its <c>ConcurrentDictionary</c> via an atomic compare-and-swap.
|
||||
/// This makes handing the reference straight to UI callers safe — a consumer can
|
||||
/// never observe a torn or half-applied update.
|
||||
/// </summary>
|
||||
public sealed record SiteHealthState
|
||||
{
|
||||
/// <summary>Gets the unique identifier of the site this state record belongs to.</summary>
|
||||
public required string SiteId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The latest full <see cref="SiteHealthReport"/> received for the site, or
|
||||
/// <c>null</c> if the site is known only via heartbeats and has not yet sent
|
||||
/// a report.
|
||||
/// </summary>
|
||||
public SiteHealthReport? LatestReport { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Time the latest full <see cref="SiteHealthReport"/> was processed, or
|
||||
/// <c>null</c> if the site is known only via heartbeats and has not yet sent
|
||||
/// a report. Used by the UI to surface report staleness during failover;
|
||||
/// the <c>null</c> case must be rendered as "no report yet" rather than as a
|
||||
/// timestamp (a <c>default</c> sentinel would display as year 0001).
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastReportReceivedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Time the most recent signal of any kind (full report OR heartbeat) was
|
||||
/// received. Drives offline detection — heartbeats from the standby keep the
|
||||
/// site marked online even when the active node is unable to produce a report
|
||||
/// (mid-failover, brief stalls). Heartbeat cadence is owned by the Cluster
|
||||
/// Infrastructure / SiteCommunicationActor (every
|
||||
/// CommunicationOptions.TransportHeartbeatInterval — 5s by default).
|
||||
/// </summary>
|
||||
public DateTimeOffset LastHeartbeatAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the instant this site was first observed by the aggregator (via a
|
||||
/// report or a heartbeat); <c>null</c> only for states not built by the
|
||||
/// aggregator (hand-constructed fixtures). Anchors metrics-staleness for a
|
||||
/// site that has NEVER delivered a report: with
|
||||
/// <see cref="LastReportReceivedAt"/> null the staleness check previously
|
||||
/// skipped the site forever. <see cref="LastHeartbeatAt"/> cannot anchor
|
||||
/// this — every heartbeat advances it.
|
||||
/// </summary>
|
||||
public DateTimeOffset? FirstSeenAt { get; init; }
|
||||
|
||||
/// <summary>Gets the sequence number of the last accepted health report, used to reject out-of-order duplicates.</summary>
|
||||
public long LastSequenceNumber { get; init; }
|
||||
/// <summary>Gets a value indicating whether the site is currently considered online.</summary>
|
||||
public bool IsOnline { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the site is online (heartbeats still
|
||||
/// arriving) but has sent no full <see cref="SiteHealthReport"/> within
|
||||
/// <c>HealthMonitoringOptions.MetricsStaleTimeout</c>. This is a signal
|
||||
/// <em>distinct</em> from liveness: a site whose HealthReportSender died keeps
|
||||
/// heartbeating and would otherwise show "online with frozen metrics forever".
|
||||
/// Cleared whenever a fresh report is processed. Heartbeats never set or clear
|
||||
/// it — they say nothing about the metrics pipeline.
|
||||
/// </summary>
|
||||
public bool IsMetricsStale { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the instant this site's LocalDb replication oplog backlog first rose above
|
||||
/// <c>HealthMonitoringOptions.LocalDbBacklogThreshold</c> and stayed there, or
|
||||
/// <c>null</c> if the backlog is not currently above it (register row 33).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The anchor for the *sustained* backlog badge: the badge shows only once
|
||||
/// <c>HealthMonitoringOptions.LocalDbBacklogSustainedFor</c> has elapsed since this
|
||||
/// instant. Holding a start time rather than a boolean is what makes the signal survive the
|
||||
/// expected daily purge burst — that spike raises the backlog for a drain's worth of
|
||||
/// reports and then clears this back to null, never reaching the dwell.
|
||||
/// <para>
|
||||
/// Null is set two different ways and they mean different things. A report whose backlog is
|
||||
/// at or below the threshold CLEARS it (the backlog demonstrably drained). A report whose
|
||||
/// backlog is <c>null</c> — an unreplicated site, or a failed poll — LEAVES IT UNCHANGED:
|
||||
/// null means "no reading", never zero, so a node that stops reporting mid-climb must not
|
||||
/// silently reset the clock, and an unreplicated site (which reports null forever) never
|
||||
/// starts one.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public DateTimeOffset? LocalDbBacklogAboveSince { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the instant of the last online↔offline flip, or <c>null</c> if the
|
||||
/// site has never changed status since it was first observed. Answers "when
|
||||
/// did this site drop / come back". Not moved by report/heartbeat traffic that
|
||||
/// leaves the online/offline status unchanged, nor by the metrics-stale flag.
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastStatusChangeAt { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user