refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)
Solution + 23 src projects + 26 test projects renamed; folders, csproj, namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated. ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated. SQL roles/logins, LDAP domains, CLI command name, and CLI config dir (~/.scadalink → ~/.scadabridge) also renamed. Build green; 5 Host.Tests fail awaiting SQL login rename in next commit. Pre-existing StaleTagMonitor timing flakes unchanged. Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,263 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
/// <summary>
|
||||
/// Central-side aggregator that receives health reports from all sites,
|
||||
/// tracks latest metrics in memory, and detects offline sites.
|
||||
/// No persistence — display-only for Central UI consumption.
|
||||
/// </summary>
|
||||
public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, SiteHealthState> _siteStates = new();
|
||||
private readonly HealthMonitoringOptions _options;
|
||||
private readonly ILogger<CentralHealthAggregator> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="CentralHealthAggregator"/>.</summary>
|
||||
/// <param name="options">Health monitoring configuration.</param>
|
||||
/// <param name="logger">Logger for aggregator diagnostics.</param>
|
||||
/// <param name="timeProvider">Optional time provider; defaults to <see cref="TimeProvider.System"/>.</param>
|
||||
public CentralHealthAggregator(
|
||||
IOptions<HealthMonitoringOptions> options,
|
||||
ILogger<CentralHealthAggregator> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ProcessReport(SiteHealthReport report)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (!_siteStates.TryGetValue(report.SiteId, out var existing))
|
||||
{
|
||||
var registered = new SiteHealthState
|
||||
{
|
||||
SiteId = report.SiteId,
|
||||
LatestReport = report,
|
||||
LastReportReceivedAt = now,
|
||||
LastHeartbeatAt = now,
|
||||
LastSequenceNumber = report.SequenceNumber,
|
||||
IsOnline = true
|
||||
};
|
||||
|
||||
if (_siteStates.TryAdd(report.SiteId, registered))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber);
|
||||
return;
|
||||
}
|
||||
|
||||
// Lost the race — another thread registered first; retry as an update.
|
||||
continue;
|
||||
}
|
||||
|
||||
if (report.SequenceNumber <= existing.LastSequenceNumber)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}",
|
||||
report.SiteId, report.SequenceNumber, existing.LastSequenceNumber);
|
||||
return;
|
||||
}
|
||||
|
||||
var updated = existing with
|
||||
{
|
||||
LatestReport = report,
|
||||
LastReportReceivedAt = now,
|
||||
LastHeartbeatAt = now,
|
||||
LastSequenceNumber = report.SequenceNumber,
|
||||
IsOnline = true
|
||||
};
|
||||
|
||||
if (_siteStates.TryUpdate(report.SiteId, updated, existing))
|
||||
{
|
||||
if (!existing.IsOnline)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// CAS lost — the entry changed under us; retry with the fresh value.
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void MarkHeartbeat(string siteId, DateTimeOffset receivedAt)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (!_siteStates.TryGetValue(siteId, out var existing))
|
||||
{
|
||||
// Unknown site — register it as online, awaiting its first
|
||||
// full report. LatestReport and LastReportReceivedAt both stay
|
||||
// null until ProcessReport runs — "no report yet" is an explicit
|
||||
// nullable state, not a year-0001 sentinel the UI must special-case.
|
||||
var registered = new SiteHealthState
|
||||
{
|
||||
SiteId = siteId,
|
||||
LatestReport = null,
|
||||
LastReportReceivedAt = null,
|
||||
LastHeartbeatAt = receivedAt,
|
||||
LastSequenceNumber = 0,
|
||||
IsOnline = true
|
||||
};
|
||||
|
||||
if (_siteStates.TryAdd(siteId, registered))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Site {SiteId} registered online via heartbeat (awaiting first report)", siteId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Lost the race — another thread registered first; retry as an update.
|
||||
continue;
|
||||
}
|
||||
|
||||
// HealthMonitoring-020: when an offline→online transition is being
|
||||
// applied, the heartbeat timestamp must reflect a fresh observation,
|
||||
// not the prior stored value. If receivedAt is older than the stored
|
||||
// LastHeartbeatAt (clock skew, an out-of-order heartbeat arriving
|
||||
// after an earlier one already advanced the field), promoting the
|
||||
// site back to online while leaving LastHeartbeatAt stale would let
|
||||
// CheckForOfflineSites flap it straight back to offline on the next
|
||||
// tick. Anchor the heartbeat to the current time provider instead,
|
||||
// so an offline-to-online transition is always backed by an
|
||||
// up-to-date heartbeat.
|
||||
DateTimeOffset newHeartbeat;
|
||||
if (!existing.IsOnline)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
newHeartbeat = receivedAt > now ? receivedAt : now;
|
||||
}
|
||||
else
|
||||
{
|
||||
newHeartbeat = receivedAt > existing.LastHeartbeatAt
|
||||
? receivedAt
|
||||
: existing.LastHeartbeatAt;
|
||||
}
|
||||
|
||||
// Nothing to change — avoid a needless swap.
|
||||
if (newHeartbeat == existing.LastHeartbeatAt && existing.IsOnline)
|
||||
return;
|
||||
|
||||
var updated = existing with
|
||||
{
|
||||
LastHeartbeatAt = newHeartbeat,
|
||||
IsOnline = true
|
||||
};
|
||||
|
||||
if (_siteStates.TryUpdate(siteId, updated, existing))
|
||||
{
|
||||
if (!existing.IsOnline)
|
||||
_logger.LogInformation("Site {SiteId} is back online (heartbeat)", siteId);
|
||||
return;
|
||||
}
|
||||
|
||||
// CAS lost — retry with the fresh value.
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates()
|
||||
{
|
||||
return new Dictionary<string, SiteHealthState>(_siteStates);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public SiteHealthState? GetSiteState(string siteId)
|
||||
{
|
||||
_siteStates.TryGetValue(siteId, out var state);
|
||||
return state;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Central health aggregator started, offline timeout {Timeout}s (central {CentralTimeout}s)",
|
||||
_options.OfflineTimeout.TotalSeconds, _options.CentralOfflineTimeout.TotalSeconds);
|
||||
|
||||
// Check at half the shorter of the two offline timeouts so detection is
|
||||
// timely for whichever site class (real or "central") has the tighter
|
||||
// window — see ComputeCheckInterval.
|
||||
using var timer = new PeriodicTimer(ComputeCheckInterval(_options));
|
||||
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
CheckForOfflineSites();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the offline-check timer cadence: half of the <em>shorter</em> of
|
||||
/// <see cref="HealthMonitoringOptions.OfflineTimeout"/> and
|
||||
/// <see cref="HealthMonitoringOptions.CentralOfflineTimeout"/>. Deriving it
|
||||
/// from the shorter timeout guarantees that whichever site class has the
|
||||
/// tighter window is still polled at least twice within it — so if an
|
||||
/// operator configures <c>CentralOfflineTimeout</c> smaller than
|
||||
/// <c>OfflineTimeout</c>, central offline detection is not delayed by up to a
|
||||
/// full <c>OfflineTimeout / 2</c>.
|
||||
/// </summary>
|
||||
/// <param name="options">The health monitoring options to derive the interval from.</param>
|
||||
internal static TimeSpan ComputeCheckInterval(HealthMonitoringOptions options)
|
||||
{
|
||||
var shorter = options.OfflineTimeout < options.CentralOfflineTimeout
|
||||
? options.OfflineTimeout
|
||||
: options.CentralOfflineTimeout;
|
||||
return TimeSpan.FromMilliseconds(shorter.TotalMilliseconds / 2);
|
||||
}
|
||||
|
||||
/// <summary>Iterates all tracked sites and marks any that have exceeded their offline timeout as offline.</summary>
|
||||
internal void CheckForOfflineSites()
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
foreach (var kvp in _siteStates)
|
||||
{
|
||||
var state = kvp.Value;
|
||||
if (!state.IsOnline) continue;
|
||||
|
||||
// Use LastHeartbeatAt — heartbeats arrive every ~5s from any
|
||||
// healthy site node (cadence owned by Cluster Infrastructure /
|
||||
// SiteCommunicationActor — CommunicationOptions.TransportHeartbeatInterval),
|
||||
// so the 60s OfflineTimeout tolerates several missed heartbeats and
|
||||
// only fires when no node can reach central, not during single-node
|
||||
// failovers.
|
||||
//
|
||||
// The synthetic "central" site has no heartbeat source — its only
|
||||
// signal is the 30s CentralHealthReportLoop self-report — so it gets
|
||||
// a longer grace window (CentralOfflineTimeout) to survive a single
|
||||
// skipped/late self-report.
|
||||
var timeout = kvp.Key == CentralHealthReportLoop.CentralSiteId
|
||||
? _options.CentralOfflineTimeout
|
||||
: _options.OfflineTimeout;
|
||||
|
||||
var elapsed = now - state.LastHeartbeatAt;
|
||||
if (elapsed <= timeout)
|
||||
continue;
|
||||
|
||||
// Atomically swap to an offline copy. If the CAS loses to a
|
||||
// concurrent report/heartbeat the site was just heard from, so
|
||||
// leaving it online is the correct outcome — no retry needed.
|
||||
var offline = state with { IsOnline = false };
|
||||
if (_siteStates.TryUpdate(kvp.Key, offline, state))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
|
||||
state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user