Phase 3B: Site I/O & Observability — Communication, DCL, Script/Alarm actors, Health, Event Logging
Communication Layer (WP-1–5): - 8 message patterns with correlation IDs, per-pattern timeouts - Central/Site communication actors, transport heartbeat config - Connection failure handling (no central buffering, debug streams killed) Data Connection Layer (WP-6–14, WP-34): - Connection actor with Become/Stash lifecycle (Connecting/Connected/Reconnecting) - OPC UA + LmxProxy adapters behind IDataConnection - Auto-reconnect, bad quality propagation, transparent re-subscribe - Write-back, tag path resolution with retry, health reporting - Protocol extensibility via DataConnectionFactory Site Runtime (WP-15–25, WP-32–33): - ScriptActor/ScriptExecutionActor (triggers, concurrent execution, blocking I/O dispatcher) - AlarmActor/AlarmExecutionActor (ValueMatch/RangeViolation/RateOfChange, in-memory state) - SharedScriptLibrary (inline execution), ScriptRuntimeContext (API) - ScriptCompilationService (Roslyn, forbidden API enforcement, execution timeout) - Recursion limit (default 10), call direction enforcement - SiteStreamManager (per-subscriber bounded buffers, fire-and-forget) - Debug view backend (snapshot + stream), concurrency serialization - Local artifact storage (4 SQLite tables) Health Monitoring (WP-26–28): - SiteHealthCollector (thread-safe counters, connection state) - HealthReportSender (30s interval, monotonic sequence numbers) - CentralHealthAggregator (offline detection 60s, online recovery) Site Event Logging (WP-29–31): - SiteEventLogger (SQLite, 6 event categories, ISO 8601 UTC) - EventLogPurgeService (30-day retention, 1GB cap) - EventLogQueryService (filters, keyword search, keyset pagination) 541 tests pass, zero warnings.
This commit is contained in:
134
src/ScadaLink.HealthMonitoring/CentralHealthAggregator.cs
Normal file
134
src/ScadaLink.HealthMonitoring/CentralHealthAggregator.cs
Normal file
@@ -0,0 +1,134 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ScadaLink.Commons.Messages.Health;
|
||||
|
||||
namespace ScadaLink.HealthMonitoring;
|
||||
|
||||
/// <summary>
|
||||
/// Central-side aggregator that receives health reports from all sites,
|
||||
/// tracks latest metrics in memory, and detects offline sites.
|
||||
/// No persistence — display-only for Central UI consumption.
|
||||
/// </summary>
|
||||
public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, SiteHealthState> _siteStates = new();
|
||||
private readonly HealthMonitoringOptions _options;
|
||||
private readonly ILogger<CentralHealthAggregator> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public CentralHealthAggregator(
|
||||
IOptions<HealthMonitoringOptions> options,
|
||||
ILogger<CentralHealthAggregator> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process an incoming health report from a site.
|
||||
/// Only replaces stored state if incoming sequence number is greater than last received.
|
||||
/// Auto-marks previously offline sites as online.
|
||||
/// </summary>
|
||||
public void ProcessReport(SiteHealthReport report)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
_siteStates.AddOrUpdate(
|
||||
report.SiteId,
|
||||
_ =>
|
||||
{
|
||||
_logger.LogInformation("Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber);
|
||||
return new SiteHealthState
|
||||
{
|
||||
SiteId = report.SiteId,
|
||||
LatestReport = report,
|
||||
LastReportReceivedAt = now,
|
||||
LastSequenceNumber = report.SequenceNumber,
|
||||
IsOnline = true
|
||||
};
|
||||
},
|
||||
(_, existing) =>
|
||||
{
|
||||
if (report.SequenceNumber <= existing.LastSequenceNumber)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}",
|
||||
report.SiteId, report.SequenceNumber, existing.LastSequenceNumber);
|
||||
return existing;
|
||||
}
|
||||
|
||||
var wasOffline = !existing.IsOnline;
|
||||
existing.LatestReport = report;
|
||||
existing.LastReportReceivedAt = now;
|
||||
existing.LastSequenceNumber = report.SequenceNumber;
|
||||
existing.IsOnline = true;
|
||||
|
||||
if (wasOffline)
|
||||
{
|
||||
_logger.LogInformation("Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber);
|
||||
}
|
||||
|
||||
return existing;
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the current health state for all known sites.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates()
|
||||
{
|
||||
return new Dictionary<string, SiteHealthState>(_siteStates);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the current health state for a specific site, or null if unknown.
|
||||
/// </summary>
|
||||
public SiteHealthState? GetSiteState(string siteId)
|
||||
{
|
||||
_siteStates.TryGetValue(siteId, out var state);
|
||||
return state;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Background task that periodically checks for offline sites.
|
||||
/// </summary>
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Central health aggregator started, offline timeout {Timeout}s",
|
||||
_options.OfflineTimeout.TotalSeconds);
|
||||
|
||||
// Check at half the offline timeout interval for timely detection
|
||||
var checkInterval = TimeSpan.FromMilliseconds(_options.OfflineTimeout.TotalMilliseconds / 2);
|
||||
using var timer = new PeriodicTimer(checkInterval);
|
||||
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
CheckForOfflineSites();
|
||||
}
|
||||
}
|
||||
|
||||
internal void CheckForOfflineSites()
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
foreach (var kvp in _siteStates)
|
||||
{
|
||||
var state = kvp.Value;
|
||||
if (!state.IsOnline) continue;
|
||||
|
||||
var elapsed = now - state.LastReportReceivedAt;
|
||||
if (elapsed > _options.OfflineTimeout)
|
||||
{
|
||||
state.IsOnline = false;
|
||||
_logger.LogWarning(
|
||||
"Site {SiteId} marked offline — no report for {Elapsed}s (timeout: {Timeout}s)",
|
||||
state.SiteId, elapsed.TotalSeconds, _options.OfflineTimeout.TotalSeconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user