Phase 3B: Site I/O & Observability — Communication, DCL, Script/Alarm actors, Health, Event Logging

Communication Layer (WP-1–5):
- 8 message patterns with correlation IDs, per-pattern timeouts
- Central/Site communication actors, transport heartbeat config
- Connection failure handling (no central buffering, debug streams killed)

Data Connection Layer (WP-6–14, WP-34):
- Connection actor with Become/Stash lifecycle (Connecting/Connected/Reconnecting)
- OPC UA + LmxProxy adapters behind IDataConnection
- Auto-reconnect, bad quality propagation, transparent re-subscribe
- Write-back, tag path resolution with retry, health reporting
- Protocol extensibility via DataConnectionFactory

Site Runtime (WP-15–25, WP-32–33):
- ScriptActor/ScriptExecutionActor (triggers, concurrent execution, blocking I/O dispatcher)
- AlarmActor/AlarmExecutionActor (ValueMatch/RangeViolation/RateOfChange, in-memory state)
- SharedScriptLibrary (inline execution), ScriptRuntimeContext (API)
- ScriptCompilationService (Roslyn, forbidden API enforcement, execution timeout)
- Recursion limit (default 10), call direction enforcement
- SiteStreamManager (per-subscriber bounded buffers, fire-and-forget)
- Debug view backend (snapshot + stream), concurrency serialization
- Local artifact storage (4 SQLite tables)

Health Monitoring (WP-26–28):
- SiteHealthCollector (thread-safe counters, connection state)
- HealthReportSender (30s interval, monotonic sequence numbers)
- CentralHealthAggregator (offline detection 60s, online recovery)

Site Event Logging (WP-29–31):
- SiteEventLogger (SQLite, 6 event categories, ISO 8601 UTC)
- EventLogPurgeService (30-day retention, 1GB cap)
- EventLogQueryService (filters, keyword search, keyset pagination)

541 tests pass, zero warnings.
This commit is contained in:
Joseph Doherty
2026-03-16 20:57:25 -04:00
parent a3bf0c43f3
commit 389f5a0378
97 changed files with 8308 additions and 127 deletions

View File

@@ -0,0 +1,134 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ScadaLink.Commons.Messages.Health;
namespace ScadaLink.HealthMonitoring;
/// <summary>
/// Central-side aggregator that receives health reports from all sites,
/// tracks latest metrics in memory, and detects offline sites.
/// No persistence — display-only for Central UI consumption.
/// </summary>
public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator
{
private readonly ConcurrentDictionary<string, SiteHealthState> _siteStates = new();
private readonly HealthMonitoringOptions _options;
private readonly ILogger<CentralHealthAggregator> _logger;
private readonly TimeProvider _timeProvider;
public CentralHealthAggregator(
IOptions<HealthMonitoringOptions> options,
ILogger<CentralHealthAggregator> logger,
TimeProvider? timeProvider = null)
{
_options = options.Value;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>
/// Process an incoming health report from a site.
/// Only replaces stored state if incoming sequence number is greater than last received.
/// Auto-marks previously offline sites as online.
/// </summary>
public void ProcessReport(SiteHealthReport report)
{
var now = _timeProvider.GetUtcNow();
_siteStates.AddOrUpdate(
report.SiteId,
_ =>
{
_logger.LogInformation("Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber);
return new SiteHealthState
{
SiteId = report.SiteId,
LatestReport = report,
LastReportReceivedAt = now,
LastSequenceNumber = report.SequenceNumber,
IsOnline = true
};
},
(_, existing) =>
{
if (report.SequenceNumber <= existing.LastSequenceNumber)
{
_logger.LogDebug(
"Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}",
report.SiteId, report.SequenceNumber, existing.LastSequenceNumber);
return existing;
}
var wasOffline = !existing.IsOnline;
existing.LatestReport = report;
existing.LastReportReceivedAt = now;
existing.LastSequenceNumber = report.SequenceNumber;
existing.IsOnline = true;
if (wasOffline)
{
_logger.LogInformation("Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber);
}
return existing;
});
}
/// <summary>
/// Get the current health state for all known sites.
/// </summary>
public IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates()
{
return new Dictionary<string, SiteHealthState>(_siteStates);
}
/// <summary>
/// Get the current health state for a specific site, or null if unknown.
/// </summary>
public SiteHealthState? GetSiteState(string siteId)
{
_siteStates.TryGetValue(siteId, out var state);
return state;
}
/// <summary>
/// Background task that periodically checks for offline sites.
/// </summary>
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation(
"Central health aggregator started, offline timeout {Timeout}s",
_options.OfflineTimeout.TotalSeconds);
// Check at half the offline timeout interval for timely detection
var checkInterval = TimeSpan.FromMilliseconds(_options.OfflineTimeout.TotalMilliseconds / 2);
using var timer = new PeriodicTimer(checkInterval);
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
{
CheckForOfflineSites();
}
}
internal void CheckForOfflineSites()
{
var now = _timeProvider.GetUtcNow();
foreach (var kvp in _siteStates)
{
var state = kvp.Value;
if (!state.IsOnline) continue;
var elapsed = now - state.LastReportReceivedAt;
if (elapsed > _options.OfflineTimeout)
{
state.IsOnline = false;
_logger.LogWarning(
"Site {SiteId} marked offline — no report for {Elapsed}s (timeout: {Timeout}s)",
state.SiteId, elapsed.TotalSeconds, _options.OfflineTimeout.TotalSeconds);
}
}
}
}