Phase 3B: Site I/O & Observability — Communication, DCL, Script/Alarm actors, Health, Event Logging

Communication Layer (WP-1–5):
- 8 message patterns with correlation IDs, per-pattern timeouts
- Central/Site communication actors, transport heartbeat config
- Connection failure handling (no central buffering, debug streams killed)

Data Connection Layer (WP-6–14, WP-34):
- Connection actor with Become/Stash lifecycle (Connecting/Connected/Reconnecting)
- OPC UA + LmxProxy adapters behind IDataConnection
- Auto-reconnect, bad quality propagation, transparent re-subscribe
- Write-back, tag path resolution with retry, health reporting
- Protocol extensibility via DataConnectionFactory

Site Runtime (WP-15–25, WP-32–33):
- ScriptActor/ScriptExecutionActor (triggers, concurrent execution, blocking I/O dispatcher)
- AlarmActor/AlarmExecutionActor (ValueMatch/RangeViolation/RateOfChange, in-memory state)
- SharedScriptLibrary (inline execution), ScriptRuntimeContext (API)
- ScriptCompilationService (Roslyn, forbidden API enforcement, execution timeout)
- Recursion limit (default 10), call direction enforcement
- SiteStreamManager (per-subscriber bounded buffers, fire-and-forget)
- Debug view backend (snapshot + stream), concurrency serialization
- Local artifact storage (4 SQLite tables)

Health Monitoring (WP-26–28):
- SiteHealthCollector (thread-safe counters, connection state)
- HealthReportSender (30s interval, monotonic sequence numbers)
- CentralHealthAggregator (offline detection 60s, online recovery)

Site Event Logging (WP-29–31):
- SiteEventLogger (SQLite, 6 event categories, ISO 8601 UTC)
- EventLogPurgeService (30-day retention, 1GB cap)
- EventLogQueryService (filters, keyword search, keyset pagination)

541 tests pass, zero warnings.

src/ScadaLink.HealthMonitoring/CentralHealthAggregator.cs

@@ -0,0 +1,134 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ScadaLink.Commons.Messages.Health;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Central-side aggregator that receives health reports from all sites,
+/// tracks latest metrics in memory, and detects offline sites.
+/// No persistence — display-only for Central UI consumption.
+/// </summary>
+public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator
+{
+    private readonly ConcurrentDictionary<string, SiteHealthState> _siteStates = new();
+    private readonly HealthMonitoringOptions _options;
+    private readonly ILogger<CentralHealthAggregator> _logger;
+    private readonly TimeProvider _timeProvider;
+
+    public CentralHealthAggregator(
+        IOptions<HealthMonitoringOptions> options,
+        ILogger<CentralHealthAggregator> logger,
+        TimeProvider? timeProvider = null)
+    {
+        _options = options.Value;
+        _logger = logger;
+        _timeProvider = timeProvider ?? TimeProvider.System;
+    }
+
+    /// <summary>
+    /// Process an incoming health report from a site.
+    /// Only replaces stored state if incoming sequence number is greater than last received.
+    /// Auto-marks previously offline sites as online.
+    /// </summary>
+    public void ProcessReport(SiteHealthReport report)
+    {
+        var now = _timeProvider.GetUtcNow();
+
+        _siteStates.AddOrUpdate(
+            report.SiteId,
+            _ =>
+            {
+                _logger.LogInformation("Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber);
+                return new SiteHealthState
+                {
+                    SiteId = report.SiteId,
+                    LatestReport = report,
+                    LastReportReceivedAt = now,
+                    LastSequenceNumber = report.SequenceNumber,
+                    IsOnline = true
+                };
+            },
+            (_, existing) =>
+            {
+                if (report.SequenceNumber <= existing.LastSequenceNumber)
+                {
+                    _logger.LogDebug(
+                        "Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}",
+                        report.SiteId, report.SequenceNumber, existing.LastSequenceNumber);
+                    return existing;
+                }
+
+                var wasOffline = !existing.IsOnline;
+                existing.LatestReport = report;
+                existing.LastReportReceivedAt = now;
+                existing.LastSequenceNumber = report.SequenceNumber;
+                existing.IsOnline = true;
+
+                if (wasOffline)
+                {
+                    _logger.LogInformation("Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber);
+                }
+
+                return existing;
+            });
+    }
+
+    /// <summary>
+    /// Get the current health state for all known sites.
+    /// </summary>
+    public IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates()
+    {
+        return new Dictionary<string, SiteHealthState>(_siteStates);
+    }
+
+    /// <summary>
+    /// Get the current health state for a specific site, or null if unknown.
+    /// </summary>
+    public SiteHealthState? GetSiteState(string siteId)
+    {
+        _siteStates.TryGetValue(siteId, out var state);
+        return state;
+    }
+
+    /// <summary>
+    /// Background task that periodically checks for offline sites.
+    /// </summary>
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation(
+            "Central health aggregator started, offline timeout {Timeout}s",
+            _options.OfflineTimeout.TotalSeconds);
+
+        // Check at half the offline timeout interval for timely detection
+        var checkInterval = TimeSpan.FromMilliseconds(_options.OfflineTimeout.TotalMilliseconds / 2);
+        using var timer = new PeriodicTimer(checkInterval);
+
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
+        {
+            CheckForOfflineSites();
+        }
+    }
+
+    internal void CheckForOfflineSites()
+    {
+        var now = _timeProvider.GetUtcNow();
+
+        foreach (var kvp in _siteStates)
+        {
+            var state = kvp.Value;
+            if (!state.IsOnline) continue;
+
+            var elapsed = now - state.LastReportReceivedAt;
+            if (elapsed > _options.OfflineTimeout)
+            {
+                state.IsOnline = false;
+                _logger.LogWarning(
+                    "Site {SiteId} marked offline — no report for {Elapsed}s (timeout: {Timeout}s)",
+                    state.SiteId, elapsed.TotalSeconds, _options.OfflineTimeout.TotalSeconds);
+            }
+        }
+    }
+}

src/ScadaLink.HealthMonitoring/HealthReportSender.cs

@@ -0,0 +1,69 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ScadaLink.Commons.Messages.Health;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Periodically collects a SiteHealthReport and sends it to central via Akka remoting.
+/// Sequence numbers are monotonic, starting at 1, and reset on service restart.
+/// </summary>
+public class HealthReportSender : BackgroundService
+{
+    private readonly ISiteHealthCollector _collector;
+    private readonly IHealthReportTransport _transport;
+    private readonly HealthMonitoringOptions _options;
+    private readonly ILogger<HealthReportSender> _logger;
+    private readonly string _siteId;
+    private long _sequenceNumber;
+
+    public HealthReportSender(
+        ISiteHealthCollector collector,
+        IHealthReportTransport transport,
+        IOptions<HealthMonitoringOptions> options,
+        ILogger<HealthReportSender> logger,
+        ISiteIdentityProvider siteIdentityProvider)
+    {
+        _collector = collector;
+        _transport = transport;
+        _options = options.Value;
+        _logger = logger;
+        _siteId = siteIdentityProvider.SiteId;
+    }
+
+    /// <summary>
+    /// Current sequence number (for testing).
+    /// </summary>
+    public long CurrentSequenceNumber => Interlocked.Read(ref _sequenceNumber);
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation(
+            "Health report sender starting for site {SiteId}, interval {Interval}s",
+            _siteId, _options.ReportInterval.TotalSeconds);
+
+        using var timer = new PeriodicTimer(_options.ReportInterval);
+
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
+        {
+            try
+            {
+                var seq = Interlocked.Increment(ref _sequenceNumber);
+                var report = _collector.CollectReport(_siteId);
+
+                // Replace the placeholder sequence number with our monotonic one
+                var reportWithSeq = report with { SequenceNumber = seq };
+
+                _transport.Send(reportWithSeq);
+
+                _logger.LogDebug("Sent health report #{Seq} for site {SiteId}", seq, _siteId);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to send health report for site {SiteId}", _siteId);
+                // Continue sending — don't let a single failure stop reporting
+            }
+        }
+    }
+}

src/ScadaLink.HealthMonitoring/ICentralHealthAggregator.cs

@@ -0,0 +1,14 @@
+using ScadaLink.Commons.Messages.Health;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Interface for central-side health aggregation.
+/// Consumed by Central UI to display site health dashboards.
+/// </summary>
+public interface ICentralHealthAggregator
+{
+    void ProcessReport(SiteHealthReport report);
+    IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates();
+    SiteHealthState? GetSiteState(string siteId);
+}

src/ScadaLink.HealthMonitoring/IHealthReportTransport.cs

@@ -0,0 +1,12 @@
+using ScadaLink.Commons.Messages.Health;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Abstraction for sending health reports to central.
+/// In production, implemented via Akka remoting (Tell, fire-and-forget).
+/// </summary>
+public interface IHealthReportTransport
+{
+    void Send(SiteHealthReport report);
+}

src/ScadaLink.HealthMonitoring/ISiteHealthCollector.cs

@@ -0,0 +1,19 @@
+using ScadaLink.Commons.Messages.Health;
+using ScadaLink.Commons.Types.Enums;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Interface for site-side health metric collection.
+/// Consumed by Site Runtime actors to report errors, and by DCL to report connection health.
+/// </summary>
+public interface ISiteHealthCollector
+{
+    void IncrementScriptError();
+    void IncrementAlarmError();
+    void IncrementDeadLetter();
+    void UpdateConnectionHealth(string connectionName, ConnectionHealth health);
+    void RemoveConnection(string connectionName);
+    void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved);
+    SiteHealthReport CollectReport(string siteId);
+}

src/ScadaLink.HealthMonitoring/ISiteIdentityProvider.cs

@@ -0,0 +1,10 @@
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Provides the identity of the current site.
+/// Implemented by the Host component to supply configuration-driven site ID.
+/// </summary>
+public interface ISiteIdentityProvider
+{
+    string SiteId { get; }
+}

src/ScadaLink.HealthMonitoring/ScadaLink.HealthMonitoring.csproj

@@ -9,6 +9,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.5" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.5" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.5" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="10.0.5" />
   </ItemGroup>
 
@@ -16,4 +18,8 @@
     <ProjectReference Include="../ScadaLink.Commons/ScadaLink.Commons.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="ScadaLink.HealthMonitoring.Tests" />
+  </ItemGroup>
+
 </Project>

src/ScadaLink.HealthMonitoring/ServiceCollectionExtensions.cs

@@ -4,15 +4,30 @@ namespace ScadaLink.HealthMonitoring;
 
 public static class ServiceCollectionExtensions
 {
+    /// <summary>
+    /// Register site-side health monitoring services.
+    /// </summary>
     public static IServiceCollection AddHealthMonitoring(this IServiceCollection services)
     {
-        // Phase 0: skeleton only
+        services.AddSingleton<ISiteHealthCollector, SiteHealthCollector>();
+        services.AddHostedService<HealthReportSender>();
+        return services;
+    }
+
+    /// <summary>
+    /// Register central-side health aggregation services.
+    /// </summary>
+    public static IServiceCollection AddCentralHealthAggregation(this IServiceCollection services)
+    {
+        services.AddSingleton<CentralHealthAggregator>();
+        services.AddSingleton<ICentralHealthAggregator>(sp => sp.GetRequiredService<CentralHealthAggregator>());
+        services.AddHostedService(sp => sp.GetRequiredService<CentralHealthAggregator>());
         return services;
     }
 
     public static IServiceCollection AddHealthMonitoringActors(this IServiceCollection services)
     {
-        // Phase 0: placeholder for Akka actor registration
+        // Placeholder for Akka actor registration (Phase 4+)
         return services;
     }
 }

src/ScadaLink.HealthMonitoring/SiteHealthCollector.cs

@@ -0,0 +1,101 @@
+using System.Collections.Concurrent;
+using ScadaLink.Commons.Messages.Health;
+using ScadaLink.Commons.Types.Enums;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// Collects health metrics from all site subsystems.
+/// Thread-safe: counters use Interlocked operations, connection/tag data uses ConcurrentDictionary.
+/// </summary>
+public class SiteHealthCollector : ISiteHealthCollector
+{
+    private int _scriptErrorCount;
+    private int _alarmErrorCount;
+    private int _deadLetterCount;
+    private readonly ConcurrentDictionary<string, ConnectionHealth> _connectionStatuses = new();
+    private readonly ConcurrentDictionary<string, TagResolutionStatus> _tagResolutionCounts = new();
+
+    /// <summary>
+    /// Increment the script error counter. Covers unhandled exceptions,
+    /// timeouts, and recursion limit violations.
+    /// </summary>
+    public void IncrementScriptError()
+    {
+        Interlocked.Increment(ref _scriptErrorCount);
+    }
+
+    /// <summary>
+    /// Increment the alarm evaluation error counter.
+    /// </summary>
+    public void IncrementAlarmError()
+    {
+        Interlocked.Increment(ref _alarmErrorCount);
+    }
+
+    /// <summary>
+    /// Increment the dead letter counter for this reporting interval.
+    /// </summary>
+    public void IncrementDeadLetter()
+    {
+        Interlocked.Increment(ref _deadLetterCount);
+    }
+
+    /// <summary>
+    /// Update the health status for a named data connection.
+    /// Called by DCL when connection state changes.
+    /// </summary>
+    public void UpdateConnectionHealth(string connectionName, ConnectionHealth health)
+    {
+        _connectionStatuses[connectionName] = health;
+    }
+
+    /// <summary>
+    /// Remove a connection from tracking (e.g., on connection disposal).
+    /// </summary>
+    public void RemoveConnection(string connectionName)
+    {
+        _connectionStatuses.TryRemove(connectionName, out _);
+        _tagResolutionCounts.TryRemove(connectionName, out _);
+    }
+
+    /// <summary>
+    /// Update tag resolution counts for a named data connection.
+    /// Called by DCL after tag resolution attempts.
+    /// </summary>
+    public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved)
+    {
+        _tagResolutionCounts[connectionName] = new TagResolutionStatus(totalSubscribed, successfullyResolved);
+    }
+
+    /// <summary>
+    /// Collect the current health report for the site and reset interval counters.
+    /// Connection statuses and tag resolution counts are NOT reset (they reflect current state).
+    /// Script errors, alarm errors, and dead letters ARE reset (they are per-interval counts).
+    /// </summary>
+    public SiteHealthReport CollectReport(string siteId)
+    {
+        // Atomically read and reset the counters
+        var scriptErrors = Interlocked.Exchange(ref _scriptErrorCount, 0);
+        var alarmErrors = Interlocked.Exchange(ref _alarmErrorCount, 0);
+        var deadLetters = Interlocked.Exchange(ref _deadLetterCount, 0);
+
+        // Snapshot current connection and tag resolution state
+        var connectionStatuses = new Dictionary<string, ConnectionHealth>(_connectionStatuses);
+        var tagResolution = new Dictionary<string, TagResolutionStatus>(_tagResolutionCounts);
+
+        // S&F buffer depth: placeholder (Phase 3C)
+        var sfBufferDepths = new Dictionary<string, int>();
+
+        return new SiteHealthReport(
+            SiteId: siteId,
+            SequenceNumber: 0, // Caller (HealthReportSender) assigns the sequence number
+            ReportTimestamp: DateTimeOffset.UtcNow,
+            DataConnectionStatuses: connectionStatuses,
+            TagResolutionCounts: tagResolution,
+            ScriptErrorCount: scriptErrors,
+            AlarmEvaluationErrorCount: alarmErrors,
+            StoreAndForwardBufferDepths: sfBufferDepths,
+            DeadLetterCount: deadLetters);
+    }
+}

src/ScadaLink.HealthMonitoring/SiteHealthState.cs

@@ -0,0 +1,15 @@
+using ScadaLink.Commons.Messages.Health;
+
+namespace ScadaLink.HealthMonitoring;
+
+/// <summary>
+/// In-memory state for a single site's health, stored by the central aggregator.
+/// </summary>
+public class SiteHealthState
+{
+    public required string SiteId { get; init; }
+    public SiteHealthReport LatestReport { get; set; } = null!;
+    public DateTimeOffset LastReportReceivedAt { get; set; }
+    public long LastSequenceNumber { get; set; }
+    public bool IsOnline { get; set; }
+}