Phase 3B: Site I/O & Observability — Communication, DCL, Script/Alarm actors, Health, Event Logging
Communication Layer (WP-1–5): - 8 message patterns with correlation IDs, per-pattern timeouts - Central/Site communication actors, transport heartbeat config - Connection failure handling (no central buffering, debug streams killed) Data Connection Layer (WP-6–14, WP-34): - Connection actor with Become/Stash lifecycle (Connecting/Connected/Reconnecting) - OPC UA + LmxProxy adapters behind IDataConnection - Auto-reconnect, bad quality propagation, transparent re-subscribe - Write-back, tag path resolution with retry, health reporting - Protocol extensibility via DataConnectionFactory Site Runtime (WP-15–25, WP-32–33): - ScriptActor/ScriptExecutionActor (triggers, concurrent execution, blocking I/O dispatcher) - AlarmActor/AlarmExecutionActor (ValueMatch/RangeViolation/RateOfChange, in-memory state) - SharedScriptLibrary (inline execution), ScriptRuntimeContext (API) - ScriptCompilationService (Roslyn, forbidden API enforcement, execution timeout) - Recursion limit (default 10), call direction enforcement - SiteStreamManager (per-subscriber bounded buffers, fire-and-forget) - Debug view backend (snapshot + stream), concurrency serialization - Local artifact storage (4 SQLite tables) Health Monitoring (WP-26–28): - SiteHealthCollector (thread-safe counters, connection state) - HealthReportSender (30s interval, monotonic sequence numbers) - CentralHealthAggregator (offline detection 60s, online recovery) Site Event Logging (WP-29–31): - SiteEventLogger (SQLite, 6 event categories, ISO 8601 UTC) - EventLogPurgeService (30-day retention, 1GB cap) - EventLogQueryService (filters, keyword search, keyset pagination) 541 tests pass, zero warnings.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,180 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ScadaLink.Commons.Messages.Health;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.HealthMonitoring.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// A simple fake TimeProvider for testing that allows advancing time manually.
|
||||
/// </summary>
|
||||
internal sealed class TestTimeProvider : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _utcNow;
|
||||
|
||||
public TestTimeProvider(DateTimeOffset startTime)
|
||||
{
|
||||
_utcNow = startTime;
|
||||
}
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _utcNow;
|
||||
|
||||
public void Advance(TimeSpan duration) => _utcNow += duration;
|
||||
}
|
||||
|
||||
public class CentralHealthAggregatorTests
|
||||
{
|
||||
private readonly TestTimeProvider _timeProvider;
|
||||
private readonly CentralHealthAggregator _aggregator;
|
||||
|
||||
public CentralHealthAggregatorTests()
|
||||
{
|
||||
_timeProvider = new TestTimeProvider(DateTimeOffset.UtcNow);
|
||||
var options = Options.Create(new HealthMonitoringOptions
|
||||
{
|
||||
OfflineTimeout = TimeSpan.FromSeconds(60)
|
||||
});
|
||||
_aggregator = new CentralHealthAggregator(
|
||||
options,
|
||||
NullLogger<CentralHealthAggregator>.Instance,
|
||||
_timeProvider);
|
||||
}
|
||||
|
||||
private static SiteHealthReport MakeReport(string siteId, long seq) =>
|
||||
new(
|
||||
SiteId: siteId,
|
||||
SequenceNumber: seq,
|
||||
ReportTimestamp: DateTimeOffset.UtcNow,
|
||||
DataConnectionStatuses: new Dictionary<string, ConnectionHealth>(),
|
||||
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>(),
|
||||
ScriptErrorCount: 0,
|
||||
AlarmEvaluationErrorCount: 0,
|
||||
StoreAndForwardBufferDepths: new Dictionary<string, int>(),
|
||||
DeadLetterCount: 0);
|
||||
|
||||
[Fact]
|
||||
public void ProcessReport_StoresState_ForNewSite()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
|
||||
var state = _aggregator.GetSiteState("site-1");
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state.IsOnline);
|
||||
Assert.Equal(1, state.LastSequenceNumber);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessReport_UpdatesState_WhenSequenceIncreases()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 2));
|
||||
|
||||
var state = _aggregator.GetSiteState("site-1");
|
||||
Assert.Equal(2, state!.LastSequenceNumber);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessReport_RejectsStaleReport_WhenSequenceNotGreater()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 5));
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 3));
|
||||
|
||||
var state = _aggregator.GetSiteState("site-1");
|
||||
Assert.Equal(5, state!.LastSequenceNumber);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessReport_RejectsEqualSequence()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 5));
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 5));
|
||||
|
||||
var state = _aggregator.GetSiteState("site-1");
|
||||
Assert.Equal(5, state!.LastSequenceNumber);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineDetection_SiteGoesOffline_WhenNoReportWithinTimeout()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
Assert.True(_aggregator.GetSiteState("site-1")!.IsOnline);
|
||||
|
||||
// Advance past the offline timeout
|
||||
_timeProvider.Advance(TimeSpan.FromSeconds(61));
|
||||
_aggregator.CheckForOfflineSites();
|
||||
|
||||
Assert.False(_aggregator.GetSiteState("site-1")!.IsOnline);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OnlineRecovery_SiteComesBackOnline_WhenReportReceived()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
|
||||
// Go offline
|
||||
_timeProvider.Advance(TimeSpan.FromSeconds(61));
|
||||
_aggregator.CheckForOfflineSites();
|
||||
Assert.False(_aggregator.GetSiteState("site-1")!.IsOnline);
|
||||
|
||||
// Receive new report → back online
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 2));
|
||||
Assert.True(_aggregator.GetSiteState("site-1")!.IsOnline);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineDetection_SiteRemainsOnline_WhenReportWithinTimeout()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
|
||||
_timeProvider.Advance(TimeSpan.FromSeconds(30));
|
||||
_aggregator.CheckForOfflineSites();
|
||||
|
||||
Assert.True(_aggregator.GetSiteState("site-1")!.IsOnline);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetAllSiteStates_ReturnsAllKnownSites()
|
||||
{
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
_aggregator.ProcessReport(MakeReport("site-2", 1));
|
||||
|
||||
var states = _aggregator.GetAllSiteStates();
|
||||
Assert.Equal(2, states.Count);
|
||||
Assert.Contains("site-1", states.Keys);
|
||||
Assert.Contains("site-2", states.Keys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetSiteState_ReturnsNull_ForUnknownSite()
|
||||
{
|
||||
var state = _aggregator.GetSiteState("nonexistent");
|
||||
Assert.Null(state);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessReport_StoresLatestReport()
|
||||
{
|
||||
var report = MakeReport("site-1", 1) with { ScriptErrorCount = 42 };
|
||||
_aggregator.ProcessReport(report);
|
||||
|
||||
var state = _aggregator.GetSiteState("site-1");
|
||||
Assert.Equal(42, state!.LatestReport.ScriptErrorCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SequenceNumberReset_RejectedUntilExceedsPrevMax()
|
||||
{
|
||||
// Site sends seq 10, then restarts and sends seq 1.
|
||||
// Per design: sequence resets on singleton restart.
|
||||
// The aggregator will reject seq 1 < 10 — expected behavior.
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 10));
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
||||
|
||||
var state = _aggregator.GetSiteState("site-1");
|
||||
Assert.Equal(10, state!.LastSequenceNumber);
|
||||
|
||||
// Once it exceeds the old max, it works again
|
||||
_aggregator.ProcessReport(MakeReport("site-1", 11));
|
||||
Assert.Equal(11, state.LastSequenceNumber);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ScadaLink.Commons.Messages.Health;
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.HealthMonitoring.Tests;
|
||||
|
||||
public class HealthReportSenderTests
|
||||
{
|
||||
private class FakeTransport : IHealthReportTransport
|
||||
{
|
||||
public List<SiteHealthReport> SentReports { get; } = [];
|
||||
public void Send(SiteHealthReport report) => SentReports.Add(report);
|
||||
}
|
||||
|
||||
private class FakeSiteIdentityProvider : ISiteIdentityProvider
|
||||
{
|
||||
public string SiteId { get; set; } = "test-site";
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendsReportsWithMonotonicSequenceNumbers()
|
||||
{
|
||||
var transport = new FakeTransport();
|
||||
var collector = new SiteHealthCollector();
|
||||
var options = Options.Create(new HealthMonitoringOptions
|
||||
{
|
||||
ReportInterval = TimeSpan.FromMilliseconds(50)
|
||||
});
|
||||
|
||||
var sender = new HealthReportSender(
|
||||
collector,
|
||||
transport,
|
||||
options,
|
||||
NullLogger<HealthReportSender>.Instance,
|
||||
new FakeSiteIdentityProvider { SiteId = "site-A" });
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(300));
|
||||
try
|
||||
{
|
||||
await sender.StartAsync(cts.Token);
|
||||
await Task.Delay(280, CancellationToken.None);
|
||||
await sender.StopAsync(CancellationToken.None);
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
|
||||
// Should have sent several reports
|
||||
Assert.True(transport.SentReports.Count >= 2,
|
||||
$"Expected at least 2 reports, got {transport.SentReports.Count}");
|
||||
|
||||
// Verify monotonic sequence numbers starting at 1
|
||||
for (int i = 0; i < transport.SentReports.Count; i++)
|
||||
{
|
||||
Assert.Equal(i + 1, transport.SentReports[i].SequenceNumber);
|
||||
Assert.Equal("site-A", transport.SentReports[i].SiteId);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SequenceNumberStartsAtOne()
|
||||
{
|
||||
var transport = new FakeTransport();
|
||||
var collector = new SiteHealthCollector();
|
||||
var options = Options.Create(new HealthMonitoringOptions
|
||||
{
|
||||
ReportInterval = TimeSpan.FromMilliseconds(50)
|
||||
});
|
||||
|
||||
var sender = new HealthReportSender(
|
||||
collector,
|
||||
transport,
|
||||
options,
|
||||
NullLogger<HealthReportSender>.Instance,
|
||||
new FakeSiteIdentityProvider());
|
||||
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(150));
|
||||
try
|
||||
{
|
||||
await sender.StartAsync(cts.Token);
|
||||
await Task.Delay(120, CancellationToken.None);
|
||||
await sender.StopAsync(CancellationToken.None);
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
|
||||
Assert.True(transport.SentReports.Count >= 1);
|
||||
Assert.Equal(1, transport.SentReports[0].SequenceNumber);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReportsIncludeUtcTimestamp()
|
||||
{
|
||||
var transport = new FakeTransport();
|
||||
var collector = new SiteHealthCollector();
|
||||
var options = Options.Create(new HealthMonitoringOptions
|
||||
{
|
||||
ReportInterval = TimeSpan.FromMilliseconds(50)
|
||||
});
|
||||
|
||||
var sender = new HealthReportSender(
|
||||
collector,
|
||||
transport,
|
||||
options,
|
||||
NullLogger<HealthReportSender>.Instance,
|
||||
new FakeSiteIdentityProvider());
|
||||
|
||||
var before = DateTimeOffset.UtcNow;
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(150));
|
||||
try
|
||||
{
|
||||
await sender.StartAsync(cts.Token);
|
||||
await Task.Delay(120, CancellationToken.None);
|
||||
await sender.StopAsync(CancellationToken.None);
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
var after = DateTimeOffset.UtcNow;
|
||||
|
||||
Assert.True(transport.SentReports.Count >= 1);
|
||||
foreach (var report in transport.SentReports)
|
||||
{
|
||||
Assert.InRange(report.ReportTimestamp, before, after);
|
||||
Assert.Equal(TimeSpan.Zero, report.ReportTimestamp.Offset);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InitialSequenceNumberIsZero()
|
||||
{
|
||||
var transport = new FakeTransport();
|
||||
var collector = new SiteHealthCollector();
|
||||
var options = Options.Create(new HealthMonitoringOptions());
|
||||
|
||||
var sender = new HealthReportSender(
|
||||
collector,
|
||||
transport,
|
||||
options,
|
||||
NullLogger<HealthReportSender>.Instance,
|
||||
new FakeSiteIdentityProvider());
|
||||
|
||||
Assert.Equal(0, sender.CurrentSequenceNumber);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
@@ -10,6 +10,8 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.5" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||
@@ -23,4 +25,4 @@
|
||||
<ProjectReference Include="../../src/ScadaLink.HealthMonitoring/ScadaLink.HealthMonitoring.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
using ScadaLink.Commons.Types.Enums;
|
||||
|
||||
namespace ScadaLink.HealthMonitoring.Tests;
|
||||
|
||||
public class SiteHealthCollectorTests
|
||||
{
|
||||
private readonly SiteHealthCollector _collector = new();
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_ReturnsZeroCounters_WhenNoErrorsRecorded()
|
||||
{
|
||||
var report = _collector.CollectReport("site-1");
|
||||
|
||||
Assert.Equal("site-1", report.SiteId);
|
||||
Assert.Equal(0, report.ScriptErrorCount);
|
||||
Assert.Equal(0, report.AlarmEvaluationErrorCount);
|
||||
Assert.Equal(0, report.DeadLetterCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncrementScriptError_AccumulatesBetweenReports()
|
||||
{
|
||||
_collector.IncrementScriptError();
|
||||
_collector.IncrementScriptError();
|
||||
_collector.IncrementScriptError();
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(3, report.ScriptErrorCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncrementAlarmError_AccumulatesBetweenReports()
|
||||
{
|
||||
_collector.IncrementAlarmError();
|
||||
_collector.IncrementAlarmError();
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(2, report.AlarmEvaluationErrorCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncrementDeadLetter_AccumulatesBetweenReports()
|
||||
{
|
||||
_collector.IncrementDeadLetter();
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(1, report.DeadLetterCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_ResetsCounters_AfterCollection()
|
||||
{
|
||||
_collector.IncrementScriptError();
|
||||
_collector.IncrementAlarmError();
|
||||
_collector.IncrementDeadLetter();
|
||||
|
||||
var first = _collector.CollectReport("site-1");
|
||||
Assert.Equal(1, first.ScriptErrorCount);
|
||||
Assert.Equal(1, first.AlarmEvaluationErrorCount);
|
||||
Assert.Equal(1, first.DeadLetterCount);
|
||||
|
||||
var second = _collector.CollectReport("site-1");
|
||||
Assert.Equal(0, second.ScriptErrorCount);
|
||||
Assert.Equal(0, second.AlarmEvaluationErrorCount);
|
||||
Assert.Equal(0, second.DeadLetterCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpdateConnectionHealth_ReflectedInReport()
|
||||
{
|
||||
_collector.UpdateConnectionHealth("opc-1", ConnectionHealth.Connected);
|
||||
_collector.UpdateConnectionHealth("opc-2", ConnectionHealth.Disconnected);
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
|
||||
Assert.Equal(2, report.DataConnectionStatuses.Count);
|
||||
Assert.Equal(ConnectionHealth.Connected, report.DataConnectionStatuses["opc-1"]);
|
||||
Assert.Equal(ConnectionHealth.Disconnected, report.DataConnectionStatuses["opc-2"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConnectionHealth_NotResetAfterCollect()
|
||||
{
|
||||
_collector.UpdateConnectionHealth("opc-1", ConnectionHealth.Connected);
|
||||
|
||||
_collector.CollectReport("site-1");
|
||||
var second = _collector.CollectReport("site-1");
|
||||
|
||||
Assert.Single(second.DataConnectionStatuses);
|
||||
Assert.Equal(ConnectionHealth.Connected, second.DataConnectionStatuses["opc-1"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveConnection_RemovesFromReport()
|
||||
{
|
||||
_collector.UpdateConnectionHealth("opc-1", ConnectionHealth.Connected);
|
||||
_collector.UpdateTagResolution("opc-1", 10, 8);
|
||||
_collector.RemoveConnection("opc-1");
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Empty(report.DataConnectionStatuses);
|
||||
Assert.Empty(report.TagResolutionCounts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpdateTagResolution_ReflectedInReport()
|
||||
{
|
||||
_collector.UpdateTagResolution("opc-1", 50, 45);
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
|
||||
Assert.Single(report.TagResolutionCounts);
|
||||
Assert.Equal(50, report.TagResolutionCounts["opc-1"].TotalSubscribed);
|
||||
Assert.Equal(45, report.TagResolutionCounts["opc-1"].SuccessfullyResolved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StoreAndForwardBufferDepths_IsEmptyPlaceholder()
|
||||
{
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Empty(report.StoreAndForwardBufferDepths);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_IncludesUtcTimestamp()
|
||||
{
|
||||
var before = DateTimeOffset.UtcNow;
|
||||
var report = _collector.CollectReport("site-1");
|
||||
var after = DateTimeOffset.UtcNow;
|
||||
|
||||
Assert.InRange(report.ReportTimestamp, before, after);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_SequenceNumberIsZero_CallerAssignsIt()
|
||||
{
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(0, report.SequenceNumber);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ThreadSafety_ConcurrentIncrements()
|
||||
{
|
||||
const int iterations = 10_000;
|
||||
var tasks = new[]
|
||||
{
|
||||
Task.Run(() => { for (int i = 0; i < iterations; i++) _collector.IncrementScriptError(); }),
|
||||
Task.Run(() => { for (int i = 0; i < iterations; i++) _collector.IncrementAlarmError(); }),
|
||||
Task.Run(() => { for (int i = 0; i < iterations; i++) _collector.IncrementDeadLetter(); })
|
||||
};
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(iterations, report.ScriptErrorCount);
|
||||
Assert.Equal(iterations, report.AlarmEvaluationErrorCount);
|
||||
Assert.Equal(iterations, report.DeadLetterCount);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
namespace ScadaLink.HealthMonitoring.Tests;
|
||||
|
||||
public class UnitTest1
|
||||
{
|
||||
[Fact]
|
||||
public void Test1()
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user