Communication Layer (WP-1–5): - 8 message patterns with correlation IDs, per-pattern timeouts - Central/Site communication actors, transport heartbeat config - Connection failure handling (no central buffering, debug streams killed) Data Connection Layer (WP-6–14, WP-34): - Connection actor with Become/Stash lifecycle (Connecting/Connected/Reconnecting) - OPC UA + LmxProxy adapters behind IDataConnection - Auto-reconnect, bad quality propagation, transparent re-subscribe - Write-back, tag path resolution with retry, health reporting - Protocol extensibility via DataConnectionFactory Site Runtime (WP-15–25, WP-32–33): - ScriptActor/ScriptExecutionActor (triggers, concurrent execution, blocking I/O dispatcher) - AlarmActor/AlarmExecutionActor (ValueMatch/RangeViolation/RateOfChange, in-memory state) - SharedScriptLibrary (inline execution), ScriptRuntimeContext (API) - ScriptCompilationService (Roslyn, forbidden API enforcement, execution timeout) - Recursion limit (default 10), call direction enforcement - SiteStreamManager (per-subscriber bounded buffers, fire-and-forget) - Debug view backend (snapshot + stream), concurrency serialization - Local artifact storage (4 SQLite tables) Health Monitoring (WP-26–28): - SiteHealthCollector (thread-safe counters, connection state) - HealthReportSender (30s interval, monotonic sequence numbers) - CentralHealthAggregator (offline detection 60s, online recovery) Site Event Logging (WP-29–31): - SiteEventLogger (SQLite, 6 event categories, ISO 8601 UTC) - EventLogPurgeService (30-day retention, 1GB cap) - EventLogQueryService (filters, keyword search, keyset pagination) 541 tests pass, zero warnings.
181 lines
5.7 KiB
C#
181 lines
5.7 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ScadaLink.Commons.Messages.Health;
|
|
using ScadaLink.Commons.Types.Enums;
|
|
|
|
namespace ScadaLink.HealthMonitoring.Tests;
|
|
|
|
/// <summary>
|
|
/// A simple fake TimeProvider for testing that allows advancing time manually.
|
|
/// </summary>
|
|
internal sealed class TestTimeProvider : TimeProvider
|
|
{
|
|
private DateTimeOffset _utcNow;
|
|
|
|
public TestTimeProvider(DateTimeOffset startTime)
|
|
{
|
|
_utcNow = startTime;
|
|
}
|
|
|
|
public override DateTimeOffset GetUtcNow() => _utcNow;
|
|
|
|
public void Advance(TimeSpan duration) => _utcNow += duration;
|
|
}
|
|
|
|
public class CentralHealthAggregatorTests
|
|
{
|
|
private readonly TestTimeProvider _timeProvider;
|
|
private readonly CentralHealthAggregator _aggregator;
|
|
|
|
public CentralHealthAggregatorTests()
|
|
{
|
|
_timeProvider = new TestTimeProvider(DateTimeOffset.UtcNow);
|
|
var options = Options.Create(new HealthMonitoringOptions
|
|
{
|
|
OfflineTimeout = TimeSpan.FromSeconds(60)
|
|
});
|
|
_aggregator = new CentralHealthAggregator(
|
|
options,
|
|
NullLogger<CentralHealthAggregator>.Instance,
|
|
_timeProvider);
|
|
}
|
|
|
|
private static SiteHealthReport MakeReport(string siteId, long seq) =>
|
|
new(
|
|
SiteId: siteId,
|
|
SequenceNumber: seq,
|
|
ReportTimestamp: DateTimeOffset.UtcNow,
|
|
DataConnectionStatuses: new Dictionary<string, ConnectionHealth>(),
|
|
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>(),
|
|
ScriptErrorCount: 0,
|
|
AlarmEvaluationErrorCount: 0,
|
|
StoreAndForwardBufferDepths: new Dictionary<string, int>(),
|
|
DeadLetterCount: 0);
|
|
|
|
[Fact]
|
|
public void ProcessReport_StoresState_ForNewSite()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
|
|
var state = _aggregator.GetSiteState("site-1");
|
|
Assert.NotNull(state);
|
|
Assert.True(state.IsOnline);
|
|
Assert.Equal(1, state.LastSequenceNumber);
|
|
}
|
|
|
|
[Fact]
|
|
public void ProcessReport_UpdatesState_WhenSequenceIncreases()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
_aggregator.ProcessReport(MakeReport("site-1", 2));
|
|
|
|
var state = _aggregator.GetSiteState("site-1");
|
|
Assert.Equal(2, state!.LastSequenceNumber);
|
|
}
|
|
|
|
[Fact]
|
|
public void ProcessReport_RejectsStaleReport_WhenSequenceNotGreater()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 5));
|
|
_aggregator.ProcessReport(MakeReport("site-1", 3));
|
|
|
|
var state = _aggregator.GetSiteState("site-1");
|
|
Assert.Equal(5, state!.LastSequenceNumber);
|
|
}
|
|
|
|
[Fact]
|
|
public void ProcessReport_RejectsEqualSequence()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 5));
|
|
_aggregator.ProcessReport(MakeReport("site-1", 5));
|
|
|
|
var state = _aggregator.GetSiteState("site-1");
|
|
Assert.Equal(5, state!.LastSequenceNumber);
|
|
}
|
|
|
|
[Fact]
|
|
public void OfflineDetection_SiteGoesOffline_WhenNoReportWithinTimeout()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
Assert.True(_aggregator.GetSiteState("site-1")!.IsOnline);
|
|
|
|
// Advance past the offline timeout
|
|
_timeProvider.Advance(TimeSpan.FromSeconds(61));
|
|
_aggregator.CheckForOfflineSites();
|
|
|
|
Assert.False(_aggregator.GetSiteState("site-1")!.IsOnline);
|
|
}
|
|
|
|
[Fact]
|
|
public void OnlineRecovery_SiteComesBackOnline_WhenReportReceived()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
|
|
// Go offline
|
|
_timeProvider.Advance(TimeSpan.FromSeconds(61));
|
|
_aggregator.CheckForOfflineSites();
|
|
Assert.False(_aggregator.GetSiteState("site-1")!.IsOnline);
|
|
|
|
// Receive new report → back online
|
|
_aggregator.ProcessReport(MakeReport("site-1", 2));
|
|
Assert.True(_aggregator.GetSiteState("site-1")!.IsOnline);
|
|
}
|
|
|
|
[Fact]
|
|
public void OfflineDetection_SiteRemainsOnline_WhenReportWithinTimeout()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
|
|
_timeProvider.Advance(TimeSpan.FromSeconds(30));
|
|
_aggregator.CheckForOfflineSites();
|
|
|
|
Assert.True(_aggregator.GetSiteState("site-1")!.IsOnline);
|
|
}
|
|
|
|
[Fact]
|
|
public void GetAllSiteStates_ReturnsAllKnownSites()
|
|
{
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
_aggregator.ProcessReport(MakeReport("site-2", 1));
|
|
|
|
var states = _aggregator.GetAllSiteStates();
|
|
Assert.Equal(2, states.Count);
|
|
Assert.Contains("site-1", states.Keys);
|
|
Assert.Contains("site-2", states.Keys);
|
|
}
|
|
|
|
[Fact]
|
|
public void GetSiteState_ReturnsNull_ForUnknownSite()
|
|
{
|
|
var state = _aggregator.GetSiteState("nonexistent");
|
|
Assert.Null(state);
|
|
}
|
|
|
|
[Fact]
|
|
public void ProcessReport_StoresLatestReport()
|
|
{
|
|
var report = MakeReport("site-1", 1) with { ScriptErrorCount = 42 };
|
|
_aggregator.ProcessReport(report);
|
|
|
|
var state = _aggregator.GetSiteState("site-1");
|
|
Assert.Equal(42, state!.LatestReport.ScriptErrorCount);
|
|
}
|
|
|
|
[Fact]
|
|
public void SequenceNumberReset_RejectedUntilExceedsPrevMax()
|
|
{
|
|
// Site sends seq 10, then restarts and sends seq 1.
|
|
// Per design: sequence resets on singleton restart.
|
|
// The aggregator will reject seq 1 < 10 — expected behavior.
|
|
_aggregator.ProcessReport(MakeReport("site-1", 10));
|
|
_aggregator.ProcessReport(MakeReport("site-1", 1));
|
|
|
|
var state = _aggregator.GetSiteState("site-1");
|
|
Assert.Equal(10, state!.LastSequenceNumber);
|
|
|
|
// Once it exceeds the old max, it works again
|
|
_aggregator.ProcessReport(MakeReport("site-1", 11));
|
|
Assert.Equal(11, state.LastSequenceNumber);
|
|
}
|
|
}
|