Joseph Doherty
abbbcd1a8d
Defect: the hourly KPI rollup fold (KpiHistoryRepository.FoldHourlyRollupsAsync) sums every Rate-classified sample recorded in an hour, correct only if each sample is a non-overlapping delta covering exactly one recorder tick (KpiHistoryOptions.SampleInterval). That invariant was an unenforced silent assumption across three independently-owned config keys — SampleInterval, NotificationOutboxOptions.DeliveredKpiWindow, SiteCallAuditOptions.KpiInterval — so widening DeliveredKpiWindow to e.g. 15 min would silently over-count the rollup ~15x. Robust fix: NotificationOutboxKpiSampleSource and SiteCallAuditKpiSampleSource now rescale their Rate-classified samples (deliveredLastInterval, failedLastInterval) by SampleInterval / (DeliveredKpiWindow or KpiInterval) before recording, so the rollup's per-hour Sum is correct regardless of how the other two keys are tuned — the live dashboard tile's own cutoff/window is untouched. Both sources gain a project reference to KpiHistory to read SampleInterval; no cycle (KpiHistory references only Commons). Eager cross-validation (defense-in-depth): KpiRateFoldIntervalCrossValidator (Host, the one composition root referencing all three component projects) fails boot naming all three config keys when they diverge, protecting the separate (still-real) invariant that the recorded/charted KPI-history sample equals the live tile at the same instant — which the rescale trades away whenever the scale factor isn't 1. Rides KpiHistoryOptions' existing ValidateOnStart trigger. Also: NotificationOutboxOptions had no registered validator at all before this (the gap this task's "whichever options class owns DeliveredKpiWindow" file scope pointed at) — added NotificationOutboxOptionsValidator mirroring SiteCallAuditOptionsValidator's positive-duration/count convention, wired with ValidateOnStart. Tests: new rescale tests in both sample-source suites prove the fold is numerically correct at a non-default SampleInterval (30/40 counted over a 15/20-minute window correctly divides down to a 1-minute-tick delta); new KpiRateFoldIntervalCrossValidatorTests prove divergent config fails with all three keys named; new NotificationOutboxOptionsValidatorTests. Every new test proven to fail against the unfixed computation first (temporarily short-circuited the scale factor / cross-check to the pre-fix behavior, confirmed failure, restored) — not via git stash (repo-wide, shared across concurrent worktrees this session) but via file-copy swap. Full suites green: Host.Tests (527), ConfigurationDatabase.Tests (401), KpiHistory.Tests (54), NotificationOutbox.Tests (163), SiteCallAudit.Tests (67).
255 lines
13 KiB
C#
255 lines
13 KiB
C#
using NSubstitute;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Notifications;
|
||
using ZB.MOM.WW.ScadaBridge.KpiHistory;
|
||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Kpi;
|
||
|
||
namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.Kpi;
|
||
|
||
/// <summary>
|
||
/// Tests for <see cref="NotificationOutboxKpiSampleSource"/> — the M6 KPI sample source that
|
||
/// snapshots the Notification Outbox delivery KPIs (global / per-site / per-node) into the
|
||
/// central KPI-history store.
|
||
/// </summary>
|
||
public class NotificationOutboxKpiSampleSourceTests
|
||
{
|
||
private static readonly DateTime CapturedAt = new(2026, 6, 15, 12, 0, 0, DateTimeKind.Utc);
|
||
|
||
private static readonly NotificationOutboxOptions Options = new()
|
||
{
|
||
StuckAgeThreshold = TimeSpan.FromMinutes(10),
|
||
DeliveredKpiWindow = TimeSpan.FromMinutes(1),
|
||
};
|
||
|
||
// Matches Options.DeliveredKpiWindow (1 minute) so the default-config tests below see
|
||
// an identity rescale factor (1.0) and their pre-existing raw-count assertions hold.
|
||
private static readonly KpiHistoryOptions DefaultKpiHistoryOptions = new()
|
||
{
|
||
SampleInterval = TimeSpan.FromMinutes(1),
|
||
};
|
||
|
||
private static NotificationOutboxKpiSampleSource CreateSource(INotificationOutboxRepository repository) =>
|
||
CreateSource(repository, DefaultKpiHistoryOptions);
|
||
|
||
private static NotificationOutboxKpiSampleSource CreateSource(
|
||
INotificationOutboxRepository repository, KpiHistoryOptions kpiHistoryOptions) =>
|
||
CreateSource(repository, Options, kpiHistoryOptions);
|
||
|
||
private static NotificationOutboxKpiSampleSource CreateSource(
|
||
INotificationOutboxRepository repository,
|
||
NotificationOutboxOptions options,
|
||
KpiHistoryOptions kpiHistoryOptions) =>
|
||
new(
|
||
repository,
|
||
Microsoft.Extensions.Options.Options.Create(options),
|
||
Microsoft.Extensions.Options.Options.Create(kpiHistoryOptions));
|
||
|
||
[Fact]
|
||
public void Source_IsNotificationOutbox()
|
||
{
|
||
var source = CreateSource(Substitute.For<INotificationOutboxRepository>());
|
||
|
||
Assert.Equal(KpiSources.NotificationOutbox, source.Source);
|
||
}
|
||
|
||
// C4 (arch-review 04): the two metric names that were inline string literals now
|
||
// live in the shared KpiMetrics catalog. Their VALUES are persisted in the KpiSample
|
||
// store and rendered by the trend UI, so a rename would silently orphan history —
|
||
// pin the exact historical strings here.
|
||
[Fact]
|
||
public void KpiMetricCatalog_NotificationOutbox_PreservesHistoricalStringValues()
|
||
{
|
||
Assert.Equal("queueDepth", KpiMetrics.NotificationOutbox.QueueDepth);
|
||
Assert.Equal("stuckCount", KpiMetrics.NotificationOutbox.StuckCount);
|
||
Assert.Equal("parkedCount", KpiMetrics.NotificationOutbox.ParkedCount);
|
||
Assert.Equal("deliveredLastInterval", KpiMetrics.NotificationOutbox.DeliveredLastInterval);
|
||
Assert.Equal("oldestPendingAgeSeconds", KpiMetrics.NotificationOutbox.OldestPendingAgeSeconds);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_PassesCutoffsAnchoredOnCapturedAt()
|
||
{
|
||
var repository = Substitute.For<INotificationOutboxRepository>();
|
||
StubEmptySnapshots(repository);
|
||
|
||
var source = CreateSource(repository);
|
||
await source.CollectAsync(CapturedAt);
|
||
|
||
var expectedStuckCutoff = new DateTimeOffset(CapturedAt, TimeSpan.Zero) - Options.StuckAgeThreshold;
|
||
var expectedDeliveredSince = new DateTimeOffset(CapturedAt, TimeSpan.Zero) - Options.DeliveredKpiWindow;
|
||
|
||
await repository.Received(1).ComputeKpisAsync(
|
||
expectedStuckCutoff, expectedDeliveredSince, Arg.Any<CancellationToken>());
|
||
await repository.Received(1).ComputePerSiteKpisAsync(
|
||
expectedStuckCutoff, expectedDeliveredSince, Arg.Any<CancellationToken>());
|
||
await repository.Received(1).ComputePerNodeKpisAsync(
|
||
expectedStuckCutoff, expectedDeliveredSince, Arg.Any<CancellationToken>());
|
||
}
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_EmitsGlobalSiteAndNodeSamples_WithExpectedTuples()
|
||
{
|
||
var repository = Substitute.For<INotificationOutboxRepository>();
|
||
repository.ComputeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(new NotificationKpiSnapshot(
|
||
QueueDepth: 5,
|
||
StuckCount: 2,
|
||
ParkedCount: 1,
|
||
DeliveredLastInterval: 7,
|
||
OldestPendingAge: TimeSpan.FromSeconds(90)));
|
||
repository.ComputePerSiteKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(new[]
|
||
{
|
||
new SiteNotificationKpiSnapshot(
|
||
SourceSiteId: "site-a",
|
||
QueueDepth: 3,
|
||
StuckCount: 1,
|
||
ParkedCount: 0,
|
||
DeliveredLastInterval: 4,
|
||
OldestPendingAge: TimeSpan.FromSeconds(30)),
|
||
});
|
||
repository.ComputePerNodeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(new[]
|
||
{
|
||
new NodeNotificationKpiSnapshot(
|
||
SourceNode: "node-a",
|
||
QueueDepth: 2,
|
||
StuckCount: 1,
|
||
ParkedCount: 1,
|
||
DeliveredLastInterval: 3,
|
||
OldestPendingAge: TimeSpan.FromSeconds(60)),
|
||
});
|
||
|
||
var source = CreateSource(repository);
|
||
var samples = await source.CollectAsync(CapturedAt);
|
||
|
||
// 3 scopes × 5 metrics (all ages non-null) = 15 samples.
|
||
Assert.Equal(15, samples.Count);
|
||
Assert.All(samples, s => Assert.Equal(KpiSources.NotificationOutbox, s.Source));
|
||
Assert.All(samples, s => Assert.Equal(CapturedAt, s.CapturedAtUtc));
|
||
|
||
// Global — null ScopeKey.
|
||
AssertSample(samples, "queueDepth", KpiScopes.Global, null, 5);
|
||
AssertSample(samples, "stuckCount", KpiScopes.Global, null, 2);
|
||
AssertSample(samples, "parkedCount", KpiScopes.Global, null, 1);
|
||
AssertSample(samples, "deliveredLastInterval", KpiScopes.Global, null, 7);
|
||
AssertSample(samples, "oldestPendingAgeSeconds", KpiScopes.Global, null, 90);
|
||
|
||
// Site — ScopeKey == site id.
|
||
AssertSample(samples, "queueDepth", KpiScopes.Site, "site-a", 3);
|
||
AssertSample(samples, "stuckCount", KpiScopes.Site, "site-a", 1);
|
||
AssertSample(samples, "parkedCount", KpiScopes.Site, "site-a", 0);
|
||
AssertSample(samples, "deliveredLastInterval", KpiScopes.Site, "site-a", 4);
|
||
AssertSample(samples, "oldestPendingAgeSeconds", KpiScopes.Site, "site-a", 30);
|
||
|
||
// Node — ScopeKey == node name.
|
||
AssertSample(samples, "queueDepth", KpiScopes.Node, "node-a", 2);
|
||
AssertSample(samples, "stuckCount", KpiScopes.Node, "node-a", 1);
|
||
AssertSample(samples, "parkedCount", KpiScopes.Node, "node-a", 1);
|
||
AssertSample(samples, "deliveredLastInterval", KpiScopes.Node, "node-a", 3);
|
||
AssertSample(samples, "oldestPendingAgeSeconds", KpiScopes.Node, "node-a", 60);
|
||
}
|
||
|
||
// P5-5 (arch-review Low): the KPI-history rollup fold sums every deliveredLastInterval
|
||
// sample recorded in an hour, which over-counts unless each sample is a non-overlapping
|
||
// delta covering exactly one recorder tick (KpiHistoryOptions.SampleInterval). Before the
|
||
// fix, this class recorded the RAW DeliveredKpiWindow-windowed count verbatim — with a
|
||
// 15-minute DeliveredKpiWindow sampled every 1-minute SampleInterval, 60 overlapping
|
||
// 15-minute counts summed to ~15x the true hourly total. Proven against the unfixed
|
||
// computation: temporarily short-circuiting DeliveredIntervalScaleFactor to always return
|
||
// 1.0 (i.e. "record the raw count verbatim", the pre-P5-5 behavior) makes this test fail
|
||
// (30 != 2); restoring the real scale factor passes it.
|
||
[Fact]
|
||
public async Task CollectAsync_RescalesDeliveredLastInterval_OntoKpiHistorySampleInterval()
|
||
{
|
||
var repository = Substitute.For<INotificationOutboxRepository>();
|
||
repository.ComputeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(new NotificationKpiSnapshot(
|
||
QueueDepth: 0, StuckCount: 0, ParkedCount: 0,
|
||
DeliveredLastInterval: 30, OldestPendingAge: null));
|
||
repository.ComputePerSiteKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(Array.Empty<SiteNotificationKpiSnapshot>());
|
||
repository.ComputePerNodeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(Array.Empty<NodeNotificationKpiSnapshot>());
|
||
|
||
// DeliveredKpiWindow (15 min) is 15x KpiHistoryOptions.SampleInterval (1 min) — a
|
||
// non-default SampleInterval configuration deliberately chosen to prove the rescale
|
||
// math, not just the identity (factor == 1) default-config case the other tests use.
|
||
var options = new NotificationOutboxOptions { DeliveredKpiWindow = TimeSpan.FromMinutes(15) };
|
||
var kpiHistoryOptions = new KpiHistoryOptions { SampleInterval = TimeSpan.FromMinutes(1) };
|
||
var source = CreateSource(repository, options, kpiHistoryOptions);
|
||
|
||
var samples = await source.CollectAsync(CapturedAt);
|
||
|
||
var delivered = Assert.Single(samples, s => s.Metric == "deliveredLastInterval");
|
||
// 30 counted over a 15-minute window, rescaled onto a 1-minute recorder tick: 30 / 15 = 2.
|
||
// Summing 60 such 1-minute-tick samples across an hour therefore reconstructs the true
|
||
// hourly total instead of the ~15x over-count a raw (unscaled) 15-minute count would
|
||
// produce if simply summed 60 times.
|
||
Assert.Equal(2.0, delivered.Value, precision: 6);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_OmitsOldestPendingAge_WhenNull()
|
||
{
|
||
var repository = Substitute.For<INotificationOutboxRepository>();
|
||
repository.ComputeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(new NotificationKpiSnapshot(
|
||
QueueDepth: 0,
|
||
StuckCount: 0,
|
||
ParkedCount: 0,
|
||
DeliveredLastInterval: 0,
|
||
OldestPendingAge: null));
|
||
repository.ComputePerSiteKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(Array.Empty<SiteNotificationKpiSnapshot>());
|
||
repository.ComputePerNodeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(Array.Empty<NodeNotificationKpiSnapshot>());
|
||
|
||
var source = CreateSource(repository);
|
||
var samples = await source.CollectAsync(CapturedAt);
|
||
|
||
// Only the global snapshot, age omitted -> the four count metrics.
|
||
Assert.Equal(4, samples.Count);
|
||
Assert.DoesNotContain(samples, s => s.Metric == "oldestPendingAgeSeconds");
|
||
AssertSample(samples, "queueDepth", KpiScopes.Global, null, 0);
|
||
AssertSample(samples, "stuckCount", KpiScopes.Global, null, 0);
|
||
AssertSample(samples, "parkedCount", KpiScopes.Global, null, 0);
|
||
AssertSample(samples, "deliveredLastInterval", KpiScopes.Global, null, 0);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_ReturnsEmptyList_NeverNull_WhenNothingToReport()
|
||
{
|
||
// ComputeKpisAsync always returns a global snapshot; the only way the list is empty is
|
||
// a guard that produces no samples. Confirm an all-zero global with null age still yields
|
||
// the four count metrics (i.e. the list is never null even at idle).
|
||
var repository = Substitute.For<INotificationOutboxRepository>();
|
||
StubEmptySnapshots(repository);
|
||
|
||
var source = CreateSource(repository);
|
||
var samples = await source.CollectAsync(CapturedAt);
|
||
|
||
Assert.NotNull(samples);
|
||
}
|
||
|
||
private static void StubEmptySnapshots(INotificationOutboxRepository repository)
|
||
{
|
||
repository.ComputeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(new NotificationKpiSnapshot(0, 0, 0, 0, null));
|
||
repository.ComputePerSiteKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(Array.Empty<SiteNotificationKpiSnapshot>());
|
||
repository.ComputePerNodeKpisAsync(Arg.Any<DateTimeOffset>(), Arg.Any<DateTimeOffset>(), Arg.Any<CancellationToken>())
|
||
.Returns(Array.Empty<NodeNotificationKpiSnapshot>());
|
||
}
|
||
|
||
private static void AssertSample(
|
||
IReadOnlyList<KpiSample> samples, string metric, string scope, string? scopeKey, double value)
|
||
{
|
||
var match = Assert.Single(
|
||
samples, s => s.Metric == metric && s.Scope == scope && s.ScopeKey == scopeKey);
|
||
Assert.Equal(value, match.Value);
|
||
}
|
||
}
|