abbbcd1a8d
Defect: the hourly KPI rollup fold (KpiHistoryRepository.FoldHourlyRollupsAsync) sums every Rate-classified sample recorded in an hour, correct only if each sample is a non-overlapping delta covering exactly one recorder tick (KpiHistoryOptions.SampleInterval). That invariant was an unenforced silent assumption across three independently-owned config keys — SampleInterval, NotificationOutboxOptions.DeliveredKpiWindow, SiteCallAuditOptions.KpiInterval — so widening DeliveredKpiWindow to e.g. 15 min would silently over-count the rollup ~15x. Robust fix: NotificationOutboxKpiSampleSource and SiteCallAuditKpiSampleSource now rescale their Rate-classified samples (deliveredLastInterval, failedLastInterval) by SampleInterval / (DeliveredKpiWindow or KpiInterval) before recording, so the rollup's per-hour Sum is correct regardless of how the other two keys are tuned — the live dashboard tile's own cutoff/window is untouched. Both sources gain a project reference to KpiHistory to read SampleInterval; no cycle (KpiHistory references only Commons). Eager cross-validation (defense-in-depth): KpiRateFoldIntervalCrossValidator (Host, the one composition root referencing all three component projects) fails boot naming all three config keys when they diverge, protecting the separate (still-real) invariant that the recorded/charted KPI-history sample equals the live tile at the same instant — which the rescale trades away whenever the scale factor isn't 1. Rides KpiHistoryOptions' existing ValidateOnStart trigger. Also: NotificationOutboxOptions had no registered validator at all before this (the gap this task's "whichever options class owns DeliveredKpiWindow" file scope pointed at) — added NotificationOutboxOptionsValidator mirroring SiteCallAuditOptionsValidator's positive-duration/count convention, wired with ValidateOnStart. Tests: new rescale tests in both sample-source suites prove the fold is numerically correct at a non-default SampleInterval (30/40 counted over a 15/20-minute window correctly divides down to a 1-minute-tick delta); new KpiRateFoldIntervalCrossValidatorTests prove divergent config fails with all three keys named; new NotificationOutboxOptionsValidatorTests. Every new test proven to fail against the unfixed computation first (temporarily short-circuited the scale factor / cross-check to the pre-fix behavior, confirmed failure, restored) — not via git stash (repo-wide, shared across concurrent worktrees this session) but via file-copy swap. Full suites green: Host.Tests (527), ConfigurationDatabase.Tests (401), KpiHistory.Tests (54), NotificationOutbox.Tests (163), SiteCallAudit.Tests (67).
364 lines
16 KiB
C#
364 lines
16 KiB
C#
using Microsoft.Extensions.Options;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||
using ZB.MOM.WW.ScadaBridge.KpiHistory;
|
||
using ZB.MOM.WW.ScadaBridge.SiteCallAudit.Kpi;
|
||
|
||
namespace ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests.Kpi;
|
||
|
||
/// <summary>
|
||
/// Unit tests for <see cref="SiteCallAuditKpiSampleSource"/> (M6 KPI History,
|
||
/// #K7). A hand-rolled <see cref="ISiteCallAuditRepository"/> fake (matching the
|
||
/// project's existing test-double style — no mocking library) returns known
|
||
/// global + per-site + per-node snapshots; the assertions pin the exact
|
||
/// <c>(Metric, Scope, ScopeKey, Value)</c> tuples the recorder will persist, the
|
||
/// cutoffs derived from <see cref="SiteCallAuditOptions"/>, and the
|
||
/// oldest-pending-age omission when its source is <c>null</c>.
|
||
/// </summary>
|
||
public class SiteCallAuditKpiSampleSourceTests
|
||
{
|
||
private static readonly DateTime CapturedAt =
|
||
new(2026, 6, 15, 12, 0, 0, DateTimeKind.Utc);
|
||
|
||
private static SiteCallAuditOptions Options(
|
||
TimeSpan? stuckAge = null, TimeSpan? kpiInterval = null) => new()
|
||
{
|
||
StuckAgeThreshold = stuckAge ?? TimeSpan.FromMinutes(10),
|
||
KpiInterval = kpiInterval ?? TimeSpan.FromMinutes(1),
|
||
};
|
||
|
||
// Matches Options().KpiInterval (1 minute) so the default-config tests below see an
|
||
// identity rescale factor (1.0) and their pre-existing raw-count assertions hold.
|
||
private static readonly KpiHistoryOptions DefaultKpiHistoryOptions = new()
|
||
{
|
||
SampleInterval = TimeSpan.FromMinutes(1),
|
||
};
|
||
|
||
private static SiteCallAuditKpiSampleSource CreateSource(
|
||
ISiteCallAuditRepository repo,
|
||
SiteCallAuditOptions? options = null,
|
||
KpiHistoryOptions? kpiHistoryOptions = null) =>
|
||
new(
|
||
repo,
|
||
OptionsWrap(options ?? Options()),
|
||
Microsoft.Extensions.Options.Options.Create(kpiHistoryOptions ?? DefaultKpiHistoryOptions));
|
||
|
||
private static IOptions<SiteCallAuditOptions> OptionsWrap(SiteCallAuditOptions o) =>
|
||
Microsoft.Extensions.Options.Options.Create(o);
|
||
|
||
// ---------------------------------------------------------------------
|
||
// 1. Source identifier is the canonical SiteCallAudit constant.
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public void Source_IsSiteCallAuditConstant()
|
||
{
|
||
var source = CreateSource(new StubRepo());
|
||
Assert.Equal(KpiSources.SiteCallAudit, source.Source);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// 2. Full snapshot: global + per-site + per-node, exact tuples.
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_EmitsExpectedTuples_ForGlobalSiteAndNode()
|
||
{
|
||
var repo = new StubRepo
|
||
{
|
||
Global = new SiteCallKpiSnapshot(
|
||
BufferedCount: 5,
|
||
ParkedCount: 2,
|
||
FailedLastInterval: 1,
|
||
DeliveredLastInterval: 9,
|
||
OldestPendingAge: TimeSpan.FromSeconds(42),
|
||
StuckCount: 3),
|
||
PerSite =
|
||
[
|
||
new SiteCallSiteKpiSnapshot(
|
||
SourceSite: "site-a",
|
||
BufferedCount: 4,
|
||
ParkedCount: 1,
|
||
FailedLastInterval: 0,
|
||
DeliveredLastInterval: 7,
|
||
OldestPendingAge: TimeSpan.FromSeconds(30),
|
||
StuckCount: 2),
|
||
],
|
||
PerNode =
|
||
[
|
||
new SiteCallNodeKpiSnapshot(
|
||
SourceNode: "node-a",
|
||
BufferedCount: 3,
|
||
ParkedCount: 1,
|
||
FailedLastInterval: 1,
|
||
DeliveredLastInterval: 5,
|
||
OldestPendingAge: TimeSpan.FromSeconds(20),
|
||
StuckCount: 1),
|
||
],
|
||
};
|
||
|
||
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
|
||
|
||
// Every sample carries Source + CapturedAtUtc.
|
||
Assert.All(samples, s =>
|
||
{
|
||
Assert.Equal(KpiSources.SiteCallAudit, s.Source);
|
||
Assert.Equal(CapturedAt, s.CapturedAtUtc);
|
||
});
|
||
|
||
// Global (null ScopeKey): six metrics (age present).
|
||
AssertHas(samples, "buffered", KpiScopes.Global, null, 5);
|
||
AssertHas(samples, "parked", KpiScopes.Global, null, 2);
|
||
AssertHas(samples, "failedLastInterval", KpiScopes.Global, null, 1);
|
||
AssertHas(samples, "deliveredLastInterval", KpiScopes.Global, null, 9);
|
||
AssertHas(samples, "oldestPendingAgeSeconds", KpiScopes.Global, null, 42);
|
||
AssertHas(samples, "stuck", KpiScopes.Global, null, 3);
|
||
|
||
// Per-site (ScopeKey = site id).
|
||
AssertHas(samples, "buffered", KpiScopes.Site, "site-a", 4);
|
||
AssertHas(samples, "parked", KpiScopes.Site, "site-a", 1);
|
||
AssertHas(samples, "failedLastInterval", KpiScopes.Site, "site-a", 0);
|
||
AssertHas(samples, "deliveredLastInterval", KpiScopes.Site, "site-a", 7);
|
||
AssertHas(samples, "oldestPendingAgeSeconds", KpiScopes.Site, "site-a", 30);
|
||
AssertHas(samples, "stuck", KpiScopes.Site, "site-a", 2);
|
||
|
||
// Per-node (ScopeKey = node name).
|
||
AssertHas(samples, "buffered", KpiScopes.Node, "node-a", 3);
|
||
AssertHas(samples, "parked", KpiScopes.Node, "node-a", 1);
|
||
AssertHas(samples, "failedLastInterval", KpiScopes.Node, "node-a", 1);
|
||
AssertHas(samples, "deliveredLastInterval", KpiScopes.Node, "node-a", 5);
|
||
AssertHas(samples, "oldestPendingAgeSeconds", KpiScopes.Node, "node-a", 20);
|
||
AssertHas(samples, "stuck", KpiScopes.Node, "node-a", 1);
|
||
|
||
// 6 metrics × 3 scopes, all ages present.
|
||
Assert.Equal(18, samples.Count);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// 3. Null oldest-pending-age is omitted (not written as zero).
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_OmitsOldestPendingAge_WhenNull()
|
||
{
|
||
var repo = new StubRepo
|
||
{
|
||
Global = new SiteCallKpiSnapshot(0, 0, 0, 0, OldestPendingAge: null, 0),
|
||
PerSite =
|
||
[
|
||
new SiteCallSiteKpiSnapshot("site-a", 0, 0, 0, 0, OldestPendingAge: null, 0),
|
||
],
|
||
PerNode =
|
||
[
|
||
new SiteCallNodeKpiSnapshot("node-a", 0, 0, 0, 0, OldestPendingAge: null, 0),
|
||
],
|
||
};
|
||
|
||
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
|
||
|
||
// No oldestPendingAgeSeconds rows at any scope when the age is null.
|
||
Assert.DoesNotContain(samples, s => s.Metric == "oldestPendingAgeSeconds");
|
||
|
||
// The five count metrics are still present at each of the 3 scopes.
|
||
Assert.Equal(15, samples.Count);
|
||
AssertHas(samples, "buffered", KpiScopes.Global, null, 0);
|
||
AssertHas(samples, "stuck", KpiScopes.Node, "node-a", 0);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// 4. Empty per-site/per-node: only the global six metrics emitted.
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_GlobalOnly_WhenNoPerSiteOrPerNodeRows()
|
||
{
|
||
var repo = new StubRepo
|
||
{
|
||
Global = new SiteCallKpiSnapshot(1, 0, 0, 0, TimeSpan.FromSeconds(5), 0),
|
||
PerSite = [],
|
||
PerNode = [],
|
||
};
|
||
|
||
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
|
||
|
||
Assert.All(samples, s => Assert.Equal(KpiScopes.Global, s.Scope));
|
||
Assert.Equal(6, samples.Count);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// 5. Cutoffs are anchored on capturedAtUtc using the options windows.
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_DerivesCutoffs_FromOptionsAnchoredOnCapturedAt()
|
||
{
|
||
var repo = new StubRepo();
|
||
var options = Options(
|
||
stuckAge: TimeSpan.FromMinutes(10),
|
||
kpiInterval: TimeSpan.FromMinutes(2));
|
||
|
||
await CreateSource(repo, options).CollectAsync(CapturedAt);
|
||
|
||
var expectedStuck = CapturedAt - TimeSpan.FromMinutes(10);
|
||
var expectedSince = CapturedAt - TimeSpan.FromMinutes(2);
|
||
|
||
// Every repository call must have received the identical anchored cutoffs.
|
||
Assert.NotEmpty(repo.Calls);
|
||
Assert.All(repo.Calls, c =>
|
||
{
|
||
Assert.Equal(expectedStuck, c.StuckCutoff);
|
||
Assert.Equal(expectedSince, c.IntervalSince);
|
||
});
|
||
|
||
// All three KPI methods were invoked.
|
||
Assert.Equal(3, repo.Calls.Count);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// P5-5 (arch-review Low): failedLastInterval/deliveredLastInterval are rescaled onto
|
||
// KpiHistoryOptions.SampleInterval before recording, so the central hourly rollup fold's
|
||
// per-hour Sum stays correct regardless of how KpiInterval is tuned. Before the fix, this
|
||
// class recorded the RAW KpiInterval-windowed counts verbatim; with a 20-minute
|
||
// KpiInterval sampled every 1-minute SampleInterval, summing 60 overlapping 20-minute
|
||
// counts across an hour would over-count ~20x. Proven against the unfixed computation:
|
||
// temporarily short-circuiting IntervalScaleFactor to always return 1.0 (i.e. "record the
|
||
// raw count verbatim", the pre-P5-5 behavior) makes this test fail (40/1 != 2/0.05);
|
||
// restoring the real scale factor passes it.
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_RescalesFailedAndDeliveredLastInterval_OntoKpiHistorySampleInterval()
|
||
{
|
||
var repo = new StubRepo
|
||
{
|
||
Global = new SiteCallKpiSnapshot(
|
||
BufferedCount: 0,
|
||
ParkedCount: 0,
|
||
FailedLastInterval: 1,
|
||
DeliveredLastInterval: 40,
|
||
OldestPendingAge: null,
|
||
StuckCount: 0),
|
||
};
|
||
|
||
// KpiInterval (20 min) is 20x KpiHistoryOptions.SampleInterval (1 min) — a
|
||
// non-default SampleInterval configuration deliberately chosen to prove the rescale
|
||
// math, not just the identity (factor == 1) default-config case the other tests use.
|
||
var options = Options(kpiInterval: TimeSpan.FromMinutes(20));
|
||
var kpiHistoryOptions = new KpiHistoryOptions { SampleInterval = TimeSpan.FromMinutes(1) };
|
||
|
||
var samples = await CreateSource(repo, options, kpiHistoryOptions).CollectAsync(CapturedAt);
|
||
|
||
// 1 failure / 40 deliveries counted over a 20-minute window, rescaled onto a
|
||
// 1-minute recorder tick: divide by 20. Summing 60 such 1-minute-tick samples across
|
||
// an hour therefore reconstructs the true hourly totals instead of the ~20x
|
||
// over-count a raw (unscaled) 20-minute count would produce if simply summed 60 times.
|
||
AssertHas(samples, "failedLastInterval", KpiScopes.Global, null, 0.05);
|
||
AssertHas(samples, "deliveredLastInterval", KpiScopes.Global, null, 2.0);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// 6. Multiple sites/nodes are all emitted with distinct ScopeKeys.
|
||
// ---------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task CollectAsync_EmitsAllSitesAndNodes()
|
||
{
|
||
var repo = new StubRepo
|
||
{
|
||
Global = new SiteCallKpiSnapshot(0, 0, 0, 0, null, 0),
|
||
PerSite =
|
||
[
|
||
new SiteCallSiteKpiSnapshot("site-a", 1, 0, 0, 0, null, 0),
|
||
new SiteCallSiteKpiSnapshot("site-b", 2, 0, 0, 0, null, 0),
|
||
],
|
||
PerNode =
|
||
[
|
||
new SiteCallNodeKpiSnapshot("node-a", 0, 0, 0, 0, null, 0),
|
||
new SiteCallNodeKpiSnapshot("node-b", 0, 0, 0, 0, null, 0),
|
||
],
|
||
};
|
||
|
||
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
|
||
|
||
AssertHas(samples, "buffered", KpiScopes.Site, "site-a", 1);
|
||
AssertHas(samples, "buffered", KpiScopes.Site, "site-b", 2);
|
||
Assert.Contains(samples, s => s is { Scope: "Node", ScopeKey: "node-a" });
|
||
Assert.Contains(samples, s => s is { Scope: "Node", ScopeKey: "node-b" });
|
||
}
|
||
|
||
// ---------------------------------------------------------------------
|
||
// Helpers
|
||
// ---------------------------------------------------------------------
|
||
|
||
private static void AssertHas(
|
||
IReadOnlyList<KpiSample> samples,
|
||
string metric, string scope, string? scopeKey, double value)
|
||
{
|
||
var match = samples.SingleOrDefault(s =>
|
||
s.Metric == metric && s.Scope == scope && s.ScopeKey == scopeKey);
|
||
Assert.True(
|
||
match is not null,
|
||
$"expected a sample for ({metric}, {scope}, {scopeKey ?? "<null>"})");
|
||
Assert.Equal(value, match!.Value);
|
||
}
|
||
|
||
/// <summary>Captured arguments of one KPI computation call.</summary>
|
||
private readonly record struct KpiCall(DateTime StuckCutoff, DateTime IntervalSince);
|
||
|
||
/// <summary>
|
||
/// Hand-rolled <see cref="ISiteCallAuditRepository"/> stub returning
|
||
/// configurable snapshots and recording the cutoffs each KPI method received.
|
||
/// Non-KPI members are inert (this source only reads KPIs).
|
||
/// </summary>
|
||
private sealed class StubRepo : ISiteCallAuditRepository
|
||
{
|
||
public SiteCallKpiSnapshot Global { get; set; } =
|
||
new(0, 0, 0, 0, null, 0);
|
||
public IReadOnlyList<SiteCallSiteKpiSnapshot> PerSite { get; set; } =
|
||
Array.Empty<SiteCallSiteKpiSnapshot>();
|
||
public IReadOnlyList<SiteCallNodeKpiSnapshot> PerNode { get; set; } =
|
||
Array.Empty<SiteCallNodeKpiSnapshot>();
|
||
|
||
public List<KpiCall> Calls { get; } = new();
|
||
|
||
public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
|
||
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
|
||
{
|
||
Calls.Add(new KpiCall(stuckCutoff, intervalSince));
|
||
return Task.FromResult(Global);
|
||
}
|
||
|
||
public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
|
||
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
|
||
{
|
||
Calls.Add(new KpiCall(stuckCutoff, intervalSince));
|
||
return Task.FromResult(PerSite);
|
||
}
|
||
|
||
public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
|
||
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
|
||
{
|
||
Calls.Add(new KpiCall(stuckCutoff, intervalSince));
|
||
return Task.FromResult(PerNode);
|
||
}
|
||
|
||
// ── Inert non-KPI members ──
|
||
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
|
||
Task.CompletedTask;
|
||
|
||
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
|
||
Task.FromResult<SiteCall?>(null);
|
||
|
||
public Task<IReadOnlyList<SiteCall>> QueryAsync(
|
||
SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
|
||
Task.FromResult<IReadOnlyList<SiteCall>>(Array.Empty<SiteCall>());
|
||
|
||
public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
|
||
Task.FromResult(0);
|
||
}
|
||
}
|