Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests/Kpi/SiteCallAuditKpiSampleSourceTests.cs
T
Joseph Doherty abbbcd1a8d fix(kpihistory)+fix(notificationoutbox)+fix(sitecallaudit)+fix(host): P5-5 — rate-fold interval triple made self-consistent + cross-validated
Defect: the hourly KPI rollup fold (KpiHistoryRepository.FoldHourlyRollupsAsync)
sums every Rate-classified sample recorded in an hour, correct only if each
sample is a non-overlapping delta covering exactly one recorder tick
(KpiHistoryOptions.SampleInterval). That invariant was an unenforced silent
assumption across three independently-owned config keys — SampleInterval,
NotificationOutboxOptions.DeliveredKpiWindow, SiteCallAuditOptions.KpiInterval —
so widening DeliveredKpiWindow to e.g. 15 min would silently over-count the
rollup ~15x.

Robust fix: NotificationOutboxKpiSampleSource and SiteCallAuditKpiSampleSource
now rescale their Rate-classified samples (deliveredLastInterval,
failedLastInterval) by SampleInterval / (DeliveredKpiWindow or KpiInterval)
before recording, so the rollup's per-hour Sum is correct regardless of how
the other two keys are tuned — the live dashboard tile's own cutoff/window is
untouched. Both sources gain a project reference to KpiHistory to read
SampleInterval; no cycle (KpiHistory references only Commons).

Eager cross-validation (defense-in-depth): KpiRateFoldIntervalCrossValidator
(Host, the one composition root referencing all three component projects)
fails boot naming all three config keys when they diverge, protecting the
separate (still-real) invariant that the recorded/charted KPI-history sample
equals the live tile at the same instant — which the rescale trades away
whenever the scale factor isn't 1. Rides KpiHistoryOptions' existing
ValidateOnStart trigger.

Also: NotificationOutboxOptions had no registered validator at all before this
(the gap this task's "whichever options class owns DeliveredKpiWindow" file
scope pointed at) — added NotificationOutboxOptionsValidator mirroring
SiteCallAuditOptionsValidator's positive-duration/count convention, wired with
ValidateOnStart.

Tests: new rescale tests in both sample-source suites prove the fold is
numerically correct at a non-default SampleInterval (30/40 counted over a
15/20-minute window correctly divides down to a 1-minute-tick delta); new
KpiRateFoldIntervalCrossValidatorTests prove divergent config fails with all
three keys named; new NotificationOutboxOptionsValidatorTests. Every new test
proven to fail against the unfixed computation first (temporarily
short-circuited the scale factor / cross-check to the pre-fix behavior,
confirmed failure, restored) — not via git stash (repo-wide, shared across
concurrent worktrees this session) but via file-copy swap.

Full suites green: Host.Tests (527), ConfigurationDatabase.Tests (401),
KpiHistory.Tests (54), NotificationOutbox.Tests (163),
SiteCallAudit.Tests (67).
2026-08-19 16:41:04 -04:00

364 lines
16 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
using ZB.MOM.WW.ScadaBridge.KpiHistory;
using ZB.MOM.WW.ScadaBridge.SiteCallAudit.Kpi;
namespace ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests.Kpi;
/// <summary>
/// Unit tests for <see cref="SiteCallAuditKpiSampleSource"/> (M6 KPI History,
/// #K7). A hand-rolled <see cref="ISiteCallAuditRepository"/> fake (matching the
/// project's existing test-double style — no mocking library) returns known
/// global + per-site + per-node snapshots; the assertions pin the exact
/// <c>(Metric, Scope, ScopeKey, Value)</c> tuples the recorder will persist, the
/// cutoffs derived from <see cref="SiteCallAuditOptions"/>, and the
/// oldest-pending-age omission when its source is <c>null</c>.
/// </summary>
public class SiteCallAuditKpiSampleSourceTests
{
private static readonly DateTime CapturedAt =
new(2026, 6, 15, 12, 0, 0, DateTimeKind.Utc);
private static SiteCallAuditOptions Options(
TimeSpan? stuckAge = null, TimeSpan? kpiInterval = null) => new()
{
StuckAgeThreshold = stuckAge ?? TimeSpan.FromMinutes(10),
KpiInterval = kpiInterval ?? TimeSpan.FromMinutes(1),
};
// Matches Options().KpiInterval (1 minute) so the default-config tests below see an
// identity rescale factor (1.0) and their pre-existing raw-count assertions hold.
private static readonly KpiHistoryOptions DefaultKpiHistoryOptions = new()
{
SampleInterval = TimeSpan.FromMinutes(1),
};
private static SiteCallAuditKpiSampleSource CreateSource(
ISiteCallAuditRepository repo,
SiteCallAuditOptions? options = null,
KpiHistoryOptions? kpiHistoryOptions = null) =>
new(
repo,
OptionsWrap(options ?? Options()),
Microsoft.Extensions.Options.Options.Create(kpiHistoryOptions ?? DefaultKpiHistoryOptions));
private static IOptions<SiteCallAuditOptions> OptionsWrap(SiteCallAuditOptions o) =>
Microsoft.Extensions.Options.Options.Create(o);
// ---------------------------------------------------------------------
// 1. Source identifier is the canonical SiteCallAudit constant.
// ---------------------------------------------------------------------
[Fact]
public void Source_IsSiteCallAuditConstant()
{
var source = CreateSource(new StubRepo());
Assert.Equal(KpiSources.SiteCallAudit, source.Source);
}
// ---------------------------------------------------------------------
// 2. Full snapshot: global + per-site + per-node, exact tuples.
// ---------------------------------------------------------------------
[Fact]
public async Task CollectAsync_EmitsExpectedTuples_ForGlobalSiteAndNode()
{
var repo = new StubRepo
{
Global = new SiteCallKpiSnapshot(
BufferedCount: 5,
ParkedCount: 2,
FailedLastInterval: 1,
DeliveredLastInterval: 9,
OldestPendingAge: TimeSpan.FromSeconds(42),
StuckCount: 3),
PerSite =
[
new SiteCallSiteKpiSnapshot(
SourceSite: "site-a",
BufferedCount: 4,
ParkedCount: 1,
FailedLastInterval: 0,
DeliveredLastInterval: 7,
OldestPendingAge: TimeSpan.FromSeconds(30),
StuckCount: 2),
],
PerNode =
[
new SiteCallNodeKpiSnapshot(
SourceNode: "node-a",
BufferedCount: 3,
ParkedCount: 1,
FailedLastInterval: 1,
DeliveredLastInterval: 5,
OldestPendingAge: TimeSpan.FromSeconds(20),
StuckCount: 1),
],
};
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
// Every sample carries Source + CapturedAtUtc.
Assert.All(samples, s =>
{
Assert.Equal(KpiSources.SiteCallAudit, s.Source);
Assert.Equal(CapturedAt, s.CapturedAtUtc);
});
// Global (null ScopeKey): six metrics (age present).
AssertHas(samples, "buffered", KpiScopes.Global, null, 5);
AssertHas(samples, "parked", KpiScopes.Global, null, 2);
AssertHas(samples, "failedLastInterval", KpiScopes.Global, null, 1);
AssertHas(samples, "deliveredLastInterval", KpiScopes.Global, null, 9);
AssertHas(samples, "oldestPendingAgeSeconds", KpiScopes.Global, null, 42);
AssertHas(samples, "stuck", KpiScopes.Global, null, 3);
// Per-site (ScopeKey = site id).
AssertHas(samples, "buffered", KpiScopes.Site, "site-a", 4);
AssertHas(samples, "parked", KpiScopes.Site, "site-a", 1);
AssertHas(samples, "failedLastInterval", KpiScopes.Site, "site-a", 0);
AssertHas(samples, "deliveredLastInterval", KpiScopes.Site, "site-a", 7);
AssertHas(samples, "oldestPendingAgeSeconds", KpiScopes.Site, "site-a", 30);
AssertHas(samples, "stuck", KpiScopes.Site, "site-a", 2);
// Per-node (ScopeKey = node name).
AssertHas(samples, "buffered", KpiScopes.Node, "node-a", 3);
AssertHas(samples, "parked", KpiScopes.Node, "node-a", 1);
AssertHas(samples, "failedLastInterval", KpiScopes.Node, "node-a", 1);
AssertHas(samples, "deliveredLastInterval", KpiScopes.Node, "node-a", 5);
AssertHas(samples, "oldestPendingAgeSeconds", KpiScopes.Node, "node-a", 20);
AssertHas(samples, "stuck", KpiScopes.Node, "node-a", 1);
// 6 metrics × 3 scopes, all ages present.
Assert.Equal(18, samples.Count);
}
// ---------------------------------------------------------------------
// 3. Null oldest-pending-age is omitted (not written as zero).
// ---------------------------------------------------------------------
[Fact]
public async Task CollectAsync_OmitsOldestPendingAge_WhenNull()
{
var repo = new StubRepo
{
Global = new SiteCallKpiSnapshot(0, 0, 0, 0, OldestPendingAge: null, 0),
PerSite =
[
new SiteCallSiteKpiSnapshot("site-a", 0, 0, 0, 0, OldestPendingAge: null, 0),
],
PerNode =
[
new SiteCallNodeKpiSnapshot("node-a", 0, 0, 0, 0, OldestPendingAge: null, 0),
],
};
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
// No oldestPendingAgeSeconds rows at any scope when the age is null.
Assert.DoesNotContain(samples, s => s.Metric == "oldestPendingAgeSeconds");
// The five count metrics are still present at each of the 3 scopes.
Assert.Equal(15, samples.Count);
AssertHas(samples, "buffered", KpiScopes.Global, null, 0);
AssertHas(samples, "stuck", KpiScopes.Node, "node-a", 0);
}
// ---------------------------------------------------------------------
// 4. Empty per-site/per-node: only the global six metrics emitted.
// ---------------------------------------------------------------------
[Fact]
public async Task CollectAsync_GlobalOnly_WhenNoPerSiteOrPerNodeRows()
{
var repo = new StubRepo
{
Global = new SiteCallKpiSnapshot(1, 0, 0, 0, TimeSpan.FromSeconds(5), 0),
PerSite = [],
PerNode = [],
};
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
Assert.All(samples, s => Assert.Equal(KpiScopes.Global, s.Scope));
Assert.Equal(6, samples.Count);
}
// ---------------------------------------------------------------------
// 5. Cutoffs are anchored on capturedAtUtc using the options windows.
// ---------------------------------------------------------------------
[Fact]
public async Task CollectAsync_DerivesCutoffs_FromOptionsAnchoredOnCapturedAt()
{
var repo = new StubRepo();
var options = Options(
stuckAge: TimeSpan.FromMinutes(10),
kpiInterval: TimeSpan.FromMinutes(2));
await CreateSource(repo, options).CollectAsync(CapturedAt);
var expectedStuck = CapturedAt - TimeSpan.FromMinutes(10);
var expectedSince = CapturedAt - TimeSpan.FromMinutes(2);
// Every repository call must have received the identical anchored cutoffs.
Assert.NotEmpty(repo.Calls);
Assert.All(repo.Calls, c =>
{
Assert.Equal(expectedStuck, c.StuckCutoff);
Assert.Equal(expectedSince, c.IntervalSince);
});
// All three KPI methods were invoked.
Assert.Equal(3, repo.Calls.Count);
}
// ---------------------------------------------------------------------
// P5-5 (arch-review Low): failedLastInterval/deliveredLastInterval are rescaled onto
// KpiHistoryOptions.SampleInterval before recording, so the central hourly rollup fold's
// per-hour Sum stays correct regardless of how KpiInterval is tuned. Before the fix, this
// class recorded the RAW KpiInterval-windowed counts verbatim; with a 20-minute
// KpiInterval sampled every 1-minute SampleInterval, summing 60 overlapping 20-minute
// counts across an hour would over-count ~20x. Proven against the unfixed computation:
// temporarily short-circuiting IntervalScaleFactor to always return 1.0 (i.e. "record the
// raw count verbatim", the pre-P5-5 behavior) makes this test fail (40/1 != 2/0.05);
// restoring the real scale factor passes it.
// ---------------------------------------------------------------------
[Fact]
public async Task CollectAsync_RescalesFailedAndDeliveredLastInterval_OntoKpiHistorySampleInterval()
{
var repo = new StubRepo
{
Global = new SiteCallKpiSnapshot(
BufferedCount: 0,
ParkedCount: 0,
FailedLastInterval: 1,
DeliveredLastInterval: 40,
OldestPendingAge: null,
StuckCount: 0),
};
// KpiInterval (20 min) is 20x KpiHistoryOptions.SampleInterval (1 min) — a
// non-default SampleInterval configuration deliberately chosen to prove the rescale
// math, not just the identity (factor == 1) default-config case the other tests use.
var options = Options(kpiInterval: TimeSpan.FromMinutes(20));
var kpiHistoryOptions = new KpiHistoryOptions { SampleInterval = TimeSpan.FromMinutes(1) };
var samples = await CreateSource(repo, options, kpiHistoryOptions).CollectAsync(CapturedAt);
// 1 failure / 40 deliveries counted over a 20-minute window, rescaled onto a
// 1-minute recorder tick: divide by 20. Summing 60 such 1-minute-tick samples across
// an hour therefore reconstructs the true hourly totals instead of the ~20x
// over-count a raw (unscaled) 20-minute count would produce if simply summed 60 times.
AssertHas(samples, "failedLastInterval", KpiScopes.Global, null, 0.05);
AssertHas(samples, "deliveredLastInterval", KpiScopes.Global, null, 2.0);
}
// ---------------------------------------------------------------------
// 6. Multiple sites/nodes are all emitted with distinct ScopeKeys.
// ---------------------------------------------------------------------
[Fact]
public async Task CollectAsync_EmitsAllSitesAndNodes()
{
var repo = new StubRepo
{
Global = new SiteCallKpiSnapshot(0, 0, 0, 0, null, 0),
PerSite =
[
new SiteCallSiteKpiSnapshot("site-a", 1, 0, 0, 0, null, 0),
new SiteCallSiteKpiSnapshot("site-b", 2, 0, 0, 0, null, 0),
],
PerNode =
[
new SiteCallNodeKpiSnapshot("node-a", 0, 0, 0, 0, null, 0),
new SiteCallNodeKpiSnapshot("node-b", 0, 0, 0, 0, null, 0),
],
};
var samples = await CreateSource(repo).CollectAsync(CapturedAt);
AssertHas(samples, "buffered", KpiScopes.Site, "site-a", 1);
AssertHas(samples, "buffered", KpiScopes.Site, "site-b", 2);
Assert.Contains(samples, s => s is { Scope: "Node", ScopeKey: "node-a" });
Assert.Contains(samples, s => s is { Scope: "Node", ScopeKey: "node-b" });
}
// ---------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------
private static void AssertHas(
IReadOnlyList<KpiSample> samples,
string metric, string scope, string? scopeKey, double value)
{
var match = samples.SingleOrDefault(s =>
s.Metric == metric && s.Scope == scope && s.ScopeKey == scopeKey);
Assert.True(
match is not null,
$"expected a sample for ({metric}, {scope}, {scopeKey ?? "<null>"})");
Assert.Equal(value, match!.Value);
}
/// <summary>Captured arguments of one KPI computation call.</summary>
private readonly record struct KpiCall(DateTime StuckCutoff, DateTime IntervalSince);
/// <summary>
/// Hand-rolled <see cref="ISiteCallAuditRepository"/> stub returning
/// configurable snapshots and recording the cutoffs each KPI method received.
/// Non-KPI members are inert (this source only reads KPIs).
/// </summary>
private sealed class StubRepo : ISiteCallAuditRepository
{
public SiteCallKpiSnapshot Global { get; set; } =
new(0, 0, 0, 0, null, 0);
public IReadOnlyList<SiteCallSiteKpiSnapshot> PerSite { get; set; } =
Array.Empty<SiteCallSiteKpiSnapshot>();
public IReadOnlyList<SiteCallNodeKpiSnapshot> PerNode { get; set; } =
Array.Empty<SiteCallNodeKpiSnapshot>();
public List<KpiCall> Calls { get; } = new();
public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
{
Calls.Add(new KpiCall(stuckCutoff, intervalSince));
return Task.FromResult(Global);
}
public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
{
Calls.Add(new KpiCall(stuckCutoff, intervalSince));
return Task.FromResult(PerSite);
}
public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
{
Calls.Add(new KpiCall(stuckCutoff, intervalSince));
return Task.FromResult(PerNode);
}
// ── Inert non-KPI members ──
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
Task.CompletedTask;
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
Task.FromResult<SiteCall?>(null);
public Task<IReadOnlyList<SiteCall>> QueryAsync(
SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<SiteCall>>(Array.Empty<SiteCall>());
public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
Task.FromResult(0);
}
}