8da60f57ac
LocalDbOplogBacklog was reported and Prometheus-exported with nothing consuming it: not sampled into KPI history, so a purge-burst spike could not be reviewed after the fact, and no threshold treatment anywhere. Both halves shipped. KPI SAMPLING. SiteHealthKpiSampleSource now emits localDbOplogBacklog and localDbReplicationConnected (0/1), and the backlog is charted on the Health dashboard's per-site trend panel. Charting is the point: a point-in-time tile can never answer 'did that spike drain?', which is the only thing separating an expected purge burst from a stalled peer. Both are CONDITIONALLY emitted rather than coerced with ?? 0 like the auditBacklogPending metric beside them. Both health-report fields are nullable and null means 'no reading' — replication not configured (rig site-b/site-c) or a failed poll — never zero. A fabricated 0 would render a pair that cannot read its own oplog, or one deliberately unreplicated, as perfectly healthy once a minute forever, burying the real series. So a site emits 12 metrics when unreplicated and 14 when replicated, and a GAP in these two series truthfully means 'not replicated / not reporting'. Note false and null differ for the connected flag: a configured link that is DOWN samples as 0; no link samples as nothing. Both fold as Gauge (summing an oplog depth, or a 0/1 flag, per hour is meaningless). THRESHOLD SEMANTICS — sustained, not instantaneous. This is the design, not a refinement. The site_events retention purge is CDC-captured like any other write, so the backlog jumps by the deleted batch and drains as the peer acks it; a bare 'backlog > threshold' badge would fire on every replicated site every day and be ignored within a week. Healthy is defined by the backlog RETURNING to ~0, which only a dwell can express. CentralHealthAggregator holds LocalDbBacklogAboveSince — the instant the backlog first rose above LocalDbBacklogThreshold (default 1000) — and the badge appears only after LocalDbBacklogSustainedFor (default 15 min). A report at or below the threshold clears the anchor; one above leaves the ORIGINAL instant intact, so the window is sustained rather than sliding. The null case is deliberately a third branch: a null backlog leaves the anchor UNCHANGED rather than clearing it. Null is evidence neither of a drain nor of a backlog, so a node going blind mid-climb must not silently reset its own dwell, and an unreplicated site (null forever) never accumulates an anchor and can never be badged. Validation rejects a non-positive threshold (0 would badge every replicated site permanently, since an idle backlog already sits at it) and a dwell shorter than one ReportInterval (satisfiable by a single sample, which collapses the check back into the bare threshold it replaces). DOC CORRECTION worth flagging: this row's premise was 'mirror the SiteAuditBacklog configurable threshold badge'. That badge does not exist — Component-HealthMonitoring.md described it in two places but no option, no comparison and no markup were ever implemented. The two stale claims are corrected here rather than left to send the next reader looking for it; the LocalDb badge is built fresh on the Metrics-stale badge's pattern, which is a real configurable-threshold badge. Tests: HealthMonitoring 115 (9 new aggregator tests incl. purge-burst-drains, unreplicated-never-anchors, null-mid-climb, exact-threshold, stale-report; 5 new validator tests; 4 new sampler tests incl. the null-semantics and false-vs-null cases); CentralUI 1042 (5 new badge/chart tests incl. the purge-burst negative control); Commons Kpi 93; KpiHistory 54.
154 lines
5.4 KiB
C#
154 lines
5.4 KiB
C#
using Microsoft.Extensions.Options;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests;
|
|
|
|
/// <summary>
|
|
/// HealthMonitoring-014 regression: <see cref="HealthMonitoringOptions"/> intervals
|
|
/// are fed straight into <c>new PeriodicTimer(...)</c>, which throws
|
|
/// <see cref="ArgumentOutOfRangeException"/> for a zero/negative period. A
|
|
/// misconfigured <c>appsettings.json</c> must be rejected by an
|
|
/// <see cref="IValidateOptions{TOptions}"/> with a clear, key-naming message
|
|
/// rather than crashing the hosted service with an opaque exception.
|
|
/// </summary>
|
|
public class HealthMonitoringOptionsValidatorTests
|
|
{
|
|
private static ValidateOptionsResult Validate(HealthMonitoringOptions options) =>
|
|
new HealthMonitoringOptionsValidator().Validate(Options.DefaultName, options);
|
|
|
|
[Fact]
|
|
public void DefaultOptions_AreValid()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions());
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void ZeroReportInterval_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions { ReportInterval = TimeSpan.Zero });
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("ReportInterval", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void NegativeReportInterval_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions { ReportInterval = TimeSpan.FromSeconds(-1) });
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("ReportInterval", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void ZeroOfflineTimeout_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions { OfflineTimeout = TimeSpan.Zero });
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("OfflineTimeout", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void ZeroCentralOfflineTimeout_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions { CentralOfflineTimeout = TimeSpan.Zero });
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("CentralOfflineTimeout", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void CentralOfflineTimeout_ShorterThanOfflineTimeout_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions
|
|
{
|
|
OfflineTimeout = TimeSpan.FromSeconds(60),
|
|
CentralOfflineTimeout = TimeSpan.FromSeconds(30)
|
|
});
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("CentralOfflineTimeout", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void ZeroMetricsStaleTimeout_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions { MetricsStaleTimeout = TimeSpan.Zero });
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("MetricsStaleTimeout", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void MetricsStaleTimeout_ShorterThanReportInterval_IsRejected()
|
|
{
|
|
// A stale window shorter than one report interval flags every site.
|
|
var result = Validate(new HealthMonitoringOptions
|
|
{
|
|
ReportInterval = TimeSpan.FromSeconds(30),
|
|
MetricsStaleTimeout = TimeSpan.FromSeconds(10)
|
|
});
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("MetricsStaleTimeout", result.FailureMessage);
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// LocalDb sustained-backlog badge (register row 33)
|
|
// ------------------------------------------------------------------
|
|
|
|
[Theory]
|
|
[InlineData(0)]
|
|
[InlineData(-1)]
|
|
public void NonPositiveLocalDbBacklogThreshold_IsRejected(long threshold)
|
|
{
|
|
// An idle backlog of 0 already sits at or above a zero/negative threshold, so this
|
|
// would badge every replicated site permanently.
|
|
var result = Validate(new HealthMonitoringOptions { LocalDbBacklogThreshold = threshold });
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("LocalDbBacklogThreshold", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void ZeroLocalDbBacklogSustainedFor_IsRejected()
|
|
{
|
|
var result = Validate(new HealthMonitoringOptions
|
|
{
|
|
LocalDbBacklogSustainedFor = TimeSpan.Zero,
|
|
});
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("LocalDbBacklogSustainedFor", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void LocalDbBacklogSustainedFor_ShorterThanReportInterval_IsRejected()
|
|
{
|
|
// A dwell shorter than one report interval can be satisfied by a single sample, which
|
|
// collapses the sustained check back into the bare threshold it replaced — and that
|
|
// fires on every daily purge burst.
|
|
var result = Validate(new HealthMonitoringOptions
|
|
{
|
|
ReportInterval = TimeSpan.FromSeconds(30),
|
|
LocalDbBacklogSustainedFor = TimeSpan.FromSeconds(10),
|
|
});
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("LocalDbBacklogSustainedFor", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void DefaultSustainedWindow_ComfortablyOutlastsAPurgeDrain()
|
|
{
|
|
// Guards the intent of the default rather than its digits: the dwell must be long
|
|
// enough that an expected purge spike drains inside it, or the badge becomes daily
|
|
// noise and gets ignored.
|
|
var options = new HealthMonitoringOptions();
|
|
|
|
Assert.True(options.LocalDbBacklogSustainedFor >= TimeSpan.FromMinutes(5));
|
|
Assert.True(options.LocalDbBacklogSustainedFor >= options.ReportInterval);
|
|
}
|
|
}
|