using System.Diagnostics.Metrics; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; using ZB.MOM.WW.Overview.Observability; using ZB.MOM.WW.Overview.Polling; using ZB.MOM.WW.Overview.Registry; namespace ZB.MOM.WW.Overview.Tests; /// /// The dashboard watching itself: the gauge it exports about the fleet, and the two checks that /// decide whether its own /health/ready is telling the truth. /// public class ObservabilityTests { private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero); private static OverviewOptions Registry( int pollIntervalSeconds = 10, int timeoutSeconds = 3, params ApplicationEntry[] applications) => new() { PollIntervalSeconds = pollIntervalSeconds, TimeoutSeconds = timeoutSeconds, StaleAfterSeconds = 45, Applications = applications.Length > 0 ? applications : [new ApplicationEntry { Name = "App", Instances = [new InstanceEntry { Name = "a", BaseUrl = "http://a:8080" }], }], }; private static HealthCheckContext Context() => new(); // ---------------------------------------------------------------- registry-loaded [Fact] public async Task RegistryLoaded_ReportsTheCountsItBound() { var options = Registry(applications: [ new ApplicationEntry { Name = "ScadaBridge", Instances = [ new InstanceEntry { Name = "central-a", BaseUrl = "http://central-a:9000" }, new InstanceEntry { Name = "central-b", BaseUrl = "http://central-b:9100" }, ], }, new ApplicationEntry { Name = "OtOpcUa", Instances = [new InstanceEntry { Name = "admin", BaseUrl = "http://admin:9200" }], }, ]); var result = await new RegistryLoadedHealthCheck(Options.Create(options)) .CheckHealthAsync(Context()); Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal(2, result.Data["applications"]); Assert.Equal(3, result.Data["instances"]); } [Fact] public async Task RegistryLoaded_EmptyRegistry_IsUnhealthy() { // Unreachable through the real startup path — preflight and the validator both refuse it — // but the check must still be able to say so, or it is only ever decorative. var options = new OverviewOptions { Applications = [] }; var result = await new RegistryLoadedHealthCheck(Options.Create(options)) .CheckHealthAsync(Context()); Assert.Equal(HealthStatus.Unhealthy, result.Status); Assert.Equal(0, result.Data["instances"]); } // ---------------------------------------------------------------- grace window [Fact] public void Grace_IsTwiceTheFastestIntervalPlusTheSlowestTimeout() { var options = Registry(pollIntervalSeconds: 10, timeoutSeconds: 3); Assert.Equal(TimeSpan.FromSeconds(26), PollCycleHealthCheck.GraceFor(options)); } [Fact] public void Grace_HonoursPerInstanceOverrides() { // The sweep loop ticks at the FASTEST configured interval, and the slowest timeout bounds // how long any one sweep can take — so the window has to be derived from both extremes, // not from the global defaults. var options = Registry(pollIntervalSeconds: 30, timeoutSeconds: 3, applications: [ new ApplicationEntry { Name = "App", Instances = [ new InstanceEntry { Name = "fast", BaseUrl = "http://fast:8080", PollIntervalSeconds = 5 }, new InstanceEntry { Name = "slow", BaseUrl = "http://slow:8080", TimeoutSeconds = 20 }, ], }, ]); Assert.Equal(TimeSpan.FromSeconds(50), PollCycleHealthCheck.GraceFor(options)); } // ---------------------------------------------------------------- last-poll-cycle-completed [Fact] public async Task PollCycle_NoCycleYetButStillStarting_IsDegradedNotUnhealthy() { // Every deployment passes through this state. Failing readiness here would make the // container look broken for the first seconds of its life, every single restart. var time = new FakeTimeProvider(Start); var check = new PollCycleHealthCheck(new PollCycleHeartbeat(time), Options.Create(Registry()), time); time.Advance(TimeSpan.FromSeconds(5)); var result = await check.CheckHealthAsync(Context()); Assert.Equal(HealthStatus.Degraded, result.Status); } [Fact] public async Task PollCycle_NeverSwept_EventuallyGoesUnhealthy() { // A poller that never started must not hide behind "still starting" forever. var time = new FakeTimeProvider(Start); var check = new PollCycleHealthCheck(new PollCycleHeartbeat(time), Options.Create(Registry()), time); time.Advance(TimeSpan.FromMinutes(5)); var result = await check.CheckHealthAsync(Context()); Assert.Equal(HealthStatus.Unhealthy, result.Status); Assert.Contains("since startup", result.Description, StringComparison.Ordinal); } [Fact] public async Task PollCycle_RecentSweep_IsHealthyAndPublishesItsAge() { var time = new FakeTimeProvider(Start); var heartbeat = new PollCycleHeartbeat(time); var check = new PollCycleHealthCheck(heartbeat, Options.Create(Registry()), time); heartbeat.RecordCycleCompleted(Start); time.Advance(TimeSpan.FromSeconds(4)); var result = await check.CheckHealthAsync(Context()); Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal(4d, result.Data["ageSeconds"]); Assert.Equal(26d, result.Data["graceSeconds"]); } [Fact] public async Task PollCycle_StoppedSweeping_IsUnhealthy() { // The failure this check exists for: Kestrel is fine, the page still renders, and every // card on it is quietly frozen. var time = new FakeTimeProvider(Start); var heartbeat = new PollCycleHeartbeat(time); var check = new PollCycleHealthCheck(heartbeat, Options.Create(Registry()), time); heartbeat.RecordCycleCompleted(Start); time.Advance(TimeSpan.FromMinutes(2)); var result = await check.CheckHealthAsync(Context()); Assert.Equal(HealthStatus.Unhealthy, result.Status); Assert.Contains("stale", result.Description, StringComparison.Ordinal); } [Fact] public void Heartbeat_ReportsNothingUntilACycleCompletes() { var heartbeat = new PollCycleHeartbeat(new FakeTimeProvider(Start)); Assert.Null(heartbeat.LastCompletedUtc); Assert.Equal(Start, heartbeat.StartedUtc); heartbeat.RecordCycleCompleted(Start.AddSeconds(7)); Assert.Equal(Start.AddSeconds(7), heartbeat.LastCompletedUtc); } // ---------------------------------------------------------------- the fleet gauge [Fact] public void Gauge_ReportsOneMeasurementPerProbedInstance() { var store = new OverviewSnapshotStore(); store.Publish(SnapshotOf( Snapshots.Instance("central-a", group: "central", status: InstanceStatus.Up), Snapshots.Instance("central-b", group: "central", status: InstanceStatus.Degraded), Snapshots.Instance("site-a-a", group: "site-a", status: InstanceStatus.Unreachable))); using var factory = new DummyMeterFactory(); using var metrics = new OverviewMetrics(factory, store); var measured = CollectStatusGauge(factory); Assert.Equal(3, measured.Count); Assert.Equal(0, measured.Single(m => m.Tags["node"] as string == "central-a").Value); Assert.Equal(1, measured.Single(m => m.Tags["node"] as string == "central-b").Value); Assert.Equal(3, measured.Single(m => m.Tags["node"] as string == "site-a-a").Value); Assert.Equal("central", measured.First().Tags["group"]); Assert.Equal("App", measured.First().Tags["application"]); } [Fact] public void Gauge_SkipsInstancesThatHaveNeverBeenProbed() { // The enum's zero ordinal is Up, so emitting anything for an unprobed instance would // publish the single most reassuring answer about a node nobody has heard from. var store = new OverviewSnapshotStore(); store.Publish(SnapshotOf( Snapshots.Instance("probed", status: InstanceStatus.Down), Snapshots.Instance("never-probed", status: null))); using var factory = new DummyMeterFactory(); using var metrics = new OverviewMetrics(factory, store); var measured = CollectStatusGauge(factory); Assert.Equal("probed", Assert.Single(measured).Tags["node"]); } [Fact] public void Gauge_TracksTheStoreRatherThanASnapshotTakenAtConstruction() { // It is an OBSERVABLE gauge: read at scrape time from whatever the store currently holds, // so the metric and the page can never disagree about what the dashboard believes. var store = new OverviewSnapshotStore(); store.Publish(SnapshotOf(Snapshots.Instance("a", status: InstanceStatus.Up))); using var factory = new DummyMeterFactory(); using var metrics = new OverviewMetrics(factory, store); Assert.Equal(0, Assert.Single(CollectStatusGauge(factory)).Value); store.Publish(SnapshotOf(Snapshots.Instance("a", status: InstanceStatus.Down))); Assert.Equal(2, Assert.Single(CollectStatusGauge(factory)).Value); } [Fact] public async Task Sweep_StampsTheHeartbeatSoTheSelfCheckCanSeeIt() { // Ties the two halves together: without this the health check would be measuring a value // nothing ever writes, and would report a healthy poller forever. var time = new FakeTimeProvider(Start); var store = new OverviewSnapshotStore(); var heartbeat = new PollCycleHeartbeat(time); var handler = new RoutingHandler() .Json("http://a:8080/health/ready", System.Net.HttpStatusCode.OK, HealthBody.Ready("Healthy")); using var factory = new DummyMeterFactory(); var poller = new OverviewPollerService( Options.Create(Registry()), new StubHttpClientFactory(handler), store, time, new OverviewMetrics(factory, store), heartbeat, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); Assert.Null(heartbeat.LastCompletedUtc); await poller.SweepAsync(); Assert.Equal(Start, heartbeat.LastCompletedUtc); } private static OverviewSnapshot SnapshotOf(params InstanceSnapshot[] instances) => new(Start, [new ApplicationSnapshot("App", instances, [])]); private static List<(int Value, Dictionary Tags)> CollectStatusGauge( DummyMeterFactory factory) { var measured = new List<(int, Dictionary)>(); using var listener = new MeterListener { InstrumentPublished = (instrument, l) => { // Scoped to THIS test's factory: meter names are process-global, so matching on // the name alone would also collect instruments from tests running in parallel. if (ReferenceEquals(instrument.Meter.Scope, factory) && instrument.Name == OverviewMetrics.InstanceStatusInstrument) { l.EnableMeasurementEvents(instrument); } }, }; listener.SetMeasurementEventCallback((_, value, tags, _) => measured.Add((value, tags.ToArray().ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal)))); listener.Start(); listener.RecordObservableInstruments(); return measured; } }