using System.Diagnostics.Metrics;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Overview.Observability;
using ZB.MOM.WW.Overview.Polling;
using ZB.MOM.WW.Overview.Registry;
namespace ZB.MOM.WW.Overview.Tests;
///
/// The dashboard watching itself: the gauge it exports about the fleet, and the two checks that
/// decide whether its own /health/ready is telling the truth.
///
public class ObservabilityTests
{
private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);
private static OverviewOptions Registry(
int pollIntervalSeconds = 10,
int timeoutSeconds = 3,
params ApplicationEntry[] applications) =>
new()
{
PollIntervalSeconds = pollIntervalSeconds,
TimeoutSeconds = timeoutSeconds,
StaleAfterSeconds = 45,
Applications = applications.Length > 0
? applications
: [new ApplicationEntry
{
Name = "App",
Instances = [new InstanceEntry { Name = "a", BaseUrl = "http://a:8080" }],
}],
};
private static HealthCheckContext Context() => new();
// ---------------------------------------------------------------- registry-loaded
[Fact]
public async Task RegistryLoaded_ReportsTheCountsItBound()
{
var options = Registry(applications:
[
new ApplicationEntry
{
Name = "ScadaBridge",
Instances =
[
new InstanceEntry { Name = "central-a", BaseUrl = "http://central-a:9000" },
new InstanceEntry { Name = "central-b", BaseUrl = "http://central-b:9100" },
],
},
new ApplicationEntry
{
Name = "OtOpcUa",
Instances = [new InstanceEntry { Name = "admin", BaseUrl = "http://admin:9200" }],
},
]);
var result = await new RegistryLoadedHealthCheck(Options.Create(options))
.CheckHealthAsync(Context());
Assert.Equal(HealthStatus.Healthy, result.Status);
Assert.Equal(2, result.Data["applications"]);
Assert.Equal(3, result.Data["instances"]);
}
[Fact]
public async Task RegistryLoaded_EmptyRegistry_IsUnhealthy()
{
// Unreachable through the real startup path — preflight and the validator both refuse it —
// but the check must still be able to say so, or it is only ever decorative.
var options = new OverviewOptions { Applications = [] };
var result = await new RegistryLoadedHealthCheck(Options.Create(options))
.CheckHealthAsync(Context());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Equal(0, result.Data["instances"]);
}
// ---------------------------------------------------------------- grace window
[Fact]
public void Grace_IsTwiceTheFastestIntervalPlusTheSlowestTimeout()
{
var options = Registry(pollIntervalSeconds: 10, timeoutSeconds: 3);
Assert.Equal(TimeSpan.FromSeconds(26), PollCycleHealthCheck.GraceFor(options));
}
[Fact]
public void Grace_HonoursPerInstanceOverrides()
{
// The sweep loop ticks at the FASTEST configured interval, and the slowest timeout bounds
// how long any one sweep can take — so the window has to be derived from both extremes,
// not from the global defaults.
var options = Registry(pollIntervalSeconds: 30, timeoutSeconds: 3, applications:
[
new ApplicationEntry
{
Name = "App",
Instances =
[
new InstanceEntry { Name = "fast", BaseUrl = "http://fast:8080", PollIntervalSeconds = 5 },
new InstanceEntry { Name = "slow", BaseUrl = "http://slow:8080", TimeoutSeconds = 20 },
],
},
]);
Assert.Equal(TimeSpan.FromSeconds(50), PollCycleHealthCheck.GraceFor(options));
}
// ---------------------------------------------------------------- last-poll-cycle-completed
[Fact]
public async Task PollCycle_NoCycleYetButStillStarting_IsDegradedNotUnhealthy()
{
// Every deployment passes through this state. Failing readiness here would make the
// container look broken for the first seconds of its life, every single restart.
var time = new FakeTimeProvider(Start);
var check = new PollCycleHealthCheck(new PollCycleHeartbeat(time), Options.Create(Registry()), time);
time.Advance(TimeSpan.FromSeconds(5));
var result = await check.CheckHealthAsync(Context());
Assert.Equal(HealthStatus.Degraded, result.Status);
}
[Fact]
public async Task PollCycle_NeverSwept_EventuallyGoesUnhealthy()
{
// A poller that never started must not hide behind "still starting" forever.
var time = new FakeTimeProvider(Start);
var check = new PollCycleHealthCheck(new PollCycleHeartbeat(time), Options.Create(Registry()), time);
time.Advance(TimeSpan.FromMinutes(5));
var result = await check.CheckHealthAsync(Context());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Contains("since startup", result.Description, StringComparison.Ordinal);
}
[Fact]
public async Task PollCycle_RecentSweep_IsHealthyAndPublishesItsAge()
{
var time = new FakeTimeProvider(Start);
var heartbeat = new PollCycleHeartbeat(time);
var check = new PollCycleHealthCheck(heartbeat, Options.Create(Registry()), time);
heartbeat.RecordCycleCompleted(Start);
time.Advance(TimeSpan.FromSeconds(4));
var result = await check.CheckHealthAsync(Context());
Assert.Equal(HealthStatus.Healthy, result.Status);
Assert.Equal(4d, result.Data["ageSeconds"]);
Assert.Equal(26d, result.Data["graceSeconds"]);
}
[Fact]
public async Task PollCycle_StoppedSweeping_IsUnhealthy()
{
// The failure this check exists for: Kestrel is fine, the page still renders, and every
// card on it is quietly frozen.
var time = new FakeTimeProvider(Start);
var heartbeat = new PollCycleHeartbeat(time);
var check = new PollCycleHealthCheck(heartbeat, Options.Create(Registry()), time);
heartbeat.RecordCycleCompleted(Start);
time.Advance(TimeSpan.FromMinutes(2));
var result = await check.CheckHealthAsync(Context());
Assert.Equal(HealthStatus.Unhealthy, result.Status);
Assert.Contains("stale", result.Description, StringComparison.Ordinal);
}
[Fact]
public void Heartbeat_ReportsNothingUntilACycleCompletes()
{
var heartbeat = new PollCycleHeartbeat(new FakeTimeProvider(Start));
Assert.Null(heartbeat.LastCompletedUtc);
Assert.Equal(Start, heartbeat.StartedUtc);
heartbeat.RecordCycleCompleted(Start.AddSeconds(7));
Assert.Equal(Start.AddSeconds(7), heartbeat.LastCompletedUtc);
}
// ---------------------------------------------------------------- the fleet gauge
[Fact]
public void Gauge_ReportsOneMeasurementPerProbedInstance()
{
var store = new OverviewSnapshotStore();
store.Publish(SnapshotOf(
Snapshots.Instance("central-a", group: "central", status: InstanceStatus.Up),
Snapshots.Instance("central-b", group: "central", status: InstanceStatus.Degraded),
Snapshots.Instance("site-a-a", group: "site-a", status: InstanceStatus.Unreachable)));
using var factory = new DummyMeterFactory();
using var metrics = new OverviewMetrics(factory, store);
var measured = CollectStatusGauge(factory);
Assert.Equal(3, measured.Count);
Assert.Equal(0, measured.Single(m => m.Tags["node"] as string == "central-a").Value);
Assert.Equal(1, measured.Single(m => m.Tags["node"] as string == "central-b").Value);
Assert.Equal(3, measured.Single(m => m.Tags["node"] as string == "site-a-a").Value);
Assert.Equal("central", measured.First().Tags["group"]);
Assert.Equal("App", measured.First().Tags["application"]);
}
[Fact]
public void Gauge_SkipsInstancesThatHaveNeverBeenProbed()
{
// The enum's zero ordinal is Up, so emitting anything for an unprobed instance would
// publish the single most reassuring answer about a node nobody has heard from.
var store = new OverviewSnapshotStore();
store.Publish(SnapshotOf(
Snapshots.Instance("probed", status: InstanceStatus.Down),
Snapshots.Instance("never-probed", status: null)));
using var factory = new DummyMeterFactory();
using var metrics = new OverviewMetrics(factory, store);
var measured = CollectStatusGauge(factory);
Assert.Equal("probed", Assert.Single(measured).Tags["node"]);
}
[Fact]
public void Gauge_TracksTheStoreRatherThanASnapshotTakenAtConstruction()
{
// It is an OBSERVABLE gauge: read at scrape time from whatever the store currently holds,
// so the metric and the page can never disagree about what the dashboard believes.
var store = new OverviewSnapshotStore();
store.Publish(SnapshotOf(Snapshots.Instance("a", status: InstanceStatus.Up)));
using var factory = new DummyMeterFactory();
using var metrics = new OverviewMetrics(factory, store);
Assert.Equal(0, Assert.Single(CollectStatusGauge(factory)).Value);
store.Publish(SnapshotOf(Snapshots.Instance("a", status: InstanceStatus.Down)));
Assert.Equal(2, Assert.Single(CollectStatusGauge(factory)).Value);
}
[Fact]
public async Task Sweep_StampsTheHeartbeatSoTheSelfCheckCanSeeIt()
{
// Ties the two halves together: without this the health check would be measuring a value
// nothing ever writes, and would report a healthy poller forever.
var time = new FakeTimeProvider(Start);
var store = new OverviewSnapshotStore();
var heartbeat = new PollCycleHeartbeat(time);
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", System.Net.HttpStatusCode.OK, HealthBody.Ready("Healthy"));
using var factory = new DummyMeterFactory();
var poller = new OverviewPollerService(
Options.Create(Registry()),
new StubHttpClientFactory(handler),
store,
time,
new OverviewMetrics(factory, store),
heartbeat,
Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
Assert.Null(heartbeat.LastCompletedUtc);
await poller.SweepAsync();
Assert.Equal(Start, heartbeat.LastCompletedUtc);
}
private static OverviewSnapshot SnapshotOf(params InstanceSnapshot[] instances) =>
new(Start, [new ApplicationSnapshot("App", instances, [])]);
private static List<(int Value, Dictionary Tags)> CollectStatusGauge(
DummyMeterFactory factory)
{
var measured = new List<(int, Dictionary)>();
using var listener = new MeterListener
{
InstrumentPublished = (instrument, l) =>
{
// Scoped to THIS test's factory: meter names are process-global, so matching on
// the name alone would also collect instruments from tests running in parallel.
if (ReferenceEquals(instrument.Meter.Scope, factory) &&
instrument.Name == OverviewMetrics.InstanceStatusInstrument)
{
l.EnableMeasurementEvents(instrument);
}
},
};
listener.SetMeasurementEventCallback((_, value, tags, _) =>
measured.Add((value, tags.ToArray().ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal))));
listener.Start();
listener.RecordObservableInstruments();
return measured;
}
}