using System.Net; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.Overview.Observability; using ZB.MOM.WW.Overview.Polling; using ZB.MOM.WW.Overview.Registry; namespace ZB.MOM.WW.Overview.Tests; /// /// Sweep behaviour, driven directly rather than through the timer: the rules worth pinning are what /// a sweep probes, what it does with the answers, and when it declines to probe at all. /// public class OverviewPollerServiceTests { private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero); private static OverviewOptions Registry(params ApplicationEntry[] applications) => new() { PollIntervalSeconds = 10, TimeoutSeconds = 3, StaleAfterSeconds = 45, Applications = applications, }; private static ApplicationEntry App(string name, params InstanceEntry[] instances) => new() { Name = name, ManagementLabel = "AdminUI", Instances = instances }; private static InstanceEntry Node( string name, string baseUrl, string? group = null, bool hasActiveRole = false, int? pollIntervalSeconds = null) => new() { Name = name, BaseUrl = baseUrl, Group = group, HasActiveRole = hasActiveRole, PollIntervalSeconds = pollIntervalSeconds, }; private static (OverviewPollerService Poller, OverviewSnapshotStore Store) Build( OverviewOptions options, HttpMessageHandler handler, TimeProvider timeProvider) { var store = new OverviewSnapshotStore(); var poller = new OverviewPollerService( Options.Create(options), new StubHttpClientFactory(handler), store, timeProvider, new OverviewMetrics(new DummyMeterFactory(), store), new PollCycleHeartbeat(timeProvider), NullLogger.Instance); return (poller, store); } [Fact] public async Task Sweep_ProbesReadyForEveryInstance() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var (poller, _) = Build( Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))), handler, new FakeTimeProvider(Start)); var snapshot = await poller.SweepAsync(); Assert.Equal(2, snapshot.AllInstances.Count()); Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status)); Assert.Equal(2, handler.Requests.Count); } [Fact] public async Task Sweep_TrailingSlashInBaseUrl_DoesNotDoubleTheSeparator() { // "//health/ready" would 404 on every instance — a registry typo class that must not matter. var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080/"))), handler, new FakeTimeProvider(Start)); var snapshot = await poller.SweepAsync(); Assert.Equal(InstanceStatus.Up, Assert.Single(snapshot.AllInstances).Status); } [Fact] public async Task Sweep_ActiveTierProbedOnlyForInstancesThatHaveTheRole() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Status("http://a:8080/health/active", HttpStatusCode.OK) .Json("http://gw:5120/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var (poller, _) = Build( Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true), Node("gw", "http://gw:5120"))), handler, new FakeTimeProvider(Start)); var snapshot = await poller.SweepAsync(); Assert.DoesNotContain("http://gw:5120/health/active", handler.Requests, StringComparer.Ordinal); var byName = snapshot.AllInstances.ToDictionary(i => i.Instance, StringComparer.Ordinal); Assert.Equal(ActiveState.Active, byName["a"].Active); Assert.Equal(ActiveState.NotApplicable, byName["gw"].Active); } [Fact] public async Task Sweep_StandbyAnswers503OnActive_AndIsStillUp() { // 503 on the ACTIVE tier is the standby's correct answer and says nothing about health — // conflating the two tiers would paint every healthy standby in the fleet as Down. var handler = new RoutingHandler() .Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Status("http://b:8080/health/active", HttpStatusCode.ServiceUnavailable); var (poller, _) = Build( Registry(App("App", Node("b", "http://b:8080", "pair", hasActiveRole: true))), handler, new FakeTimeProvider(Start)); var instance = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal(InstanceStatus.Up, instance.Status); Assert.Equal(ActiveState.Standby, instance.Active); } [Fact] public async Task Sweep_UnreachableInstance_SkipsTheActiveProbe() { // Spending a second full timeout to confirm what the first one established would double // the worst-case sweep duration for every dead node in the registry. var handler = new RoutingHandler().Fails("http://a:8080/health/ready", "Connection refused"); var (poller, _) = Build( Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))), handler, new FakeTimeProvider(Start)); await poller.SweepAsync(); Assert.DoesNotContain("http://a:8080/health/active", handler.Requests, StringComparer.Ordinal); } [Fact] public async Task Sweep_SickInstanceStillGetsItsActiveProbe() { // A 503 on ready means the app is talking. Whether the sick node is the ACTIVE half is // precisely what decides whether this is an outage or a standby nobody is using. var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy")) .Status("http://a:8080/health/active", HttpStatusCode.OK); var (poller, _) = Build( Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))), handler, new FakeTimeProvider(Start)); var instance = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal(InstanceStatus.Down, instance.Status); Assert.Equal(ActiveState.Active, instance.Active); } [Fact] public async Task Sweep_FirstFailureAfterAGoodPoll_IsDampedThenAdopted() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var time = new FakeTimeProvider(Start); var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time); Assert.Equal(InstanceStatus.Up, Assert.Single((await poller.SweepAsync()).AllInstances).Status); handler.Fails("http://a:8080/health/ready", "Connection refused"); time.Advance(TimeSpan.FromSeconds(10)); var damped = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal(InstanceStatus.Up, damped.Status); Assert.Equal(1, damped.ConsecutiveFailures); time.Advance(TimeSpan.FromSeconds(10)); var adopted = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal(InstanceStatus.Unreachable, adopted.Status); Assert.Equal(2, adopted.ConsecutiveFailures); } [Fact] public async Task Sweep_LastReachableSurvivesAFailedPoll() { // "last seen" must keep pointing at the last real contact, not reset to null the moment // the instance goes away — that timestamp is how an operator judges how long it has been out. var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var time = new FakeTimeProvider(Start); var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time); await poller.SweepAsync(); handler.Fails("http://a:8080/health/ready", "Connection refused"); time.Advance(TimeSpan.FromSeconds(10)); var instance = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal(Start, instance.LastReachableUtc); Assert.Equal(Start.AddSeconds(10), instance.LastPolledUtc); } [Fact] public async Task Sweep_BeforeAnInstanceIsDue_LeavesItAlone() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var time = new FakeTimeProvider(Start); var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time); await poller.SweepAsync(); time.Advance(TimeSpan.FromSeconds(1)); await poller.SweepAsync(); Assert.Single(handler.Requests); } [Fact] public async Task Sweep_PerInstanceInterval_IsHonoured() { // The override is only real if a slow instance actually skips ticks while a fast one runs. var handler = new RoutingHandler() .Json("http://fast:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Json("http://slow:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var time = new FakeTimeProvider(Start); var (poller, _) = Build( Registry(App( "App", Node("fast", "http://fast:8080", pollIntervalSeconds: 5), Node("slow", "http://slow:8080", pollIntervalSeconds: 30))), handler, time); for (var i = 0; i < 4; i++) { await poller.SweepAsync(); time.Advance(TimeSpan.FromSeconds(5)); } Assert.Equal(4, handler.Requests.Count(r => r.Contains("fast", StringComparison.Ordinal))); Assert.Equal(1, handler.Requests.Count(r => r.Contains("slow", StringComparison.Ordinal))); } [Fact] public async Task Sweep_DueCheckToleratesAnEarlyTick() { // A tick arriving a few ms early must not push the instance to half its configured rate. var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var time = new FakeTimeProvider(Start); var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time); await poller.SweepAsync(); time.Advance(TimeSpan.FromMilliseconds(9_900)); await poller.SweepAsync(); Assert.Equal(2, handler.Requests.Count); } [Fact] public async Task Sweep_PublishesToTheStore() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start)); OverviewSnapshot? notified = null; store.Changed += s => notified = s; var returned = await poller.SweepAsync(); Assert.Same(returned, store.Current); Assert.Same(returned, notified); } [Fact] public async Task Sweep_CarriesRegistryMetadataOntoEveryCard() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Status("http://a:8080/health/active", HttpStatusCode.OK); var application = App("ScadaBridge", Node("central-a", "http://a:8080", "central", hasActiveRole: true)); application.Instances[0].ManagementUrl = "http://a:9000/"; var (poller, _) = Build(Registry(application), handler, new FakeTimeProvider(Start)); var instance = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal("ScadaBridge", instance.Application); Assert.Equal("central", instance.Group); Assert.Equal("AdminUI", instance.ManagementLabel); Assert.Equal("http://a:9000/", instance.ManagementUrl); Assert.Equal(TimeSpan.FromSeconds(45), instance.StaleAfter); } [Fact] public async Task Sweep_ResolvesGroupLeadersIntoTheSnapshot() { const string leader = "akka.tcp://scadabridge@a:8082"; var handler = new RoutingHandler() .Json("http://a:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader))) .Status("http://a:8084/health/active", HttpStatusCode.OK) .Json("http://b:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader))) .Status("http://b:8084/health/active", HttpStatusCode.ServiceUnavailable); var (poller, _) = Build( Registry(App( "ScadaBridge", Node("site-a-a", "http://a:8084", "site-a", hasActiveRole: true), Node("site-a-b", "http://b:8084", "site-a", hasActiveRole: true))), handler, new FakeTimeProvider(Start)); var application = Assert.Single((await poller.SweepAsync()).Applications); var group = Assert.Single(application.Groups); Assert.Equal("site-a", group.Name); Assert.Equal("site-a-a", group.LeaderInstance); Assert.False(group.Disagreement); } [Fact] public async Task Sweep_WorstStatusRollsUpAcrossTheApplication() { var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Json("http://b:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy")); var (poller, _) = Build( Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))), handler, new FakeTimeProvider(Start)); var application = Assert.Single((await poller.SweepAsync()).Applications); Assert.Equal(InstanceStatus.Down, application.Worst); } [Fact] public async Task Sweep_OneSlowInstanceDoesNotDelayTheOthers() { // The fan-out must be genuinely parallel: a serial sweep over a fleet with several dead // nodes would take timeout × dead-node-count and starve the whole dashboard. var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")) .Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var options = Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))); var gate = new SemaphoreSlim(0); var (poller, _) = Build(options, new GatedHandler(handler, gate), new FakeTimeProvider(Start)); var sweep = poller.SweepAsync(); Assert.False(sweep.IsCompleted); // Both probes must already be in flight; releasing one at a time proves neither waited. gate.Release(2); var snapshot = await sweep.WaitAsync(TimeSpan.FromSeconds(10)); Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status)); } [Fact] public async Task Sweep_UnregisteredInstances_RenderAsPendingNotAsHealthy() { // Between host start and the first probe the registry is already published so the page has // its layout. Those cards must read "pending", never an unmeasured "Up". var handler = new RoutingHandler(); var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start)); Assert.Empty(store.Current.Applications); handler.Fails("http://a:8080/health/ready", "Connection refused"); var instance = Assert.Single((await poller.SweepAsync()).AllInstances); Assert.Equal(InstanceStatus.Unreachable, instance.Status); } [Fact] public void PendingSnapshot_IsNeverStale() { // Staleness on a card that has never been polled would be meaningless and alarming. var pending = Snapshots.Instance("a", status: null); Assert.False(pending.IsStale(Start.AddYears(1))); } [Fact] public void Snapshot_GoesStaleOnceItsWindowElapses() { var polled = Snapshots.Instance("a") with { LastPolledUtc = Start, StaleAfter = TimeSpan.FromSeconds(45) }; Assert.False(polled.IsStale(Start.AddSeconds(45))); Assert.True(polled.IsStale(Start.AddSeconds(46))); } [Fact] public async Task Sweep_HostShutdown_AbandonsTheSweepInsteadOfRecordingFailures() { // Writing "Unreachable" across the registry on the way out would leave a lie in the store. var handler = new RoutingHandler() .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy")); var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start)); using var shutdown = new CancellationTokenSource(); await shutdown.CancelAsync(); await Assert.ThrowsAnyAsync(() => poller.SweepAsync(shutdown.Token)); Assert.Empty(store.Current.Applications); } [Fact] public async Task EmptyRegistry_StartsAndStopsWithoutProbing() { var (poller, _) = Build(Registry(), new RoutingHandler(), new FakeTimeProvider(Start)); await poller.StartAsync(CancellationToken.None); await poller.StopAsync(CancellationToken.None); } /// Holds every request until the test releases it, proving the fan-out is concurrent. private sealed class GatedHandler(RoutingHandler inner, SemaphoreSlim gate) : HttpMessageHandler { protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { await gate.WaitAsync(cancellationToken).ConfigureAwait(false); return await inner.HandleAsync(request).ConfigureAwait(false); } } }