using System.Net;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Overview.Observability;
using ZB.MOM.WW.Overview.Polling;
using ZB.MOM.WW.Overview.Registry;
namespace ZB.MOM.WW.Overview.Tests;
///
/// Sweep behaviour, driven directly rather than through the timer: the rules worth pinning are what
/// a sweep probes, what it does with the answers, and when it declines to probe at all.
///
public class OverviewPollerServiceTests
{
private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);
private static OverviewOptions Registry(params ApplicationEntry[] applications) =>
new()
{
PollIntervalSeconds = 10,
TimeoutSeconds = 3,
StaleAfterSeconds = 45,
Applications = applications,
};
private static ApplicationEntry App(string name, params InstanceEntry[] instances) =>
new() { Name = name, ManagementLabel = "AdminUI", Instances = instances };
private static InstanceEntry Node(
string name,
string baseUrl,
string? group = null,
bool hasActiveRole = false,
int? pollIntervalSeconds = null) =>
new()
{
Name = name,
BaseUrl = baseUrl,
Group = group,
HasActiveRole = hasActiveRole,
PollIntervalSeconds = pollIntervalSeconds,
};
private static (OverviewPollerService Poller, OverviewSnapshotStore Store) Build(
OverviewOptions options,
HttpMessageHandler handler,
TimeProvider timeProvider)
{
var store = new OverviewSnapshotStore();
var poller = new OverviewPollerService(
Options.Create(options),
new StubHttpClientFactory(handler),
store,
timeProvider,
new OverviewMetrics(new DummyMeterFactory(), store),
new PollCycleHeartbeat(timeProvider),
NullLogger.Instance);
return (poller, store);
}
[Fact]
public async Task Sweep_ProbesReadyForEveryInstance()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var (poller, _) = Build(
Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))),
handler,
new FakeTimeProvider(Start));
var snapshot = await poller.SweepAsync();
Assert.Equal(2, snapshot.AllInstances.Count());
Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
Assert.Equal(2, handler.Requests.Count);
}
[Fact]
public async Task Sweep_TrailingSlashInBaseUrl_DoesNotDoubleTheSeparator()
{
// "//health/ready" would 404 on every instance — a registry typo class that must not matter.
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080/"))), handler, new FakeTimeProvider(Start));
var snapshot = await poller.SweepAsync();
Assert.Equal(InstanceStatus.Up, Assert.Single(snapshot.AllInstances).Status);
}
[Fact]
public async Task Sweep_ActiveTierProbedOnlyForInstancesThatHaveTheRole()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Status("http://a:8080/health/active", HttpStatusCode.OK)
.Json("http://gw:5120/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var (poller, _) = Build(
Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true), Node("gw", "http://gw:5120"))),
handler,
new FakeTimeProvider(Start));
var snapshot = await poller.SweepAsync();
Assert.DoesNotContain("http://gw:5120/health/active", handler.Requests, StringComparer.Ordinal);
var byName = snapshot.AllInstances.ToDictionary(i => i.Instance, StringComparer.Ordinal);
Assert.Equal(ActiveState.Active, byName["a"].Active);
Assert.Equal(ActiveState.NotApplicable, byName["gw"].Active);
}
[Fact]
public async Task Sweep_StandbyAnswers503OnActive_AndIsStillUp()
{
// 503 on the ACTIVE tier is the standby's correct answer and says nothing about health —
// conflating the two tiers would paint every healthy standby in the fleet as Down.
var handler = new RoutingHandler()
.Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Status("http://b:8080/health/active", HttpStatusCode.ServiceUnavailable);
var (poller, _) = Build(
Registry(App("App", Node("b", "http://b:8080", "pair", hasActiveRole: true))),
handler,
new FakeTimeProvider(Start));
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal(InstanceStatus.Up, instance.Status);
Assert.Equal(ActiveState.Standby, instance.Active);
}
[Fact]
public async Task Sweep_UnreachableInstance_SkipsTheActiveProbe()
{
// Spending a second full timeout to confirm what the first one established would double
// the worst-case sweep duration for every dead node in the registry.
var handler = new RoutingHandler().Fails("http://a:8080/health/ready", "Connection refused");
var (poller, _) = Build(
Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))),
handler,
new FakeTimeProvider(Start));
await poller.SweepAsync();
Assert.DoesNotContain("http://a:8080/health/active", handler.Requests, StringComparer.Ordinal);
}
[Fact]
public async Task Sweep_SickInstanceStillGetsItsActiveProbe()
{
// A 503 on ready means the app is talking. Whether the sick node is the ACTIVE half is
// precisely what decides whether this is an outage or a standby nobody is using.
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy"))
.Status("http://a:8080/health/active", HttpStatusCode.OK);
var (poller, _) = Build(
Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))),
handler,
new FakeTimeProvider(Start));
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal(InstanceStatus.Down, instance.Status);
Assert.Equal(ActiveState.Active, instance.Active);
}
[Fact]
public async Task Sweep_FirstFailureAfterAGoodPoll_IsDampedThenAdopted()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var time = new FakeTimeProvider(Start);
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
Assert.Equal(InstanceStatus.Up, Assert.Single((await poller.SweepAsync()).AllInstances).Status);
handler.Fails("http://a:8080/health/ready", "Connection refused");
time.Advance(TimeSpan.FromSeconds(10));
var damped = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal(InstanceStatus.Up, damped.Status);
Assert.Equal(1, damped.ConsecutiveFailures);
time.Advance(TimeSpan.FromSeconds(10));
var adopted = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal(InstanceStatus.Unreachable, adopted.Status);
Assert.Equal(2, adopted.ConsecutiveFailures);
}
[Fact]
public async Task Sweep_LastReachableSurvivesAFailedPoll()
{
// "last seen" must keep pointing at the last real contact, not reset to null the moment
// the instance goes away — that timestamp is how an operator judges how long it has been out.
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var time = new FakeTimeProvider(Start);
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
await poller.SweepAsync();
handler.Fails("http://a:8080/health/ready", "Connection refused");
time.Advance(TimeSpan.FromSeconds(10));
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal(Start, instance.LastReachableUtc);
Assert.Equal(Start.AddSeconds(10), instance.LastPolledUtc);
}
[Fact]
public async Task Sweep_BeforeAnInstanceIsDue_LeavesItAlone()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var time = new FakeTimeProvider(Start);
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
await poller.SweepAsync();
time.Advance(TimeSpan.FromSeconds(1));
await poller.SweepAsync();
Assert.Single(handler.Requests);
}
[Fact]
public async Task Sweep_PerInstanceInterval_IsHonoured()
{
// The override is only real if a slow instance actually skips ticks while a fast one runs.
var handler = new RoutingHandler()
.Json("http://fast:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Json("http://slow:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var time = new FakeTimeProvider(Start);
var (poller, _) = Build(
Registry(App(
"App",
Node("fast", "http://fast:8080", pollIntervalSeconds: 5),
Node("slow", "http://slow:8080", pollIntervalSeconds: 30))),
handler,
time);
for (var i = 0; i < 4; i++)
{
await poller.SweepAsync();
time.Advance(TimeSpan.FromSeconds(5));
}
Assert.Equal(4, handler.Requests.Count(r => r.Contains("fast", StringComparison.Ordinal)));
Assert.Equal(1, handler.Requests.Count(r => r.Contains("slow", StringComparison.Ordinal)));
}
[Fact]
public async Task Sweep_DueCheckToleratesAnEarlyTick()
{
// A tick arriving a few ms early must not push the instance to half its configured rate.
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var time = new FakeTimeProvider(Start);
var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);
await poller.SweepAsync();
time.Advance(TimeSpan.FromMilliseconds(9_900));
await poller.SweepAsync();
Assert.Equal(2, handler.Requests.Count);
}
[Fact]
public async Task Sweep_PublishesToTheStore()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
OverviewSnapshot? notified = null;
store.Changed += s => notified = s;
var returned = await poller.SweepAsync();
Assert.Same(returned, store.Current);
Assert.Same(returned, notified);
}
[Fact]
public async Task Sweep_CarriesRegistryMetadataOntoEveryCard()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Status("http://a:8080/health/active", HttpStatusCode.OK);
var application = App("ScadaBridge", Node("central-a", "http://a:8080", "central", hasActiveRole: true));
application.Instances[0].ManagementUrl = "http://a:9000/";
var (poller, _) = Build(Registry(application), handler, new FakeTimeProvider(Start));
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal("ScadaBridge", instance.Application);
Assert.Equal("central", instance.Group);
Assert.Equal("AdminUI", instance.ManagementLabel);
Assert.Equal("http://a:9000/", instance.ManagementUrl);
Assert.Equal(TimeSpan.FromSeconds(45), instance.StaleAfter);
}
[Fact]
public async Task Sweep_ResolvesGroupLeadersIntoTheSnapshot()
{
const string leader = "akka.tcp://scadabridge@a:8082";
var handler = new RoutingHandler()
.Json("http://a:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader)))
.Status("http://a:8084/health/active", HttpStatusCode.OK)
.Json("http://b:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader)))
.Status("http://b:8084/health/active", HttpStatusCode.ServiceUnavailable);
var (poller, _) = Build(
Registry(App(
"ScadaBridge",
Node("site-a-a", "http://a:8084", "site-a", hasActiveRole: true),
Node("site-a-b", "http://b:8084", "site-a", hasActiveRole: true))),
handler,
new FakeTimeProvider(Start));
var application = Assert.Single((await poller.SweepAsync()).Applications);
var group = Assert.Single(application.Groups);
Assert.Equal("site-a", group.Name);
Assert.Equal("site-a-a", group.LeaderInstance);
Assert.False(group.Disagreement);
}
[Fact]
public async Task Sweep_WorstStatusRollsUpAcrossTheApplication()
{
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Json("http://b:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy"));
var (poller, _) = Build(
Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))),
handler,
new FakeTimeProvider(Start));
var application = Assert.Single((await poller.SweepAsync()).Applications);
Assert.Equal(InstanceStatus.Down, application.Worst);
}
[Fact]
public async Task Sweep_OneSlowInstanceDoesNotDelayTheOthers()
{
// The fan-out must be genuinely parallel: a serial sweep over a fleet with several dead
// nodes would take timeout × dead-node-count and starve the whole dashboard.
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
.Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var options = Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080")));
var gate = new SemaphoreSlim(0);
var (poller, _) = Build(options, new GatedHandler(handler, gate), new FakeTimeProvider(Start));
var sweep = poller.SweepAsync();
Assert.False(sweep.IsCompleted);
// Both probes must already be in flight; releasing one at a time proves neither waited.
gate.Release(2);
var snapshot = await sweep.WaitAsync(TimeSpan.FromSeconds(10));
Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
}
[Fact]
public async Task Sweep_UnregisteredInstances_RenderAsPendingNotAsHealthy()
{
// Between host start and the first probe the registry is already published so the page has
// its layout. Those cards must read "pending", never an unmeasured "Up".
var handler = new RoutingHandler();
var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
Assert.Empty(store.Current.Applications);
handler.Fails("http://a:8080/health/ready", "Connection refused");
var instance = Assert.Single((await poller.SweepAsync()).AllInstances);
Assert.Equal(InstanceStatus.Unreachable, instance.Status);
}
[Fact]
public void PendingSnapshot_IsNeverStale()
{
// Staleness on a card that has never been polled would be meaningless and alarming.
var pending = Snapshots.Instance("a", status: null);
Assert.False(pending.IsStale(Start.AddYears(1)));
}
[Fact]
public void Snapshot_GoesStaleOnceItsWindowElapses()
{
var polled = Snapshots.Instance("a") with { LastPolledUtc = Start, StaleAfter = TimeSpan.FromSeconds(45) };
Assert.False(polled.IsStale(Start.AddSeconds(45)));
Assert.True(polled.IsStale(Start.AddSeconds(46)));
}
[Fact]
public async Task Sweep_HostShutdown_AbandonsTheSweepInsteadOfRecordingFailures()
{
// Writing "Unreachable" across the registry on the way out would leave a lie in the store.
var handler = new RoutingHandler()
.Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
using var shutdown = new CancellationTokenSource();
await shutdown.CancelAsync();
await Assert.ThrowsAnyAsync(() => poller.SweepAsync(shutdown.Token));
Assert.Empty(store.Current.Applications);
}
[Fact]
public async Task EmptyRegistry_StartsAndStopsWithoutProbing()
{
var (poller, _) = Build(Registry(), new RoutingHandler(), new FakeTimeProvider(Start));
await poller.StartAsync(CancellationToken.None);
await poller.StopAsync(CancellationToken.None);
}
/// Holds every request until the test releases it, proving the fan-out is concurrent.
private sealed class GatedHandler(RoutingHandler inner, SemaphoreSlim gate) : HttpMessageHandler
{
protected override async Task SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
return await inner.HandleAsync(request).ConfigureAwait(false);
}
}
}