scadaproj/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/OverviewPollerServiceTests.cs

using System.Net;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Overview.Observability;
using ZB.MOM.WW.Overview.Polling;
using ZB.MOM.WW.Overview.Registry;

namespace ZB.MOM.WW.Overview.Tests;

/// <summary>
/// Sweep behaviour, driven directly rather than through the timer: the rules worth pinning are what
/// a sweep probes, what it does with the answers, and when it declines to probe at all.
/// </summary>
public class OverviewPollerServiceTests
{
    private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);

    private static OverviewOptions Registry(params ApplicationEntry[] applications) =>
        new()
        {
            PollIntervalSeconds = 10,
            TimeoutSeconds = 3,
            StaleAfterSeconds = 45,
            Applications = applications,
        };

    private static ApplicationEntry App(string name, params InstanceEntry[] instances) =>
        new() { Name = name, ManagementLabel = "AdminUI", Instances = instances };

    private static InstanceEntry Node(
        string name,
        string baseUrl,
        string? group = null,
        bool hasActiveRole = false,
        int? pollIntervalSeconds = null) =>
        new()
        {
            Name = name,
            BaseUrl = baseUrl,
            Group = group,
            HasActiveRole = hasActiveRole,
            PollIntervalSeconds = pollIntervalSeconds,
        };

    private static (OverviewPollerService Poller, OverviewSnapshotStore Store) Build(
        OverviewOptions options,
        HttpMessageHandler handler,
        TimeProvider timeProvider)
    {
        var store = new OverviewSnapshotStore();
        var poller = new OverviewPollerService(
            Options.Create(options),
            new StubHttpClientFactory(handler),
            store,
            timeProvider,
            new OverviewMetrics(new DummyMeterFactory(), store),
            new PollCycleHeartbeat(timeProvider),
            NullLogger<OverviewPollerService>.Instance);

        return (poller, store);
    }

    [Fact]
    public async Task Sweep_ProbesReadyForEveryInstance()
    {
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var (poller, _) = Build(
            Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))),
            handler,
            new FakeTimeProvider(Start));

        var snapshot = await poller.SweepAsync();

        Assert.Equal(2, snapshot.AllInstances.Count());
        Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
        // Counted on the READY tier alone: one ready probe == one polled instance, which is what
        // this test means. A reachable instance is also probed on the active tier for its data, so
        // a raw request count would pin the number of tiers rather than the polling behaviour.
        Assert.Equal(2, ReadyProbes(handler));
    }

    [Fact]
    public async Task Sweep_TrailingSlashInBaseUrl_DoesNotDoubleTheSeparator()
    {
        // "//health/ready" would 404 on every instance — a registry typo class that must not matter.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080/"))), handler, new FakeTimeProvider(Start));

        var snapshot = await poller.SweepAsync();

        Assert.Equal(InstanceStatus.Up, Assert.Single(snapshot.AllInstances).Status);
    }

    [Fact]
    public async Task Sweep_ProbesTheActiveTierEvenWithoutTheRole_ButStillAwardsNoBadge()
    {
        // The active tier answers two separable questions: "am I the active half of a pair" and
        // "what does this tier report about me". A standalone gateway has no answer to the first
        // and plenty to say about the second (its live MXAccess session counts), so the PROBE is
        // unconditional while the BADGE stays role-gated. Declaring such a gateway hasActiveRole
        // would fetch the data at the cost of asserting a role that does not exist.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Status("http://a:8080/health/active", HttpStatusCode.OK)
            .Json("http://gw:5120/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Status("http://gw:5120/health/active", HttpStatusCode.OK);
        var (poller, _) = Build(
            Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true), Node("gw", "http://gw:5120"))),
            handler,
            new FakeTimeProvider(Start));

        var snapshot = await poller.SweepAsync();

        Assert.Contains("http://gw:5120/health/active", handler.Requests, StringComparer.Ordinal);
        var byName = snapshot.AllInstances.ToDictionary(i => i.Instance, StringComparer.Ordinal);
        Assert.Equal(ActiveState.Active, byName["a"].Active);
        Assert.Equal(ActiveState.NotApplicable, byName["gw"].Active);
    }

    [Fact]
    public async Task Sweep_RetainsTheActiveTierBody_SoItsDataCanBeRendered()
    {
        var body = HealthBody.WithData(
            "Healthy",
            ("mxaccess-sessions", "Healthy", new Dictionary<string, object>(StringComparer.Ordinal)
            {
                ["total"] = 2,
                ["ready"] = 2,
                ["faulted"] = 0,
            }));
        var handler = new RoutingHandler()
            .Json("http://gw:5120/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Json("http://gw:5120/health/active", HttpStatusCode.OK, body);
        var (poller, _) = Build(
            Registry(App("App", Node("gw", "http://gw:5120"))),
            handler,
            new FakeTimeProvider(Start));

        var instance = Assert.Single((await poller.SweepAsync()).AllInstances);

        Assert.NotNull(instance.ActiveReport);
        Assert.Equal(2, instance.ActiveReport!.Entries["mxaccess-sessions"].DataInt("total"));
        // The tiers stay separate: a check name is only unique within a tier.
        Assert.DoesNotContain("mxaccess-sessions", instance.Report!.Entries.Keys, StringComparer.Ordinal);
    }

    [Fact]
    public async Task Sweep_StandbyAnswers503OnActive_AndIsStillUp()
    {
        // 503 on the ACTIVE tier is the standby's correct answer and says nothing about health —
        // conflating the two tiers would paint every healthy standby in the fleet as Down.
        var handler = new RoutingHandler()
            .Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Status("http://b:8080/health/active", HttpStatusCode.ServiceUnavailable);
        var (poller, _) = Build(
            Registry(App("App", Node("b", "http://b:8080", "pair", hasActiveRole: true))),
            handler,
            new FakeTimeProvider(Start));

        var instance = Assert.Single((await poller.SweepAsync()).AllInstances);

        Assert.Equal(InstanceStatus.Up, instance.Status);
        Assert.Equal(ActiveState.Standby, instance.Active);
    }

    [Fact]
    public async Task Sweep_UnreachableInstance_SkipsTheActiveProbe()
    {
        // Spending a second full timeout to confirm what the first one established would double
        // the worst-case sweep duration for every dead node in the registry.
        var handler = new RoutingHandler().Fails("http://a:8080/health/ready", "Connection refused");
        var (poller, _) = Build(
            Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))),
            handler,
            new FakeTimeProvider(Start));

        await poller.SweepAsync();

        Assert.DoesNotContain("http://a:8080/health/active", handler.Requests, StringComparer.Ordinal);
    }

    [Fact]
    public async Task Sweep_SickInstanceStillGetsItsActiveProbe()
    {
        // A 503 on ready means the app is talking. Whether the sick node is the ACTIVE half is
        // precisely what decides whether this is an outage or a standby nobody is using.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy"))
            .Status("http://a:8080/health/active", HttpStatusCode.OK);
        var (poller, _) = Build(
            Registry(App("App", Node("a", "http://a:8080", "pair", hasActiveRole: true))),
            handler,
            new FakeTimeProvider(Start));

        var instance = Assert.Single((await poller.SweepAsync()).AllInstances);

        Assert.Equal(InstanceStatus.Down, instance.Status);
        Assert.Equal(ActiveState.Active, instance.Active);
    }

    [Fact]
    public async Task Sweep_FirstFailureAfterAGoodPoll_IsDampedThenAdopted()
    {
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var time = new FakeTimeProvider(Start);
        var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);

        Assert.Equal(InstanceStatus.Up, Assert.Single((await poller.SweepAsync()).AllInstances).Status);

        handler.Fails("http://a:8080/health/ready", "Connection refused");

        time.Advance(TimeSpan.FromSeconds(10));
        var damped = Assert.Single((await poller.SweepAsync()).AllInstances);
        Assert.Equal(InstanceStatus.Up, damped.Status);
        Assert.Equal(1, damped.ConsecutiveFailures);

        time.Advance(TimeSpan.FromSeconds(10));
        var adopted = Assert.Single((await poller.SweepAsync()).AllInstances);
        Assert.Equal(InstanceStatus.Unreachable, adopted.Status);
        Assert.Equal(2, adopted.ConsecutiveFailures);
    }

    [Fact]
    public async Task Sweep_LastReachableSurvivesAFailedPoll()
    {
        // "last seen" must keep pointing at the last real contact, not reset to null the moment
        // the instance goes away — that timestamp is how an operator judges how long it has been out.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var time = new FakeTimeProvider(Start);
        var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);

        await poller.SweepAsync();
        handler.Fails("http://a:8080/health/ready", "Connection refused");
        time.Advance(TimeSpan.FromSeconds(10));

        var instance = Assert.Single((await poller.SweepAsync()).AllInstances);

        Assert.Equal(Start, instance.LastReachableUtc);
        Assert.Equal(Start.AddSeconds(10), instance.LastPolledUtc);
    }

    [Fact]
    public async Task Sweep_BeforeAnInstanceIsDue_LeavesItAlone()
    {
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var time = new FakeTimeProvider(Start);
        var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);

        await poller.SweepAsync();
        time.Advance(TimeSpan.FromSeconds(1));
        await poller.SweepAsync();

        // One ready probe == one poll. The second sweep must not poll at all; counting ready
        // probes says exactly that without also asserting how many tiers a poll touches.
        Assert.Equal(1, ReadyProbes(handler));
    }

    [Fact]
    public async Task Sweep_PerInstanceInterval_IsHonoured()
    {
        // The override is only real if a slow instance actually skips ticks while a fast one runs.
        var handler = new RoutingHandler()
            .Json("http://fast:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Json("http://slow:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var time = new FakeTimeProvider(Start);
        var (poller, _) = Build(
            Registry(App(
                "App",
                Node("fast", "http://fast:8080", pollIntervalSeconds: 5),
                Node("slow", "http://slow:8080", pollIntervalSeconds: 30))),
            handler,
            time);

        for (var i = 0; i < 4; i++)
        {
            await poller.SweepAsync();
            time.Advance(TimeSpan.FromSeconds(5));
        }

        Assert.Equal(4, ReadyProbes(handler, "fast"));
        Assert.Equal(1, ReadyProbes(handler, "slow"));
    }

    [Fact]
    public async Task Sweep_DueCheckToleratesAnEarlyTick()
    {
        // A tick arriving a few ms early must not push the instance to half its configured rate.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var time = new FakeTimeProvider(Start);
        var (poller, _) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, time);

        await poller.SweepAsync();
        time.Advance(TimeSpan.FromMilliseconds(9_900));
        await poller.SweepAsync();

        Assert.Equal(2, ReadyProbes(handler));
    }

    [Fact]
    public async Task Sweep_PublishesToTheStore()
    {
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
        OverviewSnapshot? notified = null;
        store.Changed += s => notified = s;

        var returned = await poller.SweepAsync();

        Assert.Same(returned, store.Current);
        Assert.Same(returned, notified);
    }

    [Fact]
    public async Task Sweep_CarriesRegistryMetadataOntoEveryCard()
    {
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Status("http://a:8080/health/active", HttpStatusCode.OK);
        var application = App("ScadaBridge", Node("central-a", "http://a:8080", "central", hasActiveRole: true));
        application.Instances[0].ManagementUrl = "http://a:9000/";
        var (poller, _) = Build(Registry(application), handler, new FakeTimeProvider(Start));

        var instance = Assert.Single((await poller.SweepAsync()).AllInstances);

        Assert.Equal("ScadaBridge", instance.Application);
        Assert.Equal("central", instance.Group);
        Assert.Equal("AdminUI", instance.ManagementLabel);
        Assert.Equal("http://a:9000/", instance.ManagementUrl);
        Assert.Equal(TimeSpan.FromSeconds(45), instance.StaleAfter);
    }

    [Fact]
    public async Task Sweep_ResolvesGroupLeadersIntoTheSnapshot()
    {
        const string leader = "akka.tcp://scadabridge@a:8082";
        var handler = new RoutingHandler()
            .Json("http://a:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader)))
            .Status("http://a:8084/health/active", HttpStatusCode.OK)
            .Json("http://b:8084/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy", ("akka-cluster", "Healthy", leader)))
            .Status("http://b:8084/health/active", HttpStatusCode.ServiceUnavailable);
        var (poller, _) = Build(
            Registry(App(
                "ScadaBridge",
                Node("site-a-a", "http://a:8084", "site-a", hasActiveRole: true),
                Node("site-a-b", "http://b:8084", "site-a", hasActiveRole: true))),
            handler,
            new FakeTimeProvider(Start));

        var application = Assert.Single((await poller.SweepAsync()).Applications);
        var group = Assert.Single(application.Groups);

        Assert.Equal("site-a", group.Name);
        Assert.Equal("site-a-a", group.LeaderInstance);
        Assert.False(group.Disagreement);
    }

    [Fact]
    public async Task Sweep_WorstStatusRollsUpAcrossTheApplication()
    {
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Json("http://b:8080/health/ready", HttpStatusCode.ServiceUnavailable, HealthBody.Ready("Unhealthy"));
        var (poller, _) = Build(
            Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080"))),
            handler,
            new FakeTimeProvider(Start));

        var application = Assert.Single((await poller.SweepAsync()).Applications);

        Assert.Equal(InstanceStatus.Down, application.Worst);
    }

    [Fact]
    public async Task Sweep_OneSlowInstanceDoesNotDelayTheOthers()
    {
        // The fan-out must be genuinely parallel: a serial sweep over a fleet with several dead
        // nodes would take timeout × dead-node-count and starve the whole dashboard.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"))
            .Json("http://b:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var options = Registry(App("App", Node("a", "http://a:8080"), Node("b", "http://b:8080")));
        var gate = new SemaphoreSlim(0);
        var (poller, _) = Build(options, new GatedHandler(handler, gate), new FakeTimeProvider(Start));

        var sweep = poller.SweepAsync();
        Assert.False(sweep.IsCompleted);

        // Both probes must already be in flight; releasing one at a time proves neither waited.
        gate.Release(2);
        var snapshot = await sweep.WaitAsync(TimeSpan.FromSeconds(10));

        Assert.All(snapshot.AllInstances, i => Assert.Equal(InstanceStatus.Up, i.Status));
    }

    [Fact]
    public async Task Sweep_UnregisteredInstances_RenderAsPendingNotAsHealthy()
    {
        // Between host start and the first probe the registry is already published so the page has
        // its layout. Those cards must read "pending", never an unmeasured "Up".
        var handler = new RoutingHandler();
        var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));

        Assert.Empty(store.Current.Applications);

        handler.Fails("http://a:8080/health/ready", "Connection refused");
        var instance = Assert.Single((await poller.SweepAsync()).AllInstances);

        Assert.Equal(InstanceStatus.Unreachable, instance.Status);
    }

    [Fact]
    public void PendingSnapshot_IsNeverStale()
    {
        // Staleness on a card that has never been polled would be meaningless and alarming.
        var pending = Snapshots.Instance("a", status: null);

        Assert.False(pending.IsStale(Start.AddYears(1)));
    }

    [Fact]
    public void Snapshot_GoesStaleOnceItsWindowElapses()
    {
        var polled = Snapshots.Instance("a") with { LastPolledUtc = Start, StaleAfter = TimeSpan.FromSeconds(45) };

        Assert.False(polled.IsStale(Start.AddSeconds(45)));
        Assert.True(polled.IsStale(Start.AddSeconds(46)));
    }

    [Fact]
    public async Task Sweep_HostShutdown_AbandonsTheSweepInsteadOfRecordingFailures()
    {
        // Writing "Unreachable" across the registry on the way out would leave a lie in the store.
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", HttpStatusCode.OK, HealthBody.Ready("Healthy"));
        var (poller, store) = Build(Registry(App("App", Node("a", "http://a:8080"))), handler, new FakeTimeProvider(Start));
        using var shutdown = new CancellationTokenSource();
        await shutdown.CancelAsync();

        await Assert.ThrowsAnyAsync<OperationCanceledException>(() => poller.SweepAsync(shutdown.Token));

        Assert.Empty(store.Current.Applications);
    }

    [Fact]
    public async Task EmptyRegistry_StartsAndStopsWithoutProbing()
    {
        var (poller, _) = Build(Registry(), new RoutingHandler(), new FakeTimeProvider(Start));

        await poller.StartAsync(CancellationToken.None);
        await poller.StopAsync(CancellationToken.None);
    }

    /// <summary>Holds every request until the test releases it, proving the fan-out is concurrent.</summary>
    private sealed class GatedHandler(RoutingHandler inner, SemaphoreSlim gate) : HttpMessageHandler
    {
        protected override async Task<HttpResponseMessage> SendAsync(
            HttpRequestMessage request, CancellationToken cancellationToken)
        {
            await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
            return await inner.HandleAsync(request).ConfigureAwait(false);
        }
    }

    /// <summary>
    /// Ready-tier probes only - one per polled instance. Tests about polling cadence must not
    /// count total requests, which also includes the active-tier probe every reachable instance
    /// now receives.
    /// </summary>
    private static int ReadyProbes(RoutingHandler handler, string? hostContains = null) =>
        handler.Requests.Count(r =>
            r.EndsWith("/health/ready", StringComparison.Ordinal)
            && (hostContains is null || r.Contains(hostContains, StringComparison.Ordinal)));
}