scadaproj/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ObservabilityTests.cs

using System.Diagnostics.Metrics;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Overview.Observability;
using ZB.MOM.WW.Overview.Polling;
using ZB.MOM.WW.Overview.Registry;

namespace ZB.MOM.WW.Overview.Tests;

/// <summary>
/// The dashboard watching itself: the gauge it exports about the fleet, and the two checks that
/// decide whether its own <c>/health/ready</c> is telling the truth.
/// </summary>
public class ObservabilityTests
{
    private static readonly DateTimeOffset Start = new(2026, 7, 24, 12, 0, 0, TimeSpan.Zero);

    private static OverviewOptions Registry(
        int pollIntervalSeconds = 10,
        int timeoutSeconds = 3,
        params ApplicationEntry[] applications) =>
        new()
        {
            PollIntervalSeconds = pollIntervalSeconds,
            TimeoutSeconds = timeoutSeconds,
            StaleAfterSeconds = 45,
            Applications = applications.Length > 0
                ? applications
                : [new ApplicationEntry
                {
                    Name = "App",
                    Instances = [new InstanceEntry { Name = "a", BaseUrl = "http://a:8080" }],
                }],
        };

    private static HealthCheckContext Context() => new();

    // ---------------------------------------------------------------- registry-loaded

    [Fact]
    public async Task RegistryLoaded_ReportsTheCountsItBound()
    {
        var options = Registry(applications:
        [
            new ApplicationEntry
            {
                Name = "ScadaBridge",
                Instances =
                [
                    new InstanceEntry { Name = "central-a", BaseUrl = "http://central-a:9000" },
                    new InstanceEntry { Name = "central-b", BaseUrl = "http://central-b:9100" },
                ],
            },
            new ApplicationEntry
            {
                Name = "OtOpcUa",
                Instances = [new InstanceEntry { Name = "admin", BaseUrl = "http://admin:9200" }],
            },
        ]);

        var result = await new RegistryLoadedHealthCheck(Options.Create(options))
            .CheckHealthAsync(Context());

        Assert.Equal(HealthStatus.Healthy, result.Status);
        Assert.Equal(2, result.Data["applications"]);
        Assert.Equal(3, result.Data["instances"]);
    }

    [Fact]
    public async Task RegistryLoaded_EmptyRegistry_IsUnhealthy()
    {
        // Unreachable through the real startup path — preflight and the validator both refuse it —
        // but the check must still be able to say so, or it is only ever decorative.
        var options = new OverviewOptions { Applications = [] };

        var result = await new RegistryLoadedHealthCheck(Options.Create(options))
            .CheckHealthAsync(Context());

        Assert.Equal(HealthStatus.Unhealthy, result.Status);
        Assert.Equal(0, result.Data["instances"]);
    }

    // ---------------------------------------------------------------- grace window

    [Fact]
    public void Grace_IsTwiceTheFastestIntervalPlusTheSlowestTimeout()
    {
        var options = Registry(pollIntervalSeconds: 10, timeoutSeconds: 3);

        Assert.Equal(TimeSpan.FromSeconds(26), PollCycleHealthCheck.GraceFor(options));
    }

    [Fact]
    public void Grace_HonoursPerInstanceOverrides()
    {
        // The sweep loop ticks at the FASTEST configured interval, and the slowest timeout bounds
        // how long any one sweep can take — so the window has to be derived from both extremes,
        // not from the global defaults.
        var options = Registry(pollIntervalSeconds: 30, timeoutSeconds: 3, applications:
        [
            new ApplicationEntry
            {
                Name = "App",
                Instances =
                [
                    new InstanceEntry { Name = "fast", BaseUrl = "http://fast:8080", PollIntervalSeconds = 5 },
                    new InstanceEntry { Name = "slow", BaseUrl = "http://slow:8080", TimeoutSeconds = 20 },
                ],
            },
        ]);

        Assert.Equal(TimeSpan.FromSeconds(50), PollCycleHealthCheck.GraceFor(options));
    }

    // ---------------------------------------------------------------- last-poll-cycle-completed

    [Fact]
    public async Task PollCycle_NoCycleYetButStillStarting_IsDegradedNotUnhealthy()
    {
        // Every deployment passes through this state. Failing readiness here would make the
        // container look broken for the first seconds of its life, every single restart.
        var time = new FakeTimeProvider(Start);
        var check = new PollCycleHealthCheck(new PollCycleHeartbeat(time), Options.Create(Registry()), time);

        time.Advance(TimeSpan.FromSeconds(5));
        var result = await check.CheckHealthAsync(Context());

        Assert.Equal(HealthStatus.Degraded, result.Status);
    }

    [Fact]
    public async Task PollCycle_NeverSwept_EventuallyGoesUnhealthy()
    {
        // A poller that never started must not hide behind "still starting" forever.
        var time = new FakeTimeProvider(Start);
        var check = new PollCycleHealthCheck(new PollCycleHeartbeat(time), Options.Create(Registry()), time);

        time.Advance(TimeSpan.FromMinutes(5));
        var result = await check.CheckHealthAsync(Context());

        Assert.Equal(HealthStatus.Unhealthy, result.Status);
        Assert.Contains("since startup", result.Description, StringComparison.Ordinal);
    }

    [Fact]
    public async Task PollCycle_RecentSweep_IsHealthyAndPublishesItsAge()
    {
        var time = new FakeTimeProvider(Start);
        var heartbeat = new PollCycleHeartbeat(time);
        var check = new PollCycleHealthCheck(heartbeat, Options.Create(Registry()), time);

        heartbeat.RecordCycleCompleted(Start);
        time.Advance(TimeSpan.FromSeconds(4));
        var result = await check.CheckHealthAsync(Context());

        Assert.Equal(HealthStatus.Healthy, result.Status);
        Assert.Equal(4d, result.Data["ageSeconds"]);
        Assert.Equal(26d, result.Data["graceSeconds"]);
    }

    [Fact]
    public async Task PollCycle_StoppedSweeping_IsUnhealthy()
    {
        // The failure this check exists for: Kestrel is fine, the page still renders, and every
        // card on it is quietly frozen.
        var time = new FakeTimeProvider(Start);
        var heartbeat = new PollCycleHeartbeat(time);
        var check = new PollCycleHealthCheck(heartbeat, Options.Create(Registry()), time);

        heartbeat.RecordCycleCompleted(Start);
        time.Advance(TimeSpan.FromMinutes(2));
        var result = await check.CheckHealthAsync(Context());

        Assert.Equal(HealthStatus.Unhealthy, result.Status);
        Assert.Contains("stale", result.Description, StringComparison.Ordinal);
    }

    [Fact]
    public void Heartbeat_ReportsNothingUntilACycleCompletes()
    {
        var heartbeat = new PollCycleHeartbeat(new FakeTimeProvider(Start));

        Assert.Null(heartbeat.LastCompletedUtc);
        Assert.Equal(Start, heartbeat.StartedUtc);

        heartbeat.RecordCycleCompleted(Start.AddSeconds(7));

        Assert.Equal(Start.AddSeconds(7), heartbeat.LastCompletedUtc);
    }

    // ---------------------------------------------------------------- the fleet gauge

    [Fact]
    public void Gauge_ReportsOneMeasurementPerProbedInstance()
    {
        var store = new OverviewSnapshotStore();
        store.Publish(SnapshotOf(
            Snapshots.Instance("central-a", group: "central", status: InstanceStatus.Up),
            Snapshots.Instance("central-b", group: "central", status: InstanceStatus.Degraded),
            Snapshots.Instance("site-a-a", group: "site-a", status: InstanceStatus.Unreachable)));

        using var factory = new DummyMeterFactory();
        using var metrics = new OverviewMetrics(factory, store);

        var measured = CollectStatusGauge(factory);

        Assert.Equal(3, measured.Count);
        Assert.Equal(0, measured.Single(m => m.Tags["node"] as string == "central-a").Value);
        Assert.Equal(1, measured.Single(m => m.Tags["node"] as string == "central-b").Value);
        Assert.Equal(3, measured.Single(m => m.Tags["node"] as string == "site-a-a").Value);
        Assert.Equal("central", measured.First().Tags["group"]);
        Assert.Equal("App", measured.First().Tags["application"]);
    }

    [Fact]
    public void Gauge_SkipsInstancesThatHaveNeverBeenProbed()
    {
        // The enum's zero ordinal is Up, so emitting anything for an unprobed instance would
        // publish the single most reassuring answer about a node nobody has heard from.
        var store = new OverviewSnapshotStore();
        store.Publish(SnapshotOf(
            Snapshots.Instance("probed", status: InstanceStatus.Down),
            Snapshots.Instance("never-probed", status: null)));

        using var factory = new DummyMeterFactory();
        using var metrics = new OverviewMetrics(factory, store);

        var measured = CollectStatusGauge(factory);

        Assert.Equal("probed", Assert.Single(measured).Tags["node"]);
    }

    [Fact]
    public void Gauge_TracksTheStoreRatherThanASnapshotTakenAtConstruction()
    {
        // It is an OBSERVABLE gauge: read at scrape time from whatever the store currently holds,
        // so the metric and the page can never disagree about what the dashboard believes.
        var store = new OverviewSnapshotStore();
        store.Publish(SnapshotOf(Snapshots.Instance("a", status: InstanceStatus.Up)));

        using var factory = new DummyMeterFactory();
        using var metrics = new OverviewMetrics(factory, store);

        Assert.Equal(0, Assert.Single(CollectStatusGauge(factory)).Value);

        store.Publish(SnapshotOf(Snapshots.Instance("a", status: InstanceStatus.Down)));

        Assert.Equal(2, Assert.Single(CollectStatusGauge(factory)).Value);
    }

    [Fact]
    public async Task Sweep_StampsTheHeartbeatSoTheSelfCheckCanSeeIt()
    {
        // Ties the two halves together: without this the health check would be measuring a value
        // nothing ever writes, and would report a healthy poller forever.
        var time = new FakeTimeProvider(Start);
        var store = new OverviewSnapshotStore();
        var heartbeat = new PollCycleHeartbeat(time);
        var handler = new RoutingHandler()
            .Json("http://a:8080/health/ready", System.Net.HttpStatusCode.OK, HealthBody.Ready("Healthy"));

        using var factory = new DummyMeterFactory();
        var poller = new OverviewPollerService(
            Options.Create(Registry()),
            new StubHttpClientFactory(handler),
            store,
            time,
            new OverviewMetrics(factory, store),
            heartbeat,
            Microsoft.Extensions.Logging.Abstractions.NullLogger<OverviewPollerService>.Instance);

        Assert.Null(heartbeat.LastCompletedUtc);

        await poller.SweepAsync();

        Assert.Equal(Start, heartbeat.LastCompletedUtc);
    }

    private static OverviewSnapshot SnapshotOf(params InstanceSnapshot[] instances) =>
        new(Start, [new ApplicationSnapshot("App", instances, [])]);

    private static List<(int Value, Dictionary<string, object?> Tags)> CollectStatusGauge(
        DummyMeterFactory factory)
    {
        var measured = new List<(int, Dictionary<string, object?>)>();

        using var listener = new MeterListener
        {
            InstrumentPublished = (instrument, l) =>
            {
                // Scoped to THIS test's factory: meter names are process-global, so matching on
                // the name alone would also collect instruments from tests running in parallel.
                if (ReferenceEquals(instrument.Meter.Scope, factory) &&
                    instrument.Name == OverviewMetrics.InstanceStatusInstrument)
                {
                    l.EnableMeasurementEvents(instrument);
                }
            },
        };

        listener.SetMeasurementEventCallback<int>((_, value, tags, _) =>
            measured.Add((value, tags.ToArray().ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal))));

        listener.Start();
        listener.RecordObservableInstruments();

        return measured;
    }
}