ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteHealthCheckTests.cs

using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.Health.Akka;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Registration;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.ScadaBridge.Host.Health;

namespace ZB.MOM.WW.ScadaBridge.Host.Tests;

/// <summary>
/// Site-node health endpoints. Site nodes previously served no health surface at all — only gRPC on
/// the HTTP/2 listener and <c>/metrics</c> on the HTTP/1.1 one — so the family overview dashboard
/// had no uniform way to probe them. These tests pin the three checks the site now registers and
/// the tier each one lands in.
/// <para>
/// The registration assertions build the REAL container from
/// <see cref="SiteServiceRegistration.Configure"/> rather than a hand-assembled
/// <c>ServiceCollection</c>, for the reason spelled out in <see cref="SiteLocalDbWiringTests"/>:
/// this family's wiring defects have consistently been ones that only a container-built graph
/// catches. Each registration is additionally ACTIVATED through its factory, so a check that is
/// registered but cannot resolve its dependencies fails here instead of at the first live probe.
/// </para>
/// </summary>
public class SiteHealthCheckTests : IDisposable
{
    private readonly WebApplication _host;
    private readonly string _tempDbPath;
    private readonly string _tempSiteDbPath;
    private readonly string _tempTrackingDbPath;

    /// <summary>Initializes the test host with a temporary site database and the site health checks.</summary>
    public SiteHealthCheckTests()
    {
        var stamp = Guid.NewGuid();
        _tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_localdb_{stamp}.db");
        _tempSiteDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_site_{stamp}.db");
        _tempTrackingDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_health_track_{stamp}.db");

        var builder = WebApplication.CreateBuilder();
        builder.Configuration.Sources.Clear();
        builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
        {
            ["ScadaBridge:Node:Role"] = "Site",
            ["ScadaBridge:Node:NodeName"] = "node-a",
            ["ScadaBridge:Node:NodeHostname"] = "test-site",
            ["ScadaBridge:Node:SiteId"] = "TestSite",
            ["ScadaBridge:Node:RemotingPort"] = "0",
            ["ScadaBridge:Node:GrpcPort"] = "0",
            ["ScadaBridge:Database:SiteDbPath"] = _tempSiteDbPath,
            ["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={_tempTrackingDbPath}",
            ["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
            ["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
            ["LocalDb:Path"] = _tempDbPath,
        });

        builder.Services.AddGrpc();
        builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();

        SiteServiceRegistration.Configure(builder.Services, builder.Configuration);

        AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);

        _host = builder.Build();
    }

    /// <summary>Releases the resources held by this instance.</summary>
    public void Dispose()
    {
        (_host as IDisposable)?.Dispose();
        foreach (var path in new[] { _tempDbPath, _tempSiteDbPath, _tempTrackingDbPath })
        {
            try { File.Delete(path); } catch { /* best effort */ }
        }

        GC.SuppressFinalize(this);
    }

    private IReadOnlyList<HealthCheckRegistration> Registrations =>
        _host.Services.GetRequiredService<IOptions<HealthCheckServiceOptions>>().Value.Registrations.ToList();

    /// <summary>Verifies that a site registers exactly the three site health checks.</summary>
    [Fact]
    public void Site_RegistersExactlyTheThreeSiteChecks()
    {
        // Exact-set, not "contains": an extra check silently changes what /health/ready means for
        // every consumer, and a site node inheriting a central-only probe is precisely the mistake
        // this asserts against.
        Assert.Equal(
            new[] { "active-node", "akka-cluster", "localdb" },
            Registrations.Select(r => r.Name).OrderBy(n => n, StringComparer.Ordinal).ToArray());
    }

    /// <summary>Verifies that each site check carries its intended tier tag.</summary>
    /// <param name="name">The health check registration name.</param>
    /// <param name="expectedTag">The tier tag the check is expected to carry.</param>
    [Theory]
    [InlineData("akka-cluster", ZbHealthTags.Ready)]
    [InlineData("localdb", ZbHealthTags.Ready)]
    [InlineData("active-node", ZbHealthTags.Active)]
    public void Site_ChecksCarryTheirIntendedTier(string name, string expectedTag)
    {
        var registration = Assert.Single(Registrations, r => r.Name == name);

        // Exactly one tier tag each: MapZbHealth selects by tag, so a check tagged both Ready and
        // Active would put standby-ness into readiness and drop the standby out of readiness too.
        Assert.Equal(new[] { expectedTag }, registration.Tags.ToArray());
    }

    /// <summary>Verifies that each site check resolves to its intended implementation type.</summary>
    /// <param name="name">The health check registration name.</param>
    /// <param name="expected">The expected check implementation type.</param>
    [Theory]
    [InlineData("akka-cluster", typeof(AkkaClusterHealthCheck))]
    [InlineData("localdb", typeof(SiteLocalDbHealthCheck))]
    [InlineData("active-node", typeof(ActiveNodeHealthCheck))]
    public void Site_ChecksActivateToTheIntendedType(string name, Type expected)
    {
        var registration = Assert.Single(Registrations, r => r.Name == name);

        // Running the factory is the point: a type-activated check whose constructor dependencies
        // are not registered on site would throw here rather than on the first probe in production.
        var check = registration.Factory(_host.Services);

        Assert.IsType(expected, check);
    }

    /// <summary>Verifies the site active-node check answers Degraded (HTTP 200), not Unhealthy, while the ActorSystem is not yet available (ScadaBridge#97).</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Site_ActiveNodeCheck_IsDegradedNotUnhealthy_DuringStartup()
    {
        // The Health spec says a node still booting is Degraded, not a failed standby. The old
        // private site check returned Unhealthy (503) whenever no ActorSystem was up yet.
        var registration = Assert.Single(Registrations, r => r.Name == "active-node");
        using var empty = new ServiceCollection().BuildServiceProvider();
        var check = registration.Factory(empty);

        var result = await check.CheckHealthAsync(NewContext("active-node", check));

        Assert.Equal(HealthStatus.Degraded, result.Status);
    }

    /// <summary>Verifies the site active-node check reports the active node in its data and is scoped to the site's own role (ScadaBridge#97).</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Site_ActiveNodeCheck_ReportsActiveNode_AndIsScopedToTheSiteRole()
    {
        var registration = Assert.Single(Registrations, r => r.Name == "active-node");

        // A real single-node cluster that carries the site's role: it is the oldest Up member, so
        // it is Active and must name itself as activeNode with activeRole = site-TestSite.
        using var inRole = await StartSelfJoinedClusterAsync("site-TestSite");
        var check = registration.Factory(inRole.Provider);
        var result = await check.CheckHealthAsync(NewContext("active-node", check));

        Assert.Equal(HealthStatus.Healthy, result.Status);
        Assert.Equal("site-TestSite", result.Data["activeRole"]);
        Assert.Equal(inRole.SelfAddress, result.Data["activeNode"]);
        Assert.Equal(inRole.SelfAddress, result.Data["selfAddress"]);

        // A node that is the oldest member overall but carries ANOTHER site's role is not this
        // site's active node: the check is scoped, so it must not answer Healthy.
        using var otherRole = await StartSelfJoinedClusterAsync("site-Other");
        var other = registration.Factory(otherRole.Provider);
        var otherResult = await other.CheckHealthAsync(NewContext("active-node", other));

        Assert.NotEqual(HealthStatus.Healthy, otherResult.Status);
        Assert.False(otherResult.Data.ContainsKey("activeNode"));
    }

    private static async Task<ClusterHandle> StartSelfJoinedClusterAsync(string role)
    {
        var config = Akka.Configuration.ConfigurationFactory.ParseString($$"""
            akka.actor.provider = cluster
            akka.remote.dot-netty.tcp.hostname = "127.0.0.1"
            akka.remote.dot-netty.tcp.port = 0
            akka.cluster.roles = ["{{role}}"]
            akka.loglevel = OFF
            """);
        var system = Akka.Actor.ActorSystem.Create("hc" + Guid.NewGuid().ToString("N")[..8], config);
        var cluster = Akka.Cluster.Cluster.Get(system);
        cluster.Join(cluster.SelfAddress);
        var deadline = DateTime.UtcNow.AddSeconds(20);
        while (cluster.SelfMember.Status != Akka.Cluster.MemberStatus.Up && DateTime.UtcNow < deadline)
            await Task.Delay(50);
        Assert.Equal(Akka.Cluster.MemberStatus.Up, cluster.SelfMember.Status);

        var services = new ServiceCollection();
        services.AddSingleton(system);
        return new ClusterHandle(system, services.BuildServiceProvider(), cluster.SelfAddress.ToString());
    }

    private sealed class ClusterHandle(Akka.Actor.ActorSystem system, ServiceProvider provider, string selfAddress) : IDisposable
    {
        public IServiceProvider Provider { get; } = provider;
        public string SelfAddress { get; } = selfAddress;

        public void Dispose()
        {
            system.Terminate().Wait(TimeSpan.FromSeconds(10));
        }
    }

    /// <summary>Verifies that a site does not register central-only health checks.</summary>
    /// <param name="centralOnlyName">The name of a check that only central should register.</param>
    [Theory]
    [InlineData("database")]
    [InlineData("required-singletons")]
    public void Site_DoesNotRegisterCentralOnlyChecks(string centralOnlyName)
    {
        // Positive control first: if the harness ever stopped registering health checks at all,
        // every "is absent" assertion below would pass vacuously and this test would be worthless.
        Assert.Contains(Registrations, r => r.Name == "akka-cluster");

        Assert.DoesNotContain(Registrations, r => r.Name == centralOnlyName);
    }

    /// <summary>Verifies that the LocalDb check reports healthy against the real site database.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Site_LocalDbCheck_ReportsHealthyAgainstTheRealSiteDatabase()
    {
        // Behaviour, not wiring: the probe runs against the consolidated site database this
        // composition actually built, so a check that resolves but cannot query fails here.
        var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb")
            .Factory(_host.Services);

        var result = await check.CheckHealthAsync(NewContext("localdb", check));

        Assert.Equal(HealthStatus.Healthy, result.Status);
    }

    /// <summary>Verifies that the LocalDb check adds replication state without failing when replication is off by default.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Site_LocalDbCheck_EnrichesWithReplicationStateWithoutFailingOnDefaultOff()
    {
        // Replication ships default-OFF, so the unreplicated steady state must stay Healthy while
        // still surfacing the state as data — enrichment must never become a verdict.
        var check = (SiteLocalDbHealthCheck)Assert.Single(Registrations, r => r.Name == "localdb")
            .Factory(_host.Services);

        var result = await check.CheckHealthAsync(NewContext("localdb", check));

        Assert.Equal(HealthStatus.Healthy, result.Status);
        Assert.False(Assert.IsType<bool>(result.Data["replicationConnected"]));
        Assert.Equal(0L, Assert.IsType<long>(result.Data["replicationOplogBacklog"]));
    }

    /// <summary>Verifies that the LocalDb check reports unhealthy when the store is unreachable.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Site_LocalDbCheck_ReportsUnhealthyWhenTheStoreIsUnreachable()
    {
        var check = new SiteLocalDbHealthCheck(new ThrowingLocalDb());

        var result = await check.CheckHealthAsync(NewContext("localdb", check));

        Assert.Equal(HealthStatus.Unhealthy, result.Status);
    }

    private static HealthCheckContext NewContext(string name, IHealthCheck check) => new()
    {
        Registration = new HealthCheckRegistration(name, check, HealthStatus.Unhealthy, tags: null),
    };

    private sealed class ThrowingLocalDb : ILocalDb
    {
        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <param name="sql">The SQL text (ignored).</param>
        /// <param name="parameters">The statement parameters (ignored).</param>
        /// <param name="ct">A token to cancel the operation (ignored).</param>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public Task<int> ExecuteAsync(string sql, object? parameters = null, CancellationToken ct = default) =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <typeparam name="T">The row type.</typeparam>
        /// <param name="sql">The SQL text (ignored).</param>
        /// <param name="map">The row mapper (ignored).</param>
        /// <param name="parameters">The query parameters (ignored).</param>
        /// <param name="ct">A token to cancel the operation (ignored).</param>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public Task<IReadOnlyList<T>> QueryAsync<T>(
            string sql,
            Func<Microsoft.Data.Sqlite.SqliteDataReader, T> map,
            object? parameters = null,
            CancellationToken ct = default) =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <param name="ct">A token to cancel the operation (ignored).</param>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public Task<ILocalDbTransaction> BeginTransactionAsync(CancellationToken ct = default) =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public Microsoft.Data.Sqlite.SqliteConnection CreateConnection() =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <param name="tableName">The table name (ignored).</param>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public ReplicatedTable RegisterReplicated(string tableName) =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <param name="tableName">The table name (ignored).</param>
        /// <param name="baselineExistingRows">Whether to baseline existing rows (ignored).</param>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public ReplicatedTable RegisterReplicated(string tableName, bool baselineExistingRows) =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Always throws to simulate an unreachable store.</summary>
        /// <param name="tableName">The table name (ignored).</param>
        /// <returns>Never returns; always throws <see cref="InvalidOperationException"/>.</returns>
        public bool DeregisterReplicated(string tableName) =>
            throw new InvalidOperationException("store unreachable");

        /// <summary>Gets the replicated tables; always throws to simulate an unreachable store.</summary>
        public IReadOnlyDictionary<string, ReplicatedTable> ReplicatedTables =>
            throw new InvalidOperationException("store unreachable");
    }
}