ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/LocalDbSitePairHarness.cs

using System.Net;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.ScadaBridge.Host;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;

/// <summary>
/// Serializes the site-pair convergence tests against each other: each one stands up a real
/// Kestrel listener plus two SQLite files, and running them concurrently under CI
/// contention is a flakiness risk.
/// </summary>
[CollectionDefinition("LocalDbSitePairConvergence")]
public sealed class LocalDbSitePairConvergenceCollection;

/// <summary>
/// Two ScadaBridge site nodes replicating the consolidated site database over a REAL
/// loopback gRPC transport, through the REAL fail-closed auth interceptor.
/// </summary>
/// <remarks>
/// <para>
/// Extracted from the Phase 1 convergence tests once Phase 2 needed the same pair for the
/// store-and-forward buffer and the configuration tables. Everything here is fixture; the
/// derived classes hold only their own data helpers and scenarios.
/// </para>
/// <para>
/// It uses <see cref="SiteLocalDbSetup.OnReady"/>, not a hand-written schema, so the tables,
/// their primary keys, and the registration ORDER under test are the ones the host actually
/// runs. A separate schema here would prove only that the test agrees with itself.
/// </para>
/// <para>
/// Offline: no docker, no external services. Loopback Kestrel with h2c.
/// </para>
/// </remarks>
public abstract class LocalDbSitePairHarness : IAsyncLifetime
{
    private const string SharedApiKey = "site-pair-convergence-key";

    /// <summary>How long a scenario waits for the pair to agree before failing.</summary>
    protected static readonly TimeSpan ConvergeTimeout = TimeSpan.FromSeconds(30);

    private readonly string _pathA = Path.Combine(Path.GetTempPath(), $"sitepairA-{Guid.NewGuid():N}.db");
    private readonly string _pathB = Path.Combine(Path.GetTempPath(), $"sitepairB-{Guid.NewGuid():N}.db");

    // The databases are owned by the fixture, in their own providers, and registered into the
    // hosts as pre-constructed instances. MS.DI does not dispose instances it did not create,
    // so tearing a host down (the offline-peer scenario) leaves the databases intact and
    // writable — which is exactly what lets node A accumulate writes while B is down.
    private ServiceProvider _dbProviderA = null!;
    private ServiceProvider _dbProviderB = null!;

    private IHost? _serverHost;      // node B — passive
    private IHost? _initiatorHost;   // node A — dials the peer

    static LocalDbSitePairHarness() =>
        // Grpc.Net.Client dials the loopback server over HTTP/2 cleartext.
        AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);

    /// <summary>Node A — the initiator, which dials the peer.</summary>
    protected ILocalDb A => _dbProviderA.GetRequiredService<ILocalDb>();

    /// <summary>Node B — the passive node, which listens.</summary>
    protected ILocalDb B => _dbProviderB.GetRequiredService<ILocalDb>();

    /// <summary>Prepares the test class before the tests run.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public async Task InitializeAsync()
    {
        _dbProviderA = BuildDatabaseProvider(_pathA, "node-a");
        _dbProviderB = BuildDatabaseProvider(_pathB, "node-b");

        // Force construction (and therefore OnReady) before anything replicates.
        _ = A;
        _ = B;

        await StartPassiveAsync();
        await StartInitiatorAsync();
    }

    /// <summary>Releases the resources held by this test class asynchronously.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    public async Task DisposeAsync()
    {
        await StopHostAsync(_initiatorHost);
        await StopHostAsync(_serverHost);
        await _dbProviderA.DisposeAsync();
        await _dbProviderB.DisposeAsync();

        Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
        foreach (var path in new[] { _pathA, _pathB })
        {
            foreach (var suffix in new[] { "", "-wal", "-shm" })
            {
                try { File.Delete(path + suffix); } catch { /* best effort */ }
            }
        }
    }

    // ---- fixture internals ------------------------------------------------------------

    /// <summary>
    /// A provider owning one consolidated site database, initialized through the host's own
    /// <see cref="SiteLocalDbSetup.OnReady"/> — same schema, same registration order.
    /// </summary>
    private static ServiceProvider BuildDatabaseProvider(string path, string nodeName)
    {
        var config = new ConfigurationBuilder()
            .AddInMemoryCollection(new Dictionary<string, string?>
            {
                ["LocalDb:Path"] = path,
                ["ScadaBridge:Node:NodeName"] = nodeName,
                // OnReady installs the CDC capture triggers only on a node that has
                // replication configured, so the key has to be here and not only in
                // ReplicationConfig below — without it these nodes would run the sync
                // engine over an oplog nothing ever writes to, and every convergence
                // scenario would time out with two intact but unrelated databases.
                ["LocalDb:Replication:ApiKey"] = SharedApiKey,
                // Point the legacy migrators at paths that do not exist, so they no-op rather
                // than picking up stray files from the test working directory. The two Phase 2
                // defaults matter most: unlike the Phase 1 pair they resolve inside ./data/,
                // and a migration would also RENAME whatever it found.
                ["ScadaBridge:OperationTracking:ConnectionString"] =
                    $"Data Source={Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db")}",
                ["ScadaBridge:SiteEventLog:DatabasePath"] =
                    Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
                ["ScadaBridge:StoreAndForward:SqliteDbPath"] =
                    Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
                ["ScadaBridge:Database:SiteDbPath"] =
                    Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
            })
            .Build();

        return new ServiceCollection()
            .AddZbLocalDb(config, db => SiteLocalDbSetup.OnReady(db, config))
            .BuildServiceProvider();
    }

    /// <summary>
    /// The eight tables the Phase 2 cutover registers. Listed literally rather than derived
    /// from production code, so a registration that drifts fails these tests instead of
    /// agreeing with itself.
    /// </summary>
    /// <remarks>
    /// <c>notification_lists</c> and <c>smtp_configurations</c> are absent by design. They
    /// are permanently empty (no site writer since 2026-07-10, the migrator skips them, the
    /// active-node purge keeps them empty), so registering them would open a standing
    /// replication channel whose only historical payload was plaintext SMTP passwords.
    /// </remarks>
    protected static readonly string[] Phase2ReplicatedTables =
    [
        "sf_messages",
        "deployed_configurations", "static_attribute_overrides", "shared_scripts",
        "external_systems", "database_connections", "data_connection_definitions",
        "native_alarm_state",
    ];

    private static IConfiguration ReplicationConfig(string? peerAddress)
    {
        var values = new Dictionary<string, string?>
        {
            // Tight flush + bounded reconnect backoff so convergence is observable well
            // inside the poll deadline. The 60 s production default would let the doubling
            // backoff overrun it after a peer outage.
            ["LocalDb:Replication:FlushInterval"] = "00:00:00.050",
            ["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02",
            // Both nodes share one key — the interceptor is fail-closed, so a mismatch here
            // turns every scenario below red (verified by deliberately breaking it).
            ["LocalDb:Replication:ApiKey"] = SharedApiKey,
        };

        if (peerAddress is not null)
            values["LocalDb:Replication:PeerAddress"] = peerAddress;

        return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
    }

    /// <summary>Starts node B, the passive listener.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    protected async Task StartPassiveAsync()
    {
        var config = ReplicationConfig(peerAddress: null);

        _serverHost = await new HostBuilder()
            .ConfigureWebHost(web =>
            {
                web.UseKestrel(o =>
                    o.Listen(IPAddress.Loopback, 0, listen => listen.Protocols = HttpProtocols.Http2));
                web.ConfigureServices(services =>
                {
                    services.AddLogging();
                    services.AddRouting();
                    // The REAL interceptor, not a stand-in. If it rejected legitimate peer
                    // traffic, every scenario below would fail — which is the point.
                    services.AddGrpc(o => o.Interceptors.Add<LocalDbSyncAuthInterceptor>());
                    services.AddSingleton(B);
                    services.AddZbLocalDbReplication(config);
                });
                web.Configure(app =>
                {
                    app.UseRouting();
                    app.UseEndpoints(e => e.MapZbLocalDbSync());
                });
            })
            .StartAsync();
    }

    /// <summary>Starts node A, which dials the passive node.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    protected async Task StartInitiatorAsync()
    {
        var config = ReplicationConfig(PassiveAddress());

        _initiatorHost = await new HostBuilder()
            .ConfigureServices(services =>
            {
                services.AddLogging();
                services.AddSingleton(A);
                services.AddZbLocalDbReplication(config);
            })
            .StartAsync();
    }

    /// <summary>Takes node B's listener down, leaving its database intact and writable.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    protected async Task StopPassiveAsync()
    {
        await StopHostAsync(_serverHost);
        _serverHost = null;
    }

    /// <summary>
    /// Brings node B back on a NEW loopback port and re-dials from A. The initiator's channel
    /// factory re-reads the peer address on each reconnect, so this is a genuine rejoin.
    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    protected async Task RestartPairAsync()
    {
        await StartPassiveAsync();
        await StopHostAsync(_initiatorHost);
        await StartInitiatorAsync();
    }

    private string PassiveAddress()
        => _serverHost!.Services.GetRequiredService<IServer>()
            .Features.Get<IServerAddressesFeature>()!.Addresses.Single();

    private static async Task StopHostAsync(IHost? host)
    {
        if (host is null) return;
        try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
        host.Dispose();
    }

    /// <summary>Polls <paramref name="condition"/> until true or the deadline passes.</summary>
    /// <param name="condition">The asynchronous condition to poll.</param>
    /// <param name="because">The failure message used if the condition never becomes true.</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
    protected static async Task WaitUntilAsync(Func<Task<bool>> condition, string because)
    {
        var deadline = DateTime.UtcNow + ConvergeTimeout;
        while (DateTime.UtcNow < deadline)
        {
            if (await condition()) return;
            await Task.Delay(50);
        }

        Assert.Fail($"Timed out after {ConvergeTimeout.TotalSeconds:0}s waiting for: {because}");
    }
}