ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/LocalDbSitePairConvergenceTests.cs

using ZB.MOM.WW.LocalDb;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;

/// <summary>
/// Phase 1 convergence: operation tracking and site events across a real site pair.
/// </summary>
/// <remarks>
/// This is the test that answers the question Phase 1 exists to answer: does a site node
/// pair actually stop losing operation-tracking and site-event state? Everything upstream
/// of it — schema helpers, DI wiring, the interceptor — can be individually green while the
/// pair still fails to converge.
/// <para>
/// The fixture lives in <see cref="LocalDbSitePairHarness"/>, shared with the Phase 2
/// convergence suites.
/// </para>
/// </remarks>
[Collection("LocalDbSitePairConvergence")]
public sealed class LocalDbSitePairConvergenceTests : LocalDbSitePairHarness
{
    // ---- data helpers -----------------------------------------------------------------

    private static Task WriteTrackingAsync(ILocalDb db, string id, string status, string target)
        => db.ExecuteAsync(
            """
            INSERT INTO OperationTracking (
                TrackedOperationId, Kind, TargetSummary, Status, RetryCount,
                CreatedAtUtc, UpdatedAtUtc)
            VALUES (@id, 'ApiCallCached', @target, @status, 0, @now, @now)
            ON CONFLICT(TrackedOperationId) DO UPDATE SET
                Status = excluded.Status,
                TargetSummary = excluded.TargetSummary,
                UpdatedAtUtc = excluded.UpdatedAtUtc;
            """,
            new { id, status, target, now = DateTime.UtcNow.ToString("o") });

    private static Task WriteEventAsync(ILocalDb db, string id, string message)
        => db.ExecuteAsync(
            """
            INSERT INTO site_events (id, timestamp, event_type, severity, source, message)
            VALUES (@id, @ts, 'script', 'Info', 'convergence-test', @message);
            """,
            new { id, ts = DateTimeOffset.UtcNow.ToString("o"), message });

    private static async Task<string?> ReadTrackingStatusAsync(ILocalDb db, string id)
    {
        var rows = await db.QueryAsync(
            "SELECT Status FROM OperationTracking WHERE TrackedOperationId = @id",
            static r => r.GetString(0), new { id });
        return rows.Count == 0 ? null : rows[0];
    }

    private static Task<IReadOnlyList<string>> ReadEventIdsAsync(ILocalDb db)
        => db.QueryAsync("SELECT id FROM site_events ORDER BY id", static r => r.GetString(0));

    // ---- scenarios --------------------------------------------------------------------

    /// <summary>Verifies that a tracking row written on node A becomes readable on node B.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task TrackingRow_WrittenOnA_BecomesReadableOnB()
    {
        await WriteTrackingAsync(A, "op-a-1", "Submitted", "ERP.GetOrder");

        await WaitUntilAsync(
            async () => await ReadTrackingStatusAsync(B, "op-a-1") == "Submitted",
            "the tracking row written on node A to appear on node B");
    }

    /// <summary>Verifies that a tracking row written on node B becomes readable on node A.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task TrackingRow_WrittenOnB_BecomesReadableOnA()
    {
        // Replication is bidirectional even though only A dials: proving the passive node's
        // writes flow back is what makes a failover in EITHER direction safe.
        await WriteTrackingAsync(B, "op-b-1", "Submitted", "ERP.GetOrder");

        await WaitUntilAsync(
            async () => await ReadTrackingStatusAsync(A, "op-b-1") == "Submitted",
            "the tracking row written on node B to appear on node A");
    }

    /// <summary>Verifies that the same operation updated on both nodes converges to a single winning value.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task SameOperation_UpdatedOnBothNodes_ConvergesToOneWinner()
    {
        // Last-writer-wins on the primary key. The specific winner is not asserted — that is
        // the HLC's business — but the two nodes MUST agree, and must agree on a value one of
        // them actually wrote rather than a merge of both.
        await WriteTrackingAsync(A, "op-conflict", "Submitted", "ERP.GetOrder");
        await WaitUntilAsync(
            async () => await ReadTrackingStatusAsync(B, "op-conflict") is not null,
            "the conflicting row to exist on both nodes before it is updated");

        await WriteTrackingAsync(A, "op-conflict", "Delivered", "ERP.GetOrder");
        await WriteTrackingAsync(B, "op-conflict", "Parked", "ERP.GetOrder");

        await WaitUntilAsync(
            async () =>
            {
                var a = await ReadTrackingStatusAsync(A, "op-conflict");
                var b = await ReadTrackingStatusAsync(B, "op-conflict");
                return a is not null && a == b;
            },
            "both nodes to converge on one status for the contended operation");

        var winner = await ReadTrackingStatusAsync(A, "op-conflict");
        Assert.Contains(winner, new[] { "Delivered", "Parked" });
    }

    /// <summary>Verifies that events logged on both nodes converge to their union with no id collisions.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task EventsLoggedOnBothNodes_ConvergeToTheUnion_WithNoIdCollisions()
    {
        // The reason site_events moved to GUID ids. Under the old autoincrement scheme both
        // nodes would independently mint id=1, id=2, ... and last-writer-wins on the primary
        // key would silently DESTROY one node's events instead of merging them. The union
        // count is the assertion that catches that.
        var idsFromA = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList();
        var idsFromB = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList();

        foreach (var id in idsFromA) await WriteEventAsync(A, id, "from A");
        foreach (var id in idsFromB) await WriteEventAsync(B, id, "from B");

        var expected = idsFromA.Concat(idsFromB).OrderBy(x => x, StringComparer.Ordinal).ToList();

        await WaitUntilAsync(
            async () => (await ReadEventIdsAsync(A)).SequenceEqual(expected)
                        && (await ReadEventIdsAsync(B)).SequenceEqual(expected),
            "both nodes to hold the union of all 10 events");
    }

    /// <summary>Verifies that a peer that goes offline and rejoins catches up on everything it missed.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task PeerOffline_ThenRejoins_CatchesUpOnEverythingItMissed()
    {
        // The failover case that motivates Phase 1: one node is down while the other keeps
        // working, and nothing written during the outage may be lost.
        await StopPassiveAsync();

        var idsDuringOutage = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid().ToString("N")).ToList();
        foreach (var id in idsDuringOutage) await WriteEventAsync(A, id, "written while B was down");
        await WriteTrackingAsync(A, "op-during-outage", "Delivered", "ERP.GetOrder");

        // Node B's database survived the host teardown (pre-constructed instance), so this is
        // a genuine rejoin rather than a fresh node.
        await RestartPairAsync();

        await WaitUntilAsync(
            async () =>
            {
                var events = await ReadEventIdsAsync(B);
                return idsDuringOutage.All(events.Contains)
                       && await ReadTrackingStatusAsync(B, "op-during-outage") == "Delivered";
            },
            "node B to catch up on everything written while it was offline");
    }
}