ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/LocalDbStoreAndForwardConvergenceTests.cs

using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;

/// <summary>
/// Phase 2 convergence: the store-and-forward buffer across a real site pair.
/// </summary>
/// <remarks>
/// <para>
/// These are <b>specifications, written before the deletion they justify</b>. The bespoke
/// <c>ReplicationService</c> (an explicit Add/Remove/Park/Requeue operation stream over Akka)
/// is deleted at Task 14 and replaced by LocalDb's trigger-based change capture. Each test
/// here is one behaviour the old mechanism provided, restated as an outcome the replacement
/// must still deliver — so the cutover has something to be judged against other than "it
/// compiles".
/// </para>
/// <para>
/// They are written in terms of <i>rows</i>, not operations, which is the whole point:
/// under CDC there is no Add or Park message to observe, only a row that must end up in the
/// right state on both nodes.
/// </para>
/// <para>
/// <b>One intent is deliberately not ported:</b>
/// <c>ReplicationServiceTests.ReplicationOperations_AreDispatchedInIssueOrder</c> — 200
/// interleaved operations dispatched synchronously, observed in strict issue order. That
/// asserts the <i>mechanism</i> (inline fire-and-forget dispatch), not an outcome, and CDC
/// capture is asynchronous and batched by construction, so no honest port exists. Its
/// portable content is the ordering <i>outcome</i>: an add followed by a remove must never
/// converge to present. That is
/// <see cref="MessageAddedThenRemoved_NeverConvergesToPresent"/>. This paragraph exists so a
/// future reader does not conclude the test was dropped by accident.
/// </para>
/// </remarks>
[Collection("LocalDbSitePairConvergence")]
public sealed class LocalDbStoreAndForwardConvergenceTests : LocalDbSitePairHarness
{
    // ---- data helpers -----------------------------------------------------------------

    /// <summary>Buffers a message, as <c>StoreAndForwardStorage.EnqueueAsync</c> does.</summary>
    private static Task EnqueueAsync(ILocalDb db, string id, string target = "ERP.GetOrder")
        => db.ExecuteAsync(
            """
            INSERT INTO sf_messages (
                id, category, target, payload_json, retry_count, max_retries,
                retry_interval_ms, created_at, status)
            VALUES (@id, 0, @target, '{"order":1}', 0, 50, 30000, @now, @pending)
            ON CONFLICT(id) DO UPDATE SET
                status = excluded.status,
                retry_count = excluded.retry_count;
            """,
            new
            {
                id,
                target,
                now = DateTime.UtcNow.ToString("o"),
                pending = (int)StoreAndForwardMessageStatus.Pending,
            });

    /// <summary>Deletes a delivered message, as the successful-retry path does.</summary>
    private static Task DeleteAsync(ILocalDb db, string id)
        => db.ExecuteAsync("DELETE FROM sf_messages WHERE id = @id;", new { id });

    private static Task SetStatusAsync(ILocalDb db, string id, StoreAndForwardMessageStatus status, int retryCount)
        => db.ExecuteAsync(
            "UPDATE sf_messages SET status = @status, retry_count = @retryCount WHERE id = @id;",
            new { id, status = (int)status, retryCount });

    private static async Task<(int Status, int RetryCount)?> ReadAsync(ILocalDb db, string id)
    {
        var rows = await db.QueryAsync(
            "SELECT status, retry_count FROM sf_messages WHERE id = @id",
            static r => (r.GetInt32(0), r.GetInt32(1)), new { id });
        return rows.Count == 0 ? null : rows[0];
    }

    private static async Task<bool> ExistsAsync(ILocalDb db, string id)
        => await ReadAsync(db, id) is not null;

    private static async Task<bool> HasStatusAsync(
        ILocalDb db, string id, StoreAndForwardMessageStatus status)
        => await ReadAsync(db, id) is { } row && row.Status == (int)status;

    // ---- scenarios --------------------------------------------------------------------

    [Fact]
    public async Task BufferedMessage_MaterialisesOnThePeer()
    {
        // Was: BufferingAMessage_ReplicatesAnAddOperation.
        await EnqueueAsync(A, "msg-add");

        await WaitUntilAsync(
            () => ExistsAsync(B, "msg-add"),
            "a message buffered on node A to appear on node B");
    }

    [Fact]
    public async Task DeliveredMessage_DisappearsFromThePeer()
    {
        // Was: SuccessfulRetry_ReplicatesARemoveOperation. Row deletes propagate as
        // tombstones under CDC — the peer must not keep re-delivering a message that node A
        // already succeeded on.
        await EnqueueAsync(A, "msg-remove");
        await WaitUntilAsync(() => ExistsAsync(B, "msg-remove"), "the message to arrive on B first");

        await DeleteAsync(A, "msg-remove");

        await WaitUntilAsync(
            async () => !await ExistsAsync(B, "msg-remove"),
            "the delivered message to be gone from node B");
    }

    [Fact]
    public async Task ParkedMessage_ShowsAsParkedOnThePeer()
    {
        // Was: ParkedMessage_ReplicatesAParkOperation.
        await EnqueueAsync(A, "msg-park");
        await WaitUntilAsync(() => ExistsAsync(B, "msg-park"), "the message to arrive on B first");

        await SetStatusAsync(A, "msg-park", StoreAndForwardMessageStatus.Parked, retryCount: 50);

        await WaitUntilAsync(
            () => HasStatusAsync(B, "msg-park", StoreAndForwardMessageStatus.Parked),
            "the parked status to reach node B");
    }

    [Fact]
    public async Task RequeuedMessage_ResetsStatusAndRetryCountOnThePeer()
    {
        // Was: RetryingAParkedMessage_ReplicatesARequeueOperation +
        // ApplyReplicatedOperation_Requeue_MovesStandbyRowBackToPending. RetryCount matters
        // as much as status: a peer that took Pending but kept retry_count at max would park
        // the message again on its first attempt after a failover.
        await EnqueueAsync(A, "msg-requeue");
        await SetStatusAsync(A, "msg-requeue", StoreAndForwardMessageStatus.Parked, retryCount: 50);
        await WaitUntilAsync(
            () => HasStatusAsync(B, "msg-requeue", StoreAndForwardMessageStatus.Parked),
            "the message to be parked on B before it is requeued");

        await SetStatusAsync(A, "msg-requeue", StoreAndForwardMessageStatus.Pending, retryCount: 0);

        await WaitUntilAsync(
            async () => await ReadAsync(B, "msg-requeue")
                is { Status: (int)StoreAndForwardMessageStatus.Pending, RetryCount: 0 },
            "node B to show the requeued message as Pending with retry_count 0");
    }

    [Fact]
    public async Task MessageAddedThenRemoved_NeverConvergesToPresent()
    {
        // The portable intent of ReplicationOperations_AreDispatchedInIssueOrder (see the
        // class remarks). Ordering only ever mattered because a remove overtaken by its own
        // add would resurrect a delivered message and send it twice. Under LWW that
        // reordering is not preventable by dispatch discipline — it is prevented because the
        // tombstone carries the later HLC and therefore wins regardless of arrival order.
        //
        // Asserting the endpoint rather than the sequence is what makes this portable: the
        // old test would fail on any async transport even when the outcome was correct.
        await EnqueueAsync(A, "msg-ordering");
        await DeleteAsync(A, "msg-ordering");

        // A control row written in the same window. Without it this test is VACUOUS: an
        // absent row is also what a pair that replicates nothing at all looks like, so it
        // would pass with capture switched off entirely (observed — it was the only one of
        // these seven that survived unregistering sf_messages). The control converging is
        // what makes the absence of msg-ordering evidence rather than silence.
        await EnqueueAsync(A, "msg-ordering-control");

        await WaitUntilAsync(
            () => ExistsAsync(B, "msg-ordering-control"),
            "the control message to reach node B, proving the pair was replicating");

        await WaitUntilAsync(
            async () => !await ExistsAsync(B, "msg-ordering") && !await ExistsAsync(A, "msg-ordering"),
            "the add-then-remove pair to settle as absent on both nodes");

        // Absence has to persist, not just occur once: a late-arriving add would flip the row
        // back and the poll above could have observed a gap it never actually converged to.
        await Task.Delay(TimeSpan.FromSeconds(2));
        Assert.False(await ExistsAsync(B, "msg-ordering"));
        Assert.False(await ExistsAsync(A, "msg-ordering"));
        Assert.True(await ExistsAsync(B, "msg-ordering-control"));
    }

    [Fact]
    public async Task SameMessage_BufferedTwice_ConvergesToTheNewerState()
    {
        // Was: ApplyReplicatedAdd_Twice_IsIdempotent_NewestWins. Under CDC this is not a
        // property the application has to implement — LWW on the primary key gives it — but
        // it is still a property the pair must HAVE, so it is asserted rather than assumed.
        await EnqueueAsync(A, "msg-twice");
        await WaitUntilAsync(() => ExistsAsync(B, "msg-twice"), "the first write to reach B");

        await SetStatusAsync(A, "msg-twice", StoreAndForwardMessageStatus.InFlight, retryCount: 3);

        await WaitUntilAsync(
            async () => await ReadAsync(B, "msg-twice")
                is { Status: (int)StoreAndForwardMessageStatus.InFlight, RetryCount: 3 },
            "the newer state to win on node B");
    }

    [Fact]
    public async Task ParkArrivingWithoutItsAdd_StillMaterialisesTheRow()
    {
        // Was: ApplyReplicatedPark_WhenAddWasLost_MaterializesTheParkedRow. The bespoke
        // replicator needed explicit upsert semantics so a lost Add did not leave a Park with
        // nothing to update. The CDC equivalent is a peer that was offline for the add and
        // only ever sees the row in its parked state — it must still end up with the row,
        // not skip it as an update to something it never had.
        await StopPassiveAsync();

        await EnqueueAsync(A, "msg-park-no-add");
        await SetStatusAsync(A, "msg-park-no-add", StoreAndForwardMessageStatus.Parked, retryCount: 50);

        await RestartPairAsync();

        await WaitUntilAsync(
            () => HasStatusAsync(B, "msg-park-no-add", StoreAndForwardMessageStatus.Parked),
            "node B to materialise a row it only ever saw in its parked state");
    }
}