scadaproj/ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/Convergence/TombstoneResurrectionTests.cs

namespace ZB.MOM.WW.LocalDb.Tests.Convergence;

/// <summary>
/// scadaproj#16 — a deleted row must not come back when the peer was offline longer than
/// <c>TombstoneRetention</c>. Driven over the real loopback gRPC pair (<see cref="ConvergenceFixture"/>).
/// </summary>
/// <remarks>
/// The scenario: both nodes hold row 1; while the transport is down B updates it and A then deletes
/// it (A's delete is the newer HLC, so the delete must win everywhere). Eight days pass on A — the
/// oplog age cap drops A's delete from the oplog (flagging a snapshot resync) and maintenance runs
/// its tombstone prune. Before the fix the prune removed A's tombstone too, so A's snapshot had
/// nothing to say about row 1 and B's older update re-created it on A: resurrected on both nodes.
/// </remarks>
[Collection("Convergence")]
public sealed class TombstoneResurrectionTests
{
    private static readonly TimeSpan EightDays = TimeSpan.FromDays(8);

    [Fact]
    public async Task PeerOfflinePastRetention_DeleteStillWinsOnBothNodes()
    {
        await using var fx = await OfflineUpdateThenNewerDeleteAsync();

        // The maintenance tick 8 days later: age cap + tombstone prune, as the background service runs them.
        await fx.RunMaintenanceAAsync(EightDays);
        Assert.True(await IsTombstonedAsync(fx.A, 1),
            "A's undelivered tombstone was pruned while B had never seen it (scadaproj#16).");

        await fx.RestartTransportAsync();
        await fx.AssertDataConvergedAsync();

        Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
        Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
        Assert.True(await IsTombstonedAsync(fx.B, 1), "B should hold A's tombstone after the snapshot.");
    }

    // Positive control: the same scenario with the tombstone prune skipped must converge on "deleted".
    // If it did not, the test above could pass or fail for reasons unrelated to tombstone retention.
    [Fact]
    public async Task Control_SameScenarioWithoutTombstonePrune_DeleteWins()
    {
        await using var fx = await OfflineUpdateThenNewerDeleteAsync();

        await fx.RunMaintenanceAAsync(EightDays, pruneTombstones: false);

        await fx.RestartTransportAsync();
        await fx.AssertDataConvergedAsync();

        Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
        Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
    }

    // Retention still does its job once the peer HAS the tombstone: after the reconnect the pair has
    // exchanged it, so a prune a full retention later must remove it on A.
    [Fact]
    public async Task AfterPeerReceivesTombstone_ItIsPrunedOneRetentionLater()
    {
        await using var fx = await OfflineUpdateThenNewerDeleteAsync();
        await fx.RunMaintenanceAAsync(EightDays);
        await fx.RestartTransportAsync();
        await fx.AssertDataConvergedAsync();
        Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
        // The snapshot debt clears on B's SnapshotAck, which can trail the data by a moment.
        await WaitForNoSnapshotDebtAsync(fx.A);

        // Delivery restarts A's retention clock (proved with a controlled clock in OplogStoreTests);
        // one retention past it, the delivered tombstone goes.
        await fx.RunMaintenanceAAsync(EightDays);
        Assert.False(await HasRowVersionAsync(fx.A, 1), "A never pruned a delivered tombstone.");
    }

    private static async Task<ConvergenceFixture> OfflineUpdateThenNewerDeleteAsync()
    {
        var fx = new ConvergenceFixture();
        await ConvergenceFixture.UpsertAsync(fx.A, 1, "seed", 1);
        await fx.StartAsync();
        await fx.AssertConvergedAsync();

        await fx.KillTransportAsync();
        await ConvergenceFixture.UpsertAsync(fx.B, 1, "b-old", 2);
        // Wall-clock gap so A's delete is the strictly newer HLC regardless of millisecond ties.
        await Task.Delay(20);
        await ConvergenceFixture.DeleteAsync(fx.A, 1);
        return fx;
    }

    private static async Task WaitForNoSnapshotDebtAsync(ILocalDb db)
    {
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        while (DateTime.UtcNow < deadline)
        {
            var flag = await db.QueryAsync(
                "SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", static r => r.GetInt64(0));
            if (flag[0] == 0)
                return;
            await Task.Delay(25);
        }
        Assert.Fail("A's snapshot debt was never discharged.");
    }

    private static async Task<bool> IsTombstonedAsync(ILocalDb db, long id)
    {
        var rows = await db.QueryAsync(
            "SELECT is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
            static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
        return rows.Count == 1 && rows[0] == 1;
    }

    private static async Task<bool> HasRowVersionAsync(ILocalDb db, long id)
    {
        var rows = await db.QueryAsync(
            "SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
            static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
        return rows[0] == 1;
    }
}