namespace ZB.MOM.WW.LocalDb.Tests.Convergence;
///
/// scadaproj#16 — a deleted row must not come back when the peer was offline longer than
/// TombstoneRetention. Driven over the real loopback gRPC pair ().
///
///
/// The scenario: both nodes hold row 1; while the transport is down B updates it and A then deletes
/// it (A's delete is the newer HLC, so the delete must win everywhere). Eight days pass on A — the
/// oplog age cap drops A's delete from the oplog (flagging a snapshot resync) and maintenance runs
/// its tombstone prune. Before the fix the prune removed A's tombstone too, so A's snapshot had
/// nothing to say about row 1 and B's older update re-created it on A: resurrected on both nodes.
///
[Collection("Convergence")]
public sealed class TombstoneResurrectionTests
{
private static readonly TimeSpan EightDays = TimeSpan.FromDays(8);
[Fact]
public async Task PeerOfflinePastRetention_DeleteStillWinsOnBothNodes()
{
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
// The maintenance tick 8 days later: age cap + tombstone prune, as the background service runs them.
await fx.RunMaintenanceAAsync(EightDays);
Assert.True(await IsTombstonedAsync(fx.A, 1),
"A's undelivered tombstone was pruned while B had never seen it (scadaproj#16).");
await fx.RestartTransportAsync();
await fx.AssertDataConvergedAsync();
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
Assert.True(await IsTombstonedAsync(fx.B, 1), "B should hold A's tombstone after the snapshot.");
}
// Positive control: the same scenario with the tombstone prune skipped must converge on "deleted".
// If it did not, the test above could pass or fail for reasons unrelated to tombstone retention.
[Fact]
public async Task Control_SameScenarioWithoutTombstonePrune_DeleteWins()
{
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
await fx.RunMaintenanceAAsync(EightDays, pruneTombstones: false);
await fx.RestartTransportAsync();
await fx.AssertDataConvergedAsync();
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
}
// Retention still does its job once the peer HAS the tombstone: after the reconnect the pair has
// exchanged it, so a prune a full retention later must remove it on A.
[Fact]
public async Task AfterPeerReceivesTombstone_ItIsPrunedOneRetentionLater()
{
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
await fx.RunMaintenanceAAsync(EightDays);
await fx.RestartTransportAsync();
await fx.AssertDataConvergedAsync();
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
// The snapshot debt clears on B's SnapshotAck, which can trail the data by a moment.
await WaitForNoSnapshotDebtAsync(fx.A);
// Delivery restarts A's retention clock (proved with a controlled clock in OplogStoreTests);
// one retention past it, the delivered tombstone goes.
await fx.RunMaintenanceAAsync(EightDays);
Assert.False(await HasRowVersionAsync(fx.A, 1), "A never pruned a delivered tombstone.");
}
private static async Task OfflineUpdateThenNewerDeleteAsync()
{
var fx = new ConvergenceFixture();
await ConvergenceFixture.UpsertAsync(fx.A, 1, "seed", 1);
await fx.StartAsync();
await fx.AssertConvergedAsync();
await fx.KillTransportAsync();
await ConvergenceFixture.UpsertAsync(fx.B, 1, "b-old", 2);
// Wall-clock gap so A's delete is the strictly newer HLC regardless of millisecond ties.
await Task.Delay(20);
await ConvergenceFixture.DeleteAsync(fx.A, 1);
return fx;
}
private static async Task WaitForNoSnapshotDebtAsync(ILocalDb db)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (DateTime.UtcNow < deadline)
{
var flag = await db.QueryAsync(
"SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", static r => r.GetInt64(0));
if (flag[0] == 0)
return;
await Task.Delay(25);
}
Assert.Fail("A's snapshot debt was never discharged.");
}
private static async Task IsTombstonedAsync(ILocalDb db, long id)
{
var rows = await db.QueryAsync(
"SELECT is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
return rows.Count == 1 && rows[0] == 1;
}
private static async Task HasRowVersionAsync(ILocalDb db, long id)
{
var rows = await db.QueryAsync(
"SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
return rows[0] == 1;
}
}