namespace ZB.MOM.WW.LocalDb.Tests.Convergence; /// /// scadaproj#16 — a deleted row must not come back when the peer was offline longer than /// TombstoneRetention. Driven over the real loopback gRPC pair (). /// /// /// The scenario: both nodes hold row 1; while the transport is down B updates it and A then deletes /// it (A's delete is the newer HLC, so the delete must win everywhere). Eight days pass on A — the /// oplog age cap drops A's delete from the oplog (flagging a snapshot resync) and maintenance runs /// its tombstone prune. Before the fix the prune removed A's tombstone too, so A's snapshot had /// nothing to say about row 1 and B's older update re-created it on A: resurrected on both nodes. /// [Collection("Convergence")] public sealed class TombstoneResurrectionTests { private static readonly TimeSpan EightDays = TimeSpan.FromDays(8); [Fact] public async Task PeerOfflinePastRetention_DeleteStillWinsOnBothNodes() { await using var fx = await OfflineUpdateThenNewerDeleteAsync(); // The maintenance tick 8 days later: age cap + tombstone prune, as the background service runs them. await fx.RunMaintenanceAAsync(EightDays); Assert.True(await IsTombstonedAsync(fx.A, 1), "A's undelivered tombstone was pruned while B had never seen it (scadaproj#16)."); await fx.RestartTransportAsync(); await fx.AssertDataConvergedAsync(); Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A)); Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B)); Assert.True(await IsTombstonedAsync(fx.B, 1), "B should hold A's tombstone after the snapshot."); } // Positive control: the same scenario with the tombstone prune skipped must converge on "deleted". // If it did not, the test above could pass or fail for reasons unrelated to tombstone retention. [Fact] public async Task Control_SameScenarioWithoutTombstonePrune_DeleteWins() { await using var fx = await OfflineUpdateThenNewerDeleteAsync(); await fx.RunMaintenanceAAsync(EightDays, pruneTombstones: false); await fx.RestartTransportAsync(); await fx.AssertDataConvergedAsync(); Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A)); Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B)); } // Retention still does its job once the peer HAS the tombstone: after the reconnect the pair has // exchanged it, so a prune a full retention later must remove it on A. [Fact] public async Task AfterPeerReceivesTombstone_ItIsPrunedOneRetentionLater() { await using var fx = await OfflineUpdateThenNewerDeleteAsync(); await fx.RunMaintenanceAAsync(EightDays); await fx.RestartTransportAsync(); await fx.AssertDataConvergedAsync(); Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B)); // The snapshot debt clears on B's SnapshotAck, which can trail the data by a moment. await WaitForNoSnapshotDebtAsync(fx.A); // Delivery restarts A's retention clock (proved with a controlled clock in OplogStoreTests); // one retention past it, the delivered tombstone goes. await fx.RunMaintenanceAAsync(EightDays); Assert.False(await HasRowVersionAsync(fx.A, 1), "A never pruned a delivered tombstone."); } private static async Task OfflineUpdateThenNewerDeleteAsync() { var fx = new ConvergenceFixture(); await ConvergenceFixture.UpsertAsync(fx.A, 1, "seed", 1); await fx.StartAsync(); await fx.AssertConvergedAsync(); await fx.KillTransportAsync(); await ConvergenceFixture.UpsertAsync(fx.B, 1, "b-old", 2); // Wall-clock gap so A's delete is the strictly newer HLC regardless of millisecond ties. await Task.Delay(20); await ConvergenceFixture.DeleteAsync(fx.A, 1); return fx; } private static async Task WaitForNoSnapshotDebtAsync(ILocalDb db) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); while (DateTime.UtcNow < deadline) { var flag = await db.QueryAsync( "SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", static r => r.GetInt64(0)); if (flag[0] == 0) return; await Task.Delay(25); } Assert.Fail("A's snapshot debt was never discharged."); } private static async Task IsTombstonedAsync(ILocalDb db, long id) { var rows = await db.QueryAsync( "SELECT is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk", static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" }); return rows.Count == 1 && rows[0] == 1; } private static async Task HasRowVersionAsync(ILocalDb db, long id) { var rows = await db.QueryAsync( "SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk", static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" }); return rows[0] == 1; } }