fix(localdb): keep a tombstone until the peer holds it — no resurrection past TombstoneRetention (scadaproj#16)
0.3.7 (unpublished), owner-chosen option A. A tombstone is now pruned only when it is past TombstoneRetention AND the peer is known to hold it: - held while database-wide or per-table snapshot debt is outstanding (the snapshot is built from the row-version ledger, so a pruned tombstone is a delete it can no longer carry — the issue's second path); - a local tombstone held while its delete is still in the unacked oplog. Delivery (delta ack covering the delete, or snapshot debt discharged) restamps tombstone_utc so retention runs from delivery. New optional LocalDb:Replication:TombstoneHardCap (default off, >= TombstoneRetention) bounds an undelivered tombstone's life. No proto, lib_schema_version or on-disk schema change. Regression test TombstoneResurrectionTests (two real nodes over loopback gRPC): offline update on B, newer delete on A, A's clock +8 days (oplog age cap + tombstone prune) -> reconnect. Fails on 0.3.6 (row resurrected), passes now; plus a no-prune positive control and a pruned-after-delivery control, and five controlled-clock OplogStore tests. Each guard clause mutation-killed. 259/259. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -249,6 +249,20 @@ public sealed class ConvergenceFixture : IAsyncDisposable
|
||||
public Task<bool> EnforceCapsAAsync() =>
|
||||
new OplogStore(_dbA, InitiatorOptions).EnforceCapsAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Runs one maintenance pass (caps, then — unless <paramref name="pruneTombstones"/> is false —
|
||||
/// tombstone pruning) on node A with A's wall clock advanced by <paramref name="clockOffset"/>,
|
||||
/// i.e. what <see cref="MaintenanceBackgroundService"/> does on its tick once that much time has
|
||||
/// passed. Lets a test model "node B was offline for N days" without waiting N days.
|
||||
/// </summary>
|
||||
public async Task RunMaintenanceAAsync(TimeSpan clockOffset, bool pruneTombstones = true)
|
||||
{
|
||||
var store = new OplogStore(_dbA, InitiatorOptions, () => DateTimeOffset.UtcNow + clockOffset);
|
||||
await store.EnforceCapsAsync();
|
||||
if (pruneTombstones)
|
||||
await store.PruneTombstonesAsync();
|
||||
}
|
||||
|
||||
// ---- internals --------------------------------------------------------------------------
|
||||
|
||||
private static SqliteLocalDb NewDb(string path)
|
||||
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
namespace ZB.MOM.WW.LocalDb.Tests.Convergence;
|
||||
|
||||
/// <summary>
|
||||
/// scadaproj#16 — a deleted row must not come back when the peer was offline longer than
|
||||
/// <c>TombstoneRetention</c>. Driven over the real loopback gRPC pair (<see cref="ConvergenceFixture"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The scenario: both nodes hold row 1; while the transport is down B updates it and A then deletes
|
||||
/// it (A's delete is the newer HLC, so the delete must win everywhere). Eight days pass on A — the
|
||||
/// oplog age cap drops A's delete from the oplog (flagging a snapshot resync) and maintenance runs
|
||||
/// its tombstone prune. Before the fix the prune removed A's tombstone too, so A's snapshot had
|
||||
/// nothing to say about row 1 and B's older update re-created it on A: resurrected on both nodes.
|
||||
/// </remarks>
|
||||
[Collection("Convergence")]
|
||||
public sealed class TombstoneResurrectionTests
|
||||
{
|
||||
private static readonly TimeSpan EightDays = TimeSpan.FromDays(8);
|
||||
|
||||
[Fact]
|
||||
public async Task PeerOfflinePastRetention_DeleteStillWinsOnBothNodes()
|
||||
{
|
||||
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
|
||||
|
||||
// The maintenance tick 8 days later: age cap + tombstone prune, as the background service runs them.
|
||||
await fx.RunMaintenanceAAsync(EightDays);
|
||||
Assert.True(await IsTombstonedAsync(fx.A, 1),
|
||||
"A's undelivered tombstone was pruned while B had never seen it (scadaproj#16).");
|
||||
|
||||
await fx.RestartTransportAsync();
|
||||
await fx.AssertDataConvergedAsync();
|
||||
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
|
||||
Assert.True(await IsTombstonedAsync(fx.B, 1), "B should hold A's tombstone after the snapshot.");
|
||||
}
|
||||
|
||||
// Positive control: the same scenario with the tombstone prune skipped must converge on "deleted".
|
||||
// If it did not, the test above could pass or fail for reasons unrelated to tombstone retention.
|
||||
[Fact]
|
||||
public async Task Control_SameScenarioWithoutTombstonePrune_DeleteWins()
|
||||
{
|
||||
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
|
||||
|
||||
await fx.RunMaintenanceAAsync(EightDays, pruneTombstones: false);
|
||||
|
||||
await fx.RestartTransportAsync();
|
||||
await fx.AssertDataConvergedAsync();
|
||||
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
|
||||
}
|
||||
|
||||
// Retention still does its job once the peer HAS the tombstone: after the reconnect the pair has
|
||||
// exchanged it, so a prune a full retention later must remove it on A.
|
||||
[Fact]
|
||||
public async Task AfterPeerReceivesTombstone_ItIsPrunedOneRetentionLater()
|
||||
{
|
||||
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
|
||||
await fx.RunMaintenanceAAsync(EightDays);
|
||||
await fx.RestartTransportAsync();
|
||||
await fx.AssertDataConvergedAsync();
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
|
||||
// The snapshot debt clears on B's SnapshotAck, which can trail the data by a moment.
|
||||
await WaitForNoSnapshotDebtAsync(fx.A);
|
||||
|
||||
// Delivery restarts A's retention clock (proved with a controlled clock in OplogStoreTests);
|
||||
// one retention past it, the delivered tombstone goes.
|
||||
await fx.RunMaintenanceAAsync(EightDays);
|
||||
Assert.False(await HasRowVersionAsync(fx.A, 1), "A never pruned a delivered tombstone.");
|
||||
}
|
||||
|
||||
private static async Task<ConvergenceFixture> OfflineUpdateThenNewerDeleteAsync()
|
||||
{
|
||||
var fx = new ConvergenceFixture();
|
||||
await ConvergenceFixture.UpsertAsync(fx.A, 1, "seed", 1);
|
||||
await fx.StartAsync();
|
||||
await fx.AssertConvergedAsync();
|
||||
|
||||
await fx.KillTransportAsync();
|
||||
await ConvergenceFixture.UpsertAsync(fx.B, 1, "b-old", 2);
|
||||
// Wall-clock gap so A's delete is the strictly newer HLC regardless of millisecond ties.
|
||||
await Task.Delay(20);
|
||||
await ConvergenceFixture.DeleteAsync(fx.A, 1);
|
||||
return fx;
|
||||
}
|
||||
|
||||
private static async Task WaitForNoSnapshotDebtAsync(ILocalDb db)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var flag = await db.QueryAsync(
|
||||
"SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", static r => r.GetInt64(0));
|
||||
if (flag[0] == 0)
|
||||
return;
|
||||
await Task.Delay(25);
|
||||
}
|
||||
Assert.Fail("A's snapshot debt was never discharged.");
|
||||
}
|
||||
|
||||
private static async Task<bool> IsTombstonedAsync(ILocalDb db, long id)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
|
||||
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
|
||||
return rows.Count == 1 && rows[0] == 1;
|
||||
}
|
||||
|
||||
private static async Task<bool> HasRowVersionAsync(ILocalDb db, long id)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
|
||||
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
|
||||
return rows[0] == 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user