fix(localdb): keep a tombstone until the peer holds it — no resurrection past TombstoneRetention (scadaproj#16)
0.3.7 (unpublished), owner-chosen option A. A tombstone is now pruned only when it is past TombstoneRetention AND the peer is known to hold it: - held while database-wide or per-table snapshot debt is outstanding (the snapshot is built from the row-version ledger, so a pruned tombstone is a delete it can no longer carry — the issue's second path); - a local tombstone held while its delete is still in the unacked oplog. Delivery (delta ack covering the delete, or snapshot debt discharged) restamps tombstone_utc so retention runs from delivery. New optional LocalDb:Replication:TombstoneHardCap (default off, >= TombstoneRetention) bounds an undelivered tombstone's life. No proto, lib_schema_version or on-disk schema change. Regression test TombstoneResurrectionTests (two real nodes over loopback gRPC): offline update on B, newer delete on A, A's clock +8 days (oplog age cap + tombstone prune) -> reconnect. Fails on 0.3.6 (row resurrected), passes now; plus a no-prune positive control and a pruned-after-delivery control, and five controlled-clock OplogStore tests. Each guard clause mutation-killed. 259/259. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -249,6 +249,20 @@ public sealed class ConvergenceFixture : IAsyncDisposable
|
||||
public Task<bool> EnforceCapsAAsync() =>
|
||||
new OplogStore(_dbA, InitiatorOptions).EnforceCapsAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Runs one maintenance pass (caps, then — unless <paramref name="pruneTombstones"/> is false —
|
||||
/// tombstone pruning) on node A with A's wall clock advanced by <paramref name="clockOffset"/>,
|
||||
/// i.e. what <see cref="MaintenanceBackgroundService"/> does on its tick once that much time has
|
||||
/// passed. Lets a test model "node B was offline for N days" without waiting N days.
|
||||
/// </summary>
|
||||
public async Task RunMaintenanceAAsync(TimeSpan clockOffset, bool pruneTombstones = true)
|
||||
{
|
||||
var store = new OplogStore(_dbA, InitiatorOptions, () => DateTimeOffset.UtcNow + clockOffset);
|
||||
await store.EnforceCapsAsync();
|
||||
if (pruneTombstones)
|
||||
await store.PruneTombstonesAsync();
|
||||
}
|
||||
|
||||
// ---- internals --------------------------------------------------------------------------
|
||||
|
||||
private static SqliteLocalDb NewDb(string path)
|
||||
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
namespace ZB.MOM.WW.LocalDb.Tests.Convergence;
|
||||
|
||||
/// <summary>
|
||||
/// scadaproj#16 — a deleted row must not come back when the peer was offline longer than
|
||||
/// <c>TombstoneRetention</c>. Driven over the real loopback gRPC pair (<see cref="ConvergenceFixture"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The scenario: both nodes hold row 1; while the transport is down B updates it and A then deletes
|
||||
/// it (A's delete is the newer HLC, so the delete must win everywhere). Eight days pass on A — the
|
||||
/// oplog age cap drops A's delete from the oplog (flagging a snapshot resync) and maintenance runs
|
||||
/// its tombstone prune. Before the fix the prune removed A's tombstone too, so A's snapshot had
|
||||
/// nothing to say about row 1 and B's older update re-created it on A: resurrected on both nodes.
|
||||
/// </remarks>
|
||||
[Collection("Convergence")]
|
||||
public sealed class TombstoneResurrectionTests
|
||||
{
|
||||
private static readonly TimeSpan EightDays = TimeSpan.FromDays(8);
|
||||
|
||||
[Fact]
|
||||
public async Task PeerOfflinePastRetention_DeleteStillWinsOnBothNodes()
|
||||
{
|
||||
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
|
||||
|
||||
// The maintenance tick 8 days later: age cap + tombstone prune, as the background service runs them.
|
||||
await fx.RunMaintenanceAAsync(EightDays);
|
||||
Assert.True(await IsTombstonedAsync(fx.A, 1),
|
||||
"A's undelivered tombstone was pruned while B had never seen it (scadaproj#16).");
|
||||
|
||||
await fx.RestartTransportAsync();
|
||||
await fx.AssertDataConvergedAsync();
|
||||
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
|
||||
Assert.True(await IsTombstonedAsync(fx.B, 1), "B should hold A's tombstone after the snapshot.");
|
||||
}
|
||||
|
||||
// Positive control: the same scenario with the tombstone prune skipped must converge on "deleted".
|
||||
// If it did not, the test above could pass or fail for reasons unrelated to tombstone retention.
|
||||
[Fact]
|
||||
public async Task Control_SameScenarioWithoutTombstonePrune_DeleteWins()
|
||||
{
|
||||
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
|
||||
|
||||
await fx.RunMaintenanceAAsync(EightDays, pruneTombstones: false);
|
||||
|
||||
await fx.RestartTransportAsync();
|
||||
await fx.AssertDataConvergedAsync();
|
||||
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
|
||||
}
|
||||
|
||||
// Retention still does its job once the peer HAS the tombstone: after the reconnect the pair has
|
||||
// exchanged it, so a prune a full retention later must remove it on A.
|
||||
[Fact]
|
||||
public async Task AfterPeerReceivesTombstone_ItIsPrunedOneRetentionLater()
|
||||
{
|
||||
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
|
||||
await fx.RunMaintenanceAAsync(EightDays);
|
||||
await fx.RestartTransportAsync();
|
||||
await fx.AssertDataConvergedAsync();
|
||||
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
|
||||
// The snapshot debt clears on B's SnapshotAck, which can trail the data by a moment.
|
||||
await WaitForNoSnapshotDebtAsync(fx.A);
|
||||
|
||||
// Delivery restarts A's retention clock (proved with a controlled clock in OplogStoreTests);
|
||||
// one retention past it, the delivered tombstone goes.
|
||||
await fx.RunMaintenanceAAsync(EightDays);
|
||||
Assert.False(await HasRowVersionAsync(fx.A, 1), "A never pruned a delivered tombstone.");
|
||||
}
|
||||
|
||||
private static async Task<ConvergenceFixture> OfflineUpdateThenNewerDeleteAsync()
|
||||
{
|
||||
var fx = new ConvergenceFixture();
|
||||
await ConvergenceFixture.UpsertAsync(fx.A, 1, "seed", 1);
|
||||
await fx.StartAsync();
|
||||
await fx.AssertConvergedAsync();
|
||||
|
||||
await fx.KillTransportAsync();
|
||||
await ConvergenceFixture.UpsertAsync(fx.B, 1, "b-old", 2);
|
||||
// Wall-clock gap so A's delete is the strictly newer HLC regardless of millisecond ties.
|
||||
await Task.Delay(20);
|
||||
await ConvergenceFixture.DeleteAsync(fx.A, 1);
|
||||
return fx;
|
||||
}
|
||||
|
||||
private static async Task WaitForNoSnapshotDebtAsync(ILocalDb db)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var flag = await db.QueryAsync(
|
||||
"SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", static r => r.GetInt64(0));
|
||||
if (flag[0] == 0)
|
||||
return;
|
||||
await Task.Delay(25);
|
||||
}
|
||||
Assert.Fail("A's snapshot debt was never discharged.");
|
||||
}
|
||||
|
||||
private static async Task<bool> IsTombstonedAsync(ILocalDb db, long id)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
|
||||
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
|
||||
return rows.Count == 1 && rows[0] == 1;
|
||||
}
|
||||
|
||||
private static async Task<bool> HasRowVersionAsync(ILocalDb db, long id)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
|
||||
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
|
||||
return rows[0] == 1;
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,8 @@ public sealed class MaintenanceTests : IDisposable
|
||||
var (db, store) = await NewDbAsync(options);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'DOOMED', 1)");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
|
||||
// The peer has the delete (an unacked tombstone is never pruned on age — scadaproj#16).
|
||||
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync());
|
||||
// Backdate the delete-trigger's tombstone far past any retention window.
|
||||
var updated = await db.ExecuteAsync(
|
||||
"UPDATE __localdb_row_version SET tombstone_utc = '2000-01-01T00:00:00.000Z' WHERE is_tombstone = 1");
|
||||
@@ -147,6 +149,29 @@ public sealed class MaintenanceTests : IDisposable
|
||||
Assert.Contains(result.Failures, f => f.Contains("MaintenanceInterval", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validator_TombstoneHardCapBelowRetention_Fails_AtOrAboveAndUnset_Pass()
|
||||
{
|
||||
var validator = new ReplicationOptionsValidator();
|
||||
var below = validator.Validate(null, new ReplicationOptions { TombstoneHardCap = TimeSpan.FromDays(6) });
|
||||
Assert.True(below.Failed);
|
||||
Assert.Contains(below.Failures, f => f.Contains("TombstoneHardCap", StringComparison.Ordinal));
|
||||
|
||||
Assert.True(validator.Validate(null, new ReplicationOptions { TombstoneHardCap = TimeSpan.FromDays(7) }).Succeeded);
|
||||
Assert.True(validator.Validate(null, new ReplicationOptions()).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TombstoneHardCap_BindsFromConfiguration_DefaultsToOff()
|
||||
{
|
||||
var bound = new ReplicationOptions();
|
||||
new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?> { ["TombstoneHardCap"] = "30.00:00:00" })
|
||||
.Build().Bind(bound);
|
||||
Assert.Equal(TimeSpan.FromDays(30), bound.TombstoneHardCap);
|
||||
Assert.Null(new ReplicationOptions().TombstoneHardCap);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AddZbLocalDbReplication_RegistersMaintenance_BothRoles()
|
||||
{
|
||||
|
||||
@@ -122,6 +122,113 @@ public sealed class OplogStoreTests : IDisposable
|
||||
Assert.True(peer.NeedsSnapshot);
|
||||
}
|
||||
|
||||
// ---- scadaproj#16: a tombstone is pruned only once the peer holds it --------------------
|
||||
|
||||
private static async Task<long> TombstoneCount(SqliteLocalDb db) =>
|
||||
(await db.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_row_version WHERE is_tombstone = 1", static r => r.GetInt64(0)))[0];
|
||||
|
||||
[Fact]
|
||||
public async Task Prune_KeepsUndeliveredLocalTombstonePastRetention_ThenAgesItFromDelivery()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var store = new OplogStore(db, new ReplicationOptions(), () => now);
|
||||
|
||||
// Peer offline 8 days: past the 7-day retention, but the delete is still unacked.
|
||||
now += TimeSpan.FromDays(8);
|
||||
Assert.Equal(0, await store.PruneTombstonesAsync(default));
|
||||
Assert.Equal(1, await TombstoneCount(db));
|
||||
|
||||
// The peer comes back and acks it: retention restarts at the ack, not at the delete.
|
||||
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(), default);
|
||||
now += TimeSpan.FromDays(6);
|
||||
Assert.Equal(0, await store.PruneTombstonesAsync(default));
|
||||
Assert.Equal(1, await TombstoneCount(db));
|
||||
|
||||
now += TimeSpan.FromDays(2);
|
||||
Assert.Equal(1, await store.PruneTombstonesAsync(default));
|
||||
Assert.Equal(0, await TombstoneCount(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prune_KeepsTombstonesWhileDatabaseWideSnapshotDebtIsOutstanding()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var store = new OplogStore(db, new ReplicationOptions(), () => now);
|
||||
|
||||
// The age cap drops the delete from the oplog (never delivered) and flags a snapshot — the
|
||||
// snapshot is built from the ledger, so the tombstone is the only carrier of the delete.
|
||||
now += TimeSpan.FromDays(8);
|
||||
Assert.True(await store.EnforceCapsAsync(default));
|
||||
Assert.Equal(0, await store.GetMaxSeqAsync());
|
||||
Assert.Equal(0, await store.PruneTombstonesAsync(default));
|
||||
Assert.Equal(1, await TombstoneCount(db));
|
||||
|
||||
// The snapshot is acknowledged: debt cleared, tombstone restamped, aged from there.
|
||||
await store.ClearSnapshotDebtAsync([], default);
|
||||
now += TimeSpan.FromDays(6);
|
||||
Assert.Equal(0, await store.PruneTombstonesAsync(default));
|
||||
now += TimeSpan.FromDays(2);
|
||||
Assert.Equal(1, await store.PruneTombstonesAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prune_KeepsTombstonesOfATableWithPerTableSnapshotDebt()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var store = new OplogStore(db, new ReplicationOptions(), () => now);
|
||||
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(), default);
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO __localdb_snapshot_state (table_name, needs_snapshot) VALUES ('orders', 1)");
|
||||
|
||||
now += TimeSpan.FromDays(8);
|
||||
Assert.Equal(0, await store.PruneTombstonesAsync(default));
|
||||
|
||||
await store.ClearSnapshotDebtAsync(["orders"], default);
|
||||
now += TimeSpan.FromDays(8);
|
||||
Assert.Equal(1, await store.PruneTombstonesAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prune_RemoteOriginTombstone_AgesOnRetentionAlone_EvenWithLocalBacklog()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
// A local write the peer has not acked — it must not hold the PEER's own delete.
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (9, 'LOCAL', 1)");
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO __localdb_row_version (table_name, pk_json, hlc, node_id, is_tombstone, tombstone_utc) " +
|
||||
"VALUES ('orders', '{\"id\":1}', 9223372036854775000, 'the-peer', 1, @utc)",
|
||||
new { utc = DateTimeOffset.UtcNow.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat) });
|
||||
var now = DateTimeOffset.UtcNow + TimeSpan.FromDays(8);
|
||||
var store = new OplogStore(db, new ReplicationOptions(), () => now);
|
||||
|
||||
Assert.Equal(1, await store.PruneTombstonesAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prune_HardCap_DropsAnUndeliveredTombstoneOnceExceeded()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var store = new OplogStore(db, new ReplicationOptions { TombstoneHardCap = TimeSpan.FromDays(30) }, () => now);
|
||||
|
||||
now += TimeSpan.FromDays(29);
|
||||
Assert.Equal(0, await store.PruneTombstonesAsync(default));
|
||||
now += TimeSpan.FromDays(2);
|
||||
Assert.Equal(1, await store.PruneTombstonesAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prune_DropsExpiredTombstoneRowVersions_KeepsLive()
|
||||
{
|
||||
@@ -129,10 +236,13 @@ public sealed class OplogStoreTests : IDisposable
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1),(2,'B',2),(3,'C',3)");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1"); // will be backdated -> expired
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 3"); // fresh tombstone -> kept
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
// The peer has both deletes (scadaproj#16: an undelivered tombstone is never pruned on age);
|
||||
// backdate AFTER the ack, which restamps delivered tombstones.
|
||||
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(), default);
|
||||
await db.ExecuteAsync(
|
||||
"UPDATE __localdb_row_version SET tombstone_utc = '2000-01-01T00:00:00.000Z' WHERE pk_json = '{\"id\":1}'");
|
||||
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
await store.PruneTombstonesAsync(default);
|
||||
|
||||
var rv = await db.QueryAsync(
|
||||
|
||||
Reference in New Issue
Block a user