fix(localdb): keep a tombstone until the peer holds it — no resurrection past TombstoneRetention (scadaproj#16)

0.3.7 (unpublished), owner-chosen option A. A tombstone is now pruned only when
it is past TombstoneRetention AND the peer is known to hold it:
- held while database-wide or per-table snapshot debt is outstanding (the
  snapshot is built from the row-version ledger, so a pruned tombstone is a
  delete it can no longer carry — the issue's second path);
- a local tombstone held while its delete is still in the unacked oplog.
Delivery (delta ack covering the delete, or snapshot debt discharged) restamps
tombstone_utc so retention runs from delivery. New optional
LocalDb:Replication:TombstoneHardCap (default off, >= TombstoneRetention)
bounds an undelivered tombstone's life. No proto, lib_schema_version or
on-disk schema change.

Regression test TombstoneResurrectionTests (two real nodes over loopback gRPC):
offline update on B, newer delete on A, A's clock +8 days (oplog age cap +
tombstone prune) -> reconnect. Fails on 0.3.6 (row resurrected), passes now;
plus a no-prune positive control and a pruned-after-delivery control, and five
controlled-clock OplogStore tests. Each guard clause mutation-killed. 259/259.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 22:27:46 -05:00
parent 3215d51154
commit be5e9fd436
11 changed files with 423 additions and 18 deletions
@@ -249,6 +249,20 @@ public sealed class ConvergenceFixture : IAsyncDisposable
public Task<bool> EnforceCapsAAsync() =>
new OplogStore(_dbA, InitiatorOptions).EnforceCapsAsync();
/// <summary>
/// Runs one maintenance pass (caps, then — unless <paramref name="pruneTombstones"/> is false —
/// tombstone pruning) on node A with A's wall clock advanced by <paramref name="clockOffset"/>,
/// i.e. what <see cref="MaintenanceBackgroundService"/> does on its tick once that much time has
/// passed. Lets a test model "node B was offline for N days" without waiting N days.
/// </summary>
public async Task RunMaintenanceAAsync(TimeSpan clockOffset, bool pruneTombstones = true)
{
var store = new OplogStore(_dbA, InitiatorOptions, () => DateTimeOffset.UtcNow + clockOffset);
await store.EnforceCapsAsync();
if (pruneTombstones)
await store.PruneTombstonesAsync();
}
// ---- internals --------------------------------------------------------------------------
private static SqliteLocalDb NewDb(string path)
@@ -0,0 +1,116 @@
namespace ZB.MOM.WW.LocalDb.Tests.Convergence;
/// <summary>
/// scadaproj#16 — a deleted row must not come back when the peer was offline longer than
/// <c>TombstoneRetention</c>. Driven over the real loopback gRPC pair (<see cref="ConvergenceFixture"/>).
/// </summary>
/// <remarks>
/// The scenario: both nodes hold row 1; while the transport is down B updates it and A then deletes
/// it (A's delete is the newer HLC, so the delete must win everywhere). Eight days pass on A — the
/// oplog age cap drops A's delete from the oplog (flagging a snapshot resync) and maintenance runs
/// its tombstone prune. Before the fix the prune removed A's tombstone too, so A's snapshot had
/// nothing to say about row 1 and B's older update re-created it on A: resurrected on both nodes.
/// </remarks>
[Collection("Convergence")]
public sealed class TombstoneResurrectionTests
{
private static readonly TimeSpan EightDays = TimeSpan.FromDays(8);
[Fact]
public async Task PeerOfflinePastRetention_DeleteStillWinsOnBothNodes()
{
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
// The maintenance tick 8 days later: age cap + tombstone prune, as the background service runs them.
await fx.RunMaintenanceAAsync(EightDays);
Assert.True(await IsTombstonedAsync(fx.A, 1),
"A's undelivered tombstone was pruned while B had never seen it (scadaproj#16).");
await fx.RestartTransportAsync();
await fx.AssertDataConvergedAsync();
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
Assert.True(await IsTombstonedAsync(fx.B, 1), "B should hold A's tombstone after the snapshot.");
}
// Positive control: the same scenario with the tombstone prune skipped must converge on "deleted".
// If it did not, the test above could pass or fail for reasons unrelated to tombstone retention.
[Fact]
public async Task Control_SameScenarioWithoutTombstonePrune_DeleteWins()
{
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
await fx.RunMaintenanceAAsync(EightDays, pruneTombstones: false);
await fx.RestartTransportAsync();
await fx.AssertDataConvergedAsync();
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.A));
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
}
// Retention still does its job once the peer HAS the tombstone: after the reconnect the pair has
// exchanged it, so a prune a full retention later must remove it on A.
[Fact]
public async Task AfterPeerReceivesTombstone_ItIsPrunedOneRetentionLater()
{
await using var fx = await OfflineUpdateThenNewerDeleteAsync();
await fx.RunMaintenanceAAsync(EightDays);
await fx.RestartTransportAsync();
await fx.AssertDataConvergedAsync();
Assert.Equal(0, await ConvergenceFixture.CountOrdersAsync(fx.B));
// The snapshot debt clears on B's SnapshotAck, which can trail the data by a moment.
await WaitForNoSnapshotDebtAsync(fx.A);
// Delivery restarts A's retention clock (proved with a controlled clock in OplogStoreTests);
// one retention past it, the delivered tombstone goes.
await fx.RunMaintenanceAAsync(EightDays);
Assert.False(await HasRowVersionAsync(fx.A, 1), "A never pruned a delivered tombstone.");
}
private static async Task<ConvergenceFixture> OfflineUpdateThenNewerDeleteAsync()
{
var fx = new ConvergenceFixture();
await ConvergenceFixture.UpsertAsync(fx.A, 1, "seed", 1);
await fx.StartAsync();
await fx.AssertConvergedAsync();
await fx.KillTransportAsync();
await ConvergenceFixture.UpsertAsync(fx.B, 1, "b-old", 2);
// Wall-clock gap so A's delete is the strictly newer HLC regardless of millisecond ties.
await Task.Delay(20);
await ConvergenceFixture.DeleteAsync(fx.A, 1);
return fx;
}
private static async Task WaitForNoSnapshotDebtAsync(ILocalDb db)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (DateTime.UtcNow < deadline)
{
var flag = await db.QueryAsync(
"SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", static r => r.GetInt64(0));
if (flag[0] == 0)
return;
await Task.Delay(25);
}
Assert.Fail("A's snapshot debt was never discharged.");
}
private static async Task<bool> IsTombstonedAsync(ILocalDb db, long id)
{
var rows = await db.QueryAsync(
"SELECT is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
return rows.Count == 1 && rows[0] == 1;
}
private static async Task<bool> HasRowVersionAsync(ILocalDb db, long id)
{
var rows = await db.QueryAsync(
"SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
static r => r.GetInt64(0), new { pk = $"{{\"id\":{id}}}" });
return rows[0] == 1;
}
}
@@ -89,6 +89,8 @@ public sealed class MaintenanceTests : IDisposable
var (db, store) = await NewDbAsync(options);
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'DOOMED', 1)");
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
// The peer has the delete (an unacked tombstone is never pruned on age — scadaproj#16).
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync());
// Backdate the delete-trigger's tombstone far past any retention window.
var updated = await db.ExecuteAsync(
"UPDATE __localdb_row_version SET tombstone_utc = '2000-01-01T00:00:00.000Z' WHERE is_tombstone = 1");
@@ -147,6 +149,29 @@ public sealed class MaintenanceTests : IDisposable
Assert.Contains(result.Failures, f => f.Contains("MaintenanceInterval", StringComparison.Ordinal));
}
[Fact]
public void Validator_TombstoneHardCapBelowRetention_Fails_AtOrAboveAndUnset_Pass()
{
var validator = new ReplicationOptionsValidator();
var below = validator.Validate(null, new ReplicationOptions { TombstoneHardCap = TimeSpan.FromDays(6) });
Assert.True(below.Failed);
Assert.Contains(below.Failures, f => f.Contains("TombstoneHardCap", StringComparison.Ordinal));
Assert.True(validator.Validate(null, new ReplicationOptions { TombstoneHardCap = TimeSpan.FromDays(7) }).Succeeded);
Assert.True(validator.Validate(null, new ReplicationOptions()).Succeeded);
}
[Fact]
public void TombstoneHardCap_BindsFromConfiguration_DefaultsToOff()
{
var bound = new ReplicationOptions();
new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?> { ["TombstoneHardCap"] = "30.00:00:00" })
.Build().Bind(bound);
Assert.Equal(TimeSpan.FromDays(30), bound.TombstoneHardCap);
Assert.Null(new ReplicationOptions().TombstoneHardCap);
}
[Fact]
public void AddZbLocalDbReplication_RegistersMaintenance_BothRoles()
{
@@ -122,6 +122,113 @@ public sealed class OplogStoreTests : IDisposable
Assert.True(peer.NeedsSnapshot);
}
// ---- scadaproj#16: a tombstone is pruned only once the peer holds it --------------------
private static async Task<long> TombstoneCount(SqliteLocalDb db) =>
(await db.QueryAsync(
"SELECT COUNT(*) FROM __localdb_row_version WHERE is_tombstone = 1", static r => r.GetInt64(0)))[0];
[Fact]
public async Task Prune_KeepsUndeliveredLocalTombstonePastRetention_ThenAgesItFromDelivery()
{
using var db = await NewOrdersDb();
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
var now = DateTimeOffset.UtcNow;
var store = new OplogStore(db, new ReplicationOptions(), () => now);
// Peer offline 8 days: past the 7-day retention, but the delete is still unacked.
now += TimeSpan.FromDays(8);
Assert.Equal(0, await store.PruneTombstonesAsync(default));
Assert.Equal(1, await TombstoneCount(db));
// The peer comes back and acks it: retention restarts at the ack, not at the delete.
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(), default);
now += TimeSpan.FromDays(6);
Assert.Equal(0, await store.PruneTombstonesAsync(default));
Assert.Equal(1, await TombstoneCount(db));
now += TimeSpan.FromDays(2);
Assert.Equal(1, await store.PruneTombstonesAsync(default));
Assert.Equal(0, await TombstoneCount(db));
}
[Fact]
public async Task Prune_KeepsTombstonesWhileDatabaseWideSnapshotDebtIsOutstanding()
{
using var db = await NewOrdersDb();
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
var now = DateTimeOffset.UtcNow;
var store = new OplogStore(db, new ReplicationOptions(), () => now);
// The age cap drops the delete from the oplog (never delivered) and flags a snapshot — the
// snapshot is built from the ledger, so the tombstone is the only carrier of the delete.
now += TimeSpan.FromDays(8);
Assert.True(await store.EnforceCapsAsync(default));
Assert.Equal(0, await store.GetMaxSeqAsync());
Assert.Equal(0, await store.PruneTombstonesAsync(default));
Assert.Equal(1, await TombstoneCount(db));
// The snapshot is acknowledged: debt cleared, tombstone restamped, aged from there.
await store.ClearSnapshotDebtAsync([], default);
now += TimeSpan.FromDays(6);
Assert.Equal(0, await store.PruneTombstonesAsync(default));
now += TimeSpan.FromDays(2);
Assert.Equal(1, await store.PruneTombstonesAsync(default));
}
[Fact]
public async Task Prune_KeepsTombstonesOfATableWithPerTableSnapshotDebt()
{
using var db = await NewOrdersDb();
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
var now = DateTimeOffset.UtcNow;
var store = new OplogStore(db, new ReplicationOptions(), () => now);
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(), default);
await db.ExecuteAsync(
"INSERT INTO __localdb_snapshot_state (table_name, needs_snapshot) VALUES ('orders', 1)");
now += TimeSpan.FromDays(8);
Assert.Equal(0, await store.PruneTombstonesAsync(default));
await store.ClearSnapshotDebtAsync(["orders"], default);
now += TimeSpan.FromDays(8);
Assert.Equal(1, await store.PruneTombstonesAsync(default));
}
[Fact]
public async Task Prune_RemoteOriginTombstone_AgesOnRetentionAlone_EvenWithLocalBacklog()
{
using var db = await NewOrdersDb();
// A local write the peer has not acked — it must not hold the PEER's own delete.
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (9, 'LOCAL', 1)");
await db.ExecuteAsync(
"INSERT INTO __localdb_row_version (table_name, pk_json, hlc, node_id, is_tombstone, tombstone_utc) " +
"VALUES ('orders', '{\"id\":1}', 9223372036854775000, 'the-peer', 1, @utc)",
new { utc = DateTimeOffset.UtcNow.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat) });
var now = DateTimeOffset.UtcNow + TimeSpan.FromDays(8);
var store = new OplogStore(db, new ReplicationOptions(), () => now);
Assert.Equal(1, await store.PruneTombstonesAsync(default));
}
[Fact]
public async Task Prune_HardCap_DropsAnUndeliveredTombstoneOnceExceeded()
{
using var db = await NewOrdersDb();
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 1)");
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
var now = DateTimeOffset.UtcNow;
var store = new OplogStore(db, new ReplicationOptions { TombstoneHardCap = TimeSpan.FromDays(30) }, () => now);
now += TimeSpan.FromDays(29);
Assert.Equal(0, await store.PruneTombstonesAsync(default));
now += TimeSpan.FromDays(2);
Assert.Equal(1, await store.PruneTombstonesAsync(default));
}
[Fact]
public async Task Prune_DropsExpiredTombstoneRowVersions_KeepsLive()
{
@@ -129,10 +236,13 @@ public sealed class OplogStoreTests : IDisposable
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1),(2,'B',2),(3,'C',3)");
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1"); // will be backdated -> expired
await db.ExecuteAsync("DELETE FROM orders WHERE id = 3"); // fresh tombstone -> kept
var store = new OplogStore(db, new ReplicationOptions());
// The peer has both deletes (scadaproj#16: an undelivered tombstone is never pruned on age);
// backdate AFTER the ack, which restamps delivered tombstones.
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(), default);
await db.ExecuteAsync(
"UPDATE __localdb_row_version SET tombstone_utc = '2000-01-01T00:00:00.000Z' WHERE pk_json = '{\"id\":1}'");
var store = new OplogStore(db, new ReplicationOptions());
await store.PruneTombstonesAsync(default);
var rv = await db.QueryAsync(