feat(localdb): per-table needs_snapshot (0.3.0, on-disk schema v3)
Baselining one table used to set a single per-DATABASE needs_snapshot bit, and a snapshot streams every registered table in full — so opting one table into replication late re-sent all of them, in both directions at once on a pair enabling replication late. On a database holding a high-volume table (an event log at its retention cap) that is the largest transfer this library performs. 0.2.1 documented the cost and deferred the fix as needing an on-disk change. Schema v3 adds __localdb_snapshot_state (table_name PK, needs_snapshot), created by the shared DDL with IF NOT EXISTS so an older file gains it on open; the v2->v3 transition is a version stamp only. RegisterReplicated(..., baselineExistingRows: true) now flags that table alone. The database-wide flag survives unchanged for the paths that genuinely reach every table — a cap prune or an impossible peer ack loses delta continuity for the oplog as a whole. DeregisterReplicated drops a table's pending flag with its ledger rows, since nothing would ever stream (and so clear) it afterwards. Wire compatibility is additive and negotiated by CAPABILITY, not version: lib_schema_version stays 1 because ValidateHandshake compares it fail-closed for equality, so bumping it would stop 0.1.x/0.2.x pairs syncing outright. Handshake gains supports_partial_snapshot (field 5) and SnapshotBegin gains a tables scope (field 2) where EMPTY means all tables — precisely what a pre-0.3.0 sender emits and what a pre-0.3.0 receiver assumes. A peer that does not advertise the capability is served a full snapshot: a scoped one would make it advance last_applied_remote_seq to the snapshot's as-of and silently discard every delta below it for the tables left out. For the same reason a partial snapshot moves no watermark on either side. The receiver keeps last_applied_remote_seq (carrying only the HLC across, which is MAX-guarded), and the sender keeps _sentThruSeq, so the deltas the snapshot overlaps are still pumped and merged by LWW. Crash safety: snapshot debt now clears on the receiver's new SnapshotAck rather than on enqueue, so an ungraceful exit part-way through re-streams the same scope on reconnect. The ack rides the unbounded CONTROL lane like DeltaAck, so it composes with 0.2.1's bounded 64-message inbox and its out-of-handshake snapshot. Against a pre-0.3.0 peer the 0.2.x clear-on-enqueue behaviour is kept — there is no ack to wait for — with the handshake's horizon gap check as the net. Tests 199/199 (was 188): single-table baseline streams only that table, mixed- version interop (peer without the capability gets a full snapshot), debt standing until SnapshotAck, partial snapshot not advancing the receiver's watermark, no SnapshotAck emitted to an old peer, end-to-end over two real databases, plus v2->v3 upgrade and per-table flag isolation. README, ILocalDb remarks (which named this as the follow-up) and the umbrella CLAUDE.md index updated.
This commit is contained in:
Joseph Doherty
@@ -89,9 +89,19 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
return (duplex, toSession.Writer, fromSession.Reader);
|
||||
}
|
||||
|
||||
private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId, long lastAppliedRemoteSeq = 0)
|
||||
// supportsPartialSnapshot defaults to FALSE, which is exactly what a 0.1.x/0.2.x peer puts on
|
||||
// the wire (the field does not exist in its proto, so proto3 decodes the default). Scripted
|
||||
// peers are therefore old peers unless a test says otherwise.
|
||||
private static Handshake MatchingHandshake(
|
||||
SqliteLocalDb sessionDb, string peerNodeId, long lastAppliedRemoteSeq = 0, bool supportsPartialSnapshot = false)
|
||||
{
|
||||
var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = 1, LastAppliedRemoteSeq = lastAppliedRemoteSeq };
|
||||
var h = new Handshake
|
||||
{
|
||||
NodeId = peerNodeId,
|
||||
LibSchemaVersion = 1,
|
||||
LastAppliedRemoteSeq = lastAppliedRemoteSeq,
|
||||
SupportsPartialSnapshot = supportsPartialSnapshot,
|
||||
};
|
||||
foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
||||
h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest });
|
||||
return h;
|
||||
@@ -114,14 +124,30 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
|
||||
// Completes the two-phase handshake against a real session: the peer never requests a snapshot.
|
||||
private static async Task DriveHandshakeAsync(
|
||||
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession, CancellationToken ct)
|
||||
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession,
|
||||
CancellationToken ct, bool supportsPartialSnapshot = false)
|
||||
{
|
||||
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.Handshake, ct);
|
||||
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer") }, ct);
|
||||
await toSession.WriteAsync(
|
||||
new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer", supportsPartialSnapshot: supportsPartialSnapshot) }, ct);
|
||||
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.HandshakeAck, ct);
|
||||
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
|
||||
}
|
||||
|
||||
// Reads outbound messages until (and including) the first of the given kind.
|
||||
private static async Task<List<SyncMessage>> DrainThroughAsync(
|
||||
ChannelReader<SyncMessage> reader, SyncMessage.MsgOneofCase last, CancellationToken ct)
|
||||
{
|
||||
var all = new List<SyncMessage>();
|
||||
while (true)
|
||||
{
|
||||
var msg = await NextAsync(reader, ct);
|
||||
all.Add(msg);
|
||||
if (msg.MsgCase == last)
|
||||
return all;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task SwallowAsync(Task task)
|
||||
{
|
||||
try { await task; }
|
||||
@@ -159,6 +185,12 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
return r[0];
|
||||
}
|
||||
|
||||
/// <summary>The tables individually flagged as owing the peer a snapshot (schema v3).</summary>
|
||||
private static async Task<List<string>> PendingSnapshotTables(SqliteLocalDb db) =>
|
||||
(await db.QueryAsync(
|
||||
"SELECT table_name FROM __localdb_snapshot_state WHERE needs_snapshot <> 0 ORDER BY table_name",
|
||||
x => x.GetString(0))).ToList();
|
||||
|
||||
private static async Task<long> OplogCount(SqliteLocalDb db)
|
||||
{
|
||||
var r = await db.QueryAsync("SELECT COUNT(*) FROM __localdb_oplog", x => x.GetInt64(0));
|
||||
@@ -585,6 +617,261 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
// ---- Per-table snapshot scoping (0.3.0) ----------------------------------------------
|
||||
|
||||
private const string WidgetsSql = "CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)";
|
||||
|
||||
/// <summary>
|
||||
/// A node with two registered tables where exactly one — <c>orders</c> — was baselined.
|
||||
/// <c>widgets</c> is registered BEFORE its rows are written, so ordinary capture ledgers them;
|
||||
/// <c>orders</c> is registered after, so only baselining can. That asymmetry is what makes the
|
||||
/// scope observable: a full snapshot carries both tables' ledgers, a scoped one carries orders
|
||||
/// alone — and both tables hold rows either way, so an empty widgets stream cannot be mistaken
|
||||
/// for "there was nothing to send".
|
||||
/// </summary>
|
||||
private async Task<Side> NewBaselinedSideAsync(ReplicationOptions? options = null)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
||||
_disposables.Add(db);
|
||||
await db.ExecuteAsync(OrdersSql);
|
||||
await db.ExecuteAsync(WidgetsSql);
|
||||
|
||||
// Predates registration: invisible to CDC, so only baselining puts it in the ledger.
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'PRE',1),(2,'PRE',2)");
|
||||
|
||||
db.RegisterReplicated("widgets");
|
||||
await db.ExecuteAsync("INSERT INTO widgets (id, name) VALUES (1,'W1'),(2,'W2')");
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
return NewSessionOver(db, options);
|
||||
}
|
||||
|
||||
/// <summary>The peer of <see cref="NewBaselinedSideAsync"/>: the same two tables, empty, owing nothing.</summary>
|
||||
private async Task<Side> NewTwoTableSideAsync(ReplicationOptions? options = null)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
||||
_disposables.Add(db);
|
||||
await db.ExecuteAsync(OrdersSql);
|
||||
await db.ExecuteAsync(WidgetsSql);
|
||||
db.RegisterReplicated("orders");
|
||||
db.RegisterReplicated("widgets");
|
||||
return NewSessionOver(db, options);
|
||||
}
|
||||
|
||||
private static List<string> SnapshotTablesIn(IEnumerable<SyncMessage> messages) =>
|
||||
messages.Where(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBatch)
|
||||
.Select(m => m.SnapshotBatch.TableName)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task Baselining_OneTable_StreamsOnlyThatTable()
|
||||
{
|
||||
// The whole point of the per-table flag. Through 0.2.x the flag was one bit for the whole
|
||||
// database, so baselining `orders` re-streamed `widgets` (and everything else) in full — in
|
||||
// both directions at once on a pair enabling replication late, which on a database holding
|
||||
// an event log at its retention cap is the largest transfer this library performs.
|
||||
var a = await NewBaselinedSideAsync();
|
||||
Assert.Equal(["orders"], await PendingSnapshotTables(a.Db));
|
||||
Assert.Equal(0, await NeedsSnapshot(a.Db)); // NOT the database-wide flag
|
||||
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, supportsPartialSnapshot: true);
|
||||
var frames = await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
|
||||
var begin = frames.Single(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBegin).SnapshotBegin;
|
||||
Assert.Equal(["orders"], begin.Tables);
|
||||
Assert.Equal(["orders"], SnapshotTablesIn(frames));
|
||||
// The baselined rows really are in there — the scope narrowed the snapshot, it did not empty it.
|
||||
var pks = frames.Where(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBatch)
|
||||
.SelectMany(m => m.SnapshotBatch.Rows).Select(r => r.PkJson).Order(StringComparer.Ordinal);
|
||||
Assert.Equal(["{\"id\":1}", "{\"id\":2}"], pks);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Baselining_AgainstAPeerWithoutPartialSupport_StreamsEveryTable()
|
||||
{
|
||||
// Mixed-version interop, the hard constraint. A pre-0.3.0 peer ignores SnapshotBegin.tables
|
||||
// and would advance last_applied_remote_seq to as_of_seq on complete — which for a SCOPED
|
||||
// snapshot silently discards every delta below as_of for the tables it left out. There is no
|
||||
// way to express "partial" to that peer, so it is served the 0.2.x answer: everything.
|
||||
var a = await NewBaselinedSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
// supportsPartialSnapshot omitted == what a 0.1.x/0.2.x handshake decodes to.
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
||||
var frames = await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
|
||||
var begin = frames.Single(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBegin).SnapshotBegin;
|
||||
Assert.Empty(begin.Tables); // empty == full, the only scope the old wire can express
|
||||
Assert.Equal(["orders", "widgets"], SnapshotTablesIn(frames));
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PartialSnapshot_DebtClearsOnlyWhenThePeerAcknowledges()
|
||||
{
|
||||
// Crash safety: an ungraceful exit part-way through a snapshot must re-stream it, so the
|
||||
// flag cannot come down when the frames are merely enqueued. The peer's SnapshotAck is
|
||||
// written after its own durable watermark write, so it means "I hold these rows".
|
||||
var a = await NewBaselinedSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, supportsPartialSnapshot: true);
|
||||
var frames = await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
var complete = frames[^1].SnapshotComplete;
|
||||
|
||||
// The entire snapshot is on the wire and unacknowledged: the debt must still stand.
|
||||
Assert.Equal(["orders"], await PendingSnapshotTables(a.Db));
|
||||
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotAck = new SnapshotAck { AsOfSeq = complete.AsOfSeq } }, cts.Token);
|
||||
await WaitForAsync(async () => (await PendingSnapshotTables(a.Db)).Count == 0, RunTimeout);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Snapshot_ToAPeerWithoutPartialSupport_ClearsOnEnqueue()
|
||||
{
|
||||
// The 0.2.x behaviour, kept deliberately: a peer that sends no SnapshotAck would otherwise
|
||||
// leave the debt standing forever and re-stream the snapshot on every reconnect.
|
||||
var a = await NewBaselinedSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
||||
await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
|
||||
await WaitForAsync(async () => (await PendingSnapshotTables(a.Db)).Count == 0, RunTimeout);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PartialSnapshot_DoesNotAdvanceTheReceiversWatermark()
|
||||
{
|
||||
// The receiver's half of the same invariant. last_applied_remote_seq is ONE number for the
|
||||
// whole peer oplog; moving it to a partial snapshot's as_of would declare the untouched
|
||||
// tables applied through that point, and their deltas below it would never be resent.
|
||||
var a = await NewSideAsync();
|
||||
await a.Store.SetLastAppliedRemoteSeqAsync(10, 0);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, supportsPartialSnapshot: true);
|
||||
|
||||
var begin = new SnapshotBegin { AsOfSeq = 900 };
|
||||
begin.Tables.Add("orders");
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = begin }, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage
|
||||
{
|
||||
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "SNAP", 6_000_000) } },
|
||||
}, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 900 } }, cts.Token);
|
||||
|
||||
// The ack proves the complete was processed — and that a 0.3.0 receiver answers one at all.
|
||||
var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotAck, cts.Token);
|
||||
Assert.Equal(900, ack.SnapshotAck.AsOfSeq);
|
||||
|
||||
Assert.Equal(10, await LastAppliedRemoteSeq(a.Db)); // NOT 900
|
||||
Assert.Single(await ReadOrders(a.Db)); // the rows still merged
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FullSnapshot_ToAnOldPeer_DrawsNoSnapshotAck()
|
||||
{
|
||||
// Symmetry check on the negotiation: a 0.3.0 receiver must not put a message a pre-0.3.0
|
||||
// sender has never heard of on the wire back to it.
|
||||
var a = await NewSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 300 } }, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage
|
||||
{
|
||||
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "SNAP", 6_000_000) } },
|
||||
}, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 300 } }, cts.Token);
|
||||
|
||||
// A full snapshot from an old peer still advances the watermark — that is the 0.2.x contract.
|
||||
await WaitForAsync(async () => await LastAppliedRemoteSeq(a.Db) == 300, RunTimeout);
|
||||
// Barrier: an ack for this delta proves everything before it was processed, so an absent
|
||||
// SnapshotAck is absent rather than merely late.
|
||||
await toSession.WriteAsync(new SyncMessage
|
||||
{
|
||||
DeltaBatch = new DeltaBatch { Entries = { new OplogEntry
|
||||
{
|
||||
Seq = 301, TableName = "orders", PkJson = "{\"id\":2}",
|
||||
RowJson = "{\"id\":2,\"sku\":\"BARRIER\",\"qty\":1}", Hlc = 6_000_001, NodeId = "peer", IsTombstone = false,
|
||||
} } },
|
||||
}, cts.Token);
|
||||
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
||||
Assert.False(fromSession.TryPeek(out var stray) && stray.MsgCase == SyncMessage.MsgOneofCase.SnapshotAck);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BaselinedTable_ReachesARealPeer_WithoutRestreamingTheOthers()
|
||||
{
|
||||
// End to end over two real databases: the baselined table's pre-existing rows arrive via the
|
||||
// scoped snapshot, the untouched table's rows arrive as ordinary deltas, and the debt clears.
|
||||
var options = new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) };
|
||||
var a = await NewBaselinedSideAsync(options);
|
||||
var b = await NewTwoTableSideAsync(options);
|
||||
|
||||
var (da, db) = DuplexPair();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var runA = a.Session.RunAsync(da, cts.Token);
|
||||
var runB = b.Session.RunAsync(db, cts.Token);
|
||||
|
||||
async Task<bool> Converged() =>
|
||||
(await ReadOrders(b.Db)).Count == 2
|
||||
&& (await b.Db.QueryAsync("SELECT COUNT(*) FROM widgets", x => x.GetInt64(0)))[0] == 2;
|
||||
|
||||
await WaitForAsync(Converged, RunTimeout);
|
||||
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
|
||||
await WaitForAsync(async () => (await PendingSnapshotTables(a.Db)).Count == 0, RunTimeout);
|
||||
|
||||
// A partial snapshot speaks for nothing outside its tables, so neither side may skip the
|
||||
// deltas it covered: B's watermark tracks the oplog, not the snapshot's as-of.
|
||||
Assert.Equal(0, await NeedsSnapshot(a.Db));
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(runA);
|
||||
await SwallowAsync(runB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sender_ClearsNeedsSnapshotFlag()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user