dohertj2/scadaproj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(localdb): snapshot review fixes — MAX-guarded complete watermark, keyset pagination, phantom-row guard

Browse Source 
Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
2026-07-17 23:50:42 -04:00
parent 130e9c918f
commit 8befebd476
5 changed files with 135 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/Internal/SnapshotStreamer.cs
+40 -16
View File
@@ -26,7 +26,8 @@ internal interface ISnapshotApplier
/// </summary>
internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, ReplicationOptions options, ILogger logger)
{
public async Task SendAsync(Func<SyncMessage, CancellationToken, Task> send, CancellationToken ct)
/// <summary>Streams the full snapshot; returns the as-of oplog seq it covers, so the session can skip already-covered tail deltas.</summary>
public async Task<long> SendAsync(Func<SyncMessage, CancellationToken, Task> send, CancellationToken ct)
{
var asOfSeq = await store.GetMaxSeqAsync(ct);
await send(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = asOfSeq } }, ct);
@@ -35,26 +36,40 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
foreach (var table in db.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
{
var sql = BuildPageSql(table);
var offset = 0;
// Keyset cursor (pk_json > last seen), not LIMIT/OFFSET: concurrent inserts/prunes
// between pages cannot shift the window, so no ledger row is skipped or re-sent.
var lastPk = "";
while (true)
{
var rows = await db.QueryAsync(
sql,
// Column 4 is NULL for tombstones; a proto optional string cannot be assigned null,
// so carry it as a nullable and only set row_json when present.
// Column 4 is NULL for tombstones AND for live rows whose joined table row is
// missing; a proto optional string cannot be assigned null, so carry it as a
// nullable and only set row_json when present.
static r => (
PkJson: r.GetString(0),
Hlc: r.GetInt64(1),
NodeId: r.GetString(2),
IsTombstone: r.GetInt64(3) != 0,
RowJson: r.IsDBNull(4) ? null : r.GetString(4)),
new { t = table.Name, page = options.MaxBatchSize, off = offset }, ct);
new { t = table.Name, page = options.MaxBatchSize, lastPk }, ct);
if (rows.Count == 0)
break;
lastPk = rows[^1].PkJson;
var batch = new SnapshotBatch { TableName = table.Name };
foreach (var row in rows)
{
if (!row.IsTombstone && row.RowJson is null)
{
// Ledger says live but the LEFT JOIN found no table row: ledger/table
// divergence (e.g. an unreplicated manual delete). Emitting {"col":null,...}
// would fabricate a phantom row on the peer, so skip it defensively.
logger.LogWarning(
"Snapshot skipping live-ledger row {Table} {PkJson}: no matching table row (ledger/table divergence).",
batch.TableName, row.PkJson);
continue;
}
var snapshotRow = new SnapshotRow
{
PkJson = row.PkJson,
@@ -66,10 +81,10 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
snapshotRow.RowJson = row.RowJson;
batch.Rows.Add(snapshotRow);
}
await send(new SyncMessage { SnapshotBatch = batch }, ct);
totalRows += rows.Count;
if (batch.Rows.Count > 0)
await send(new SyncMessage { SnapshotBatch = batch }, ct);
totalRows += batch.Rows.Count;
offset += rows.Count;
if (rows.Count < options.MaxBatchSize)
break;
}
@@ -77,9 +92,16 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
await send(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = asOfSeq } }, ct);
// The sender owed this snapshot (its own needs_snapshot); clearing it here — after the whole
// snapshot is on the wire — stops it re-streaming on the next reconnect.
// snapshot is on the wire — stops it re-streaming on the next reconnect. Narrow window: the
// messages are enqueued on the data lane, not yet transport-delivered, so a crash/drop after
// this clear could lose the snapshot with the flag already down. The reconnect handshake's
// gap check (peer watermark below the pruned horizon) re-detects that in every case where the
// oplog is non-empty; the sole undetectable corner is an EMPTY oplog (nothing to gap-check),
// where the peer stays behind until new local writes recreate a horizon. Accepted: closing it
// would need transport-level delivery acks for snapshot frames.
await store.SetNeedsSnapshotAsync(false, ct);
logger.LogInformation("Snapshot sent (as_of_seq {AsOfSeq}, {Rows} rows).", asOfSeq, totalRows);
return asOfSeq;
}
// Pages one table's rows by joining the row-version ledger to the live table on the PK columns
@@ -93,11 +115,15 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
var join = string.Join(" AND ", table.PkColumns.Select(p => $"t.{Ident(p)} = json_extract(rv.pk_json, {JsonPath(p)})"));
var jsonObject = "json_object(" +
string.Join(", ", table.Columns.Select(c => $"{StringLiteral(c.Name)}, t.{Ident(c.Name)}")) + ")";
// A live ledger row whose join found no table row (first PK column NULL — PK columns are
// never NULL on a real join hit) also yields NULL row json, so the caller can detect the
// divergence instead of shipping a fabricated all-NULL row.
var joinMiss = $"t.{Ident(table.PkColumns[0])} IS NULL";
return
"SELECT rv.pk_json, rv.hlc, rv.node_id, rv.is_tombstone, " +
$"CASE WHEN rv.is_tombstone = 1 THEN NULL ELSE {jsonObject} END " +
$"CASE WHEN rv.is_tombstone = 1 OR {joinMiss} THEN NULL ELSE {jsonObject} END " +
$"FROM __localdb_row_version rv LEFT JOIN {name} t ON {join} " +
"WHERE rv.table_name = @t ORDER BY rv.pk_json LIMIT @page OFFSET @off";
"WHERE rv.table_name = @t AND rv.pk_json > @lastPk ORDER BY rv.pk_json LIMIT @page";
}
private static string Ident(string name) => "\"" + name.Replace("\"", "\"\"") + "\"";
@@ -161,11 +187,9 @@ internal sealed class SnapshotApplier : ISnapshotApplier
public async Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct)
{
// Preserve last_seen_hlc against a concurrently-applied delta that observed a higher HLC:
// SetLastAppliedRemoteSeqAsync overwrites the column, so pass the max of what we saw and
// what is already recorded.
var lastSeenHlc = Math.Max(_maxHlc, (await _store.GetPeerStateAsync(ct)).LastSeenHlc);
await _store.SetLastAppliedRemoteSeqAsync(complete.AsOfSeq, lastSeenHlc, ct);
// SetLastAppliedRemoteSeqAsync is MAX-guarded, so a delta applied mid-snapshot with a seq
// above the snapshot's as-of (or an HLC above what the snapshot carried) is never regressed.
await _store.SetLastAppliedRemoteSeqAsync(complete.AsOfSeq, _maxHlc, ct);
_logger.LogInformation("Snapshot receive complete (as_of_seq {AsOfSeq}, {Rows} rows).", complete.AsOfSeq, _rows);
}
}