using Microsoft.Extensions.Logging; using ZB.MOM.WW.LocalDb.Contracts; using ZB.MOM.WW.LocalDb.Internal; using ZB.MOM.WW.LocalDb.Registration; namespace ZB.MOM.WW.LocalDb.Replication.Internal; /// /// Consumes an inbound full-snapshot the receive loop routes to it: SnapshotBegin opens a receive, /// each SnapshotBatch merges through the SAME last-writer-wins path as deltas (never truncating — /// the receiver may hold newer local writes), and SnapshotComplete advances the durable watermark /// so incremental deltas resume above it. /// internal interface ISnapshotApplier { Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct); Task OnBatchAsync(SnapshotBatch batch, CancellationToken ct); Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct); } /// /// Which tables a snapshot covers. An EMPTY is the wire's own encoding of /// "all registered tables" (see SnapshotBegin.tables) and is what every pre-0.3.0 peer both /// sends and expects, so it is deliberately the same value here rather than a separate flag. /// /// /// The distinction is not cosmetic: a FULL snapshot carries every row state as of its /// as_of_seq, so the receiver may jump its watermark there and the sender may skip the /// deltas it covered. A PARTIAL one carries only the listed tables, so neither side may move a /// watermark that also speaks for the tables it left out. /// internal sealed record SnapshotScope(IReadOnlyList Tables) { /// Every registered table — the only scope a pre-0.3.0 peer can be sent. public static readonly SnapshotScope Full = new([]); public bool IsPartial => Tables.Count > 0; public bool Includes(string tableName) => Tables.Count == 0 || Tables.Contains(tableName, StringComparer.Ordinal); } /// /// Streams a snapshot of the tables in the requested — every registered /// table for a peer that has fallen behind the pruned oplog horizon, or just the baselined ones for /// a scoped resync. Reads the authoritative per-row version from __localdb_row_version joined /// to each live table, pages it into messages through the gated data /// lane, and brackets them with SnapshotBegin/SnapshotComplete carrying the max oplog seq as-of. /// /// /// Clearing the snapshot debt is deliberately NOT done here: whether the flags may be cleared at all /// depends on the peer's capability (a 0.3.0 peer acknowledges the snapshot, a pre-0.3.0 one cannot), /// which is session state. owns that decision; this type only streams. /// internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, ReplicationOptions options, ILogger logger) { /// /// Streams the snapshot for ; returns the as-of oplog seq it covers, so /// the session can skip already-covered tail deltas (a FULL snapshot only — a partial one speaks /// for nothing outside its tables). /// public async Task SendAsync(SnapshotScope scope, Func send, CancellationToken ct) { // The as-of is the highest seq this snapshot can stand in for. MAX(seq) alone is wrong once // ack-pruning has emptied the oplog: it reads 0, and a receiver whose watermark genuinely IS // 0 (a rebuilt node) is then never lifted by the snapshot — its MAX-guarded watermark stays // 0, the empty-oplog horizon rule (streamable from last_acked_seq + 1) sees a gap on the // next handshake, and the whole database is re-streamed on every reconnect until this node // happens to write again. Every delta pruned below last_acked_seq is already folded into the // rows this snapshot carries, so last_acked_seq is the floor the as-of must carry. var asOfSeq = Math.Max( await store.GetMaxSeqAsync(ct), (await store.GetPeerStateAsync(ct)).LastAckedSeq); var begin = new SnapshotBegin { AsOfSeq = asOfSeq }; begin.Tables.AddRange(scope.Tables); await send(new SyncMessage { SnapshotBegin = begin }, ct); var totalRows = 0L; foreach (var table in db.ReplicatedTables.Values .Where(t => scope.Includes(t.Name)) .OrderBy(t => t.Name, StringComparer.Ordinal)) { var sql = BuildPageSql(table); // Keyset cursor (pk_json > last seen), not LIMIT/OFFSET: concurrent inserts/prunes // between pages cannot shift the window, so no ledger row is skipped or re-sent. var lastPk = ""; while (true) { var rows = await db.QueryAsync( sql, // Column 4 is NULL for tombstones AND for live rows whose joined table row is // missing; a proto optional string cannot be assigned null, so carry it as a // nullable and only set row_json when present. static r => ( PkJson: r.GetString(0), Hlc: r.GetInt64(1), NodeId: r.GetString(2), IsTombstone: r.GetInt64(3) != 0, RowJson: r.IsDBNull(4) ? null : r.GetString(4)), new { t = table.Name, page = options.MaxBatchSize, lastPk }, ct); if (rows.Count == 0) break; lastPk = rows[^1].PkJson; // One DB page may span several messages: the page is bounded by ROW COUNT, but a // message is bounded by BYTES (ReplicationOptions.MaxBatchBytes) — the same budget // the delta pump honours, and for the same reason. Wide rows are exactly why a // snapshot is the easiest way to exceed the transport's message limit: it ships // whole rows, all of them, back to back. var batch = new SnapshotBatch { TableName = table.Name }; var bytes = 0L; foreach (var row in rows) { if (!row.IsTombstone && row.RowJson is null) { // Ledger says live but the LEFT JOIN found no table row: ledger/table // divergence (e.g. an unreplicated manual delete). Emitting {"col":null,...} // would fabricate a phantom row on the peer, so skip it defensively. logger.LogWarning( "Snapshot skipping live-ledger row {Table} {PkJson}: no matching table row (ledger/table divergence).", batch.TableName, row.PkJson); continue; } var snapshotRow = new SnapshotRow { PkJson = row.PkJson, Hlc = row.Hlc, NodeId = row.NodeId, IsTombstone = row.IsTombstone, }; if (row.RowJson is not null) snapshotRow.RowJson = row.RowJson; var size = RowSize(snapshotRow); // Flush before adding, never after: a message keeps at least one row however // large, so an oversized row goes out alone rather than stalling the snapshot. if (batch.Rows.Count > 0 && bytes + size > options.MaxBatchBytes) { await send(new SyncMessage { SnapshotBatch = batch }, ct); totalRows += batch.Rows.Count; batch = new SnapshotBatch { TableName = table.Name }; bytes = 0; } batch.Rows.Add(snapshotRow); bytes += size; } if (batch.Rows.Count > 0) { await send(new SyncMessage { SnapshotBatch = batch }, ct); totalRows += batch.Rows.Count; } if (rows.Count < options.MaxBatchSize) break; } } await send(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = asOfSeq } }, ct); logger.LogInformation( "Snapshot sent (as_of_seq {AsOfSeq}, {Rows} rows, scope {Scope}).", asOfSeq, totalRows, scope.IsPartial ? string.Join(",", scope.Tables) : "all"); return asOfSeq; } /// /// Serialized bytes one row contributes to its batch message: the row's own encoding plus the /// repeated field's tag and length prefix, as protobuf itself computes them. /// private static long RowSize(SnapshotRow row) => 1 + Google.Protobuf.CodedOutputStream.ComputeMessageSize(row); // Pages one table's rows by joining the row-version ledger to the live table on the PK columns // extracted from pk_json. Rows present in the live table but absent from __localdb_row_version // (data written BEFORE the table was registered for replication) are NOT snapshotted: they were // never captured, consistent with CDC semantics. // // That is a real hole for a node that starts replicating LATE — its whole pre-existing dataset is // invisible to the peer, silently and permanently — and the supported way to close it is to // register with `ILocalDb.RegisterReplicated(table, baselineExistingRows: true)`, which seeds the // ledger at an LWW-safe floor and flags the resync that carries the rows here. Nothing changes on // this side: baselined rows are ordinary ledger rows and page like any other. private static string BuildPageSql(ReplicatedTable table) { var name = Ident(table.Name); var join = string.Join(" AND ", table.PkColumns.Select(p => $"t.{Ident(p)} = json_extract(rv.pk_json, {JsonPath(p)})")); var jsonObject = "json_object(" + string.Join(", ", table.Columns.Select(c => $"{StringLiteral(c.Name)}, t.{Ident(c.Name)}")) + ")"; // A live ledger row whose join found no table row (first PK column NULL — PK columns are // never NULL on a real join hit) also yields NULL row json, so the caller can detect the // divergence instead of shipping a fabricated all-NULL row. var joinMiss = $"t.{Ident(table.PkColumns[0])} IS NULL"; return "SELECT rv.pk_json, rv.hlc, rv.node_id, rv.is_tombstone, " + $"CASE WHEN rv.is_tombstone = 1 OR {joinMiss} THEN NULL ELSE {jsonObject} END " + $"FROM __localdb_row_version rv LEFT JOIN {name} t ON {join} " + "WHERE rv.table_name = @t AND rv.pk_json > @lastPk ORDER BY rv.pk_json LIMIT @page"; } private static string Ident(string name) => "\"" + name.Replace("\"", "\"\"") + "\""; private static string StringLiteral(string value) => "'" + value.Replace("'", "''") + "'"; // A single-quoted JSON path with a quoted key member so a column name with reserved characters // still resolves: $."col". private static string JsonPath(string column) => "'$.\"" + column.Replace("\"", "\\\"") + "\"'"; } /// /// The inbound-snapshot consumer wired into . Each batch merges through the /// shared (seq 0 — snapshot rows carry no oplog seq — so the applier's /// MAX-guarded watermark update is a no-op, and per-row LWW protects any newer local write). On /// complete, the remote watermark is set to the snapshot's as-of seq so deltas resume above it. /// internal sealed class SnapshotApplier : ISnapshotApplier { private readonly LwwApplier _applier; private readonly OplogStore _store; private readonly ILogger _logger; private long _asOfSeq; private long _rows; private long _maxHlc; private bool _partial; // db is part of the collaborator's canonical shape (mirrors the streamer) even though the merge // reaches the database through the applier and store; kept for construction symmetry in the factory. public SnapshotApplier(SqliteLocalDb db, LwwApplier applier, OplogStore store, ILogger logger) { _ = db; _applier = applier; _store = store; _logger = logger; } public Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct) { _asOfSeq = begin.AsOfSeq; _rows = 0; _maxHlc = 0; // A non-empty table list scopes the snapshot; empty (which is all a pre-0.3.0 sender can // emit) means every registered table. Read here rather than on complete because // SnapshotComplete does not repeat it — the receive loop guarantees a Begin precedes both. _partial = begin.Tables.Count > 0; _logger.LogInformation( "Snapshot receive begun (as_of_seq {AsOfSeq}, scope {Scope}).", _asOfSeq, _partial ? string.Join(",", begin.Tables) : "all"); return Task.CompletedTask; } public async Task OnBatchAsync(SnapshotBatch batch, CancellationToken ct) { if (batch.Rows.Count == 0) return; var entries = new List(batch.Rows.Count); foreach (var row in batch.Rows) { if (row.Hlc > _maxHlc) _maxHlc = row.Hlc; entries.Add(new OplogEntryRecord( 0, batch.TableName, row.PkJson, row.HasRowJson ? row.RowJson : null, row.Hlc, row.NodeId, row.IsTombstone)); } await _applier.ApplyBatchAsync(entries, ct); _rows += batch.Rows.Count; } public async Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct) { // A PARTIAL snapshot may not move the inbound seq watermark. That watermark is one number // for the whole peer oplog, and jumping it to as_of_seq would declare every table applied // through that point — including the ones this snapshot deliberately left out, whose deltas // below as_of_seq the sender would then never resend. Only the HLC is carried across (seq 0 // is a no-op against the MAX guard), so the clock still cannot regress below what the peer // has stamped. The sender makes the mirror-image concession: it does not advance its own // sent-thru watermark past a partial snapshot, so those deltas are still pumped normally // and merely re-deliver rows the snapshot already carried, which LWW absorbs. var seq = _partial ? 0 : complete.AsOfSeq; // SetLastAppliedRemoteSeqAsync is MAX-guarded, so a delta applied mid-snapshot with a seq // above the snapshot's as-of (or an HLC above what the snapshot carried) is never regressed. await _store.SetLastAppliedRemoteSeqAsync(seq, _maxHlc, ct); _logger.LogInformation( "Snapshot receive complete (as_of_seq {AsOfSeq}, {Rows} rows, partial {Partial}).", complete.AsOfSeq, _rows, _partial); } }