using System.Globalization; using ZB.MOM.WW.LocalDb.Hlc; namespace ZB.MOM.WW.LocalDb.Replication.Internal; /// A single oplog delta as read for outbound replication. is null for tombstones. internal sealed record OplogEntryRecord( long Seq, string TableName, string PkJson, string? RowJson, long Hlc, string NodeId, bool IsTombstone); /// The persisted peer-sync watermarks and flags from __localdb_peer_state. internal sealed record PeerState( string? PeerNodeId, long LastAckedSeq, long LastAppliedRemoteSeq, long LastSeenHlc, bool NeedsSnapshot, string? LastSyncUtc); /// /// DB-side operations over the replication bookkeeping tables: batched oplog reads above a /// watermark, peer ack + prune, backlog/age cap enforcement, tombstone retention, and peer state. /// internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func? utcNow = null) { // Must match the delete trigger's strftime('%Y-%m-%dT%H:%M:%fZ') output so string comparison is chronological. internal const string TombstoneUtcFormat = "yyyy-MM-ddTHH:mm:ss.fffZ"; private readonly Func _utcNow = utcNow ?? (() => DateTimeOffset.UtcNow); /// /// Reads up to oplog entries with seq > afterSeq in ascending /// seq order. Caller contract: check needs_snapshot before trusting incremental reads — /// after cap-pruning, reads silently resume past a gap. /// public Task> ReadBatchAboveAsync(long afterSeq, int maxBatch, CancellationToken ct = default) => db.QueryAsync( "SELECT seq, table_name, pk_json, row_json, hlc, node_id, is_tombstone FROM __localdb_oplog " + "WHERE seq > @afterSeq ORDER BY seq LIMIT @maxBatch", static r => new OplogEntryRecord( r.GetInt64(0), r.GetString(1), r.GetString(2), r.IsDBNull(3) ? null : r.GetString(3), r.GetInt64(4), r.GetString(5), r.GetInt64(6) != 0), new { afterSeq, maxBatch }, ct); /// /// Records the peer's delivery ack and prunes oplog rows at or below the watermark, in one /// transaction. The watermark is monotonic (a late out-of-order ack never regresses it) and /// clamped to the highest seq this node has ever produced: an ack beyond that is a protocol /// violation, so needs_snapshot is set to force a resync rather than trust the peer. /// public async Task RecordPeerAckAsync(long ackedSeq, CancellationToken ct = default) { await using var tx = await db.BeginTransactionAsync(ct); // Known-max includes last_acked_seq: acked rows are already pruned, so a duplicate ack of // a pruned seq is benign — only an ack beyond anything ever produced is a violation. var knownMax = (await tx.QueryAsync( "SELECT MAX((SELECT COALESCE(MAX(seq), 0) FROM __localdb_oplog), " + "(SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1))", static r => r.GetInt64(0), null, ct))[0]; // Pruning up to the clamped ack may delete rows the peer never actually received; that is // safe only because needs_snapshot = 1 commits in the SAME transaction, forcing a resync. if (ackedSeq > knownMax) await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 1 WHERE id = 1", null, ct); await tx.ExecuteAsync( "UPDATE __localdb_peer_state SET last_acked_seq = MAX(last_acked_seq, @acked), last_sync_utc = @now WHERE id = 1", new { acked = Math.Min(ackedSeq, knownMax), now = FormatUtc(_utcNow()) }, ct); await tx.ExecuteAsync( "DELETE FROM __localdb_oplog WHERE seq <= (SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1)", null, ct); await tx.CommitAsync(ct); } /// Deletes row-version tombstones older than ; live rows and fresh tombstones are kept. public Task PruneTombstonesAsync(CancellationToken ct = default) => db.ExecuteAsync( "DELETE FROM __localdb_row_version WHERE is_tombstone = 1 AND tombstone_utc < @cutoff", new { cutoff = FormatUtc(_utcNow() - options.TombstoneRetention) }, ct); /// /// The unacked backlog — oplog entries above the peer's last-acked watermark — in O(1). /// /// /// /// Why not COUNT(*). The obvious query walks every unacked row, and this value is /// read once a minute by maintenance AND on every metrics scrape. A node whose peer has been /// down for a day therefore pays a full scan of a million-row backlog, repeatedly, precisely /// when it is already under stress. /// /// /// The formula. MAX(seq) - MAX(last_acked_seq, MIN(seq) - 1), floored at 0. Both /// MIN and MAX over seq are single b-tree seeks (it is the rowid, and SQLite's min/max /// optimization turns each into one seek), and the peer-state read is a one-row lookup — so the /// whole thing is three seeks regardless of backlog size. /// /// /// Why it is exact. seq is AUTOINCREMENT, so retained rows are a contiguous run /// wherever nothing has punched a hole in the middle — and no ordinary path punches holes: ack /// pruning deletes seq <= last_acked_seq (a prefix), the row cap deletes below /// MAX(seq) - keep + 1 (a prefix), and the age cap deletes below an HLC cutoff, which is /// a prefix because HLCs are stamped in seq order. Clamping the floor to MIN(seq) - 1 is /// what keeps the answer exact after a prune has moved the horizon above the ack watermark; /// without it a cap-pruned oplog would report its whole seq range forever and re-flag the cap on /// every tick. /// /// /// The one inexact case is ILocalDb.DeregisterReplicated, which deletes one /// table's rows from wherever they sit and so can leave holes. The estimate then over-reports by /// at most the number of holes, until the watermark advances past them. Over-reporting is the /// safe direction: it can only trigger a snapshot resync earlier, never suppress one. /// /// public async Task GetOplogDepthAsync(CancellationToken ct = default) { var rows = await db.QueryAsync(DepthSql, static r => r.GetInt64(0), null, ct); return rows[0]; } /// /// The unacked backlog read synchronously on a dedicated short-lived connection, for the /// metric-observation callbacks (gauge scrape / status poll); mirrors /// 's query. Deliberate tradeoff: synchronous I/O on the /// metrics-collection thread is acceptable here because WAL-mode readers never block (or are /// blocked by) writers, the query is three b-tree seeks, and the worst case is bounded by /// BusyTimeoutMs — all at scrape frequency, not on any data path. May throw if the db /// is disposing — callers on the observation path swallow and drop the sample. /// public long GetOplogDepthSync() { using var conn = db.CreateConnection(); using var cmd = conn.CreateCommand(); cmd.CommandText = DepthSql; return (long)cmd.ExecuteScalar()!; } // Both depth accessors run exactly this; keeping it in one place is what makes "mirrors the // async query" a fact rather than a comment that can rot. internal const string DepthSql = "SELECT MAX(0, " + "COALESCE((SELECT MAX(seq) FROM __localdb_oplog), 0) - " + "MAX((SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1), " + "COALESCE((SELECT MIN(seq) FROM __localdb_oplog), 1) - 1))"; /// /// The highest seq this node has ever produced: the oplog's max, or — once acked rows have been /// pruned away — the ack watermark that pruned them. Nothing we emit can exceed it, so a peer /// claiming to have applied more than this is not talking about the stream we are producing now. /// public async Task GetKnownMaxSeqAsync(CancellationToken ct = default) { var rows = await db.QueryAsync( "SELECT MAX((SELECT COALESCE(MAX(seq), 0) FROM __localdb_oplog), " + "(SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1))", static r => r.GetInt64(0), null, ct); return rows[0]; } /// The highest oplog seq present, or 0 when the oplog is empty. public async Task GetMaxSeqAsync(CancellationToken ct = default) { var rows = await db.QueryAsync( "SELECT COALESCE(MAX(seq), 0) FROM __localdb_oplog", static r => r.GetInt64(0), null, ct); return rows[0]; } /// /// Enforces the backlog caps in one transaction; returns true if a snapshot resync was flagged. /// Rows cap: backlog above prunes to the newest /// MaxOplogRows entries. Age cap: an oldest entry whose HLC physical time exceeds /// prunes every entry past the age cutoff — disk /// safety over delta continuity. Either trigger sets needs_snapshot. /// public async Task EnforceCapsAsync(CancellationToken ct = default) { var ageCutoffMs = (_utcNow() - options.MaxOplogAge).ToUnixTimeMilliseconds(); // Condition reads and prunes share the transaction so the decision cannot go stale (TOCTOU). await using var tx = await db.BeginTransactionAsync(ct); // Same O(1) expression as GetOplogDepthAsync — the caps check is the per-minute reader that // made the old COUNT(*) expensive in the first place. var depth = (await tx.QueryAsync(DepthSql, static r => r.GetInt64(0), null, ct))[0]; var overRows = depth > options.MaxOplogRows; var oldest = await tx.QueryAsync( "SELECT hlc FROM __localdb_oplog ORDER BY seq LIMIT 1", static r => r.GetInt64(0), null, ct); var overAge = oldest.Count > 0 && HybridLogicalClock.PhysicalMs(oldest[0]) < ageCutoffMs; if (!overRows && !overAge) return false; await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 1 WHERE id = 1", null, ct); if (overRows) await tx.ExecuteAsync( "DELETE FROM __localdb_oplog WHERE seq < ((SELECT MAX(seq) FROM __localdb_oplog) - @keep + 1)", new { keep = options.MaxOplogRows }, ct); if (overAge) await tx.ExecuteAsync( "DELETE FROM __localdb_oplog WHERE (hlc >> 16) < @ageCutoffMs", new { ageCutoffMs }, ct); await tx.CommitAsync(ct); return true; } /// Reads the singleton peer-state row. public async Task GetPeerStateAsync(CancellationToken ct = default) { var rows = await db.QueryAsync( "SELECT peer_node_id, last_acked_seq, last_applied_remote_seq, last_seen_hlc, needs_snapshot, last_sync_utc " + "FROM __localdb_peer_state WHERE id = 1", static r => new PeerState( r.IsDBNull(0) ? null : r.GetString(0), r.GetInt64(1), r.GetInt64(2), r.GetInt64(3), r.GetInt64(4) != 0, r.IsDBNull(5) ? null : r.GetString(5)), null, ct); return rows[0]; } /// Records the identity of the peer node this database syncs with. public Task SetPeerNodeIdAsync(string peerNodeId, CancellationToken ct = default) => db.ExecuteAsync("UPDATE __localdb_peer_state SET peer_node_id = @v WHERE id = 1", new { v = peerNodeId }, ct); /// /// Forgets everything we knew about the previous peer's stream: the inbound seq watermark and /// its observed clock. Called when the peer's node id changes. /// /// /// /// last_applied_remote_seq is a watermark in the PEER's seq space, and a rebuilt peer is /// a new database whose oplog numbers from 1. Carrying the old peer's watermark across the /// swap makes the new peer's early writes look already-applied, so they are dropped — silently, /// and for as many writes as the old watermark was high. /// /// /// last_acked_seq is deliberately NOT reset: it describes OUR oplog (what has been /// pruned), not the peer's, and it is what tells a rebuilt peer's handshake that it needs a /// snapshot. Clearing it would make an already-pruned node look like it had never written /// anything and turn the back-fill back off. /// /// public Task ResetInboundWatermarkAsync(CancellationToken ct = default) => db.ExecuteAsync( "UPDATE __localdb_peer_state SET last_applied_remote_seq = 0, last_seen_hlc = 0 WHERE id = 1", null, ct); /// /// Sets or clears the DATABASE-WIDE snapshot-resync flag — the one that means "every registered /// table". Set by the oplog-horizon paths (cap prune, impossible peer ack), which invalidate /// delta continuity for all tables at once. Per-table flags live in /// __localdb_snapshot_state; see . /// public Task SetNeedsSnapshotAsync(bool value, CancellationToken ct = default) => db.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = @v WHERE id = 1", new { v = value ? 1 : 0 }, ct); /// /// The tables individually flagged as owing the peer a snapshot (schema v3), in ordinal name /// order. Independent of the database-wide flag: baselining one table flags only that table. /// public Task> GetTablesNeedingSnapshotAsync(CancellationToken ct = default) => db.QueryAsync( "SELECT table_name FROM __localdb_snapshot_state WHERE needs_snapshot <> 0 ORDER BY table_name", static r => r.GetString(0), null, ct); /// /// Clears the snapshot debt a completed snapshot discharged: the named tables when /// is non-empty, otherwise everything — the database-wide flag /// and every per-table row, because a full snapshot subsumes each of them. /// /// /// /// Rows are deleted rather than zeroed: absence is the resting state, and keeping a row per /// table that has ever been baselined would accumulate bookkeeping that means nothing. Runs in /// one transaction so a full snapshot's two clears cannot half-apply and leave the pair /// re-snapshotting on every reconnect. /// /// /// A scoped clear deliberately does NOT touch the database-wide flag. If the cap enforcer /// flagged a whole-database resync while this snapshot was in flight, that debt is real and /// unrelated — the snapshot only carried some of the tables — so it must survive. The full /// clear may take it down because a full snapshot carries every row state as of its as-of seq, /// which is exactly what the pruned deltas would have delivered. /// /// public async Task ClearSnapshotDebtAsync(IReadOnlyList tables, CancellationToken ct = default) { await using var tx = await db.BeginTransactionAsync(ct); if (tables.Count == 0) { await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 0 WHERE id = 1", null, ct); await tx.ExecuteAsync("DELETE FROM __localdb_snapshot_state", null, ct); } else { foreach (var table in tables) await tx.ExecuteAsync( "DELETE FROM __localdb_snapshot_state WHERE table_name = @t", new { t = table }, ct); } await tx.CommitAsync(ct); } /// /// Records the highest remote seq applied locally plus the peer HLC observed with it; also stamps /// last_sync_utc. Both watermarks are MAX-guarded (monotonic): a snapshot completing with a /// lower as-of seq than a concurrently-applied delta must never regress the durable watermark. /// public Task SetLastAppliedRemoteSeqAsync(long seq, long lastSeenHlc, CancellationToken ct = default) => db.ExecuteAsync( "UPDATE __localdb_peer_state SET last_applied_remote_seq = MAX(last_applied_remote_seq, @seq), " + "last_seen_hlc = MAX(last_seen_hlc, @hlc), last_sync_utc = @now WHERE id = 1", new { seq, hlc = lastSeenHlc, now = FormatUtc(_utcNow()) }, ct); private static string FormatUtc(DateTimeOffset value) => value.UtcDateTime.ToString(TombstoneUtcFormat, CultureInfo.InvariantCulture); }