fix(localdb): fault live sync session on cap-prune; keepalive + newest-wins sync guard (0.3.2, ScadaBridge arch-review P4-1 + P4-3)

ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/Internal/OplogStore.cs

@@ -24,8 +24,9 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
 
     /// <summary>
     /// Reads up to <paramref name="maxBatch"/> oplog entries with <c>seq &gt; afterSeq</c> in ascending
-    /// seq order. Caller contract: check <c>needs_snapshot</c> before trusting incremental reads —
-    /// after cap-pruning, reads silently resume past a gap.
+    /// seq order. Caller contract: this read resumes SILENTLY past a cap-pruned gap, so a caller
+    /// streaming incrementally must first confirm its position is still below the horizon — see
+    /// <see cref="GetMinSeqAsync"/>, which <c>SyncSession</c>'s pump checks on every iteration.
     /// </summary>
     public Task<IReadOnlyList<OplogEntryRecord>> ReadBatchAboveAsync(long afterSeq, int maxBatch, CancellationToken ct = default) =>
         db.QueryAsync(
@@ -148,6 +149,24 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
         return rows[0];
     }
 
+    /// <summary>
+    /// The oldest oplog seq still present — the delta horizon — or 0 when the oplog is empty.
+    /// </summary>
+    /// <remarks>
+    /// One b-tree seek (<c>seq</c> is the rowid and SQLite optimizes MIN into a seek), which is what
+    /// makes it affordable for the pump to check on every flush interval. Compared against a
+    /// session's sent watermark it answers the only question the pump has: is everything I have not
+    /// yet sent still here? Ack-pruning deletes <c>seq &lt;= last_acked_seq</c> and the peer can only
+    /// ack what it received, so in a healthy session the horizon never climbs past
+    /// <c>sentThruSeq + 1</c>; a higher one means a cap prune ate unsent deltas.
+    /// </remarks>
+    public async Task<long> GetMinSeqAsync(CancellationToken ct = default)
+    {
+        var rows = await db.QueryAsync(
+            "SELECT COALESCE(MIN(seq), 0) FROM __localdb_oplog", static r => r.GetInt64(0), null, ct);
+        return rows[0];
+    }
+
     /// <summary>The highest oplog seq present, or 0 when the oplog is empty.</summary>
     public async Task<long> GetMaxSeqAsync(CancellationToken ct = default)
     {

ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/Internal/SyncSession.cs

@@ -356,9 +356,14 @@ internal sealed class SyncSession
         while (true)
         {
             ct.ThrowIfCancellationRequested();
+            // BEFORE the read, because the read itself cannot tell a pruned gap from an idle oplog.
+            ThrowIfPruneGap(await _store.GetMinSeqAsync(ct));
             var batch = await _store.ReadBatchAboveAsync(_sentThruSeq, _options.MaxBatchSize, ct);
             if (batch.Count > 0)
             {
+                // The exact guard on the rows about to go out, for a prune that landed between the
+                // probe and the read. Free (the seq is already in hand) and strictly tighter.
+                ThrowIfPruneGap(batch[0].Seq);
                 var index = 0;
                 while (index < batch.Count)
                 {
@@ -390,6 +395,40 @@ internal sealed class SyncSession
         }
     }
 
+    /// <summary>
+    /// Faults the session when the oplog horizon has climbed above this session's read position —
+    /// i.e. a cap prune deleted deltas between <see cref="_sentThruSeq"/> and
+    /// <paramref name="oldestAvailableSeq"/>. An empty oplog (0) is not a gap.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// <b>Why a live session has to die for this.</b> <c>EnforceCapsAsync</c> sets
+    /// <c>needs_snapshot</c> in the same transaction as the prune, so a RECONNECT heals — the
+    /// handshake serves a full snapshot instead of deltas. Nothing, however, made the live session
+    /// notice: the pump reads <c>seq &gt; _sentThruSeq</c>, which silently resumes above the hole, and
+    /// the peer acks those later seqs as if the pair were converged. On a stable LAN with a slow but
+    /// alive peer that session can run for days, and the pair is durably divergent for all of it —
+    /// the flag that would fix it is only ever read at handshake time. Tearing down is what turns a
+    /// persisted "needs snapshot" into an actual snapshot.
+    /// </para>
+    /// <para>
+    /// <b>Why the horizon, not the flag.</b> <c>needs_snapshot</c> is also set for reasons that do
+    /// not invalidate THIS session's position (and a full snapshot this session just streamed clears
+    /// it only when the peer's <c>SnapshotAck</c> lands, which is after the pump has started).
+    /// Comparing the horizon against the sent watermark asks the precise question instead, so a
+    /// session is faulted when — and only when — rows it still owes the peer are gone.
+    /// </para>
+    /// </remarks>
+    private void ThrowIfPruneGap(long oldestAvailableSeq)
+    {
+        if (oldestAvailableSeq == 0 || oldestAvailableSeq <= _sentThruSeq + 1)
+            return;
+        throw new LocalDbOplogGapException(
+            $"Oplog deltas above the sent watermark {_sentThruSeq} were pruned (oldest surviving seq " +
+            $"{oldestAvailableSeq}): this session can no longer stream incrementally without skipping " +
+            "them. Ending the session so the reconnect handshake serves a snapshot resync.");
+    }
+
     /// <summary>
     /// Serialized bytes one entry contributes to its batch message: the entry's own encoding plus
     /// the repeated field's tag and length prefix, as protobuf itself computes them.