dohertj2/scadaproj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(localdb): fault live sync session on cap-prune; keepalive + newest-wins sync guard (0.3.2, ScadaBridge arch-review P4-1 + P4-3)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-19 14:49:31 -04:00
parent dbcbc5bd41
commit f62b017cc7
13 changed files with 574 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SnapshotResyncTests.cs
+93 -2
View File
@@ -125,11 +125,15 @@ public sealed class SnapshotResyncTests : IDisposable
// Completes the two-phase handshake against a real session: the peer never requests a snapshot.
private static async Task DriveHandshakeAsync(
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession,
CancellationToken ct, bool supportsPartialSnapshot = false)
CancellationToken ct, bool supportsPartialSnapshot = false, long lastAppliedRemoteSeq = 0)
{
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.Handshake, ct);
await toSession.WriteAsync(
new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer", supportsPartialSnapshot: supportsPartialSnapshot) }, ct);
new SyncMessage
{
Handshake = MatchingHandshake(
sessionDb, "peer", lastAppliedRemoteSeq, supportsPartialSnapshot),
}, ct);
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.HandshakeAck, ct);
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
}
@@ -373,6 +377,93 @@ public sealed class SnapshotResyncTests : IDisposable
await SwallowAsync(runB);
}
[Fact]
public async Task CapPrune_IntoALiveSessionsUnsentRange_FaultsIt_AndTheReconnectSnapshots()
{
// P4-1. The pump reads `seq > sentThruSeq`, which resumes SILENTLY above a hole a cap prune
// punched — so it keeps shipping later deltas, the peer acks them, and the pair is durably
// divergent. `needs_snapshot` (set in the prune's own transaction) heals only a RECONNECT,
// and nothing forced one: on a stable LAN with a slow-but-alive peer the session, and the
// divergence, could last for days. Faulting is what turns the persisted flag into a snapshot.
var a = await NewSideAsync();
for (var i = 1; i <= 6; i++)
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
var run = a.Session.RunAsync(duplex, cts.Token);
// The peer holds everything so far: no snapshot is owed, and the pump idles at seq 6.
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, lastAppliedRemoteSeq: 6);
// Write three more rows and prune all but the newest, in ONE transaction — which is what a
// cap prune is, and what makes this deterministic: the pump's reader can only ever see the
// before state (nothing above seq 6) or the after state (seqs 7 and 8 already gone). It can
// never catch rows 7/8 in between and legitimately advance past them.
await using (var tx = await a.Db.BeginTransactionAsync(cts.Token))
{
await tx.ExecuteAsync(
"INSERT INTO orders (id, sku, qty) VALUES (7,'PRUNED',7),(8,'PRUNED',8),(9,'KEPT',9)", null, cts.Token);
await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 1 WHERE id = 1", null, cts.Token);
await tx.ExecuteAsync(
"DELETE FROM __localdb_oplog WHERE seq < (SELECT MAX(seq) FROM __localdb_oplog)", null, cts.Token);
await tx.CommitAsync(cts.Token);
}
var fault = await Assert.ThrowsAsync<LocalDbOplogGapException>(() => run);
Assert.Contains("6", fault.Message, StringComparison.Ordinal);
// And it faulted BEFORE shipping anything from above the hole: a delta for seq 9 would have
// moved the peer's watermark past rows 7 and 8, which is the divergence itself.
while (fromSession.TryRead(out var stray))
Assert.NotEqual(SyncMessage.MsgOneofCase.DeltaBatch, stray.MsgCase);
// The reconnect is what heals: the prune's needs_snapshot makes the handshake owe a full
// snapshot, which carries the pruned rows' current state to a real peer.
var a2 = NewSessionOver(a.Db);
var b = await NewSideAsync();
var (da, db) = DuplexPair();
var run2A = a2.Session.RunAsync(da, cts.Token);
var run2B = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 9, RunTimeout);
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
cts.Cancel();
await SwallowAsync(run2A);
await SwallowAsync(run2B);
}
[Fact]
public async Task AckPruningToEmpty_DoesNotFaultALiveSession()
{
// The other half of the horizon check: ordinary ack-pruning deletes seq <= last_acked_seq,
// which by construction sits at or below the sent watermark, so a converged pair that has
// pruned its whole oplog must keep streaming. A gap test written against `needs_snapshot`,
// or one that treated any risen horizon as a fault, would kill healthy sessions here.
var a = await NewSideAsync();
var b = await NewSideAsync();
for (var i = 1; i <= 4; i++)
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
var (da, db) = DuplexPair();
using var cts = new CancellationTokenSource(RunTimeout);
var runA = a.Session.RunAsync(da, cts.Token);
var runB = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 4, RunTimeout);
await WaitForAsync(async () => await OplogCount(a.Db) == 0, RunTimeout);
// Still live, still incremental: a post-prune write reaches B as an ordinary delta.
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (5, 'POST', 5)");
await WaitForAsync(async () => (await ReadOrders(b.Db)).Any(r => r.Id == 5), RunTimeout);
Assert.False(runA.IsFaulted, runA.Exception?.ToString());
cts.Cancel();
await SwallowAsync(runA);
await SwallowAsync(runB);
}
[Fact]
public async Task Snapshot_MergesNotTruncates()
{