dohertj2/scadaproj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(localdb): snapshot resync (LWW-merging, tombstone-carrying, delta resume)

Browse Source 
Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
2026-07-17 23:35:50 -04:00
parent e92cdc6d3a
commit 130e9c918f
5 changed files with 628 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SnapshotResyncTests.cs
+388
View File
@@ -0,0 +1,388 @@
using System.Threading.Channels;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.LocalDb.Contracts;
using ZB.MOM.WW.LocalDb.Internal;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.LocalDb.Replication.Internal;
namespace ZB.MOM.WW.LocalDb.Tests;
/// <summary>
/// Full-stack snapshot-resync coverage: a node whose peer fell behind the pruned oplog horizon
/// streams a full snapshot of its registered tables; the receiver MERGES it through LWW (never
/// truncates), tombstones are carried, and incremental deltas resume afterwards.
/// </summary>
public sealed class SnapshotResyncTests : IDisposable
{
private const string OrdersSql = "CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER)";
private static readonly TimeSpan RunTimeout = TimeSpan.FromSeconds(15);
private readonly List<string> _paths = [];
private readonly List<IDisposable> _disposables = [];
public void Dispose()
{
foreach (var d in _disposables)
d.Dispose();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (var p in _paths)
{
if (File.Exists(p)) File.Delete(p);
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
}
}
private sealed record Side(SqliteLocalDb Db, OplogStore Store, LwwApplier Applier, SyncSession Session);
// Builds a full replication stack with the snapshot hooks wired exactly as SyncSessionFactory does.
private async Task<Side> NewSideAsync(ReplicationOptions? options = null)
{
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
_paths.Add(path);
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
_disposables.Add(db);
await db.ExecuteAsync(OrdersSql);
db.RegisterReplicated("orders");
options ??= new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) };
var store = new OplogStore(db, options);
var applier = new LwwApplier(db);
var session = new SyncSession(db, store, applier, options, NullLogger.Instance);
var streamer = new SnapshotStreamer(db, store, options, NullLogger.Instance);
var snapshotApplier = new SnapshotApplier(db, applier, store, NullLogger.Instance);
session.SnapshotSender = streamer.SendAsync;
session.SnapshotApplier = snapshotApplier;
return new Side(db, store, applier, session);
}
private static (SyncDuplex A, SyncDuplex B) DuplexPair()
{
var aToB = Channel.CreateUnbounded<SyncMessage>();
var bToA = Channel.CreateUnbounded<SyncMessage>();
var a = new SyncDuplex { Send = (m, ct) => aToB.Writer.WriteAsync(m, ct).AsTask(), Inbox = bToA.Reader };
var b = new SyncDuplex { Send = (m, ct) => bToA.Writer.WriteAsync(m, ct).AsTask(), Inbox = aToB.Reader };
return (a, b);
}
private static (SyncDuplex Duplex, ChannelWriter<SyncMessage> ToSession, ChannelReader<SyncMessage> FromSession) ScriptedPeer()
{
var toSession = Channel.CreateUnbounded<SyncMessage>();
var fromSession = Channel.CreateUnbounded<SyncMessage>();
var duplex = new SyncDuplex { Send = (m, ct) => fromSession.Writer.WriteAsync(m, ct).AsTask(), Inbox = toSession.Reader };
return (duplex, toSession.Writer, fromSession.Reader);
}
private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId, long lastAppliedRemoteSeq = 0)
{
var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = 1, LastAppliedRemoteSeq = lastAppliedRemoteSeq };
foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest });
return h;
}
private static async Task<SyncMessage> NextAsync(ChannelReader<SyncMessage> reader, CancellationToken ct)
{
await reader.WaitToReadAsync(ct);
reader.TryRead(out var msg);
return msg!;
}
private static async Task<SyncMessage> NextOfCaseAsync(
ChannelReader<SyncMessage> reader, SyncMessage.MsgOneofCase msgCase, CancellationToken ct)
{
SyncMessage msg;
do { msg = await NextAsync(reader, ct); } while (msg.MsgCase != msgCase);
return msg;
}
// Completes the two-phase handshake against a real session: the peer never requests a snapshot.
private static async Task DriveHandshakeAsync(
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession, CancellationToken ct)
{
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.Handshake, ct);
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer") }, ct);
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.HandshakeAck, ct);
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
}
private static async Task SwallowAsync(Task task)
{
try { await task; }
catch (OperationCanceledException) { }
}
private static async Task WaitForAsync(Func<Task<bool>> predicate, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (await predicate()) return;
await Task.Delay(20);
}
throw new TimeoutException("Condition not reached within " + timeout);
}
private static async Task<List<(long Id, string? Sku, long? Qty)>> ReadOrders(SqliteLocalDb db)
{
var rows = await db.QueryAsync(
"SELECT id, sku, qty FROM orders ORDER BY id",
x => (x.GetInt64(0), x.IsDBNull(1) ? null : x.GetString(1), (long?)(x.IsDBNull(2) ? null : x.GetInt64(2))));
return rows.ToList();
}
private static async Task<long> LastAppliedRemoteSeq(SqliteLocalDb db)
{
var r = await db.QueryAsync("SELECT last_applied_remote_seq FROM __localdb_peer_state WHERE id = 1", x => x.GetInt64(0));
return r[0];
}
private static async Task<long> NeedsSnapshot(SqliteLocalDb db)
{
var r = await db.QueryAsync("SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", x => x.GetInt64(0));
return r[0];
}
private static SnapshotRow SnapRow(long id, string sku, long hlc, string nodeId = "peer") =>
new()
{
PkJson = $"{{\"id\":{id}}}",
RowJson = $"{{\"id\":{id},\"sku\":\"{sku}\",\"qty\":1}}",
Hlc = hlc,
NodeId = nodeId,
IsTombstone = false,
};
[Fact]
public async Task PrunedPeer_GetsSnapshot_ThenConverges()
{
var a = await NewSideAsync(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), MaxOplogRows = 2 });
var b = await NewSideAsync();
for (var i = 1; i <= 6; i++)
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
// Prune A's oplog below B's (empty) watermark and flag a resync: incremental deltas alone
// can no longer carry rows 1..4 to B.
Assert.True(await a.Store.EnforceCapsAsync());
var (da, db) = DuplexPair();
using var cts = new CancellationTokenSource(RunTimeout);
var runA = a.Session.RunAsync(da, cts.Token);
var runB = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 6, RunTimeout);
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
// Deltas resume after the snapshot: a post-snapshot write on A reaches B incrementally.
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (7, 'POST', 7)");
await WaitForAsync(async () => (await ReadOrders(b.Db)).Any(r => r.Id == 7), RunTimeout);
cts.Cancel();
await SwallowAsync(runA);
await SwallowAsync(runB);
}
[Fact]
public async Task Snapshot_MergesNotTruncates()
{
var a = await NewSideAsync(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), MaxOplogRows = 2 });
var b = await NewSideAsync();
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A_OLD', 1)");
await Task.Delay(5);
// B's row for the same pk is written later -> strictly higher HLC, so it must survive the merge.
await b.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'B_NEW', 9)");
// A owes B a snapshot; a single-row oplog would not trip the cap, so flag it directly.
await a.Store.SetNeedsSnapshotAsync(true);
var (da, db) = DuplexPair();
using var cts = new CancellationTokenSource(RunTimeout);
var runA = a.Session.RunAsync(da, cts.Token);
var runB = b.Session.RunAsync(db, cts.Token);
async Task<bool> Converged()
{
var ra = await ReadOrders(a.Db);
var rb = await ReadOrders(b.Db);
return ra.Count == 1 && rb.Count == 1 && ra[0].Sku == "B_NEW" && rb[0].Sku == "B_NEW";
}
await WaitForAsync(Converged, RunTimeout);
cts.Cancel();
await SwallowAsync(runA);
await SwallowAsync(runB);
}
[Fact]
public async Task Snapshot_CarriesTombstones()
{
var a = await NewSideAsync(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), MaxOplogRows = 2 });
var b = await NewSideAsync();
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A1', 1), (2, 'A2', 2)");
await b.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'B1', 1), (2, 'B2', 2)");
await Task.Delay(5);
// A deletes id=1 last (newest HLC), so the snapshot's tombstone must win over B's live id=1.
await a.Db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
Assert.True(await a.Store.EnforceCapsAsync());
var (da, db) = DuplexPair();
using var cts = new CancellationTokenSource(RunTimeout);
var runA = a.Session.RunAsync(da, cts.Token);
var runB = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () =>
{
var rb = await ReadOrders(b.Db);
return rb.Count == 1 && rb[0].Id == 2;
}, RunTimeout);
Assert.DoesNotContain(await ReadOrders(b.Db), r => r.Id == 1);
Assert.DoesNotContain(await ReadOrders(a.Db), r => r.Id == 1);
cts.Cancel();
await SwallowAsync(runA);
await SwallowAsync(runB);
}
[Fact]
public async Task Snapshot_WatermarkNoOp_WhenSeqZero()
{
// Snapshot rows carry seq 0; applying them must not regress an already-advanced watermark.
var a = await NewSideAsync();
await a.Store.SetLastAppliedRemoteSeqAsync(50, 12345);
var entries = new List<OplogEntryRecord>
{
new(0, "orders", "{\"id\":1}", "{\"id\":1,\"sku\":\"S\",\"qty\":1}", 6_000_000, "peer", false),
};
await a.Applier.ApplyBatchAsync(entries, default);
Assert.Equal(50, await LastAppliedRemoteSeq(a.Db)); // MAX(50, 0) -> unchanged
Assert.Single(await ReadOrders(a.Db));
}
[Fact]
public async Task SnapshotComplete_AdvancesWatermark_DeltasResume()
{
var a = await NewSideAsync();
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
var run = a.Session.RunAsync(duplex, cts.Token);
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 100 } }, cts.Token);
await toSession.WriteAsync(new SyncMessage
{
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "SNAP", 6_000_000) } },
}, cts.Token);
await toSession.WriteAsync(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 100 } }, cts.Token);
await WaitForAsync(async () => await LastAppliedRemoteSeq(a.Db) == 100, RunTimeout);
Assert.Single(await ReadOrders(a.Db));
// A normal delta above the snapshot horizon resumes incrementally.
await toSession.WriteAsync(new SyncMessage
{
DeltaBatch = new DeltaBatch { Entries = { new OplogEntry
{
Seq = 101, TableName = "orders", PkJson = "{\"id\":2}",
RowJson = "{\"id\":2,\"sku\":\"D\",\"qty\":1}", Hlc = 6_000_001, NodeId = "peer", IsTombstone = false,
} } },
}, cts.Token);
var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
Assert.Equal(101, ack.DeltaAck.AppliedThruSeq);
Assert.Equal(101, await LastAppliedRemoteSeq(a.Db));
Assert.Equal(2, (await ReadOrders(a.Db)).Count);
cts.Cancel();
await SwallowAsync(run);
}
[Fact]
public async Task SnapshotBatch_WithoutBegin_IsProtocolError()
{
var a = await NewSideAsync();
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
var run = a.Session.RunAsync(duplex, cts.Token);
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
await toSession.WriteAsync(new SyncMessage
{
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "ORPHAN", 6_000_000) } },
}, cts.Token);
await Assert.ThrowsAsync<InvalidOperationException>(() => run);
}
[Fact]
public async Task DeltaDuringSnapshot_AppliesSafely()
{
var a = await NewSideAsync();
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
var run = a.Session.RunAsync(duplex, cts.Token);
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 200 } }, cts.Token);
await toSession.WriteAsync(new SyncMessage
{
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "SNAP1", 6_000_000) } },
}, cts.Token);
// A normal delta interleaved MID-snapshot must still apply (LWW makes interleaving safe).
await toSession.WriteAsync(new SyncMessage
{
DeltaBatch = new DeltaBatch { Entries = { new OplogEntry
{
Seq = 205, TableName = "orders", PkJson = "{\"id\":2}",
RowJson = "{\"id\":2,\"sku\":\"DELTA\",\"qty\":1}", Hlc = 6_000_010, NodeId = "peer", IsTombstone = false,
} } },
}, cts.Token);
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
await toSession.WriteAsync(new SyncMessage
{
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(3, "SNAP3", 6_000_020) } },
}, cts.Token);
await toSession.WriteAsync(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 200 } }, cts.Token);
await WaitForAsync(async () => (await ReadOrders(a.Db)).Count == 3, RunTimeout);
var rows = await ReadOrders(a.Db);
Assert.Equal([1L, 2L, 3L], rows.Select(r => r.Id));
cts.Cancel();
await SwallowAsync(run);
}
[Fact]
public async Task Sender_ClearsNeedsSnapshotFlag()
{
var a = await NewSideAsync(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), MaxOplogRows = 2 });
var b = await NewSideAsync();
for (var i = 1; i <= 5; i++)
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
Assert.True(await a.Store.EnforceCapsAsync());
Assert.Equal(1, await NeedsSnapshot(a.Db));
var (da, db) = DuplexPair();
using var cts = new CancellationTokenSource(RunTimeout);
var runA = a.Session.RunAsync(da, cts.Token);
var runB = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 5, RunTimeout);
await WaitForAsync(async () => await NeedsSnapshot(a.Db) == 0, RunTimeout);
cts.Cancel();
await SwallowAsync(runA);
await SwallowAsync(runB);
}
}
ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SyncSessionTests.cs
+28 -14
View File
@@ -546,10 +546,11 @@ public sealed class SyncSessionTests : IDisposable
[Fact]
public async Task PeerNeedsSnapshot_InvokesSnapshotHooks()
{
// (a) peer's HandshakeAck.snapshot_required = true -> our SnapshotSender fires, and its
// gated send delegate routes messages through the writer onto the wire.
// (a) THIS node owes the peer a snapshot (its own needs_snapshot) -> our SnapshotSender fires,
// and its gated send delegate routes messages through the writer onto the wire.
{
var a = await NewSideAsync();
await a.Store.SetNeedsSnapshotAsync(true, default);
var (duplex, toSession, fromSession) = ScriptedPeer();
var sent = new TaskCompletionSource();
a.Session.SnapshotSender = (send, token) =>
@@ -564,7 +565,7 @@ public sealed class SyncSessionTests : IDisposable
await NextAsync(fromSession, cts.Token); // session Handshake
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(a.Db, "peer") }, cts.Token);
await NextAsync(fromSession, cts.Token); // session HandshakeAck
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = true } }, cts.Token);
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, cts.Token);
await sent.Task.WaitAsync(RunTimeout);
var complete = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
@@ -574,16 +575,12 @@ public sealed class SyncSessionTests : IDisposable
await SwallowAsync(run);
}
// (b) inbound SnapshotBegin -> our SnapshotReceiver fires with the triggering message.
// (b) inbound SnapshotBegin -> our ISnapshotApplier.OnBeginAsync fires with the triggering message.
{
var a = await NewSideAsync();
var (duplex, toSession, fromSession) = ScriptedPeer();
var received = new TaskCompletionSource<long>();
a.Session.SnapshotReceiver = (begin, _, _) =>
{
received.TrySetResult(begin.AsOfSeq);
return Task.CompletedTask;
};
var applier = new CapturingApplier();
a.Session.SnapshotApplier = applier;
using var cts = new CancellationTokenSource(RunTimeout);
var run = a.Session.RunAsync(duplex, cts.Token);
@@ -591,14 +588,16 @@ public sealed class SyncSessionTests : IDisposable
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 42 } }, cts.Token);
Assert.Equal(42, await received.Task.WaitAsync(RunTimeout));
Assert.Equal(42, await applier.BeganAsOf.Task.WaitAsync(RunTimeout));
cts.Cancel();
await SwallowAsync(run);
}
// (c) snapshot required but no sender hook -> NotSupportedException.
// (c) this node owes a snapshot but no sender hook -> NotSupportedException.
{
var a = await NewSideAsync();
await a.Store.SetNeedsSnapshotAsync(true, default);
a.Session.SnapshotSender = null;
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
@@ -607,14 +606,15 @@ public sealed class SyncSessionTests : IDisposable
await NextAsync(fromSession, cts.Token);
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(a.Db, "peer") }, cts.Token);
await NextAsync(fromSession, cts.Token);
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = true } }, cts.Token);
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, cts.Token);
await Assert.ThrowsAsync<NotSupportedException>(() => run);
}
// (d) inbound SnapshotBegin but no receiver hook -> NotSupportedException.
// (d) inbound SnapshotBegin but no applier hook -> NotSupportedException.
{
var a = await NewSideAsync();
a.Session.SnapshotApplier = null;
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
@@ -627,6 +627,20 @@ public sealed class SyncSessionTests : IDisposable
}
}
private sealed class CapturingApplier : ISnapshotApplier
{
public TaskCompletionSource<long> BeganAsOf { get; } = new();
public Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct)
{
BeganAsOf.TrySetResult(begin.AsOfSeq);
return Task.CompletedTask;
}
public Task OnBatchAsync(SnapshotBatch batch, CancellationToken ct) => Task.CompletedTask;
public Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct) => Task.CompletedTask;
}
[Fact]
public async Task SecondHandshake_MidStream_IsProtocolError()
{