fix(localdb): sync session concurrency fixes — two-lane writer, teardown fault fidelity, per-entry drift, gated snapshot seam

2026-07-17 22:44:42 -04:00
parent 925623b878
commit 92ccbac8db
3 changed files with 367 additions and 105 deletions
@@ -20,7 +20,7 @@ public sealed class SyncSessionTests : IDisposable
    {
        foreach (var d in _disposables)
            d.Dispose();
-        SqliteConnection_ClearAllPools();
+        Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
        foreach (var p in _paths)
        {
            if (File.Exists(p)) File.Delete(p);
@@ -29,8 +29,6 @@ public sealed class SyncSessionTests : IDisposable
        }
    }

-    private static void SqliteConnection_ClearAllPools() => Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
-
    private sealed record Side(
        SqliteLocalDb Db, OplogStore Store, LwwApplier Applier, SyncSession Session);

@@ -55,10 +53,13 @@ public sealed class SyncSessionTests : IDisposable
        return new Side(db, store, applier, session);
    }

-    private static (SyncDuplex A, SyncDuplex B) DuplexPair()
+    private static (SyncDuplex A, SyncDuplex B) DuplexPair(int? capacity = null)
    {
-        var aToB = Channel.CreateUnbounded<SyncMessage>();
-        var bToA = Channel.CreateUnbounded<SyncMessage>();
+        Channel<SyncMessage> New() => capacity is int c
+            ? Channel.CreateBounded<SyncMessage>(c)
+            : Channel.CreateUnbounded<SyncMessage>();
+        var aToB = New();
+        var bToA = New();
        var a = new SyncDuplex
        {
            Send = (m, ct) => aToB.Writer.WriteAsync(m, ct).AsTask(),
@@ -99,6 +100,14 @@ public sealed class SyncSessionTests : IDisposable
        return msg!;
    }

+    private static async Task<SyncMessage> NextOfCaseAsync(
+        ChannelReader<SyncMessage> reader, SyncMessage.MsgOneofCase msgCase, CancellationToken ct)
+    {
+        SyncMessage msg;
+        do { msg = await NextAsync(reader, ct); } while (msg.MsgCase != msgCase);
+        return msg;
+    }
+
    private static async Task SwallowAsync(Task task)
    {
        try { await task; }
@@ -136,6 +145,21 @@ public sealed class SyncSessionTests : IDisposable
        return r[0];
    }

+    private static OplogEntry ProtoEntry(long seq, long id, string sku, long hlc, string nodeId = "peer") =>
+        new()
+        {
+            Seq = seq,
+            TableName = "orders",
+            PkJson = $"{{\"id\":{id}}}",
+            RowJson = $"{{\"id\":{id},\"sku\":\"{sku}\",\"qty\":1}}",
+            Hlc = hlc,
+            NodeId = nodeId,
+            IsTombstone = false,
+        };
+
+    private static long AheadHlc(TimeSpan ahead) =>
+        DateTimeOffset.UtcNow.Add(ahead).ToUnixTimeMilliseconds() << 16;
+
    [Fact]
    public async Task Handshake_Exchanged_BothProceed()
    {
@@ -255,6 +279,101 @@ public sealed class SyncSessionTests : IDisposable
        await SwallowAsync(runB);
    }

+    [Fact]
+    public async Task Backpressure_BoundedDuplex_NoDeadlock()
+    {
+        // Capacity-1 crosswise channels with backlog in BOTH directions: under the old shared
+        // send-gate design each side's pump held the gate while blocked on the full channel, and
+        // each side's receive loop then blocked on the gate trying to send its ack — a circular
+        // wait. The two-lane writer breaks it: receiving enqueues acks without ever blocking.
+        var options = new ReplicationOptions { MaxBatchSize = 2, FlushInterval = TimeSpan.FromMilliseconds(20) };
+        var a = await NewSideAsync(options: options);
+        var b = await NewSideAsync(options: options);
+
+        const int rowsPerSide = 40;
+        for (var i = 1; i <= rowsPerSide; i++)
+        {
+            await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
+            await b.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'B', @id)", new { id = 100 + i });
+        }
+
+        var (da, db) = DuplexPair(capacity: 1);
+        using var cts = new CancellationTokenSource(RunTimeout);
+        var runA = a.Session.RunAsync(da, cts.Token);
+        var runB = b.Session.RunAsync(db, cts.Token);
+
+        await WaitForAsync(
+            async () => (await ReadOrders(a.Db)).Count == 2 * rowsPerSide && (await ReadOrders(b.Db)).Count == 2 * rowsPerSide,
+            RunTimeout);
+
+        cts.Cancel();
+        await SwallowAsync(runA);
+        await SwallowAsync(runB);
+    }
+
+    [Fact]
+    public async Task OutOfOrderAck_DoesNotRegress()
+    {
+        var a = await NewSideAsync();
+        for (var i = 1; i <= 5; i++)
+            await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i });
+
+        var (duplex, toSession, fromSession) = ScriptedPeer();
+        using var cts = new CancellationTokenSource(RunTimeout);
+        var run = a.Session.RunAsync(duplex, cts.Token);
+
+        await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
+
+        await toSession.WriteAsync(new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = 5 } }, cts.Token);
+        await toSession.WriteAsync(new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = 2 } }, cts.Token);
+        // Barrier: the session applies inbound messages in order, so its DeltaAck for this batch
+        // proves both out-of-order acks above were already processed.
+        await toSession.WriteAsync(new SyncMessage
+        {
+            DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(1, 100, "BARRIER", 5_000_000) } },
+        }, cts.Token);
+        await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
+
+        Assert.Equal(5, a.Session.PeerAckedSeq);
+        Assert.Equal(5, await LastAckedSeq(a.Db));
+        var snap = await a.Db.QueryAsync("SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", x => x.GetInt64(0));
+        Assert.Equal(0, snap[0]);
+
+        cts.Cancel();
+        await SwallowAsync(run);
+    }
+
+    [Fact]
+    public async Task Teardown_PrimaryFaultSurfaces()
+    {
+        var a = await NewSideAsync();
+        // Backlog keeps the pump loop mid-work while the receive loop faults.
+        for (var i = 1; i <= 10; i++)
+            await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i });
+
+        var (duplex, toSession, fromSession) = ScriptedPeer();
+        using var cts = new CancellationTokenSource(RunTimeout);
+        var run = a.Session.RunAsync(duplex, cts.Token);
+
+        await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
+        await toSession.WriteAsync(new SyncMessage
+        {
+            DeltaBatch = new DeltaBatch
+            {
+                Entries = { new OplogEntry
+                {
+                    Seq = 1, TableName = "widgets", PkJson = "{\"id\":9}",
+                    RowJson = "{\"id\":9}", Hlc = 5_000_000, NodeId = "peer", IsTombstone = false,
+                } },
+            },
+        }, cts.Token);
+
+        // The receive loop's schema fault must surface — not an OperationCanceledException from
+        // the torn-down pump/writer siblings.
+        var ex = await Assert.ThrowsAsync<LocalDbSchemaMismatchException>(() => run);
+        Assert.Contains("widgets", ex.Message);
+    }
+
    [Fact]
    public async Task HlcDriftAhead_Warns()
    {
@@ -269,33 +388,30 @@ public sealed class SyncSessionTests : IDisposable

        await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);

-        var aheadHlc = (DateTimeOffset.UtcNow.AddMinutes(10).ToUnixTimeMilliseconds() << 16);
+        // Two separate drifted batches: both apply, but only ONE warning is logged per session.
        await toSession.WriteAsync(new SyncMessage
        {
-            DeltaBatch = new DeltaBatch
-            {
-                Entries = { new OplogEntry
-                {
-                    Seq = 1, TableName = "orders", PkJson = "{\"id\":1}",
-                    RowJson = "{\"id\":1,\"sku\":\"AHEAD\",\"qty\":9}", Hlc = aheadHlc, NodeId = "peer", IsTombstone = false,
-                } },
-            },
+            DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(1, 1, "AHEAD1", AheadHlc(TimeSpan.FromMinutes(10))) } },
        }, cts.Token);
+        var ack1 = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
+        Assert.Equal(1, ack1.DeltaAck.AppliedThruSeq);

-        // Drain the DeltaAck the session emits after applying.
-        SyncMessage ack;
-        do { ack = await NextAsync(fromSession, cts.Token); } while (ack.MsgCase != SyncMessage.MsgOneofCase.DeltaAck);
-        Assert.Equal(1, ack.DeltaAck.AppliedThruSeq);
+        await toSession.WriteAsync(new SyncMessage
+        {
+            DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(2, 2, "AHEAD2", AheadHlc(TimeSpan.FromMinutes(11))) } },
+        }, cts.Token);
+        var ack2 = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
+        Assert.Equal(2, ack2.DeltaAck.AppliedThruSeq);

-        Assert.Single(await ReadOrders(a.Db)); // applied despite drift
-        Assert.Contains(logger.Entries, e => e.Level == LogLevel.Warning);
+        Assert.Equal(2, (await ReadOrders(a.Db)).Count); // applied despite drift
+        Assert.Equal(1, logger.Entries.Count(e => e.Level == LogLevel.Warning));

        cts.Cancel();
        await SwallowAsync(run);
    }

    [Fact]
-    public async Task HlcDriftAhead_FailClosed_Throws()
+    public async Task HlcDriftAhead_FailClosed_DeadLetters()
    {
        var a = await NewSideAsync(
            options: new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), FailClosedOnDrift = true });
@@ -306,32 +422,59 @@ public sealed class SyncSessionTests : IDisposable

        await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);

-        var aheadHlc = (DateTimeOffset.UtcNow.AddMinutes(10).ToUnixTimeMilliseconds() << 16);
+        // One batch: seq 1 drifted (dead-lettered), seq 2 clean (applied). Watermark covers both.
        await toSession.WriteAsync(new SyncMessage
        {
            DeltaBatch = new DeltaBatch
            {
-                Entries = { new OplogEntry
+                Entries =
                {
-                    Seq = 1, TableName = "orders", PkJson = "{\"id\":1}",
-                    RowJson = "{\"id\":1,\"sku\":\"AHEAD\",\"qty\":9}", Hlc = aheadHlc, NodeId = "peer", IsTombstone = false,
-                } },
+                    ProtoEntry(1, 1, "DRIFTED", AheadHlc(TimeSpan.FromMinutes(10))),
+                    ProtoEntry(2, 2, "CLEAN", 5_000_000),
+                },
            },
        }, cts.Token);
+        var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
+        Assert.Equal(2, ack.DeltaAck.AppliedThruSeq);

-        await Assert.ThrowsAsync<InvalidOperationException>(() => run);
-        Assert.Empty(await ReadOrders(a.Db)); // rejected, not applied
+        var rows = await ReadOrders(a.Db);
+        Assert.Single(rows);
+        Assert.Equal(2, rows[0].Id);
+
+        var dead = await a.Db.QueryAsync(
+            "SELECT pk_json, error FROM __localdb_dead_letter",
+            x => (Pk: x.GetString(0), Err: x.GetString(1)));
+        Assert.Single(dead);
+        Assert.Equal("{\"id\":1}", dead[0].Pk);
+        Assert.Contains("drift", dead[0].Err, StringComparison.OrdinalIgnoreCase);
+
+        // The session stays alive after dead-lettering: a follow-up clean batch still applies.
+        await toSession.WriteAsync(new SyncMessage
+        {
+            DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(3, 3, "AFTER", 5_000_001) } },
+        }, cts.Token);
+        var ack3 = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
+        Assert.Equal(3, ack3.DeltaAck.AppliedThruSeq);
+        Assert.Equal(2, (await ReadOrders(a.Db)).Count);
+
+        cts.Cancel();
+        await SwallowAsync(run);
    }

    [Fact]
    public async Task PeerNeedsSnapshot_InvokesSnapshotHooks()
    {
-        // (a) peer's HandshakeAck.snapshot_required = true -> our SnapshotSender fires.
+        // (a) peer's HandshakeAck.snapshot_required = true -> our SnapshotSender fires, and its
+        // gated send delegate routes messages through the writer onto the wire.
        {
            var a = await NewSideAsync();
            var (duplex, toSession, fromSession) = ScriptedPeer();
            var sent = new TaskCompletionSource();
-            a.Session.SnapshotSender = (_, _) => { sent.TrySetResult(); return Task.CompletedTask; };
+            a.Session.SnapshotSender = (send, token) =>
+            {
+                sent.TrySetResult();
+                return send(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 7 } }, token);
+            };

            using var cts = new CancellationTokenSource(RunTimeout);
            var run = a.Session.RunAsync(duplex, cts.Token);
@@ -342,24 +485,31 @@ public sealed class SyncSessionTests : IDisposable
            await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = true } }, cts.Token);

            await sent.Task.WaitAsync(RunTimeout);
+            var complete = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
+            Assert.Equal(7, complete.SnapshotComplete.AsOfSeq);
+
            cts.Cancel();
            await SwallowAsync(run);
        }

-        // (b) inbound SnapshotBegin -> our SnapshotReceiver fires.
+        // (b) inbound SnapshotBegin -> our SnapshotReceiver fires with the triggering message.
        {
            var a = await NewSideAsync();
            var (duplex, toSession, fromSession) = ScriptedPeer();
-            var received = new TaskCompletionSource();
-            a.Session.SnapshotReceiver = (_, _, _) => { received.TrySetResult(); return Task.CompletedTask; };
+            var received = new TaskCompletionSource<long>();
+            a.Session.SnapshotReceiver = (begin, _, _) =>
+            {
+                received.TrySetResult(begin.AsOfSeq);
+                return Task.CompletedTask;
+            };

            using var cts = new CancellationTokenSource(RunTimeout);
            var run = a.Session.RunAsync(duplex, cts.Token);

            await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
-            await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 0 } }, cts.Token);
+            await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 42 } }, cts.Token);

-            await received.Task.WaitAsync(RunTimeout);
+            Assert.Equal(42, await received.Task.WaitAsync(RunTimeout));
            cts.Cancel();
            await SwallowAsync(run);
        }