Joseph Doherty
693 lines
28 KiB
C#
693 lines
28 KiB
C#
using System.Threading.Channels;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.LocalDb.Contracts;
|
|
using ZB.MOM.WW.LocalDb.Internal;
|
|
using ZB.MOM.WW.LocalDb.Replication;
|
|
using ZB.MOM.WW.LocalDb.Replication.Internal;
|
|
|
|
namespace ZB.MOM.WW.LocalDb.Tests;
|
|
|
|
public sealed class SyncSessionTests : IDisposable
|
|
{
|
|
private const string OrdersSql = "CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER)";
|
|
private static readonly TimeSpan RunTimeout = TimeSpan.FromSeconds(15);
|
|
|
|
private readonly List<string> _paths = [];
|
|
private readonly List<IDisposable> _disposables = [];
|
|
|
|
public void Dispose()
|
|
{
|
|
foreach (var d in _disposables)
|
|
d.Dispose();
|
|
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
|
foreach (var p in _paths)
|
|
{
|
|
if (File.Exists(p)) File.Delete(p);
|
|
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
|
|
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
|
|
}
|
|
}
|
|
|
|
private sealed record Side(
|
|
SqliteLocalDb Db, OplogStore Store, LwwApplier Applier, SyncSession Session);
|
|
|
|
private async Task<Side> NewSideAsync(
|
|
string createSql = OrdersSql,
|
|
string table = "orders",
|
|
ReplicationOptions? options = null,
|
|
ILogger? logger = null,
|
|
Func<DateTimeOffset>? utcNow = null)
|
|
{
|
|
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
|
_paths.Add(path);
|
|
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
|
_disposables.Add(db);
|
|
await db.ExecuteAsync(createSql);
|
|
db.RegisterReplicated(table);
|
|
|
|
options ??= new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) };
|
|
var store = new OplogStore(db, options, utcNow);
|
|
var applier = new LwwApplier(db, utcNow);
|
|
var session = new SyncSession(db, store, applier, options, logger ?? NullLogger.Instance, utcNow);
|
|
return new Side(db, store, applier, session);
|
|
}
|
|
|
|
private static (SyncDuplex A, SyncDuplex B) DuplexPair(int? capacity = null)
|
|
{
|
|
Channel<SyncMessage> New() => capacity is int c
|
|
? Channel.CreateBounded<SyncMessage>(c)
|
|
: Channel.CreateUnbounded<SyncMessage>();
|
|
var aToB = New();
|
|
var bToA = New();
|
|
var a = new SyncDuplex
|
|
{
|
|
Send = (m, ct) => aToB.Writer.WriteAsync(m, ct).AsTask(),
|
|
Inbox = bToA.Reader,
|
|
};
|
|
var b = new SyncDuplex
|
|
{
|
|
Send = (m, ct) => bToA.Writer.WriteAsync(m, ct).AsTask(),
|
|
Inbox = aToB.Reader,
|
|
};
|
|
return (a, b);
|
|
}
|
|
|
|
private static (SyncDuplex Duplex, ChannelWriter<SyncMessage> ToSession, ChannelReader<SyncMessage> FromSession) ScriptedPeer()
|
|
{
|
|
var toSession = Channel.CreateUnbounded<SyncMessage>();
|
|
var fromSession = Channel.CreateUnbounded<SyncMessage>();
|
|
var duplex = new SyncDuplex
|
|
{
|
|
Send = (m, ct) => fromSession.Writer.WriteAsync(m, ct).AsTask(),
|
|
Inbox = toSession.Reader,
|
|
};
|
|
return (duplex, toSession.Writer, fromSession.Reader);
|
|
}
|
|
|
|
private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId, uint version = 1, long lastAppliedRemoteSeq = 0)
|
|
{
|
|
var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = version, LastAppliedRemoteSeq = lastAppliedRemoteSeq };
|
|
foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
|
h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest });
|
|
return h;
|
|
}
|
|
|
|
private static async Task<SyncMessage> NextAsync(ChannelReader<SyncMessage> reader, CancellationToken ct)
|
|
{
|
|
await reader.WaitToReadAsync(ct);
|
|
reader.TryRead(out var msg);
|
|
return msg!;
|
|
}
|
|
|
|
private static async Task<SyncMessage> NextOfCaseAsync(
|
|
ChannelReader<SyncMessage> reader, SyncMessage.MsgOneofCase msgCase, CancellationToken ct)
|
|
{
|
|
SyncMessage msg;
|
|
do { msg = await NextAsync(reader, ct); } while (msg.MsgCase != msgCase);
|
|
return msg;
|
|
}
|
|
|
|
private static async Task SwallowAsync(Task task)
|
|
{
|
|
try { await task; }
|
|
catch (OperationCanceledException) { }
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<Task<bool>> predicate, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (await predicate()) return;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException("Condition not reached within " + timeout);
|
|
}
|
|
|
|
private static async Task<List<(long Id, string? Sku, long? Qty)>> ReadOrders(SqliteLocalDb db)
|
|
{
|
|
var rows = await db.QueryAsync(
|
|
"SELECT id, sku, qty FROM orders ORDER BY id",
|
|
x => (x.GetInt64(0), x.IsDBNull(1) ? null : x.GetString(1), (long?)(x.IsDBNull(2) ? null : x.GetInt64(2))));
|
|
return rows.ToList();
|
|
}
|
|
|
|
private static async Task<long> OplogCount(SqliteLocalDb db)
|
|
{
|
|
var r = await db.QueryAsync("SELECT COUNT(*) FROM __localdb_oplog", x => x.GetInt64(0));
|
|
return r[0];
|
|
}
|
|
|
|
private static async Task<long> LastAckedSeq(SqliteLocalDb db)
|
|
{
|
|
var r = await db.QueryAsync("SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1", x => x.GetInt64(0));
|
|
return r[0];
|
|
}
|
|
|
|
private static async Task<long> LastAppliedRemoteSeq(SqliteLocalDb db)
|
|
{
|
|
var r = await db.QueryAsync("SELECT last_applied_remote_seq FROM __localdb_peer_state WHERE id = 1", x => x.GetInt64(0));
|
|
return r[0];
|
|
}
|
|
|
|
private static OplogEntry ProtoEntry(long seq, long id, string sku, long hlc, string nodeId = "peer") =>
|
|
new()
|
|
{
|
|
Seq = seq,
|
|
TableName = "orders",
|
|
PkJson = $"{{\"id\":{id}}}",
|
|
RowJson = $"{{\"id\":{id},\"sku\":\"{sku}\",\"qty\":1}}",
|
|
Hlc = hlc,
|
|
NodeId = nodeId,
|
|
IsTombstone = false,
|
|
};
|
|
|
|
private static long AheadHlc(TimeSpan ahead) =>
|
|
DateTimeOffset.UtcNow.Add(ahead).ToUnixTimeMilliseconds() << 16;
|
|
|
|
[Fact]
|
|
public async Task Handshake_Exchanged_BothProceed()
|
|
{
|
|
var a = await NewSideAsync();
|
|
var b = await NewSideAsync();
|
|
var (da, db) = DuplexPair();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
await WaitForAsync(() => Task.FromResult(a.Session.PeerNodeId is not null && b.Session.PeerNodeId is not null), RunTimeout);
|
|
|
|
Assert.Equal(b.Db.NodeId, a.Session.PeerNodeId);
|
|
Assert.Equal(a.Db.NodeId, b.Session.PeerNodeId);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Handshake_DigestMismatch_FailsClosed()
|
|
{
|
|
var a = await NewSideAsync();
|
|
var b = await NewSideAsync("CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER, extra TEXT)");
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 5)");
|
|
var (da, db) = DuplexPair();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
var exA = await Assert.ThrowsAsync<LocalDbSchemaMismatchException>(() => runA);
|
|
var exB = await Assert.ThrowsAsync<LocalDbSchemaMismatchException>(() => runB);
|
|
Assert.Contains("orders", exA.Message);
|
|
Assert.Contains("orders", exB.Message);
|
|
|
|
// No delta crossed the handshake: B never received A's row.
|
|
Assert.Empty(await ReadOrders(b.Db));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Handshake_SchemaVersionMismatch_FailsClosed()
|
|
{
|
|
var a = await NewSideAsync();
|
|
var (duplex, toSession, _) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(a.Db, "peer", version: 2) }, cts.Token);
|
|
|
|
await Assert.ThrowsAsync<LocalDbSchemaMismatchException>(() => run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deltas_FlowBothDirections_AcksAdvanceWatermarks()
|
|
{
|
|
var a = await NewSideAsync();
|
|
var b = await NewSideAsync();
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'FROM_A', 10)");
|
|
await b.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'FROM_B', 20)");
|
|
var (da, db) = DuplexPair();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
async Task<bool> Converged()
|
|
{
|
|
var ra = await ReadOrders(a.Db);
|
|
var rb = await ReadOrders(b.Db);
|
|
return ra.Count == 2 && rb.Count == 2
|
|
&& await OplogCount(a.Db) == 0 && await OplogCount(b.Db) == 0;
|
|
}
|
|
|
|
await WaitForAsync(Converged, RunTimeout);
|
|
|
|
var rowsA = await ReadOrders(a.Db);
|
|
var rowsB = await ReadOrders(b.Db);
|
|
Assert.Equal(rowsA, rowsB);
|
|
Assert.Equal([(1L, "FROM_A", (long?)10), (2L, "FROM_B", (long?)20)], rowsA);
|
|
Assert.True(await LastAckedSeq(a.Db) >= 1);
|
|
Assert.True(await LastAckedSeq(b.Db) >= 1);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Backlog_DrainsWithoutWaitingFlushInterval()
|
|
{
|
|
var options = new ReplicationOptions { MaxBatchSize = 5, FlushInterval = TimeSpan.FromSeconds(30) };
|
|
var a = await NewSideAsync(options: options);
|
|
var b = await NewSideAsync(options: options);
|
|
|
|
const int rows = 15; // 3 x MaxBatchSize
|
|
for (var i = 1; i <= rows; i++)
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i });
|
|
|
|
var (da, db) = DuplexPair();
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
// FlushInterval is 30 s: reaching convergence in a couple seconds proves the pump loops
|
|
// immediately on backlog instead of waiting a flush tick between batches.
|
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
|
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == rows, TimeSpan.FromSeconds(5));
|
|
sw.Stop();
|
|
Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"drain took {sw.Elapsed}");
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Backpressure_BoundedDuplex_NoDeadlock()
|
|
{
|
|
// Capacity-1 crosswise channels with backlog in BOTH directions: under the old shared
|
|
// send-gate design each side's pump held the gate while blocked on the full channel, and
|
|
// each side's receive loop then blocked on the gate trying to send its ack — a circular
|
|
// wait. The two-lane writer breaks it: receiving enqueues acks without ever blocking.
|
|
var options = new ReplicationOptions { MaxBatchSize = 2, FlushInterval = TimeSpan.FromMilliseconds(20) };
|
|
var a = await NewSideAsync(options: options);
|
|
var b = await NewSideAsync(options: options);
|
|
|
|
const int rowsPerSide = 40;
|
|
for (var i = 1; i <= rowsPerSide; i++)
|
|
{
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
|
|
await b.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'B', @id)", new { id = 100 + i });
|
|
}
|
|
|
|
var (da, db) = DuplexPair(capacity: 1);
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
await WaitForAsync(
|
|
async () => (await ReadOrders(a.Db)).Count == 2 * rowsPerSide && (await ReadOrders(b.Db)).Count == 2 * rowsPerSide,
|
|
RunTimeout);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task OutOfOrderAck_DoesNotRegress()
|
|
{
|
|
var a = await NewSideAsync();
|
|
for (var i = 1; i <= 5; i++)
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i });
|
|
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
|
|
await toSession.WriteAsync(new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = 5 } }, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = 2 } }, cts.Token);
|
|
// Barrier: the session applies inbound messages in order, so its DeltaAck for this batch
|
|
// proves both out-of-order acks above were already processed.
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(1, 100, "BARRIER", 5_000_000) } },
|
|
}, cts.Token);
|
|
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
|
|
Assert.Equal(5, a.Session.PeerAckedSeq);
|
|
Assert.Equal(5, await LastAckedSeq(a.Db));
|
|
var snap = await a.Db.QueryAsync("SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", x => x.GetInt64(0));
|
|
Assert.Equal(0, snap[0]);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Teardown_PrimaryFaultSurfaces()
|
|
{
|
|
var a = await NewSideAsync();
|
|
// Backlog keeps the pump loop mid-work while the receive loop faults.
|
|
for (var i = 1; i <= 10; i++)
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i });
|
|
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch
|
|
{
|
|
Entries = { new OplogEntry
|
|
{
|
|
Seq = 1, TableName = "widgets", PkJson = "{\"id\":9}",
|
|
RowJson = "{\"id\":9}", Hlc = 5_000_000, NodeId = "peer", IsTombstone = false,
|
|
} },
|
|
},
|
|
}, cts.Token);
|
|
|
|
// The receive loop's schema fault must surface — not an OperationCanceledException from
|
|
// the torn-down pump/writer siblings.
|
|
var ex = await Assert.ThrowsAsync<LocalDbSchemaMismatchException>(() => run);
|
|
Assert.Contains("widgets", ex.Message);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HlcDriftAhead_Warns()
|
|
{
|
|
var logger = new ListLogger();
|
|
var a = await NewSideAsync(
|
|
options: new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), FailClosedOnDrift = false },
|
|
logger: logger);
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
|
|
// Two separate drifted batches: both apply, but only ONE warning is logged per session.
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(1, 1, "AHEAD1", AheadHlc(TimeSpan.FromMinutes(10))) } },
|
|
}, cts.Token);
|
|
var ack1 = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
Assert.Equal(1, ack1.DeltaAck.AppliedThruSeq);
|
|
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(2, 2, "AHEAD2", AheadHlc(TimeSpan.FromMinutes(11))) } },
|
|
}, cts.Token);
|
|
var ack2 = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
Assert.Equal(2, ack2.DeltaAck.AppliedThruSeq);
|
|
|
|
Assert.Equal(2, (await ReadOrders(a.Db)).Count); // applied despite drift
|
|
Assert.Equal(1, logger.Entries.Count(e => e.Level == LogLevel.Warning));
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HlcDriftAhead_FailClosed_DeadLetters()
|
|
{
|
|
var a = await NewSideAsync(
|
|
options: new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), FailClosedOnDrift = true });
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
|
|
// One batch: seq 1 drifted (dead-lettered), seq 2 clean (applied). Watermark covers both.
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch
|
|
{
|
|
Entries =
|
|
{
|
|
ProtoEntry(1, 1, "DRIFTED", AheadHlc(TimeSpan.FromMinutes(10))),
|
|
ProtoEntry(2, 2, "CLEAN", 5_000_000),
|
|
},
|
|
},
|
|
}, cts.Token);
|
|
var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
Assert.Equal(2, ack.DeltaAck.AppliedThruSeq);
|
|
|
|
var rows = await ReadOrders(a.Db);
|
|
Assert.Single(rows);
|
|
Assert.Equal(2, rows[0].Id);
|
|
|
|
var dead = await a.Db.QueryAsync(
|
|
"SELECT pk_json, error FROM __localdb_dead_letter",
|
|
x => (Pk: x.GetString(0), Err: x.GetString(1)));
|
|
Assert.Single(dead);
|
|
Assert.Equal("{\"id\":1}", dead[0].Pk);
|
|
Assert.Contains("drift", dead[0].Err, StringComparison.OrdinalIgnoreCase);
|
|
|
|
// The session stays alive after dead-lettering: a follow-up clean batch still applies.
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch { Entries = { ProtoEntry(3, 3, "AFTER", 5_000_001) } },
|
|
}, cts.Token);
|
|
var ack3 = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
Assert.Equal(3, ack3.DeltaAck.AppliedThruSeq);
|
|
Assert.Equal(2, (await ReadOrders(a.Db)).Count);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TopSeqDrifted_WatermarkStillAdvances()
|
|
{
|
|
// The batch's TOP seq is dead-lettered, so the applier's watermark stops below it; the
|
|
// session's standalone MAX update must still advance the durable watermark and the ack.
|
|
var a = await NewSideAsync(
|
|
options: new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), FailClosedOnDrift = true });
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch
|
|
{
|
|
Entries =
|
|
{
|
|
ProtoEntry(1, 1, "CLEAN", 5_000_000),
|
|
ProtoEntry(2, 2, "DRIFTED", AheadHlc(TimeSpan.FromMinutes(10))),
|
|
},
|
|
},
|
|
}, cts.Token);
|
|
var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
|
|
Assert.Equal(2, ack.DeltaAck.AppliedThruSeq);
|
|
Assert.Equal(2, await LastAppliedRemoteSeq(a.Db));
|
|
var rows = await ReadOrders(a.Db);
|
|
Assert.Single(rows);
|
|
Assert.Equal(1, rows[0].Id);
|
|
var dead = await a.Db.QueryAsync("SELECT pk_json FROM __localdb_dead_letter", x => x.GetString(0));
|
|
Assert.Equal(["{\"id\":2}"], dead);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AllEntriesDrifted_WatermarkStillAdvances()
|
|
{
|
|
// Every entry dead-lettered: the applier sees an empty batch (AppliedThruSeq 0), yet the
|
|
// durable watermark and the ack must still cover the batch's max seq.
|
|
var a = await NewSideAsync(
|
|
options: new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20), FailClosedOnDrift = true });
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch
|
|
{
|
|
Entries =
|
|
{
|
|
ProtoEntry(1, 1, "DRIFTED1", AheadHlc(TimeSpan.FromMinutes(10))),
|
|
ProtoEntry(2, 2, "DRIFTED2", AheadHlc(TimeSpan.FromMinutes(11))),
|
|
},
|
|
},
|
|
}, cts.Token);
|
|
var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
|
|
Assert.Equal(2, ack.DeltaAck.AppliedThruSeq);
|
|
Assert.Equal(2, await LastAppliedRemoteSeq(a.Db));
|
|
Assert.Empty(await ReadOrders(a.Db));
|
|
var dead = await a.Db.QueryAsync("SELECT pk_json FROM __localdb_dead_letter ORDER BY pk_json", x => x.GetString(0));
|
|
Assert.Equal(["{\"id\":1}", "{\"id\":2}"], dead);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PeerNeedsSnapshot_InvokesSnapshotHooks()
|
|
{
|
|
// (a) THIS node owes the peer a snapshot (its own needs_snapshot) -> our SnapshotSender fires,
|
|
// and its gated send delegate routes messages through the writer onto the wire.
|
|
{
|
|
var a = await NewSideAsync();
|
|
await a.Store.SetNeedsSnapshotAsync(true, default);
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
var sent = new TaskCompletionSource();
|
|
a.Session.SnapshotSender = (send, token) =>
|
|
{
|
|
sent.TrySetResult();
|
|
return send(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 7 } }, token);
|
|
};
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await NextAsync(fromSession, cts.Token); // session Handshake
|
|
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(a.Db, "peer") }, cts.Token);
|
|
await NextAsync(fromSession, cts.Token); // session HandshakeAck
|
|
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, cts.Token);
|
|
|
|
await sent.Task.WaitAsync(RunTimeout);
|
|
var complete = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
|
Assert.Equal(7, complete.SnapshotComplete.AsOfSeq);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
// (b) inbound SnapshotBegin -> our ISnapshotApplier.OnBeginAsync fires with the triggering message.
|
|
{
|
|
var a = await NewSideAsync();
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
var applier = new CapturingApplier();
|
|
a.Session.SnapshotApplier = applier;
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 42 } }, cts.Token);
|
|
|
|
Assert.Equal(42, await applier.BeganAsOf.Task.WaitAsync(RunTimeout));
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
// (c) this node owes a snapshot but no sender hook -> NotSupportedException.
|
|
{
|
|
var a = await NewSideAsync();
|
|
await a.Store.SetNeedsSnapshotAsync(true, default);
|
|
a.Session.SnapshotSender = null;
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await NextAsync(fromSession, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(a.Db, "peer") }, cts.Token);
|
|
await NextAsync(fromSession, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, cts.Token);
|
|
|
|
await Assert.ThrowsAsync<NotSupportedException>(() => run);
|
|
}
|
|
|
|
// (d) inbound SnapshotBegin but no applier hook -> NotSupportedException.
|
|
{
|
|
var a = await NewSideAsync();
|
|
a.Session.SnapshotApplier = null;
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 0 } }, cts.Token);
|
|
|
|
await Assert.ThrowsAsync<NotSupportedException>(() => run);
|
|
}
|
|
}
|
|
|
|
private sealed class CapturingApplier : ISnapshotApplier
|
|
{
|
|
public TaskCompletionSource<long> BeganAsOf { get; } = new();
|
|
|
|
public Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct)
|
|
{
|
|
BeganAsOf.TrySetResult(begin.AsOfSeq);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task OnBatchAsync(SnapshotBatch batch, CancellationToken ct) => Task.CompletedTask;
|
|
public Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct) => Task.CompletedTask;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SecondHandshake_MidStream_IsProtocolError()
|
|
{
|
|
var a = await NewSideAsync();
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(a.Db, "peer") }, cts.Token);
|
|
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => run);
|
|
}
|
|
|
|
// Completes the two-phase handshake against a real session: consumes its Handshake + HandshakeAck
|
|
// and replies with a matching Handshake + a no-snapshot HandshakeAck, leaving the session in steady state.
|
|
private static async Task DriveHandshakeAsync(
|
|
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession, CancellationToken ct)
|
|
{
|
|
var hs = await NextAsync(fromSession, ct);
|
|
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, hs.MsgCase);
|
|
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer") }, ct);
|
|
var ack = await NextAsync(fromSession, ct);
|
|
Assert.Equal(SyncMessage.MsgOneofCase.HandshakeAck, ack.MsgCase);
|
|
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
|
|
}
|
|
|
|
private sealed class ListLogger : ILogger
|
|
{
|
|
public List<(LogLevel Level, string Message)> Entries { get; } = [];
|
|
|
|
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(
|
|
LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
|
{
|
|
lock (Entries)
|
|
Entries.Add((logLevel, formatter(state, exception)));
|
|
}
|
|
|
|
private sealed class NullScope : IDisposable
|
|
{
|
|
public static readonly NullScope Instance = new();
|
|
public void Dispose() { }
|
|
}
|
|
}
|
|
}
|