Joseph Doherty
493 lines
18 KiB
C#
493 lines
18 KiB
C#
using System.Diagnostics.Metrics;
|
|
using System.Threading.Channels;
|
|
using Grpc.Net.Client;
|
|
using Microsoft.AspNetCore.Builder;
|
|
using Microsoft.AspNetCore.Hosting;
|
|
using Microsoft.AspNetCore.TestHost;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.LocalDb.Contracts;
|
|
using ZB.MOM.WW.LocalDb.Internal;
|
|
using ZB.MOM.WW.LocalDb.Replication;
|
|
using ZB.MOM.WW.LocalDb.Replication.Internal;
|
|
|
|
namespace ZB.MOM.WW.LocalDb.Tests;
|
|
|
|
public sealed class MetricsTests : IAsyncLifetime
|
|
{
|
|
private const string OrdersSql = "CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER)";
|
|
private static readonly TimeSpan RunTimeout = TimeSpan.FromSeconds(30);
|
|
|
|
private readonly List<string> _paths = [];
|
|
private readonly List<IDisposable> _disposables = [];
|
|
private readonly List<IHost> _hosts = [];
|
|
private readonly List<ServiceProvider> _providers = [];
|
|
|
|
public Task InitializeAsync() => Task.CompletedTask;
|
|
|
|
public async Task DisposeAsync()
|
|
{
|
|
foreach (var provider in _providers)
|
|
await provider.DisposeAsync();
|
|
foreach (var host in _hosts)
|
|
{
|
|
try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
|
|
host.Dispose();
|
|
}
|
|
foreach (var d in _disposables)
|
|
d.Dispose();
|
|
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
|
foreach (var p in _paths)
|
|
{
|
|
if (File.Exists(p)) File.Delete(p);
|
|
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
|
|
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
|
|
}
|
|
}
|
|
|
|
// ---- in-memory duplex tests -----------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Counters_IncrementOnApply()
|
|
{
|
|
using var metricsB = new LocalDbMetrics();
|
|
using var collector = new Collector(metricsB.Meter);
|
|
|
|
var a = await NewSideAsync();
|
|
var b = await NewSideAsync(metrics: metricsB);
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'FROM_A', 10)");
|
|
|
|
var (da, db) = DuplexPair();
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 1, RunTimeout);
|
|
|
|
Assert.True(collector.Sum("localdb.sync.applied") > 0);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DeadLetter_Counter_Increments()
|
|
{
|
|
using var metrics = new LocalDbMetrics();
|
|
using var collector = new Collector(metrics.Meter);
|
|
|
|
var a = await NewSideAsync(metrics: metrics);
|
|
var (duplex, toSession, fromSession) = ScriptedPeer();
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var run = a.Session.RunAsync(duplex, cts.Token);
|
|
|
|
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
|
|
|
// A non-tombstone entry with null row_json is poison: the applier dead-letters it.
|
|
await toSession.WriteAsync(new SyncMessage
|
|
{
|
|
DeltaBatch = new DeltaBatch
|
|
{
|
|
Entries = { new OplogEntry
|
|
{
|
|
Seq = 1, TableName = "orders", PkJson = "{\"id\":1}",
|
|
Hlc = 5_000_000, NodeId = "peer", IsTombstone = false,
|
|
} },
|
|
},
|
|
}, cts.Token);
|
|
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
|
|
|
Assert.Equal(1, collector.Sum("localdb.sync.dead_lettered"));
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(run);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task OplogDepth_Gauge_ReportsBacklog()
|
|
{
|
|
using var metrics = new LocalDbMetrics();
|
|
using var collector = new Collector(metrics.Meter);
|
|
|
|
var a = await NewSideAsync();
|
|
var b = await NewSideAsync();
|
|
metrics.OplogDepthProvider = a.Store.GetOplogDepthSync;
|
|
|
|
const int rows = 4;
|
|
for (var i = 1; i <= rows; i++)
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i });
|
|
|
|
// No session running yet: the gauge reflects the unacked oplog backlog.
|
|
collector.Clear();
|
|
collector.Observe();
|
|
Assert.Equal(rows, collector.LastLong("localdb.oplog.depth"));
|
|
|
|
var (da, db) = DuplexPair();
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
await WaitForAsync(async () => await OplogCount(a.Db) == 0, RunTimeout);
|
|
|
|
collector.Clear();
|
|
collector.Observe();
|
|
Assert.Equal(0, collector.LastLong("localdb.oplog.depth"));
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SyncLag_Gauge_EmptyUntilFirstSync_ThenReports()
|
|
{
|
|
using var metrics = new LocalDbMetrics();
|
|
using var collector = new Collector(metrics.Meter);
|
|
|
|
DateTimeOffset? lastSync = null;
|
|
metrics.LastSyncUtcProvider = () => lastSync;
|
|
|
|
// Before any sync: the gauge yields no measurement (last_sync unknown).
|
|
collector.Observe();
|
|
Assert.Empty(collector.Doubles("localdb.sync.lag.seconds"));
|
|
|
|
lastSync = DateTimeOffset.UtcNow;
|
|
collector.Observe();
|
|
var lags = collector.Doubles("localdb.sync.lag.seconds");
|
|
Assert.NotEmpty(lags);
|
|
Assert.True(lags[^1] >= 0, $"lag {lags[^1]} should be non-negative");
|
|
Assert.True(lags[^1] < 60, $"lag {lags[^1]} should be small");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Metrics_NullSafe_NoMetricsRegistered()
|
|
{
|
|
// A session constructed with metrics=null and status=null must sync without an NRE.
|
|
var a = await NewSideAsync();
|
|
var b = await NewSideAsync();
|
|
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 10)");
|
|
|
|
var (da, db) = DuplexPair();
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
var runA = a.Session.RunAsync(da, cts.Token);
|
|
var runB = b.Session.RunAsync(db, cts.Token);
|
|
|
|
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 1, RunTimeout);
|
|
|
|
cts.Cancel();
|
|
await SwallowAsync(runA);
|
|
await SwallowAsync(runB);
|
|
}
|
|
|
|
// ---- gRPC host tests ------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Reconnects_Counter_Increments()
|
|
{
|
|
var serverHost = await BuildServerAsync(NewDbPath());
|
|
var handler = serverHost.GetTestServer().CreateHandler();
|
|
|
|
var calls = 0;
|
|
GrpcChannel Factory()
|
|
{
|
|
if (Interlocked.Increment(ref calls) == 1)
|
|
throw new InvalidOperationException("transient dial failure");
|
|
return ChannelOver(handler);
|
|
}
|
|
|
|
var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", Factory);
|
|
var clientDb = clientProvider.GetRequiredService<ILocalDb>();
|
|
await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)");
|
|
var metrics = clientProvider.GetRequiredService<LocalDbMetrics>();
|
|
using var collector = new Collector(metrics.Meter);
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
await bg.StartAsync(cts.Token);
|
|
|
|
var serverDb = serverHost.Services.GetRequiredService<ILocalDb>();
|
|
await WaitForAsync(async () => (await ReadOrders(serverDb)).Count == 1, RunTimeout);
|
|
|
|
Assert.True(bg.ConnectionAttempts >= 2, $"attempts {bg.ConnectionAttempts}");
|
|
Assert.True(collector.Sum("localdb.sync.reconnects") >= 1, $"reconnects {collector.Sum("localdb.sync.reconnects")}");
|
|
|
|
await bg.StopAsync(CancellationToken.None);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SyncStatus_ReflectsSessionLifecycle()
|
|
{
|
|
var serverHost = await BuildServerAsync(NewDbPath());
|
|
var serverDb = serverHost.Services.GetRequiredService<ILocalDb>();
|
|
await serverDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'S', 20)");
|
|
var handler = serverHost.GetTestServer().CreateHandler();
|
|
|
|
var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", () => ChannelOver(handler));
|
|
var clientDb = clientProvider.GetRequiredService<ILocalDb>();
|
|
await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)");
|
|
var status = clientProvider.GetRequiredService<ISyncStatus>();
|
|
|
|
Assert.False(status.Connected);
|
|
|
|
using var cts = new CancellationTokenSource(RunTimeout);
|
|
await bg.StartAsync(cts.Token);
|
|
|
|
await WaitForAsync(() => Task.FromResult(status.Connected), RunTimeout);
|
|
await WaitForAsync(
|
|
async () => (await ReadOrders(clientDb)).Count == 2 && (await ReadOrders(serverDb)).Count == 2,
|
|
RunTimeout);
|
|
|
|
Assert.NotNull(status.PeerNodeId);
|
|
Assert.NotNull(status.LastSyncUtc);
|
|
|
|
await bg.StopAsync(CancellationToken.None);
|
|
await WaitForAsync(() => Task.FromResult(!status.Connected), RunTimeout);
|
|
Assert.False(status.Connected);
|
|
}
|
|
|
|
// ---- in-memory harness ----------------------------------------------------------------
|
|
|
|
private sealed record Side(SqliteLocalDb Db, OplogStore Store, LwwApplier Applier, SyncSession Session);
|
|
|
|
private async Task<Side> NewSideAsync(LocalDbMetrics? metrics = null, SyncStatus? status = null)
|
|
{
|
|
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
|
_paths.Add(path);
|
|
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
|
_disposables.Add(db);
|
|
await db.ExecuteAsync(OrdersSql);
|
|
db.RegisterReplicated("orders");
|
|
|
|
var options = new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) };
|
|
var store = new OplogStore(db, options);
|
|
var applier = new LwwApplier(db);
|
|
var session = new SyncSession(db, store, applier, options, NullLogger.Instance, null, metrics, status);
|
|
return new Side(db, store, applier, session);
|
|
}
|
|
|
|
private static (SyncDuplex A, SyncDuplex B) DuplexPair()
|
|
{
|
|
var aToB = Channel.CreateUnbounded<SyncMessage>();
|
|
var bToA = Channel.CreateUnbounded<SyncMessage>();
|
|
var a = new SyncDuplex { Send = (m, ct) => aToB.Writer.WriteAsync(m, ct).AsTask(), Inbox = bToA.Reader };
|
|
var b = new SyncDuplex { Send = (m, ct) => bToA.Writer.WriteAsync(m, ct).AsTask(), Inbox = aToB.Reader };
|
|
return (a, b);
|
|
}
|
|
|
|
private static (SyncDuplex Duplex, ChannelWriter<SyncMessage> ToSession, ChannelReader<SyncMessage> FromSession) ScriptedPeer()
|
|
{
|
|
var toSession = Channel.CreateUnbounded<SyncMessage>();
|
|
var fromSession = Channel.CreateUnbounded<SyncMessage>();
|
|
var duplex = new SyncDuplex { Send = (m, ct) => fromSession.Writer.WriteAsync(m, ct).AsTask(), Inbox = toSession.Reader };
|
|
return (duplex, toSession.Writer, fromSession.Reader);
|
|
}
|
|
|
|
private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId)
|
|
{
|
|
var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = 1, LastAppliedRemoteSeq = 0 };
|
|
foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
|
h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest });
|
|
return h;
|
|
}
|
|
|
|
private static async Task DriveHandshakeAsync(
|
|
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession, CancellationToken ct)
|
|
{
|
|
await NextAsync(fromSession, ct);
|
|
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer") }, ct);
|
|
await NextAsync(fromSession, ct);
|
|
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
|
|
}
|
|
|
|
private static async Task<SyncMessage> NextAsync(ChannelReader<SyncMessage> reader, CancellationToken ct)
|
|
{
|
|
await reader.WaitToReadAsync(ct);
|
|
reader.TryRead(out var msg);
|
|
return msg!;
|
|
}
|
|
|
|
private static async Task<SyncMessage> NextOfCaseAsync(
|
|
ChannelReader<SyncMessage> reader, SyncMessage.MsgOneofCase msgCase, CancellationToken ct)
|
|
{
|
|
SyncMessage msg;
|
|
do { msg = await NextAsync(reader, ct); } while (msg.MsgCase != msgCase);
|
|
return msg;
|
|
}
|
|
|
|
private static async Task SwallowAsync(Task task)
|
|
{
|
|
try { await task; }
|
|
catch (OperationCanceledException) { }
|
|
}
|
|
|
|
// ---- gRPC host harness ----------------------------------------------------------------
|
|
|
|
private string NewDbPath()
|
|
{
|
|
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
|
_paths.Add(path);
|
|
return path;
|
|
}
|
|
|
|
private static void OnReady(ILocalDb db)
|
|
{
|
|
using var conn = db.CreateConnection();
|
|
using var cmd = conn.CreateCommand();
|
|
cmd.CommandText = OrdersSql;
|
|
cmd.ExecuteNonQuery();
|
|
db.RegisterReplicated("orders");
|
|
}
|
|
|
|
private static IConfiguration ConfigFor(string path, string? peerAddress)
|
|
{
|
|
var dict = new Dictionary<string, string?>
|
|
{
|
|
["LocalDb:Path"] = path,
|
|
["LocalDb:Replication:FlushInterval"] = "00:00:00.020",
|
|
};
|
|
if (peerAddress is not null) dict["LocalDb:Replication:PeerAddress"] = peerAddress;
|
|
return new ConfigurationBuilder().AddInMemoryCollection(dict).Build();
|
|
}
|
|
|
|
private static GrpcChannel ChannelOver(HttpMessageHandler handler) =>
|
|
GrpcChannel.ForAddress("http://localhost", new GrpcChannelOptions { HttpHandler = handler });
|
|
|
|
private async Task<IHost> BuildServerAsync(string path)
|
|
{
|
|
var config = ConfigFor(path, peerAddress: null);
|
|
var host = await new HostBuilder()
|
|
.ConfigureWebHost(web =>
|
|
{
|
|
web.UseTestServer();
|
|
web.ConfigureServices(services =>
|
|
{
|
|
services.AddRouting();
|
|
services.AddGrpc();
|
|
services.AddZbLocalDb(config, OnReady);
|
|
services.AddZbLocalDbReplication(config);
|
|
});
|
|
web.Configure(app =>
|
|
{
|
|
app.UseRouting();
|
|
app.UseEndpoints(e => e.MapZbLocalDbSync());
|
|
});
|
|
})
|
|
.StartAsync();
|
|
_hosts.Add(host);
|
|
return host;
|
|
}
|
|
|
|
private (ServiceProvider Provider, SyncBackgroundService Bg) BuildClient(
|
|
string path, string peerAddress, Func<GrpcChannel> channelFactory)
|
|
{
|
|
var config = ConfigFor(path, peerAddress);
|
|
var services = new ServiceCollection();
|
|
services.AddLogging();
|
|
services.AddZbLocalDb(config, OnReady);
|
|
services.AddZbLocalDbReplication(config);
|
|
var provider = services.BuildServiceProvider();
|
|
_providers.Add(provider);
|
|
|
|
var bg = provider.GetServices<IHostedService>().OfType<SyncBackgroundService>().Single();
|
|
bg.ChannelFactory = channelFactory;
|
|
return (provider, bg);
|
|
}
|
|
|
|
private static async Task<List<(long Id, string? Sku, long? Qty)>> ReadOrders(ILocalDb db)
|
|
{
|
|
var rows = await db.QueryAsync(
|
|
"SELECT id, sku, qty FROM orders ORDER BY id",
|
|
x => (x.GetInt64(0), x.IsDBNull(1) ? null : x.GetString(1), (long?)(x.IsDBNull(2) ? null : x.GetInt64(2))));
|
|
return rows.ToList();
|
|
}
|
|
|
|
private static async Task<long> OplogCount(ILocalDb db)
|
|
{
|
|
var r = await db.QueryAsync("SELECT COUNT(*) FROM __localdb_oplog", x => x.GetInt64(0));
|
|
return r[0];
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<Task<bool>> predicate, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (await predicate()) return;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException("Condition not reached within " + timeout);
|
|
}
|
|
|
|
// ---- metric collector -----------------------------------------------------------------
|
|
|
|
private sealed class Collector : IDisposable
|
|
{
|
|
private readonly MeterListener _listener;
|
|
private readonly object _lock = new();
|
|
private readonly Dictionary<string, List<long>> _longs = new(StringComparer.Ordinal);
|
|
private readonly Dictionary<string, List<double>> _doubles = new(StringComparer.Ordinal);
|
|
|
|
public Collector(Meter meter)
|
|
{
|
|
_listener = new MeterListener
|
|
{
|
|
InstrumentPublished = (inst, l) =>
|
|
{
|
|
if (ReferenceEquals(inst.Meter, meter))
|
|
l.EnableMeasurementEvents(inst);
|
|
},
|
|
};
|
|
_listener.SetMeasurementEventCallback<long>((inst, val, _, _) =>
|
|
{
|
|
lock (_lock) Append(_longs, inst.Name, val);
|
|
});
|
|
_listener.SetMeasurementEventCallback<double>((inst, val, _, _) =>
|
|
{
|
|
lock (_lock) Append(_doubles, inst.Name, val);
|
|
});
|
|
_listener.Start();
|
|
}
|
|
|
|
private static void Append<T>(Dictionary<string, List<T>> map, string name, T value)
|
|
{
|
|
if (!map.TryGetValue(name, out var list))
|
|
{
|
|
list = [];
|
|
map[name] = list;
|
|
}
|
|
list.Add(value);
|
|
}
|
|
|
|
public void Observe() => _listener.RecordObservableInstruments();
|
|
|
|
public long Sum(string name)
|
|
{
|
|
lock (_lock) return _longs.TryGetValue(name, out var l) ? l.Sum() : 0;
|
|
}
|
|
|
|
public long? LastLong(string name)
|
|
{
|
|
lock (_lock) return _longs.TryGetValue(name, out var l) && l.Count > 0 ? l[^1] : null;
|
|
}
|
|
|
|
public IReadOnlyList<double> Doubles(string name)
|
|
{
|
|
lock (_lock) return _doubles.TryGetValue(name, out var l) ? l.ToList() : [];
|
|
}
|
|
|
|
public void Clear()
|
|
{
|
|
lock (_lock)
|
|
{
|
|
_longs.Clear();
|
|
_doubles.Clear();
|
|
}
|
|
}
|
|
|
|
public void Dispose() => _listener.Dispose();
|
|
}
|
|
}
|