using System.Diagnostics.Metrics; using System.Threading.Channels; using Grpc.Net.Client; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.LocalDb.Contracts; using ZB.MOM.WW.LocalDb.Internal; using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.LocalDb.Replication.Internal; namespace ZB.MOM.WW.LocalDb.Tests; public sealed class MetricsTests : IAsyncLifetime { private const string OrdersSql = "CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER)"; private static readonly TimeSpan RunTimeout = TimeSpan.FromSeconds(30); private readonly List _paths = []; private readonly List _disposables = []; private readonly List _hosts = []; private readonly List _providers = []; public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() { foreach (var provider in _providers) await provider.DisposeAsync(); foreach (var host in _hosts) { try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ } host.Dispose(); } foreach (var d in _disposables) d.Dispose(); Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); foreach (var p in _paths) { if (File.Exists(p)) File.Delete(p); if (File.Exists(p + "-wal")) File.Delete(p + "-wal"); if (File.Exists(p + "-shm")) File.Delete(p + "-shm"); } } // ---- in-memory duplex tests ----------------------------------------------------------- [Fact] public async Task Counters_IncrementOnApply() { using var metricsB = new LocalDbMetrics(); using var collector = new Collector(metricsB.Meter); var a = await NewSideAsync(); var b = await NewSideAsync(metrics: metricsB); await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'FROM_A', 10)"); var (da, db) = DuplexPair(); using var cts = new CancellationTokenSource(RunTimeout); var runA = a.Session.RunAsync(da, cts.Token); var runB = b.Session.RunAsync(db, cts.Token); await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 1, RunTimeout); Assert.True(collector.Sum("localdb.sync.applied") > 0); cts.Cancel(); await SwallowAsync(runA); await SwallowAsync(runB); } [Fact] public async Task DeadLetter_Counter_Increments() { using var metrics = new LocalDbMetrics(); using var collector = new Collector(metrics.Meter); var a = await NewSideAsync(metrics: metrics); var (duplex, toSession, fromSession) = ScriptedPeer(); using var cts = new CancellationTokenSource(RunTimeout); var run = a.Session.RunAsync(duplex, cts.Token); await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token); // A non-tombstone entry with null row_json is poison: the applier dead-letters it. await toSession.WriteAsync(new SyncMessage { DeltaBatch = new DeltaBatch { Entries = { new OplogEntry { Seq = 1, TableName = "orders", PkJson = "{\"id\":1}", Hlc = 5_000_000, NodeId = "peer", IsTombstone = false, } }, }, }, cts.Token); await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token); Assert.Equal(1, collector.Sum("localdb.sync.dead_lettered")); cts.Cancel(); await SwallowAsync(run); } [Fact] public async Task OplogDepth_Gauge_ReportsBacklog() { using var metrics = new LocalDbMetrics(); using var collector = new Collector(metrics.Meter); var a = await NewSideAsync(); var b = await NewSideAsync(); metrics.OplogDepthProvider = a.Store.GetOplogDepthSync; const int rows = 4; for (var i = 1; i <= rows; i++) await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'R', @id)", new { id = i }); // No session running yet: the gauge reflects the unacked oplog backlog. collector.Clear(); collector.Observe(); Assert.Equal(rows, collector.LastLong("localdb.oplog.depth")); var (da, db) = DuplexPair(); using var cts = new CancellationTokenSource(RunTimeout); var runA = a.Session.RunAsync(da, cts.Token); var runB = b.Session.RunAsync(db, cts.Token); await WaitForAsync(async () => await OplogCount(a.Db) == 0, RunTimeout); collector.Clear(); collector.Observe(); Assert.Equal(0, collector.LastLong("localdb.oplog.depth")); cts.Cancel(); await SwallowAsync(runA); await SwallowAsync(runB); } [Fact] public async Task SyncLag_Gauge_EmptyUntilFirstSync_ThenReports() { using var metrics = new LocalDbMetrics(); using var collector = new Collector(metrics.Meter); DateTimeOffset? lastSync = null; metrics.LastSyncUtcProvider = () => lastSync; // Before any sync: the gauge yields no measurement (last_sync unknown). collector.Observe(); Assert.Empty(collector.Doubles("localdb.sync.lag.seconds")); lastSync = DateTimeOffset.UtcNow; collector.Observe(); var lags = collector.Doubles("localdb.sync.lag.seconds"); Assert.NotEmpty(lags); Assert.True(lags[^1] >= 0, $"lag {lags[^1]} should be non-negative"); Assert.True(lags[^1] < 60, $"lag {lags[^1]} should be small"); } [Fact] public async Task Metrics_NullSafe_NoMetricsRegistered() { // A session constructed with metrics=null and status=null must sync without an NRE. var a = await NewSideAsync(); var b = await NewSideAsync(); await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'A', 10)"); var (da, db) = DuplexPair(); using var cts = new CancellationTokenSource(RunTimeout); var runA = a.Session.RunAsync(da, cts.Token); var runB = b.Session.RunAsync(db, cts.Token); await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 1, RunTimeout); cts.Cancel(); await SwallowAsync(runA); await SwallowAsync(runB); } // ---- gRPC host tests ------------------------------------------------------------------ [Fact] public async Task Reconnects_Counter_Increments() { var serverHost = await BuildServerAsync(NewDbPath()); var handler = serverHost.GetTestServer().CreateHandler(); var calls = 0; GrpcChannel Factory() { if (Interlocked.Increment(ref calls) == 1) throw new InvalidOperationException("transient dial failure"); return ChannelOver(handler); } var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", Factory); var clientDb = clientProvider.GetRequiredService(); await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)"); var metrics = clientProvider.GetRequiredService(); using var collector = new Collector(metrics.Meter); using var cts = new CancellationTokenSource(RunTimeout); await bg.StartAsync(cts.Token); var serverDb = serverHost.Services.GetRequiredService(); await WaitForAsync(async () => (await ReadOrders(serverDb)).Count == 1, RunTimeout); Assert.True(bg.ConnectionAttempts >= 2, $"attempts {bg.ConnectionAttempts}"); Assert.True(collector.Sum("localdb.sync.reconnects") >= 1, $"reconnects {collector.Sum("localdb.sync.reconnects")}"); await bg.StopAsync(CancellationToken.None); } [Fact] public async Task SyncStatus_ReflectsSessionLifecycle() { var serverHost = await BuildServerAsync(NewDbPath()); var serverDb = serverHost.Services.GetRequiredService(); await serverDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'S', 20)"); var handler = serverHost.GetTestServer().CreateHandler(); var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", () => ChannelOver(handler)); var clientDb = clientProvider.GetRequiredService(); await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)"); var status = clientProvider.GetRequiredService(); Assert.False(status.Connected); using var cts = new CancellationTokenSource(RunTimeout); await bg.StartAsync(cts.Token); await WaitForAsync(() => Task.FromResult(status.Connected), RunTimeout); await WaitForAsync( async () => (await ReadOrders(clientDb)).Count == 2 && (await ReadOrders(serverDb)).Count == 2, RunTimeout); Assert.NotNull(status.PeerNodeId); Assert.NotNull(status.LastSyncUtc); await bg.StopAsync(CancellationToken.None); await WaitForAsync(() => Task.FromResult(!status.Connected), RunTimeout); Assert.False(status.Connected); } // ---- in-memory harness ---------------------------------------------------------------- private sealed record Side(SqliteLocalDb Db, OplogStore Store, LwwApplier Applier, SyncSession Session); private async Task NewSideAsync(LocalDbMetrics? metrics = null, SyncStatus? status = null) { var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db"); _paths.Add(path); var db = new SqliteLocalDb(new LocalDbOptions { Path = path }); _disposables.Add(db); await db.ExecuteAsync(OrdersSql); db.RegisterReplicated("orders"); var options = new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }; var store = new OplogStore(db, options); var applier = new LwwApplier(db); var session = new SyncSession(db, store, applier, options, NullLogger.Instance, null, metrics, status); return new Side(db, store, applier, session); } private static (SyncDuplex A, SyncDuplex B) DuplexPair() { var aToB = Channel.CreateUnbounded(); var bToA = Channel.CreateUnbounded(); var a = new SyncDuplex { Send = (m, ct) => aToB.Writer.WriteAsync(m, ct).AsTask(), Inbox = bToA.Reader }; var b = new SyncDuplex { Send = (m, ct) => bToA.Writer.WriteAsync(m, ct).AsTask(), Inbox = aToB.Reader }; return (a, b); } private static (SyncDuplex Duplex, ChannelWriter ToSession, ChannelReader FromSession) ScriptedPeer() { var toSession = Channel.CreateUnbounded(); var fromSession = Channel.CreateUnbounded(); var duplex = new SyncDuplex { Send = (m, ct) => fromSession.Writer.WriteAsync(m, ct).AsTask(), Inbox = toSession.Reader }; return (duplex, toSession.Writer, fromSession.Reader); } private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId) { var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = 1, LastAppliedRemoteSeq = 0 }; foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal)) h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest }); return h; } private static async Task DriveHandshakeAsync( SqliteLocalDb sessionDb, ChannelWriter toSession, ChannelReader fromSession, CancellationToken ct) { await NextAsync(fromSession, ct); await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer") }, ct); await NextAsync(fromSession, ct); await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct); } private static async Task NextAsync(ChannelReader reader, CancellationToken ct) { await reader.WaitToReadAsync(ct); reader.TryRead(out var msg); return msg!; } private static async Task NextOfCaseAsync( ChannelReader reader, SyncMessage.MsgOneofCase msgCase, CancellationToken ct) { SyncMessage msg; do { msg = await NextAsync(reader, ct); } while (msg.MsgCase != msgCase); return msg; } private static async Task SwallowAsync(Task task) { try { await task; } catch (OperationCanceledException) { } } // ---- gRPC host harness ---------------------------------------------------------------- private string NewDbPath() { var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db"); _paths.Add(path); return path; } private static void OnReady(ILocalDb db) { using var conn = db.CreateConnection(); using var cmd = conn.CreateCommand(); cmd.CommandText = OrdersSql; cmd.ExecuteNonQuery(); db.RegisterReplicated("orders"); } private static IConfiguration ConfigFor(string path, string? peerAddress) { var dict = new Dictionary { ["LocalDb:Path"] = path, ["LocalDb:Replication:FlushInterval"] = "00:00:00.020", }; if (peerAddress is not null) dict["LocalDb:Replication:PeerAddress"] = peerAddress; return new ConfigurationBuilder().AddInMemoryCollection(dict).Build(); } private static GrpcChannel ChannelOver(HttpMessageHandler handler) => GrpcChannel.ForAddress("http://localhost", new GrpcChannelOptions { HttpHandler = handler }); private async Task BuildServerAsync(string path) { var config = ConfigFor(path, peerAddress: null); var host = await new HostBuilder() .ConfigureWebHost(web => { web.UseTestServer(); web.ConfigureServices(services => { services.AddRouting(); services.AddGrpc(); services.AddZbLocalDb(config, OnReady); services.AddZbLocalDbReplication(config); }); web.Configure(app => { app.UseRouting(); app.UseEndpoints(e => e.MapZbLocalDbSync()); }); }) .StartAsync(); _hosts.Add(host); return host; } private (ServiceProvider Provider, SyncBackgroundService Bg) BuildClient( string path, string peerAddress, Func channelFactory) { var config = ConfigFor(path, peerAddress); var services = new ServiceCollection(); services.AddLogging(); services.AddZbLocalDb(config, OnReady); services.AddZbLocalDbReplication(config); var provider = services.BuildServiceProvider(); _providers.Add(provider); var bg = provider.GetServices().OfType().Single(); bg.ChannelFactory = channelFactory; return (provider, bg); } private static async Task> ReadOrders(ILocalDb db) { var rows = await db.QueryAsync( "SELECT id, sku, qty FROM orders ORDER BY id", x => (x.GetInt64(0), x.IsDBNull(1) ? null : x.GetString(1), (long?)(x.IsDBNull(2) ? null : x.GetInt64(2)))); return rows.ToList(); } private static async Task OplogCount(ILocalDb db) { var r = await db.QueryAsync("SELECT COUNT(*) FROM __localdb_oplog", x => x.GetInt64(0)); return r[0]; } private static async Task WaitForAsync(Func> predicate, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (await predicate()) return; await Task.Delay(20); } throw new TimeoutException("Condition not reached within " + timeout); } // ---- metric collector ----------------------------------------------------------------- private sealed class Collector : IDisposable { private readonly MeterListener _listener; private readonly object _lock = new(); private readonly Dictionary> _longs = new(StringComparer.Ordinal); private readonly Dictionary> _doubles = new(StringComparer.Ordinal); public Collector(Meter meter) { _listener = new MeterListener { InstrumentPublished = (inst, l) => { if (ReferenceEquals(inst.Meter, meter)) l.EnableMeasurementEvents(inst); }, }; _listener.SetMeasurementEventCallback((inst, val, _, _) => { lock (_lock) Append(_longs, inst.Name, val); }); _listener.SetMeasurementEventCallback((inst, val, _, _) => { lock (_lock) Append(_doubles, inst.Name, val); }); _listener.Start(); } private static void Append(Dictionary> map, string name, T value) { if (!map.TryGetValue(name, out var list)) { list = []; map[name] = list; } list.Add(value); } public void Observe() => _listener.RecordObservableInstruments(); public long Sum(string name) { lock (_lock) return _longs.TryGetValue(name, out var l) ? l.Sum() : 0; } public long? LastLong(string name) { lock (_lock) return _longs.TryGetValue(name, out var l) && l.Count > 0 ? l[^1] : null; } public IReadOnlyList Doubles(string name) { lock (_lock) return _doubles.TryGetValue(name, out var l) ? l.ToList() : []; } public void Clear() { lock (_lock) { _longs.Clear(); _doubles.Clear(); } } public void Dispose() => _listener.Dispose(); } }