using Grpc.Core; using Grpc.Core.Interceptors; using Grpc.Net.Client; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using ZB.MOM.WW.LocalDb.Contracts; using ZB.MOM.WW.LocalDb.Replication; namespace ZB.MOM.WW.LocalDb.Tests; public sealed class GrpcAdapterTests : IAsyncLifetime { private static readonly TimeSpan RunTimeout = TimeSpan.FromSeconds(30); private readonly List _paths = []; private readonly List _hosts = []; private readonly List _providers = []; public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() { foreach (var provider in _providers) await provider.DisposeAsync(); foreach (var host in _hosts) { try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ } host.Dispose(); } Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); foreach (var p in _paths) { if (File.Exists(p)) File.Delete(p); if (File.Exists(p + "-wal")) File.Delete(p + "-wal"); if (File.Exists(p + "-shm")) File.Delete(p + "-shm"); } } // ---- end-to-end wire tests ------------------------------------------------------------- [Fact] public async Task EndToEnd_TwoDbs_InsertOnA_AppearsOnB() { var serverHost = await BuildServerAsync(NewDbPath(), apiKey: null, sink: null); var serverDb = serverHost.Services.GetRequiredService(); await serverDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'FROM_SERVER', 20)"); var handler = serverHost.GetTestServer().CreateHandler(); var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", apiKey: null, () => ChannelOver(handler)); var clientDb = clientProvider.GetRequiredService(); await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'FROM_CLIENT', 10)"); using var cts = new CancellationTokenSource(RunTimeout); await bg.StartAsync(cts.Token); await WaitForAsync( async () => (await ReadOrders(clientDb)).Count == 2 && (await ReadOrders(serverDb)).Count == 2, RunTimeout); var clientRows = await ReadOrders(clientDb); var serverRows = await ReadOrders(serverDb); Assert.Equal(clientRows, serverRows); Assert.Equal([(1L, "FROM_CLIENT", (long?)10), (2L, "FROM_SERVER", (long?)20)], clientRows); await bg.StopAsync(CancellationToken.None); } [Fact] public async Task SecondConcurrentStream_Rejected() { var serverHost = await BuildServerAsync(NewDbPath(), apiKey: null, sink: null); var serverDb = serverHost.Services.GetRequiredService(); var handler = serverHost.GetTestServer().CreateHandler(); var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", apiKey: null, () => ChannelOver(handler)); var clientDb = clientProvider.GetRequiredService(); await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)"); using var cts = new CancellationTokenSource(RunTimeout); await bg.StartAsync(cts.Token); // Convergence proves the first stream's server handler is active and holding the single-stream guard. await WaitForAsync(async () => (await ReadOrders(serverDb)).Count == 1, RunTimeout); var client = new LocalDbSync.LocalDbSyncClient(ChannelOver(handler)); using var call2 = client.Sync(cancellationToken: cts.Token); var ex = await Assert.ThrowsAsync(() => call2.ResponseStream.MoveNext(cts.Token)); Assert.Equal(StatusCode.AlreadyExists, ex.StatusCode); await bg.StopAsync(CancellationToken.None); } // Shipped variant: the channel factory throws on its first call, then returns a live channel. // This deterministically drives the backoff/retry loop (no flaky server restart in TestHost) // and proves convergence after recovery. [Fact] public async Task ClientReconnects_AfterTransientFault() { var serverHost = await BuildServerAsync(NewDbPath(), apiKey: null, sink: null); var serverDb = serverHost.Services.GetRequiredService(); await serverDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'S', 20)"); var handler = serverHost.GetTestServer().CreateHandler(); var calls = 0; GrpcChannel Factory() { if (Interlocked.Increment(ref calls) == 1) throw new InvalidOperationException("transient dial failure"); return ChannelOver(handler); } var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", apiKey: null, Factory); var clientDb = clientProvider.GetRequiredService(); await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)"); using var cts = new CancellationTokenSource(RunTimeout); await bg.StartAsync(cts.Token); await WaitForAsync( async () => (await ReadOrders(clientDb)).Count == 2 && (await ReadOrders(serverDb)).Count == 2, RunTimeout); Assert.True(bg.ConnectionAttempts >= 2, $"expected >= 2 connection attempts, got {bg.ConnectionAttempts}"); await bg.StopAsync(CancellationToken.None); } [Fact] public async Task ApiKeyHeader_SentWhenConfigured() { var sink = new AuthCaptureSink(); var serverHost = await BuildServerAsync(NewDbPath(), apiKey: "test-key", sink: sink); var handler = serverHost.GetTestServer().CreateHandler(); var (clientProvider, bg) = BuildClient(NewDbPath(), "http://localhost", apiKey: "test-key", () => ChannelOver(handler)); var clientDb = clientProvider.GetRequiredService(); await clientDb.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'C', 10)"); using var cts = new CancellationTokenSource(RunTimeout); await bg.StartAsync(cts.Token); await WaitForAsync(() => Task.FromResult(sink.Authorization is not null), RunTimeout); Assert.Equal("Bearer test-key", sink.Authorization); await bg.StopAsync(CancellationToken.None); } // ---- validator ------------------------------------------------------------------------- [Fact] public void Validator_InvalidPeerAddress_Fails() { var result = new ReplicationOptionsValidator().Validate( null, new ReplicationOptions { PeerAddress = "not-a-uri" }); Assert.True(result.Failed); Assert.Contains(result.Failures, f => f.Contains("PeerAddress", StringComparison.Ordinal)); } [Fact] public void Validator_NonPositiveIntervals_Fail() { var result = new ReplicationOptionsValidator().Validate(null, new ReplicationOptions { FlushInterval = TimeSpan.Zero, MaxBatchSize = 0, MaxOplogRows = 0, MaxOplogAge = TimeSpan.Zero, TombstoneRetention = TimeSpan.Zero, ReconnectBackoffMax = TimeSpan.Zero, MaxHlcDriftAhead = TimeSpan.Zero, }); Assert.True(result.Failed); Assert.Contains(result.Failures, f => f.Contains("FlushInterval", StringComparison.Ordinal)); Assert.Contains(result.Failures, f => f.Contains("MaxBatchSize", StringComparison.Ordinal)); Assert.Contains(result.Failures, f => f.Contains("MaxOplogRows", StringComparison.Ordinal)); Assert.Contains(result.Failures, f => f.Contains("MaxOplogAge", StringComparison.Ordinal)); Assert.Contains(result.Failures, f => f.Contains("TombstoneRetention", StringComparison.Ordinal)); Assert.Contains(result.Failures, f => f.Contains("ReconnectBackoffMax", StringComparison.Ordinal)); Assert.Contains(result.Failures, f => f.Contains("MaxHlcDriftAhead", StringComparison.Ordinal)); } [Fact] public void Validator_EmptyPeerAddress_Allowed() { // A passive node leaves PeerAddress empty; the validator can't know the role, so it only // checks format when present. All other defaults are valid. var result = new ReplicationOptionsValidator().Validate(null, new ReplicationOptions { PeerAddress = "" }); Assert.True(result.Succeeded); } // ---- DI -------------------------------------------------------------------------------- [Fact] public void AddZbLocalDbReplication_PassiveConfig_NoHostedService() { var passive = new ServiceCollection(); passive.AddLogging(); var passiveCfg = ConfigFor(NewDbPath(), peerAddress: null, apiKey: null); passive.AddZbLocalDb(passiveCfg, OnReady); passive.AddZbLocalDbReplication(passiveCfg); var p1 = passive.BuildServiceProvider(); _providers.Add(p1); Assert.DoesNotContain(p1.GetServices(), s => s is SyncBackgroundService); Assert.NotNull(p1.GetService()); var initiator = new ServiceCollection(); initiator.AddLogging(); var initiatorCfg = ConfigFor(NewDbPath(), peerAddress: "http://localhost", apiKey: null); initiator.AddZbLocalDb(initiatorCfg, OnReady); initiator.AddZbLocalDbReplication(initiatorCfg); var p2 = initiator.BuildServiceProvider(); _providers.Add(p2); Assert.Contains(p2.GetServices(), s => s is SyncBackgroundService); } // ---- harness --------------------------------------------------------------------------- private string NewDbPath() { var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db"); _paths.Add(path); return path; } private static void OnReady(ILocalDb db) { using var conn = db.CreateConnection(); using var cmd = conn.CreateCommand(); cmd.CommandText = "CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER)"; cmd.ExecuteNonQuery(); db.RegisterReplicated("orders"); } private static IConfiguration ConfigFor(string path, string? peerAddress, string? apiKey) { var dict = new Dictionary { ["LocalDb:Path"] = path, ["LocalDb:Replication:FlushInterval"] = "00:00:00.020", }; if (peerAddress is not null) dict["LocalDb:Replication:PeerAddress"] = peerAddress; if (apiKey is not null) dict["LocalDb:Replication:ApiKey"] = apiKey; return new ConfigurationBuilder().AddInMemoryCollection(dict).Build(); } private static GrpcChannel ChannelOver(HttpMessageHandler handler) => GrpcChannel.ForAddress("http://localhost", new GrpcChannelOptions { HttpHandler = handler }); private async Task BuildServerAsync(string path, string? apiKey, AuthCaptureSink? sink) { var config = ConfigFor(path, peerAddress: null, apiKey: apiKey); var host = await new HostBuilder() .ConfigureWebHost(web => { web.UseTestServer(); web.ConfigureServices(services => { services.AddRouting(); if (sink is not null) { services.AddSingleton(sink); services.AddGrpc(o => o.Interceptors.Add()); } else { services.AddGrpc(); } services.AddZbLocalDb(config, OnReady); services.AddZbLocalDbReplication(config); }); web.Configure(app => { app.UseRouting(); app.UseEndpoints(e => e.MapZbLocalDbSync()); }); }) .StartAsync(); _hosts.Add(host); return host; } private (ServiceProvider Provider, SyncBackgroundService Bg) BuildClient( string path, string peerAddress, string? apiKey, Func channelFactory) { var config = ConfigFor(path, peerAddress, apiKey); var services = new ServiceCollection(); services.AddLogging(); services.AddZbLocalDb(config, OnReady); services.AddZbLocalDbReplication(config); var provider = services.BuildServiceProvider(); _providers.Add(provider); var bg = provider.GetServices().OfType().Single(); bg.ChannelFactory = channelFactory; return (provider, bg); } private static async Task> ReadOrders(ILocalDb db) { var rows = await db.QueryAsync( "SELECT id, sku, qty FROM orders ORDER BY id", x => (x.GetInt64(0), x.IsDBNull(1) ? null : x.GetString(1), (long?)(x.IsDBNull(2) ? null : x.GetInt64(2)))); return rows.ToList(); } private static async Task WaitForAsync(Func> predicate, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (await predicate()) return; await Task.Delay(20); } throw new TimeoutException("Condition not reached within " + timeout); } private sealed class AuthCaptureSink { public volatile string? Authorization; } private sealed class AuthCaptureInterceptor(AuthCaptureSink sink) : Interceptor { public override Task DuplexStreamingServerHandler( IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context, DuplexStreamingServerMethod continuation) { var auth = context.RequestHeaders.GetValue("authorization"); if (auth is not null) sink.Authorization = auth; return continuation(requestStream, responseStream, context); } } }