feat(localdb): gRPC sync adapters, background service, replication DI

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-17 23:01:29 -04:00
parent 183901e8ac
commit a64fc26391
7 changed files with 701 additions and 0 deletions
@@ -0,0 +1,120 @@
+using Grpc.Core;
+using Grpc.Net.Client;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.LocalDb.Contracts;
+using ZB.MOM.WW.LocalDb.Internal;
+using ZB.MOM.WW.LocalDb.Replication.Internal;
+
+namespace ZB.MOM.WW.LocalDb.Replication;
+
+/// <summary>
+/// The initiating (client) side of replication: a hosted service that dials the configured peer,
+/// runs the symmetric <see cref="SyncSession"/> over the outbound bidirectional stream, and
+/// reconnects with jittered exponential backoff whenever the session ends or faults.
+/// </summary>
+public sealed class SyncBackgroundService : BackgroundService
+{
+    private static readonly TimeSpan InitialBackoff = TimeSpan.FromSeconds(1);
+    // A session healthier than this resets backoff to the floor: the fault was transient, not a
+    // fast-failing dial, so the next reconnect should not start from an inflated delay.
+    private static readonly TimeSpan HealthySessionThreshold = TimeSpan.FromSeconds(60);
+
+    private readonly SqliteLocalDb _db;
+    private readonly ReplicationOptions _options;
+    private readonly ILoggerFactory _loggerFactory;
+    private readonly ILogger<SyncBackgroundService> _logger;
+
+    /// <summary>Test seam: supplies the gRPC channel instead of dialing <see cref="ReplicationOptions.PeerAddress"/>. A channel it returns is owned by the caller and not disposed here.</summary>
+    internal Func<GrpcChannel>? ChannelFactory { get; set; }
+
+    /// <summary>Number of connection attempts made this lifetime (one per reconnect loop iteration).</summary>
+    internal int ConnectionAttempts;
+
+    public SyncBackgroundService(ILocalDb db, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory)
+    {
+        _db = db as SqliteLocalDb
+            ?? throw new InvalidOperationException(
+                $"SyncBackgroundService requires the ILocalDb registered by AddZbLocalDb (a {nameof(SqliteLocalDb)}); " +
+                $"got {db.GetType().FullName}.");
+        _options = options.Value;
+        _loggerFactory = loggerFactory;
+        _logger = loggerFactory.CreateLogger<SyncBackgroundService>();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        var backoff = InitialBackoff;
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            Interlocked.Increment(ref ConnectionAttempts);
+            var lifetime = System.Diagnostics.Stopwatch.StartNew();
+            Exception? fault = null;
+            GrpcChannel? ownedChannel = null;
+
+            try
+            {
+                var channel = ChannelFactory is { } factory
+                    ? factory()
+                    : ownedChannel = GrpcChannel.ForAddress(_options.PeerAddress);
+
+                var client = new LocalDbSync.LocalDbSyncClient(channel);
+                var headers = new Metadata();
+                if (!string.IsNullOrEmpty(_options.ApiKey))
+                    headers.Add("Authorization", $"Bearer {_options.ApiKey}");
+
+                using var call = client.Sync(headers, cancellationToken: stoppingToken);
+                var (duplex, readerTask) = GrpcSyncDuplex.Create(call.ResponseStream, call.RequestStream, stoppingToken);
+                var session = SyncSessionFactory.Create(_db, _options, _loggerFactory);
+
+                await session.RunAsync(duplex, stoppingToken);
+                await call.RequestStream.CompleteAsync();
+                await readerTask;
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                fault = ex;
+            }
+            finally
+            {
+                lifetime.Stop();
+                ownedChannel?.Dispose();
+            }
+
+            if (IsSchemaMismatch(fault))
+                // A schema mismatch will not self-heal by reconnecting; log loudly but keep trying
+                // (an operator may fix the peer) — capped at the max backoff so it does not spin.
+                _logger.LogError(fault, "Replication to peer {Peer} rejected on a schema mismatch; retrying at max backoff.", _options.PeerAddress);
+            else if (fault is not null)
+                _logger.LogWarning(fault, "Replication session to peer {Peer} faulted; reconnecting.", _options.PeerAddress);
+
+            if (lifetime.Elapsed > HealthySessionThreshold)
+                backoff = InitialBackoff;
+            if (IsSchemaMismatch(fault))
+                backoff = _options.ReconnectBackoffMax;
+
+            try
+            {
+                await Task.Delay(Jitter(backoff), stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            backoff = TimeSpan.FromTicks(Math.Min(backoff.Ticks * 2, _options.ReconnectBackoffMax.Ticks));
+        }
+    }
+
+    private static bool IsSchemaMismatch(Exception? ex) =>
+        ex is LocalDbSchemaMismatchException
+        || (ex is RpcException rpc && rpc.StatusCode == StatusCode.FailedPrecondition);
+
+    private static TimeSpan Jitter(TimeSpan value) =>
+        value * (1.0 + (Random.Shared.NextDouble() * 0.4 - 0.2));
+}