feat(localdb): periodic oplog maintenance service (caps enforcement + tombstone pruning, both roles)

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 01:02:06 -04:00
parent 154d584190
commit c5352f3ed8
5 changed files with 302 additions and 1 deletions
@@ -16,7 +16,9 @@ public static class ReplicationServiceCollectionExtensions
    /// configuration section, registers the passive <see cref="LocalDbSyncService"/> as a singleton
    /// (its single-stream guard must span calls), and registers the initiating
    /// <see cref="SyncBackgroundService"/> as a hosted service — on a passive node (no
-    /// <see cref="ReplicationOptions.PeerAddress"/>) it starts and immediately idles. Idempotent.
+    /// <see cref="ReplicationOptions.PeerAddress"/>) it starts and immediately idles. The periodic
+    /// <see cref="MaintenanceBackgroundService"/> (oplog caps enforcement + tombstone pruning) is
+    /// also always registered and runs on BOTH roles — passive nodes need it too. Idempotent.
    /// Requires <see cref="LocalDbServiceCollectionExtensions.AddZbLocalDb"/> to have been called
    /// for the <see cref="ILocalDb"/> the adapters depend on.
    /// </summary>
@@ -59,6 +61,11 @@ public static class ReplicationServiceCollectionExtensions

        services.AddSingleton<LocalDbSyncService>();
        services.AddHostedService<SyncBackgroundService>();
+        // Factory registration because the ctor takes the internal shared OplogStore singleton.
+        services.AddHostedService(sp => new MaintenanceBackgroundService(
+            sp.GetRequiredService<OplogStore>(),
+            sp.GetRequiredService<IOptions<ReplicationOptions>>(),
+            sp.GetRequiredService<Microsoft.Extensions.Logging.ILoggerFactory>()));

        return services;
    }
@@ -0,0 +1,77 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.LocalDb.Replication.Internal;
+
+namespace ZB.MOM.WW.LocalDb.Replication;
+
+/// <summary>
+/// Periodic oplog maintenance: every <see cref="ReplicationOptions.MaintenanceInterval"/> it
+/// enforces the backlog/age caps (which flag <c>needs_snapshot</c> when exceeded) and prunes
+/// expired tombstone row-versions. Runs on BOTH roles — a passive node with a long-dead peer is
+/// exactly the case where the oplog would otherwise grow unbounded. A failing tick is logged and
+/// skipped; maintenance never dies.
+/// </summary>
+public sealed class MaintenanceBackgroundService : BackgroundService
+{
+    private readonly OplogStore _store;
+    private readonly ReplicationOptions _options;
+    private readonly ILogger<MaintenanceBackgroundService> _logger;
+
+    // Rate-limits the caps-exceeded warning to one per flag episode: EnforceCapsAsync keeps
+    // returning true every tick while the backlog stays over cap, which would otherwise flood
+    // the log at the maintenance rate. Reset when a tick comes back clean.
+    private bool _snapshotFlagWarned;
+
+    internal MaintenanceBackgroundService(
+        OplogStore store, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory)
+    {
+        _store = store;
+        _options = options.Value;
+        _logger = loggerFactory.CreateLogger<MaintenanceBackgroundService>();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // First tick lands one interval after startup (not immediately) to keep startup cheap.
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await Task.Delay(_options.MaintenanceInterval, stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            try
+            {
+                var snapshotFlagged = await _store.EnforceCapsAsync(stoppingToken);
+                await _store.PruneTombstonesAsync(stoppingToken);
+
+                if (snapshotFlagged && !_snapshotFlagWarned)
+                {
+                    _snapshotFlagWarned = true;
+                    _logger.LogWarning(
+                        "Oplog backlog/age caps exceeded: pruned to the ceiling and flagged needs_snapshot — the peer must snapshot-resync. Not re-logged until a clean tick.");
+                }
+                else if (!snapshotFlagged)
+                {
+                    _snapshotFlagWarned = false;
+                }
+
+                _logger.LogDebug("Maintenance tick complete (snapshotFlagged={SnapshotFlagged}).", snapshotFlagged);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                // Maintenance must never die: log and try again next interval.
+                _logger.LogWarning(ex, "Maintenance tick failed; retrying next interval.");
+            }
+        }
+    }
+}
@@ -29,6 +29,9 @@ public sealed class ReplicationOptions
    /// <summary>Upper bound on the exponential reconnect backoff.</summary>
    public TimeSpan ReconnectBackoffMax { get; set; } = TimeSpan.FromSeconds(60);

+    /// <summary>How often the maintenance service enforces the oplog caps and prunes expired tombstones.</summary>
+    public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromSeconds(60);
+
    /// <summary>Maximum tolerated HLC drift where a peer's clock runs ahead of ours.</summary>
    public TimeSpan MaxHlcDriftAhead { get; set; } = TimeSpan.FromMinutes(5);

@@ -36,6 +36,8 @@ public sealed class ReplicationOptionsValidator : IValidateOptions<ReplicationOp
            failures.Add($"LocalDb:Replication:ReconnectBackoffMax must be greater than zero; got {options.ReconnectBackoffMax}.");
        else if (options.ReconnectBackoffMax > ReconnectBackoffCeiling)
            failures.Add($"LocalDb:Replication:ReconnectBackoffMax must be at most {ReconnectBackoffCeiling}; got {options.ReconnectBackoffMax}.");
+        if (options.MaintenanceInterval <= TimeSpan.Zero)
+            failures.Add($"LocalDb:Replication:MaintenanceInterval must be greater than zero; got {options.MaintenanceInterval}.");
        if (options.MaxHlcDriftAhead <= TimeSpan.Zero)
            failures.Add($"LocalDb:Replication:MaxHlcDriftAhead must be greater than zero; got {options.MaxHlcDriftAhead}.");