feat(localdb): periodic oplog maintenance service (caps enforcement + tombstone pruning, both roles)

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 01:02:06 -04:00
parent 154d584190
commit c5352f3ed8
5 changed files with 302 additions and 1 deletions
@@ -0,0 +1,77 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ZB.MOM.WW.LocalDb.Replication.Internal;
+
+namespace ZB.MOM.WW.LocalDb.Replication;
+
+/// <summary>
+/// Periodic oplog maintenance: every <see cref="ReplicationOptions.MaintenanceInterval"/> it
+/// enforces the backlog/age caps (which flag <c>needs_snapshot</c> when exceeded) and prunes
+/// expired tombstone row-versions. Runs on BOTH roles — a passive node with a long-dead peer is
+/// exactly the case where the oplog would otherwise grow unbounded. A failing tick is logged and
+/// skipped; maintenance never dies.
+/// </summary>
+public sealed class MaintenanceBackgroundService : BackgroundService
+{
+    private readonly OplogStore _store;
+    private readonly ReplicationOptions _options;
+    private readonly ILogger<MaintenanceBackgroundService> _logger;
+
+    // Rate-limits the caps-exceeded warning to one per flag episode: EnforceCapsAsync keeps
+    // returning true every tick while the backlog stays over cap, which would otherwise flood
+    // the log at the maintenance rate. Reset when a tick comes back clean.
+    private bool _snapshotFlagWarned;
+
+    internal MaintenanceBackgroundService(
+        OplogStore store, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory)
+    {
+        _store = store;
+        _options = options.Value;
+        _logger = loggerFactory.CreateLogger<MaintenanceBackgroundService>();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // First tick lands one interval after startup (not immediately) to keep startup cheap.
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await Task.Delay(_options.MaintenanceInterval, stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            try
+            {
+                var snapshotFlagged = await _store.EnforceCapsAsync(stoppingToken);
+                await _store.PruneTombstonesAsync(stoppingToken);
+
+                if (snapshotFlagged && !_snapshotFlagWarned)
+                {
+                    _snapshotFlagWarned = true;
+                    _logger.LogWarning(
+                        "Oplog backlog/age caps exceeded: pruned to the ceiling and flagged needs_snapshot — the peer must snapshot-resync. Not re-logged until a clean tick.");
+                }
+                else if (!snapshotFlagged)
+                {
+                    _snapshotFlagWarned = false;
+                }
+
+                _logger.LogDebug("Maintenance tick complete (snapshotFlagged={SnapshotFlagged}).", snapshotFlagged);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                // Maintenance must never die: log and try again next interval.
+                _logger.LogWarning(ex, "Maintenance tick failed; retrying next interval.");
+            }
+        }
+    }
+}