Joseph Doherty
78 lines
3.1 KiB
C#
78 lines
3.1 KiB
C#
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.LocalDb.Replication.Internal;
|
|
|
|
namespace ZB.MOM.WW.LocalDb.Replication;
|
|
|
|
/// <summary>
|
|
/// Periodic oplog maintenance: every <see cref="ReplicationOptions.MaintenanceInterval"/> it
|
|
/// enforces the backlog/age caps (which flag <c>needs_snapshot</c> when exceeded) and prunes
|
|
/// expired tombstone row-versions. Runs on BOTH roles — a passive node with a long-dead peer is
|
|
/// exactly the case where the oplog would otherwise grow unbounded. A failing tick is logged and
|
|
/// skipped; maintenance never dies.
|
|
/// </summary>
|
|
public sealed class MaintenanceBackgroundService : BackgroundService
|
|
{
|
|
private readonly OplogStore _store;
|
|
private readonly ReplicationOptions _options;
|
|
private readonly ILogger<MaintenanceBackgroundService> _logger;
|
|
|
|
// Rate-limits the caps-exceeded warning to one per flag episode: EnforceCapsAsync keeps
|
|
// returning true every tick while the backlog stays over cap, which would otherwise flood
|
|
// the log at the maintenance rate. Reset when a tick comes back clean.
|
|
private bool _snapshotFlagWarned;
|
|
|
|
internal MaintenanceBackgroundService(
|
|
OplogStore store, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory)
|
|
{
|
|
_store = store;
|
|
_options = options.Value;
|
|
_logger = loggerFactory.CreateLogger<MaintenanceBackgroundService>();
|
|
}
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
// First tick lands one interval after startup (not immediately) to keep startup cheap.
|
|
while (!stoppingToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(_options.MaintenanceInterval, stoppingToken);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
|
|
try
|
|
{
|
|
var snapshotFlagged = await _store.EnforceCapsAsync(stoppingToken);
|
|
await _store.PruneTombstonesAsync(stoppingToken);
|
|
|
|
if (snapshotFlagged && !_snapshotFlagWarned)
|
|
{
|
|
_snapshotFlagWarned = true;
|
|
_logger.LogWarning(
|
|
"Oplog backlog/age caps exceeded: pruned to the ceiling and flagged needs_snapshot — the peer must snapshot-resync. Not re-logged until a clean tick.");
|
|
}
|
|
else if (!snapshotFlagged)
|
|
{
|
|
_snapshotFlagWarned = false;
|
|
}
|
|
|
|
_logger.LogDebug("Maintenance tick complete (snapshotFlagged={SnapshotFlagged}).", snapshotFlagged);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Maintenance must never die: log and try again next interval.
|
|
_logger.LogWarning(ex, "Maintenance tick failed; retrying next interval.");
|
|
}
|
|
}
|
|
}
|
|
}
|