perf(sitelog): batched event-log commits and sliced retention purge
This commit is contained in:
@@ -30,6 +30,9 @@ public class EventLogPurgeService : BackgroundService
|
||||
/// <summary>Number of events deleted per cap-purge batch.</summary>
|
||||
private const int CapPurgeBatchSize = 1000;
|
||||
|
||||
/// <summary>Number of events deleted per retention-purge batch — same size as <see cref="CapPurgeBatchSize"/>.</summary>
|
||||
private const int RetentionPurgeBatchSize = 1000;
|
||||
|
||||
private readonly SiteEventLogger _eventLogger;
|
||||
private readonly SiteEventLogOptions _options;
|
||||
private readonly ILogger<EventLogPurgeService> _logger;
|
||||
@@ -118,21 +121,46 @@ public class EventLogPurgeService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes events older than the retention window, in bounded
|
||||
/// <see cref="RetentionPurgeBatchSize"/>-row slices per DELETE statement rather than
|
||||
/// one unbounded DELETE for the whole expired set — mirroring the batching already
|
||||
/// used a few lines below in <see cref="PurgeByStorageCap"/> (<see cref="CapPurgeBatchSize"/>).
|
||||
/// An expired backlog in the tens/hundreds of thousands of rows would otherwise hold a
|
||||
/// single SQLite write transaction (and the shared <c>_writeLock</c>) open for the
|
||||
/// whole delete, blocking every concurrent <see cref="SiteEventLogger.LogEventAsync"/>
|
||||
/// writer-loop flush for that entire duration.
|
||||
/// </summary>
|
||||
private void PurgeByRetention()
|
||||
{
|
||||
var cutoff = DateTimeOffset.UtcNow.AddDays(-_options.RetentionDays).ToString("o");
|
||||
var totalDeleted = 0;
|
||||
|
||||
var deleted = _eventLogger.WithConnection(connection =>
|
||||
while (true)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "DELETE FROM site_events WHERE timestamp < $cutoff";
|
||||
cmd.Parameters.AddWithValue("$cutoff", cutoff);
|
||||
return cmd.ExecuteNonQuery();
|
||||
});
|
||||
var deleted = _eventLogger.WithConnection(connection =>
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = $"""
|
||||
DELETE FROM site_events WHERE id IN (
|
||||
SELECT id FROM site_events
|
||||
WHERE timestamp < $cutoff
|
||||
LIMIT {RetentionPurgeBatchSize}
|
||||
)
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$cutoff", cutoff);
|
||||
return cmd.ExecuteNonQuery();
|
||||
});
|
||||
|
||||
if (deleted > 0)
|
||||
totalDeleted += deleted;
|
||||
|
||||
if (deleted < RetentionPurgeBatchSize)
|
||||
break;
|
||||
}
|
||||
|
||||
if (totalDeleted > 0)
|
||||
{
|
||||
_logger.LogInformation("Purged {Count} events older than {Days} days", deleted, _options.RetentionDays);
|
||||
_logger.LogInformation("Purged {Count} events older than {Days} days", totalDeleted, _options.RetentionDays);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user