feat(historian): SQL Server retention purge (RetentionDays, Primary-gated, applock) (#667)

2026-10-11 08:21:30 -04:00
parent db6a53036b
commit 924eff8d3c
13 changed files with 602 additions and 2 deletions
@@ -22,6 +22,21 @@ public sealed partial class SqlServerHistorianOptions
    /// </summary>
    public bool AutoCreateSchema { get; set; } = true;

+    /// <summary>
+    ///     Days of history to keep, measured on source time. <c>null</c> (default) keeps everything. When set,
+    ///     a driver node deletes older <c>hist.Sample</c>/<c>hist.Event</c> rows every <see cref="PurgeInterval"/>.
+    /// </summary>
+    public int? RetentionDays { get; set; }
+
+    /// <summary>How often the retention purge runs. Must be positive.</summary>
+    public TimeSpan PurgeInterval { get; set; } = TimeSpan.FromHours(1);
+
+    /// <summary>
+    ///     Rows per <c>DELETE TOP(n)</c> batch. Default 4000 stays under SQL Server's 5000-lock escalation
+    ///     threshold so a purge never escalates to a table lock.
+    /// </summary>
+    public int PurgeBatchSize { get; set; } = 4000;
+
    [GeneratedRegex("^[A-Za-z_][A-Za-z0-9_]{0,127}$")]
    private static partial Regex IdentifierRegex();

@@ -41,6 +56,12 @@ public sealed partial class SqlServerHistorianOptions
            errors.Add("ServerHistorian:SqlServer:Schema must be a SQL identifier matching ^[A-Za-z_][A-Za-z0-9_]{0,127}$.");
        if (CommandTimeout <= TimeSpan.Zero)
            errors.Add("ServerHistorian:SqlServer:CommandTimeout must be greater than zero.");
+        if (RetentionDays is < 1)
+            errors.Add("ServerHistorian:SqlServer:RetentionDays must be at least 1 when set (omit it to keep history forever).");
+        if (PurgeInterval <= TimeSpan.Zero)
+            errors.Add("ServerHistorian:SqlServer:PurgeInterval must be greater than zero.");
+        if (PurgeBatchSize < 1)
+            errors.Add("ServerHistorian:SqlServer:PurgeBatchSize must be at least 1.");
        return errors;
    }
 }
@@ -96,5 +96,14 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
        return new SqlServerHistorianDataUpdater(s.Conn, s.Catalog);
    }

+    /// <inheritdoc />
+    public IHistorianRetention? CreateRetention(IServiceProvider sp)
+    {
+        if (_options.RetentionDays is not { } days) return null;
+        var s = Get(sp);
+        return new SqlServerHistorianRetention(s.Conn, s.Bootstrap, _options.PurgeInterval, days,
+            _options.PurgeBatchSize, Loggers(sp).CreateLogger<SqlServerHistorianRetention>());
+    }
+
    private sealed record Shared(SqlHistorianConnection Conn, SqlServerSchemaBootstrapper Bootstrap, SqlServerTagCatalog Catalog);
 }
@@ -0,0 +1,118 @@
+using System.Data;
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
+
+/// <summary>
+///     Deletes <c>hist.Sample</c> / <c>hist.Event</c> rows older than the retention window, in small
+///     <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
+///     delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
+///     A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
+///     To purge another table later, add it to <see cref="PurgeAsync"/>.
+/// </summary>
+internal sealed class SqlServerHistorianRetention : IHistorianRetention
+{
+    private readonly SqlHistorianConnection _conn;
+    private readonly SqlServerSchemaBootstrapper _bootstrap;
+    private readonly int _days;
+    private readonly int _batch;
+    private readonly TimeProvider _time;
+    private readonly ILogger _log;
+
+    public SqlServerHistorianRetention(SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap,
+        TimeSpan interval, int retentionDays, int batchSize, ILogger? log = null, TimeProvider? time = null)
+    {
+        _conn = conn;
+        _bootstrap = bootstrap;
+        Interval = interval;
+        _days = retentionDays;
+        _batch = batchSize;
+        _log = log ?? NullLogger.Instance;
+        _time = time ?? TimeProvider.System;
+    }
+
+    public TimeSpan Interval { get; }
+
+    public async Task<IReadOnlyList<RetentionPurgeResult>> PurgeAsync(CancellationToken ct)
+    {
+        await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
+        var cutoff = _time.GetUtcNow().UtcDateTime.AddDays(-_days);
+
+        await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
+        if (!await TryLockAsync(conn, ct).ConfigureAwait(false))
+        {
+            _log.LogDebug("Historian retention purge skipped: another purger holds the applock.");
+            return [];
+        }
+
+        try
+        {
+            long samples = 0;
+            var tagIds = new List<int>();
+            await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
+            await using (var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false))
+                while (await r.ReadAsync(ct).ConfigureAwait(false)) tagIds.Add(r.GetInt32(0));
+
+            foreach (var id in tagIds)
+            {
+                samples += await DeleteLoopAsync(conn,
+                    $"DELETE TOP (@n) FROM {_conn.Q("Sample")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
+                    cutoff, id, ct).ConfigureAwait(false);
+            }
+
+            var events = await DeleteLoopAsync(conn,
+                $"DELETE TOP (@n) FROM {_conn.Q("Event")} WHERE EventTimeUtc < @cutoff",
+                cutoff, null, ct).ConfigureAwait(false);
+
+            return [new RetentionPurgeResult("Sample", samples), new RetentionPurgeResult("Event", events)];
+        }
+        finally
+        {
+            await ReleaseAsync(conn).ConfigureAwait(false);
+        }
+    }
+
+    private async Task<long> DeleteLoopAsync(SqlConnection conn, string sql, DateTime cutoff, int? tagId, CancellationToken ct)
+    {
+        long total = 0;
+        while (true)
+        {
+            ct.ThrowIfCancellationRequested();
+            await using var cmd = _conn.CreateCommand(conn, sql);
+            cmd.Parameters.Add(new SqlParameter("@n", SqlDbType.Int) { Value = _batch });
+            if (tagId is { } id) cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
+            SqlHistorianConnection.AddDateTime2(cmd, "@cutoff", cutoff);
+            var n = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
+            total += n;
+            if (n < _batch) return total;
+        }
+    }
+
+    private string LockResource => $"OtOpcUa.Historian.Retention.{_conn.Schema}";
+
+    private async Task<bool> TryLockAsync(SqlConnection conn, CancellationToken ct)
+    {
+        await using var cmd = _conn.CreateCommand(conn,
+            "DECLARE @r int; EXEC @r = sp_getapplock @Resource = @res, @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = 0; SELECT @r;");
+        cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
+        return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) >= 0;
+    }
+
+    private async Task ReleaseAsync(SqlConnection conn)
+    {
+        try
+        {
+            await using var cmd = _conn.CreateCommand(conn,
+                "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';");
+            cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
+            await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _log.LogDebug(ex, "Releasing the retention applock failed.");
+        }
+    }
+}