feat(historian): SQL Server retention purge (RetentionDays, Primary-gated, applock) (#667)

2026-10-11 08:21:30 -04:00
parent db6a53036b
commit 924eff8d3c
13 changed files with 602 additions and 2 deletions
@@ -0,0 +1,118 @@
+using System.Data;
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
+
+/// <summary>
+///     Deletes <c>hist.Sample</c> / <c>hist.Event</c> rows older than the retention window, in small
+///     <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
+///     delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
+///     A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
+///     To purge another table later, add it to <see cref="PurgeAsync"/>.
+/// </summary>
+internal sealed class SqlServerHistorianRetention : IHistorianRetention
+{
+    private readonly SqlHistorianConnection _conn;
+    private readonly SqlServerSchemaBootstrapper _bootstrap;
+    private readonly int _days;
+    private readonly int _batch;
+    private readonly TimeProvider _time;
+    private readonly ILogger _log;
+
+    public SqlServerHistorianRetention(SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap,
+        TimeSpan interval, int retentionDays, int batchSize, ILogger? log = null, TimeProvider? time = null)
+    {
+        _conn = conn;
+        _bootstrap = bootstrap;
+        Interval = interval;
+        _days = retentionDays;
+        _batch = batchSize;
+        _log = log ?? NullLogger.Instance;
+        _time = time ?? TimeProvider.System;
+    }
+
+    public TimeSpan Interval { get; }
+
+    public async Task<IReadOnlyList<RetentionPurgeResult>> PurgeAsync(CancellationToken ct)
+    {
+        await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
+        var cutoff = _time.GetUtcNow().UtcDateTime.AddDays(-_days);
+
+        await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
+        if (!await TryLockAsync(conn, ct).ConfigureAwait(false))
+        {
+            _log.LogDebug("Historian retention purge skipped: another purger holds the applock.");
+            return [];
+        }
+
+        try
+        {
+            long samples = 0;
+            var tagIds = new List<int>();
+            await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
+            await using (var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false))
+                while (await r.ReadAsync(ct).ConfigureAwait(false)) tagIds.Add(r.GetInt32(0));
+
+            foreach (var id in tagIds)
+            {
+                samples += await DeleteLoopAsync(conn,
+                    $"DELETE TOP (@n) FROM {_conn.Q("Sample")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
+                    cutoff, id, ct).ConfigureAwait(false);
+            }
+
+            var events = await DeleteLoopAsync(conn,
+                $"DELETE TOP (@n) FROM {_conn.Q("Event")} WHERE EventTimeUtc < @cutoff",
+                cutoff, null, ct).ConfigureAwait(false);
+
+            return [new RetentionPurgeResult("Sample", samples), new RetentionPurgeResult("Event", events)];
+        }
+        finally
+        {
+            await ReleaseAsync(conn).ConfigureAwait(false);
+        }
+    }
+
+    private async Task<long> DeleteLoopAsync(SqlConnection conn, string sql, DateTime cutoff, int? tagId, CancellationToken ct)
+    {
+        long total = 0;
+        while (true)
+        {
+            ct.ThrowIfCancellationRequested();
+            await using var cmd = _conn.CreateCommand(conn, sql);
+            cmd.Parameters.Add(new SqlParameter("@n", SqlDbType.Int) { Value = _batch });
+            if (tagId is { } id) cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
+            SqlHistorianConnection.AddDateTime2(cmd, "@cutoff", cutoff);
+            var n = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
+            total += n;
+            if (n < _batch) return total;
+        }
+    }
+
+    private string LockResource => $"OtOpcUa.Historian.Retention.{_conn.Schema}";
+
+    private async Task<bool> TryLockAsync(SqlConnection conn, CancellationToken ct)
+    {
+        await using var cmd = _conn.CreateCommand(conn,
+            "DECLARE @r int; EXEC @r = sp_getapplock @Resource = @res, @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = 0; SELECT @r;");
+        cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
+        return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) >= 0;
+    }
+
+    private async Task ReleaseAsync(SqlConnection conn)
+    {
+        try
+        {
+            await using var cmd = _conn.CreateCommand(conn,
+                "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';");
+            cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
+            await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _log.LogDebug(ex, "Releasing the retention applock failed.");
+        }
+    }
+}