dohertj2/lmxopcua
1
0
Fork 0
Code Issues 36 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf(historian): SQL Server retention deletes only for tags with eligible rows (lmxopcua#746)

Browse Source 
The SqlServer purge read every TagId and issued at least one DELETE TOP per
table per tag on every pass, even when nothing was eligible: 50000 tags made
100000 statements per hourly purge, serialised on one connection under the
applock.

One pre-filter query per table (EXISTS seek on the clustered key) now picks
the tags holding a row older than the cutoff; only those get deletes. A pass
with nothing eligible sends just the Event delete. The mid-command
cancellation test now accepts the pre-filter SELECT as the blocked command,
since it is the first statement to touch Sample.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 16:56:57 -04:00
parent 74afc3f90a
commit d44e58dcd6
3 changed files with 75 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianRetention.cs
+29 -12
View File
@@ -9,7 +9,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>
/// Deletes <c>hist.Sample</c> / <c>hist.SampleModified</c> / <c>hist.Event</c> rows older than the retention window, in small
/// <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
/// delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
/// delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c> — and only for the tags a single pre-filter query
/// finds holding an eligible row, so a pass with nothing to delete costs one query per table, not two per tag; events
/// seek on <c>CX_Event_Time</c>.
/// A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
/// To purge another table later, add it to <see cref="PurgeAsync"/>. On a v1 schema (no Part 11 trail)
/// the <c>SampleModified</c> purge is skipped and absent from the results.
@@ -86,22 +88,24 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
// A v1 schema has no SampleModified table — skip its purge rather than fail the whole pass.
var trail = _bootstrap.SupportsModifiedTrail;
long samples = 0, modified = 0;
var tagIds = new List<int>();
await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
await using (var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false))
while (await r.ReadAsync(ct).ConfigureAwait(false)) tagIds.Add(r.GetInt32(0));
foreach (var id in tagIds)
// Pre-filter: only the tags holding at least one eligible row get delete statements — one statement, a key seek
// per tag on (TagId, SourceTimeUtc). Deleting blindly per tag sent two DELETE TOP statements per tag per pass
// (100 000 for 50 000 tags) on a database with nothing to delete, all under the applock.
var sampleTags = await EligibleTagsAsync(conn, "Sample", cutoff, ct).ConfigureAwait(false);
foreach (var id in sampleTags)
{
samples += await DeleteLoopAsync(conn,
$"DELETE TOP (@n) FROM {_conn.Q("Sample")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
"DELETE TOP (@n) FROM " + _conn.Q("Sample") + " WHERE TagId = @id AND SourceTimeUtc < @cutoff",
cutoff, id, ct).ConfigureAwait(false);
// The Part 11 modification trail ages out on the SAME source-time cutoff as the samples it
// describes; a purge never writes to it (no trigger, no OUTPUT INTO).
if (trail)
}
// The Part 11 modification trail ages out on the SAME source-time cutoff as the samples it
// describes; a purge never writes to it (no trigger, no OUTPUT INTO).
if (trail)
{
foreach (var id in await EligibleTagsAsync(conn, "SampleModified", cutoff, ct).ConfigureAwait(false))
{
modified += await DeleteLoopAsync(conn,
$"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
"DELETE TOP (@n) FROM " + _conn.Q("SampleModified") + " WHERE TagId = @id AND SourceTimeUtc < @cutoff",
cutoff, id, ct).ConfigureAwait(false);
}
}
@@ -131,6 +135,19 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
}
}
/// <summary>The tags with at least one <paramref name="table"/> row older than <paramref name="cutoff"/>.</summary>
private async Task<List<int>> EligibleTagsAsync(SqlConnection conn, string table, DateTime cutoff, CancellationToken ct)
{
var ids = new List<int>();
await using var cmd = _conn.CreateCommand(conn,
"SELECT t.TagId FROM " + _conn.Q("Tag") + " t WHERE EXISTS (SELECT 1 FROM " + _conn.Q(table) +
" s WHERE s.TagId = t.TagId AND s.SourceTimeUtc < @cutoff)");
SqlHistorianConnection.AddDateTime2(cmd, "@cutoff", cutoff);
await using var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await r.ReadAsync(ct).ConfigureAwait(false)) ids.Add(r.GetInt32(0));
return ids;
}
private async Task<long> DeleteLoopAsync(SqlConnection conn, string sql, DateTime cutoff, int? tagId, CancellationToken ct)
{
long total = 0;