dohertj2/lmxopcua
1
0
Fork 0
Code Issues 25 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

perf(historian): SQL Server retention deletes only for tags with eligible rows (lmxopcua#746)

Browse Source 
The SqlServer purge read every TagId and issued at least one DELETE TOP per
table per tag on every pass, even when nothing was eligible: 50000 tags made
100000 statements per hourly purge, serialised on one connection under the
applock.

One pre-filter query per table (EXISTS seek on the clustered key) now picks
the tags holding a row older than the cutoff; only those get deletes. A pass
with nothing eligible sends just the Event delete. The mid-command
cancellation test now accepts the pre-filter SELECT as the blocked command,
since it is the first statement to touch Sample.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 16:56:57 -04:00
parent 74afc3f90a
commit d44e58dcd6
3 changed files with 75 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs
+42 -4
View File
@@ -92,6 +92,43 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Tag")).ShouldBe(3);
}
[Fact]
public async Task Only_tags_with_eligible_rows_get_delete_statements()
{
// 50 tags with only fresh data + 1 tag with old samples and an old trail row: the pre-filter must send deletes for
// that one tag only — it used to issue two DELETE TOP statements per tag per pass even with nothing to delete.
fx.SkipIfUnavailable();
var (c, b, _, s) = Make();
await b.EnsureAsync(CancellationToken.None);
var old = Now.AddDays(-40).ToString("yyyy-MM-ddTHH:mm:ss");
var fresh = Now.AddDays(-1).ToString("yyyy-MM-ddTHH:mm:ss");
await ExecAsync(
$"DECLARE @t int = 0; WHILE @t <= 50 BEGIN INSERT [{s}].Tag (Name, DataType, CreatedUtc, UpdatedUtc) VALUES (CONCAT(N'tag', @t), 1, SYSUTCDATETIME(), SYSUTCDATETIME()); SET @t += 1; END;" +
$"INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, '{fresh}', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag;" +
$"INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, '{old}', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag WHERE Name = N'tag7';" +
$"INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal) SELECT TagId, '{old}', SYSUTCDATETIME(), 1, N'p' FROM [{s}].Tag WHERE Name = N'tag9';");
var deletes = 0;
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now))
{
AfterBatch = () => deletes++,
};
var results = await purger.PurgeAsync(CancellationToken.None);
results.ShouldBe([
new RetentionPurgeResult("Sample", 1),
new RetentionPurgeResult("SampleModified", 1),
new RetentionPurgeResult("Event", 0),
]);
deletes.ShouldBe(3, "one Sample delete (tag7), one SampleModified delete (tag9), one Event delete — not 2 × 51 + 1");
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(51);
// A pass with nothing eligible sends only the Event delete.
deletes = 0;
await purger.PurgeAsync(CancellationToken.None);
deletes.ShouldBe(1);
}
[Fact]
public async Task A_second_purge_is_a_no_op()
{
@@ -207,14 +244,15 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
var purge = purger.PurgeAsync(cts.Token);
// Cancel only once the server reports a DELETE actually blocked behind the holder — a fixed delay could
// cancel during bootstrap/applock acquisition on a slow host and never exercise the mid-command path.
// Cancel only once the server reports a purge command actually blocked behind the holder — a fixed delay could
// cancel during bootstrap/applock acquisition on a slow host and never exercise the mid-command path. The first
// command to touch Sample is the eligible-tags pre-filter (a SELECT), then the DELETEs; either is mid-command.
var deadline = DateTime.UtcNow.AddSeconds(15);
while (await CountAsync(
$"SELECT COUNT(*) FROM sys.dm_exec_requests WHERE blocking_session_id = {holderSpid} AND command = 'DELETE'") == 0)
$"SELECT COUNT(*) FROM sys.dm_exec_requests WHERE blocking_session_id = {holderSpid} AND command IN ('DELETE', 'SELECT')") == 0)
{
purge.IsCompleted.ShouldBeFalse("the purge finished without ever blocking on the holder's table lock");
DateTime.UtcNow.ShouldBeLessThan(deadline, "the purge's DELETE never blocked behind the holder");
DateTime.UtcNow.ShouldBeLessThan(deadline, "the purge never blocked behind the holder");
await Task.Delay(50, TestContext.Current.CancellationToken);
}
cts.Cancel();