Joseph Doherty
d44e58dcd6
The SqlServer purge read every TagId and issued at least one DELETE TOP per table per tag on every pass, even when nothing was eligible: 50000 tags made 100000 statements per hourly purge, serialised on one connection under the applock. One pre-filter query per table (EXISTS seek on the clustered key) now picks the tags holding a row older than the cutoff; only those get deletes. A pass with nothing eligible sends just the Event delete. The mid-command cancellation test now accepts the pre-filter SELECT as the blocked command, since it is the first statement to touch Sample. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
364 lines
18 KiB
C#
364 lines
18 KiB
C#
using Microsoft.Data.SqlClient;
|
||
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
|
||
|
||
[Collection(SqlHistorianCollection.Name)]
|
||
public sealed class RetentionTests(SqlHistorianFixture fx)
|
||
{
|
||
private static readonly DateTime Now = new(2026, 10, 11, 12, 0, 0, DateTimeKind.Utc);
|
||
|
||
private sealed class FixedTime(DateTime utc) : TimeProvider
|
||
{
|
||
public override DateTimeOffset GetUtcNow() => new(utc, TimeSpan.Zero);
|
||
}
|
||
|
||
private (SqlHistorianConnection C, SqlServerSchemaBootstrapper B, SqlServerHistorianOptions O, string S) Make()
|
||
{
|
||
var s = SqlHistorianFixture.UniqueSchema();
|
||
var o = new SqlServerHistorianOptions { ConnectionString = fx.ConnectionString, Schema = s };
|
||
var c = new SqlHistorianConnection(o);
|
||
return (c, new SqlServerSchemaBootstrapper(c, o), o, s);
|
||
}
|
||
|
||
private async Task ExecAsync(string sql)
|
||
{
|
||
await using var c = new SqlConnection(fx.ConnectionString);
|
||
await c.OpenAsync();
|
||
await using var cmd = c.CreateCommand();
|
||
cmd.CommandText = sql;
|
||
await cmd.ExecuteNonQueryAsync();
|
||
}
|
||
|
||
private async Task<long> CountAsync(string sql)
|
||
{
|
||
await using var c = new SqlConnection(fx.ConnectionString);
|
||
await c.OpenAsync();
|
||
await using var cmd = c.CreateCommand();
|
||
cmd.CommandText = sql;
|
||
return Convert.ToInt64(await cmd.ExecuteScalarAsync());
|
||
}
|
||
|
||
/// <summary>3 tags x (5 old + 2 new) samples, plus 4 old + 1 new events. Retention 30 days at <see cref="Now"/>.</summary>
|
||
private async Task SeedAsync(SqlServerSchemaBootstrapper b, string s)
|
||
{
|
||
await b.EnsureAsync(CancellationToken.None);
|
||
var old = Now.AddDays(-40).ToString("yyyy-MM-ddTHH:mm:ss");
|
||
var fresh = Now.AddDays(-1).ToString("yyyy-MM-ddTHH:mm:ss");
|
||
var sql = new System.Text.StringBuilder();
|
||
for (var t = 1; t <= 3; t++)
|
||
{
|
||
sql.Append($"INSERT [{s}].Tag (Name, DataType, CreatedUtc, UpdatedUtc) VALUES (N'tag{t}', 1, SYSUTCDATETIME(), SYSUTCDATETIME());");
|
||
}
|
||
sql.Append($"DECLARE @i int = 0; WHILE @i < 5 BEGIN INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{old}'), 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
|
||
sql.Append($"SET @i = 0; WHILE @i < 2 BEGIN INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{fresh}'), 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
|
||
sql.Append($"SET @i = 0; WHILE @i < 4 BEGIN INSERT [{s}].Event (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, EventTimeUtc, ReceivedTimeUtc) VALUES (CONCAT('old', @i), 's', 'a', 'n', 't', 1, 'Activated', DATEADD(minute, @i, '{old}'), SYSUTCDATETIME()); SET @i += 1; END;");
|
||
sql.Append($"INSERT [{s}].Event (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, EventTimeUtc, ReceivedTimeUtc) VALUES ('new0', 's', 'a', 'n', 't', 1, 'Activated', '{fresh}', SYSUTCDATETIME());");
|
||
// #668 modification trail: 2 old + 1 fresh row per tag (by SOURCE time — the modification time is "now" on all).
|
||
sql.Append($"SET @i = 0; WHILE @i < 2 BEGIN INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{old}'), SYSUTCDATETIME(), 2, N'p', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
|
||
sql.Append($"INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal) SELECT TagId, '{fresh}', SYSUTCDATETIME(), 1, N'p' FROM [{s}].Tag;");
|
||
await ExecAsync(sql.ToString());
|
||
}
|
||
|
||
[Theory]
|
||
[InlineData(4000)]
|
||
[InlineData(1)] // forces the delete loop to iterate once per row
|
||
public async Task Purge_removes_only_rows_older_than_the_window(int batch)
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
var identBefore = await CountAsync($"SELECT IDENT_CURRENT('[{s}].SampleModified')");
|
||
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, batch, time: new FixedTime(Now));
|
||
var results = await purger.PurgeAsync(CancellationToken.None);
|
||
|
||
// #668: a purge never WRITES to the modification trail. The identity seed proves no row was ever inserted —
|
||
// a row-count check could not, since the trail purge would delete an old-source-time audit row again.
|
||
(await CountAsync($"SELECT IDENT_CURRENT('[{s}].SampleModified')")).ShouldBe(identBefore);
|
||
|
||
results.ShouldBe([
|
||
new RetentionPurgeResult("Sample", 15),
|
||
new RetentionPurgeResult("SampleModified", 6),
|
||
new RetentionPurgeResult("Event", 4),
|
||
]);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].SampleModified")).ShouldBe(3);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].SampleModified WHERE SourceTimeUtc < '{Now.AddDays(-30):yyyy-MM-ddTHH:mm:ss}'")).ShouldBe(0);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(6);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample WHERE SourceTimeUtc < '{Now.AddDays(-30):yyyy-MM-ddTHH:mm:ss}'")).ShouldBe(0);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Event")).ShouldBe(1);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Tag")).ShouldBe(3);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Only_tags_with_eligible_rows_get_delete_statements()
|
||
{
|
||
// 50 tags with only fresh data + 1 tag with old samples and an old trail row: the pre-filter must send deletes for
|
||
// that one tag only — it used to issue two DELETE TOP statements per tag per pass even with nothing to delete.
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await b.EnsureAsync(CancellationToken.None);
|
||
var old = Now.AddDays(-40).ToString("yyyy-MM-ddTHH:mm:ss");
|
||
var fresh = Now.AddDays(-1).ToString("yyyy-MM-ddTHH:mm:ss");
|
||
await ExecAsync(
|
||
$"DECLARE @t int = 0; WHILE @t <= 50 BEGIN INSERT [{s}].Tag (Name, DataType, CreatedUtc, UpdatedUtc) VALUES (CONCAT(N'tag', @t), 1, SYSUTCDATETIME(), SYSUTCDATETIME()); SET @t += 1; END;" +
|
||
$"INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, '{fresh}', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag;" +
|
||
$"INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, '{old}', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag WHERE Name = N'tag7';" +
|
||
$"INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal) SELECT TagId, '{old}', SYSUTCDATETIME(), 1, N'p' FROM [{s}].Tag WHERE Name = N'tag9';");
|
||
|
||
var deletes = 0;
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now))
|
||
{
|
||
AfterBatch = () => deletes++,
|
||
};
|
||
var results = await purger.PurgeAsync(CancellationToken.None);
|
||
|
||
results.ShouldBe([
|
||
new RetentionPurgeResult("Sample", 1),
|
||
new RetentionPurgeResult("SampleModified", 1),
|
||
new RetentionPurgeResult("Event", 0),
|
||
]);
|
||
deletes.ShouldBe(3, "one Sample delete (tag7), one SampleModified delete (tag9), one Event delete — not 2 × 51 + 1");
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(51);
|
||
|
||
// A pass with nothing eligible sends only the Event delete.
|
||
deletes = 0;
|
||
await purger.PurgeAsync(CancellationToken.None);
|
||
deletes.ShouldBe(1);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_second_purge_is_a_no_op()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
|
||
await purger.PurgeAsync(CancellationToken.None);
|
||
var again = await purger.PurgeAsync(CancellationToken.None);
|
||
again.ShouldBe([
|
||
new RetentionPurgeResult("Sample", 0),
|
||
new RetentionPurgeResult("SampleModified", 0),
|
||
new RetentionPurgeResult("Event", 0),
|
||
]);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_purger_skips_while_another_holds_the_applock()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
|
||
// Hold the same session applock the purger takes, on a separate connection.
|
||
await using var holder = new SqlConnection(fx.ConnectionString);
|
||
await holder.OpenAsync(TestContext.Current.CancellationToken);
|
||
await using (var cmd = holder.CreateCommand())
|
||
{
|
||
cmd.CommandText = $"EXEC sp_getapplock @Resource = N'OtOpcUa.Historian.Retention.{s}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = 0;";
|
||
await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken);
|
||
}
|
||
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
|
||
(await purger.PurgeAsync(CancellationToken.None)).ShouldBeEmpty();
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(21);
|
||
|
||
await using (var cmd = holder.CreateCommand())
|
||
{
|
||
cmd.CommandText = $"EXEC sp_releaseapplock @Resource = N'OtOpcUa.Historian.Retention.{s}', @LockOwner = 'Session';";
|
||
await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken);
|
||
}
|
||
(await purger.PurgeAsync(CancellationToken.None)).ShouldNotBeEmpty();
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(6);
|
||
}
|
||
|
||
/// <summary>True when a fresh, non-pooled session can take the retention applock (i.e. nobody holds it).</summary>
|
||
private async Task<bool> LockIsFreeAsync(string s)
|
||
{
|
||
var cs = new SqlConnectionStringBuilder(fx.ConnectionString) { Pooling = false }.ConnectionString;
|
||
await using var c = new SqlConnection(cs);
|
||
await c.OpenAsync();
|
||
await using var cmd = c.CreateCommand();
|
||
cmd.CommandText = $"DECLARE @r int; EXEC @r = sp_getapplock @Resource = N'OtOpcUa.Historian.Retention.{s}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = 0; SELECT @r;";
|
||
return Convert.ToInt32(await cmd.ExecuteScalarAsync()) >= 0;
|
||
}
|
||
|
||
[Fact]
|
||
public async Task The_applock_is_released_after_a_purge()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
|
||
await purger.PurgeAsync(CancellationToken.None);
|
||
// A pooled-session re-entry would hide a missing release; a separate non-pooled session cannot.
|
||
(await LockIsFreeAsync(s)).ShouldBeTrue();
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Cancelling_between_batches_stops_the_purge_and_still_releases_the_applock()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
using var cts = new CancellationTokenSource();
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 1, time: new FixedTime(Now));
|
||
var batches = 0;
|
||
purger.AfterBatch = () => { if (++batches == 2) cts.Cancel(); };
|
||
|
||
await Should.ThrowAsync<OperationCanceledException>(() => purger.PurgeAsync(cts.Token));
|
||
|
||
batches.ShouldBe(2);
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(19); // 21 - two batches of 1
|
||
(await LockIsFreeAsync(s)).ShouldBeTrue();
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Cancelling_mid_command_while_the_delete_is_blocked_still_releases_the_applock()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
|
||
// A second session holds an exclusive table lock, so the purger's first DELETE blocks server-side.
|
||
await using var holder = new SqlConnection(fx.ConnectionString);
|
||
await holder.OpenAsync(TestContext.Current.CancellationToken);
|
||
await using var tx = (SqlTransaction)await holder.BeginTransactionAsync(TestContext.Current.CancellationToken);
|
||
await using (var cmd = holder.CreateCommand())
|
||
{
|
||
cmd.Transaction = tx;
|
||
cmd.CommandText = $"SELECT COUNT(*) FROM [{s}].Sample WITH (TABLOCKX, HOLDLOCK)";
|
||
await cmd.ExecuteScalarAsync(TestContext.Current.CancellationToken);
|
||
}
|
||
short holderSpid;
|
||
await using (var cmd = holder.CreateCommand())
|
||
{
|
||
cmd.Transaction = tx;
|
||
cmd.CommandText = "SELECT CAST(@@SPID AS smallint)";
|
||
holderSpid = (short)(await cmd.ExecuteScalarAsync(TestContext.Current.CancellationToken))!;
|
||
}
|
||
|
||
using var cts = new CancellationTokenSource();
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
|
||
var purge = purger.PurgeAsync(cts.Token);
|
||
|
||
// Cancel only once the server reports a purge command actually blocked behind the holder — a fixed delay could
|
||
// cancel during bootstrap/applock acquisition on a slow host and never exercise the mid-command path. The first
|
||
// command to touch Sample is the eligible-tags pre-filter (a SELECT), then the DELETEs; either is mid-command.
|
||
var deadline = DateTime.UtcNow.AddSeconds(15);
|
||
while (await CountAsync(
|
||
$"SELECT COUNT(*) FROM sys.dm_exec_requests WHERE blocking_session_id = {holderSpid} AND command IN ('DELETE', 'SELECT')") == 0)
|
||
{
|
||
purge.IsCompleted.ShouldBeFalse("the purge finished without ever blocking on the holder's table lock");
|
||
DateTime.UtcNow.ShouldBeLessThan(deadline, "the purge never blocked behind the holder");
|
||
await Task.Delay(50, TestContext.Current.CancellationToken);
|
||
}
|
||
cts.Cancel();
|
||
|
||
// SqlClient surfaces the attention as OperationCanceledException or as a SqlException.
|
||
var ex = await Should.ThrowAsync<Exception>(() => purge);
|
||
(ex is OperationCanceledException or SqlException).ShouldBeTrue($"unexpected {ex.GetType().Name}: {ex.Message}");
|
||
|
||
await tx.RollbackAsync(TestContext.Current.CancellationToken);
|
||
(await LockIsFreeAsync(s)).ShouldBeTrue();
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(21);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_failing_release_falls_back_to_clearing_the_pool_so_the_applock_is_freed()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now))
|
||
{
|
||
ReleaseSql = "EXEC sp_this_procedure_does_not_exist;",
|
||
};
|
||
|
||
await purger.PurgeAsync(CancellationToken.None);
|
||
|
||
(await LockIsFreeAsync(s)).ShouldBeTrue();
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Falsification_control_a_failing_release_with_a_no_op_ClearPool_leaves_the_applock_held()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now))
|
||
{
|
||
ReleaseSql = "EXEC sp_this_procedure_does_not_exist;",
|
||
ClearPool = _ => { },
|
||
};
|
||
|
||
await purger.PurgeAsync(CancellationToken.None);
|
||
|
||
try
|
||
{
|
||
// The pooled session still owns the Session-scoped lock: proves the fallback test above is not vacuous.
|
||
(await LockIsFreeAsync(s)).ShouldBeFalse();
|
||
}
|
||
finally
|
||
{
|
||
SqlConnection.ClearAllPools(); // drop the leaked session so the lock does not outlive the test
|
||
}
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_throwing_purge_releases_the_applock()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 1, time: new FixedTime(Now));
|
||
purger.AfterBatch = () => throw new InvalidOperationException("boom");
|
||
await Should.ThrowAsync<InvalidOperationException>(() => purger.PurgeAsync(CancellationToken.None));
|
||
(await LockIsFreeAsync(s)).ShouldBeTrue();
|
||
}
|
||
|
||
private sealed class CapturingLogger : Microsoft.Extensions.Logging.ILogger
|
||
{
|
||
public readonly List<Microsoft.Extensions.Logging.LogLevel> Levels = [];
|
||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
|
||
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
|
||
TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||
{
|
||
lock (Levels) Levels.Add(logLevel);
|
||
}
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_skipped_pass_logs_no_warning_and_does_not_clear_the_pool()
|
||
{
|
||
fx.SkipIfUnavailable();
|
||
var (c, b, _, s) = Make();
|
||
await SeedAsync(b, s);
|
||
|
||
await using var holder = new SqlConnection(fx.ConnectionString);
|
||
await holder.OpenAsync(TestContext.Current.CancellationToken);
|
||
await using (var cmd = holder.CreateCommand())
|
||
{
|
||
cmd.CommandText = $"EXEC sp_getapplock @Resource = N'OtOpcUa.Historian.Retention.{s}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = 0;";
|
||
await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken);
|
||
}
|
||
|
||
var log = new CapturingLogger();
|
||
var clears = 0;
|
||
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, log, new FixedTime(Now))
|
||
{
|
||
ClearPool = _ => Interlocked.Increment(ref clears),
|
||
};
|
||
|
||
for (var i = 0; i < 3; i++) (await purger.PurgeAsync(CancellationToken.None)).ShouldBeEmpty();
|
||
|
||
lock (log.Levels) log.Levels.ShouldNotContain(l => l >= Microsoft.Extensions.Logging.LogLevel.Warning);
|
||
clears.ShouldBe(0);
|
||
// The shared pool is still healthy.
|
||
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(21);
|
||
}
|
||
}
|