Joseph Doherty
9c3c9b0d61
DeleteRaw deleted an unbounded range in one statement and transaction with OUTPUT INTO the modification trail. Over years of a 1 Hz tag that is tens of millions of rows deleted and audited at once: lock escalation on Sample (blocking the live writer), transaction-log growth, and a 30 s CommandTimeout that rolled the whole thing back. It now runs DELETE TOP (4000) batches, each its own transaction with its trail rows in the same statement (every deleted row is still audited), cancellable between batches. A failure part-way leaves the committed batches deleted and reports the failure; re-issuing completes it (documented, same as QuestDB's paged tombstoning). The existing atomicity tests still hold per batch. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
72 lines
4.0 KiB
C#
72 lines
4.0 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// DeleteRaw is paged: <c>DELETE TOP (n)</c> batches, each with its trail rows, instead of one unbounded statement and
|
|
/// transaction (lock escalation on Sample, log growth, a CommandTimeout rolling the whole range back). The result, the
|
|
/// window bounds and the Part 11 trail are unchanged.
|
|
/// </summary>
|
|
[Collection(SqlHistorianCollection.Name)]
|
|
public sealed class DeleteRawPagingTests(SqlHistorianFixture fx)
|
|
{
|
|
private static readonly DateTime T0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
|
|
|
private static async Task<long> ScalarAsync(ReadTestDb db, string sql)
|
|
{
|
|
await using var c = await db.Conn.OpenAsync(CancellationToken.None);
|
|
await using var cmd = db.Conn.CreateCommand(c, sql);
|
|
return Convert.ToInt64(await cmd.ExecuteScalarAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_range_larger_than_a_batch_is_deleted_in_batches_with_every_trail_row()
|
|
{
|
|
var db = await ReadTestDb.CreateAsync(fx);
|
|
var u = new SqlServerHistorianDataUpdater(db.Conn, db.Boot, new SqlServerTagCatalog(db.Conn, db.Boot)) { DeleteRawBatchSize = 40 };
|
|
// Seeded with plain INSERTs (a 1,000-row HistoryUpdate MERGE is not what this test is about).
|
|
await db.SeedAsync("t", Enumerable.Range(0, 102)
|
|
.Select(i => (T0.AddSeconds(i), HistorianValue.FromDouble(i), ReadTestDb.Good)).ToArray());
|
|
var trailBefore = await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")}");
|
|
|
|
// [T0 + 1 s, T0 + 101 s): 100 rows — the first and the last sample lie outside the window.
|
|
(await u.DeleteRawAsync("t", T0.AddSeconds(1), T0.AddSeconds(101), ConformanceData.Editor, CancellationToken.None))
|
|
.ShouldBe(HistorianStatusCodes.Good);
|
|
|
|
u.DeleteRawBatches.ShouldBe(3, "40 + 40 + 20 — paged, not one statement");
|
|
(await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("Sample")}")).ShouldBe(2);
|
|
(await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")}")).ShouldBe(trailBefore + 100,
|
|
"every deleted row has its Delete trail row");
|
|
(await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")} WHERE UpdateType = {(byte)HistoryModificationType.Delete}"))
|
|
.ShouldBe(100);
|
|
|
|
// An empty range is still BadNoData, and an exact multiple of the batch size ends after one short batch.
|
|
(await u.DeleteRawAsync("t", T0.AddSeconds(1), T0.AddSeconds(101), ConformanceData.Editor, CancellationToken.None))
|
|
.ShouldBe(HistorianStatusCodes.BadNoData);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_cancelled_delete_between_batches_stops_and_what_it_deleted_stays_audited()
|
|
{
|
|
var db = await ReadTestDb.CreateAsync(fx);
|
|
var u = new SqlServerHistorianDataUpdater(db.Conn, db.Boot, new SqlServerTagCatalog(db.Conn, db.Boot)) { DeleteRawBatchSize = 10 };
|
|
await db.SeedAsync("t", Enumerable.Range(0, 100)
|
|
.Select(i => (T0.AddSeconds(i), HistorianValue.FromDouble(i), ReadTestDb.Good)).ToArray());
|
|
|
|
using var cts = new CancellationTokenSource();
|
|
await cts.CancelAsync();
|
|
await Should.ThrowAsync<OperationCanceledException>(
|
|
() => u.DeleteRawAsync("t", T0, T0.AddSeconds(100), ConformanceData.Editor, cts.Token));
|
|
(await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("Sample")}")).ShouldBe(100, "cancelled before the first batch");
|
|
|
|
// Positive control: the same delete on a live token removes everything, auditing each row.
|
|
(await u.DeleteRawAsync("t", T0, T0.AddSeconds(100), ConformanceData.Editor, CancellationToken.None)).ShouldBe(HistorianStatusCodes.Good);
|
|
(await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("Sample")}")).ShouldBe(0);
|
|
u.DeleteRawBatches.ShouldBe(11);
|
|
}
|
|
}
|