From 9c3c9b0d6193199c752243ac92bb143457b25d85 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 17:11:43 -0400 Subject: [PATCH] perf(historian): page SQL Server HistoryUpdate DeleteRaw (lmxopcua#746) DeleteRaw deleted an unbounded range in one statement and transaction with OUTPUT INTO the modification trail. Over years of a 1 Hz tag that is tens of millions of rows deleted and audited at once: lock escalation on Sample (blocking the live writer), transaction-log growth, and a 30 s CommandTimeout that rolled the whole thing back. It now runs DELETE TOP (4000) batches, each its own transaction with its trail rows in the same statement (every deleted row is still audited), cancellable between batches. A failure part-way leaves the committed batches deleted and reports the failure; re-issuing completes it (documented, same as QuestDB's paged tombstoning). The existing atomicity tests still hold per batch. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- docs/Historian.md | 7 ++ .../SqlServerHistorianDataUpdater.cs | 52 ++++++++++---- .../DeleteRawPagingTests.cs | 71 +++++++++++++++++++ 3 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/DeleteRawPagingTests.cs diff --git a/docs/Historian.md b/docs/Historian.md index 14cceeb48..fad1348c9 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -2222,6 +2222,13 @@ server to answer `BadTypeMismatch` for a value of the wrong type; this server de **`DeleteRawModifiedDetails` needs both bounds:** a `StartTime` or `EndTime` of `DateTime.MinValue` (the wire's "unspecified") is `BadInvalidArgument` for that node and never reaches the backend. +**DeleteRaw is paged on both SQL backends.** `SqlServer` deletes in `DELETE TOP (4000)` batches, each its own transaction with its trail +rows (`OUTPUT … INTO` on the same statement, so every deleted row is still audited); it used to delete — and audit — the whole range in +one statement and transaction, which over years of a 1 Hz tag meant lock escalation on `hist.Sample` (blocking the live writer), +transaction-log growth, and a `CommandTimeout` that rolled everything back. QuestDB tombstones 1,000 rows per page. ⚠️ On both, a +DeleteRaw that fails part-way (timeout, cancellation, outage) has already deleted the batches before the failure — each with its trail +rows — and reports the failure; re-issuing the same DeleteRaw completes it. + **Duplicate timestamps inside one request** (compared at 100 ns): *Insert* — the first occurrence is applied, later ones are `BadEntryExists`; *Replace* / *Update* — the last one wins and every earlier occurrence reports the winner's code; *DeleteAtTime* — the first reports the real outcome, later ones `BadNoEntryExists`. **Unknown tag:** Insert / Update create the tag row; Replace and the deletes report `BadNoEntryExists` diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs index 6b542163e..697b463a5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs @@ -5,7 +5,7 @@ using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer; /// -/// OPC UA HistoryUpdate against the SQL Server sample table (spec §4 "Updates"). One transaction per call; +/// OPC UA HistoryUpdate against the SQL Server sample table (spec §4 "Updates"). One transaction per call (DeleteRaw: per batch); /// exactly one status per input item, in input order. /// Duplicate timestamps within one request (compared at full 100 ns tick precision): /// Insert — the first occurrence is applied, later ones are BadEntryExists; Replace/Update — last wins, @@ -22,6 +22,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer; /// is written. /// Unknown tag. Insert/Update create it via ; Replace and the /// deletes report BadNoEntryExists (DeleteRaw: BadNoData) without creating anything. +/// DeleteRaw is paged ( rows per batch, one transaction each, the trail +/// rows in the same statement), so a range over years of data never runs as one giant statement/transaction. /// Modified-value history (Part 11). Every row the MERGE inserts or updates and every row a /// delete removes is copied into SampleModified by an OUTPUT … INTO on the SAME statement — the prior /// value from deleted.* (NULL for an insert), the caller's , the @@ -96,23 +98,49 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument; var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false); if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0); + // Paged: DELETE TOP (n) batches, each its own transaction with its trail rows (OUTPUT INTO on the same statement, so + // every deleted row still has its audit row). One statement over years of a 1 Hz tag deleted — and audited — tens + // of millions of rows in one transaction: lock escalation on Sample (blocking the live writer), transaction-log + // growth, and a CommandTimeout that rolled the whole thing back. A failure part-way leaves the batches already + // committed deleted (each with its trail rows) and is reported as the call's failure; re-issuing the same DeleteRaw + // completes it. + long rows = 0; await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false); - await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false); - int rows; - await using (var cmd = _conn.CreateCommand(conn, - $"DELETE FROM {_conn.Q("Sample")} {AuditInto(HistoryModificationType.Delete)} " + - "WHERE TagId = @tagId AND SourceTimeUtc >= @s AND SourceTimeUtc < @e", tx)) + while (true) { - cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value }); - AddAuditParameters(cmd, context); - SqlHistorianConnection.AddDateTime2(cmd, "@s", startUtc); - SqlHistorianConnection.AddDateTime2(cmd, "@e", endUtc); - rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + ct.ThrowIfCancellationRequested(); + int batch; + await using (var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false)) + { + await using (var cmd = _conn.CreateCommand(conn, + $"DELETE TOP (@n) FROM {_conn.Q("Sample")} {AuditInto(HistoryModificationType.Delete)} " + + "WHERE TagId = @tagId AND SourceTimeUtc >= @s AND SourceTimeUtc < @e", tx)) + { + cmd.Parameters.Add(new SqlParameter("@n", System.Data.SqlDbType.Int) { Value = DeleteRawBatchSize }); + cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value }); + AddAuditParameters(cmd, context); + SqlHistorianConnection.AddDateTime2(cmd, "@s", startUtc); + SqlHistorianConnection.AddDateTime2(cmd, "@e", endUtc); + batch = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + await tx.CommitAsync(ct).ConfigureAwait(false); + } + rows += batch; + DeleteRawBatches++; + if (batch < DeleteRawBatchSize) break; } - await tx.CommitAsync(ct).ConfigureAwait(false); return HistoryUpdatePlanner.DeleteRawResult(tagKnown: true, rows); } + /// + /// Rows per DeleteRaw batch: under SQL Server's ~5,000-lock escalation threshold (each deleted row also inserts its + /// trail row), so a range delete never takes a table lock. Settable for tests. + /// + internal int DeleteRawBatchSize { get; set; } = 4000; + + /// DeleteRaw batches run so far (lifetime) — test/diagnostic. + internal int DeleteRawBatches { get; private set; } + /// public async Task> DeleteAtTimeAsync(string tag, IReadOnlyList timesUtc, HistoryUpdateContext context, CancellationToken ct) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/DeleteRawPagingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/DeleteRawPagingTests.cs new file mode 100644 index 000000000..fe603a863 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/DeleteRawPagingTests.cs @@ -0,0 +1,71 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance; +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests; + +/// +/// DeleteRaw is paged: DELETE TOP (n) batches, each with its trail rows, instead of one unbounded statement and +/// transaction (lock escalation on Sample, log growth, a CommandTimeout rolling the whole range back). The result, the +/// window bounds and the Part 11 trail are unchanged. +/// +[Collection(SqlHistorianCollection.Name)] +public sealed class DeleteRawPagingTests(SqlHistorianFixture fx) +{ + private static readonly DateTime T0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + private static async Task ScalarAsync(ReadTestDb db, string sql) + { + await using var c = await db.Conn.OpenAsync(CancellationToken.None); + await using var cmd = db.Conn.CreateCommand(c, sql); + return Convert.ToInt64(await cmd.ExecuteScalarAsync()); + } + + [Fact] + public async Task A_range_larger_than_a_batch_is_deleted_in_batches_with_every_trail_row() + { + var db = await ReadTestDb.CreateAsync(fx); + var u = new SqlServerHistorianDataUpdater(db.Conn, db.Boot, new SqlServerTagCatalog(db.Conn, db.Boot)) { DeleteRawBatchSize = 40 }; + // Seeded with plain INSERTs (a 1,000-row HistoryUpdate MERGE is not what this test is about). + await db.SeedAsync("t", Enumerable.Range(0, 102) + .Select(i => (T0.AddSeconds(i), HistorianValue.FromDouble(i), ReadTestDb.Good)).ToArray()); + var trailBefore = await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")}"); + + // [T0 + 1 s, T0 + 101 s): 100 rows — the first and the last sample lie outside the window. + (await u.DeleteRawAsync("t", T0.AddSeconds(1), T0.AddSeconds(101), ConformanceData.Editor, CancellationToken.None)) + .ShouldBe(HistorianStatusCodes.Good); + + u.DeleteRawBatches.ShouldBe(3, "40 + 40 + 20 — paged, not one statement"); + (await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("Sample")}")).ShouldBe(2); + (await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")}")).ShouldBe(trailBefore + 100, + "every deleted row has its Delete trail row"); + (await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")} WHERE UpdateType = {(byte)HistoryModificationType.Delete}")) + .ShouldBe(100); + + // An empty range is still BadNoData, and an exact multiple of the batch size ends after one short batch. + (await u.DeleteRawAsync("t", T0.AddSeconds(1), T0.AddSeconds(101), ConformanceData.Editor, CancellationToken.None)) + .ShouldBe(HistorianStatusCodes.BadNoData); + } + + [Fact] + public async Task A_cancelled_delete_between_batches_stops_and_what_it_deleted_stays_audited() + { + var db = await ReadTestDb.CreateAsync(fx); + var u = new SqlServerHistorianDataUpdater(db.Conn, db.Boot, new SqlServerTagCatalog(db.Conn, db.Boot)) { DeleteRawBatchSize = 10 }; + await db.SeedAsync("t", Enumerable.Range(0, 100) + .Select(i => (T0.AddSeconds(i), HistorianValue.FromDouble(i), ReadTestDb.Good)).ToArray()); + + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + await Should.ThrowAsync( + () => u.DeleteRawAsync("t", T0, T0.AddSeconds(100), ConformanceData.Editor, cts.Token)); + (await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("Sample")}")).ShouldBe(100, "cancelled before the first batch"); + + // Positive control: the same delete on a live token removes everything, auditing each row. + (await u.DeleteRawAsync("t", T0, T0.AddSeconds(100), ConformanceData.Editor, CancellationToken.None)).ShouldBe(HistorianStatusCodes.Good); + (await ScalarAsync(db, $"SELECT COUNT(*) FROM {db.Conn.Q("Sample")}")).ShouldBe(0); + u.DeleteRawBatches.ShouldBe(11); + } +}