From 910befc84c22418b8bdc5b675c838af4e279a4b9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 12 Oct 2026 10:33:28 -0400 Subject: [PATCH] perf(historian): #683 SqlServer StartEnd join-back on PK Start/End/Delta: one GROUP BY computes the counts and each bucket's four pick times (MIN/MAX SourceTimeUtc, any quality and usable); each pick's columns are then fetched by a seek on the clustered key (TagId, SourceTimeUtc). Replaces the windowed ROW_NUMBER form (live-gate 8 F1), whose four window sorts spilled. Output identical (EXCEPT both ways empty, 6M rows, forward + reverse incl. a sub-microsecond reverse start). 6M rows / 30 days, avg of 3 x 2 rounds: 30d@1h 18.6-31.7 s -> 3.4 s (reverse 18.1-29.9 s -> 3.4-5.2 s) 7d@1d 5.2-6.1 s -> 0.8 s 1d@1min 0.31-0.40 s -> 0.16 s (1d@7min reverse 0.46 -> 0.12 s) The seeks are separate reads within one statement: under READ COMMITTED a HistoryUpdate delete between the aggregate and the seek nulls that pick; the null-tolerant reader (2ceed561) makes that bucket's Start/End/Delta BadNoData, transiently. Unit test pins it; documented in the query remarks + Historian.md. QuestDB single-scan StartEnd unchanged. --- docs/Historian.md | 20 ++++++- .../SqlAggregateQueries.cs | 52 +++++++++++-------- .../BucketRowMappingTests.cs | 19 +++++++ 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/docs/Historian.md b/docs/Historian.md index f8cde2e6..d9041c78 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -1602,8 +1602,24 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t query now reads the table **once**: one `GROUP BY` with `first()`/`last()` for the first/last sample of any quality and `arg_min`/`arg_max` keyed on the usable rows' time for the first/last usable one (~15 % faster than before; the usable picks carry no `str_value` — `arg_min` has no VARCHAR form, and only Delta reads them, never with a usable string in the bucket). On `SqlServer` - (READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was reshaped the same - way with a windowed pass + conditional `MAX` — 1.4–3× *faster* than the old five-reference form on a 1 M-row tag. Min/MaxActualTime + (READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was first reshaped + into a windowed pass + conditional `MAX`; **#683 replaced that** — its four `ROW_NUMBER` sorts spilled and it measured *slower* than + the five-reference form on long windows. It is now **aggregate, then join back on the key**: one `GROUP BY` computes the counts and each + bucket's four pick *times* (`MIN`/`MAX(SourceTimeUtc)`, any quality and usable), and each pick's columns are fetched by a seek on the + clustered key `(TagId, SourceTimeUtc)`. Output is identical (`EXCEPT` both ways empty on 6 M rows, forward and reverse); measured on + 6 M rows / 30 days, one tag, average of 3 runs over 2 rounds: + + | Window | windowed form | join-back | + |---|---|---| + | 30 d @ 1 h (720 buckets) | 18.6–31.7 s | 3.4 s | + | 30 d @ 1 h, reverse | 18.1–29.9 s | 3.4–5.2 s | + | 7 d @ 1 d (7 buckets) | 5.2–6.1 s | 0.79–0.80 s | + | 1 d @ 1 min (1440 buckets) | 0.31–0.40 s | 0.16 s | + | 1 d @ 7 min, reverse, sub-µs start | 0.46–0.47 s | 0.12 s | + + (The dev SQL container is slow in absolute terms — the ratio is the point.) ⚠️ The seeks are separate reads of the table within one + statement, so a sample **deleted** by HistoryUpdate between the aggregate and its seek leaves that pick NULL while the counts include + it: the null-tolerant reader below answers that bucket's Start/End/Delta `BadNoData`, transiently. Min/MaxActualTime keep their counts + filtered-rank form on both backends: every single-scan variant measured 1.2–2.4× slower, and QuestDB's `arg_max` cannot express the earliest-time tie-break. So, on both backends, a row whose picks are absent anyway maps to "no such sample" — `BadNoData` for that interval, never an exception and never a value-less Good. `LiveWriteReadRaceStressTests` (QuestDB diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs index d57120e5..50bbdf0d 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs @@ -78,36 +78,42 @@ internal static class SqlAggregateQueries /// StatusCode) and 22..27 the last (same order) — present for every returned (non-empty) bucket. /// /// - /// One scan (live-gate 8 F1). The counts and all four picked samples come from one reference to s: - /// a windowed pass ranks each row within its bucket (rfa/rla over every sample, rfu/rlu - /// within its usable / not-usable partition) and the outer GROUP BY both counts and lifts each picked row's - /// columns with a conditional MAX (one row per bucket matches each pick). The previous form referenced the - /// CTE five times; SQL Server expands each reference into its own scan, and under READ COMMITTED nothing makes those - /// scans see the same rows, so a sample committed between two of them could be counted but not picked (QuestDB hit - /// exactly that). It is also fewer scans of the key range. + /// + /// Aggregate, then join back on the key (#683). One GROUP BY over the window computes the counts and + /// each bucket's four pick TIMES — MIN/MAX(SourceTimeUtc) over every sample (any quality) and over + /// the usable ones — and each pick's columns are then fetched by a seek on the clustered key + /// (TagId, SourceTimeUtc) (unique, so one row per pick; a NULL usable time joins nothing ⇒ NULL columns, + /// as before). Output is identical to the previous windowed form (one ROW_NUMBER pass + conditional + /// MAX, live-gate 8 F1) — EXCEPT both ways is empty on 6M rows, forward and reverse — without that + /// form's four window sorts, which spilled: on 6M rows / 30 days, 30 d @ 1 h ~3.4 s vs 18–32 s, 7 d @ 1 d + /// ~0.8 s vs 5.2–6.1 s, 1 d @ 1 min ~0.16 s vs 0.31–0.40 s (docs/Historian.md). + /// + /// + /// ⚠️ The key seeks are separate reads of the table within one statement, so under READ COMMITTED a sample + /// deleted (HistoryUpdate DeleteRaw/DeleteAtTime) between the aggregate and its seek leaves that pick NULL while + /// the counts still include it. The reader maps a NULL pick to "absent" (ReadStoredOrNull, live-gate 8 F1) + /// and the evaluator answers that bucket's Start/End/Delta BadNoData — transient, until the next read. A + /// sample replaced in between is picked with its new value (same time); a sample inserted in between is in + /// neither half (the pick times come from the aggregate). + /// /// /// Schema-qualified Sample table name. /// The SQL text. /// True for a reverse read (window (@start, @end]). public static string StartEnd(string sample, bool reverse = false) { - static string Pick(string when, bool any) => - (any ? $"MAX(CASE WHEN {when} THEN SourceTimeUtc END), " : "") + - $"MAX(CASE WHEN {when} THEN ValueKind END), MAX(CASE WHEN {when} THEN NumValue END), " + - $"MAX(CASE WHEN {when} THEN IntValue END), MAX(CASE WHEN {when} THEN StrValue END), " + - (any ? $"MAX(CASE WHEN {when} THEN StatusCode END)" : $"MAX(CASE WHEN {when} THEN SourceTimeUtc END)"); + // A pick's columns from its joined row, in the established order: usable picks (kind, num, int, str, time), any- + // quality picks (time, kind, num, int, str, status). + static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc"; + static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode"; + string Join(string a, string time) => $"LEFT JOIN {sample} {a} ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time} "; return Classified(sample, reverse) + - ", w AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, u, g, " + - "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rfa, " + - "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rla, " + - "ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc ASC) AS rfu, " + - "ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc DESC) AS rlu FROM s) " + - "SELECT b, " + CommonCounts() + ", " + - Pick("u = 1 AND rfu = 1", any: false) + ", " + - Pick("u = 1 AND rlu = 1", any: false) + ", " + - Pick("rfa = 1", any: true) + ", " + - Pick("rla = 1", any: true) + " " + - "FROM w GROUP BY b ORDER BY b;"; + ", st AS (SELECT b, " + CommonCounts() + ", " + + "MIN(CASE WHEN u = 1 THEN SourceTimeUtc END) AS fu, MAX(CASE WHEN u = 1 THEN SourceTimeUtc END) AS lu, " + + "MIN(SourceTimeUtc) AS fa, MAX(SourceTimeUtc) AS la FROM s GROUP BY b) " + + "SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " + + Usable("x") + ", " + Usable("y") + ", " + Any("p") + ", " + Any("q") + " FROM st " + + Join("x", "fu") + Join("y", "lu") + Join("p", "fa") + Join("q", "la") + "ORDER BY st.b;"; } /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs index 24f40663..626db115 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs @@ -76,6 +76,25 @@ public sealed class BucketRowMappingTests AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc); } + [Fact] + public void A_pick_nulled_by_a_delete_racing_the_key_join_is_no_data_for_that_end_only() + { + // #683 join-back: the bucket's first sample (usable, so both first picks) was deleted between the GROUP BY and its + // key seek — the counts still include it, its pick columns are NULL; the last picks joined normally. + var t = StartEndTable(); + var row = Counts(t, 0, 2); + row["k1"] = (byte)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1); + row["at1"] = T.AddSeconds(1); row["ak1"] = (byte)1; row["an1"] = 20d; row["st1"] = 0; + t.Rows.Add(row); + + var b = Map(t, AggregateFamily.StartEnd); + b.FirstAny.ShouldBeNull(); + b.First.Kind.ShouldBe(HistorianValueKind.Empty); + AggregateEvaluator.Evaluate(HistoryAggregateType.Start, b, T, T).StatusCode.ShouldBe(NoDataCalc); + AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).StatusCode.ShouldBe(NoDataCalc); + AggregateEvaluator.Evaluate(HistoryAggregateType.End, b, T, T).Value.ShouldBe(20d); + } + [Fact] public void A_consistent_row_still_maps_every_pick() {