diff --git a/docs/Historian.md b/docs/Historian.md
index f8cde2e6..d9041c78 100644
--- a/docs/Historian.md
+++ b/docs/Historian.md
@@ -1602,8 +1602,24 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t
query now reads the table **once**: one `GROUP BY` with `first()`/`last()` for the first/last sample of any quality and
`arg_min`/`arg_max` keyed on the usable rows' time for the first/last usable one (~15 % faster than before; the usable picks carry
no `str_value` — `arg_min` has no VARCHAR form, and only Delta reads them, never with a usable string in the bucket). On `SqlServer`
- (READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was reshaped the same
- way with a windowed pass + conditional `MAX` — 1.4–3× *faster* than the old five-reference form on a 1 M-row tag. Min/MaxActualTime
+ (READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was first reshaped
+ into a windowed pass + conditional `MAX`; **#683 replaced that** — its four `ROW_NUMBER` sorts spilled and it measured *slower* than
+ the five-reference form on long windows. It is now **aggregate, then join back on the key**: one `GROUP BY` computes the counts and each
+ bucket's four pick *times* (`MIN`/`MAX(SourceTimeUtc)`, any quality and usable), and each pick's columns are fetched by a seek on the
+ clustered key `(TagId, SourceTimeUtc)`. Output is identical (`EXCEPT` both ways empty on 6 M rows, forward and reverse); measured on
+ 6 M rows / 30 days, one tag, average of 3 runs over 2 rounds:
+
+ | Window | windowed form | join-back |
+ |---|---|---|
+ | 30 d @ 1 h (720 buckets) | 18.6–31.7 s | 3.4 s |
+ | 30 d @ 1 h, reverse | 18.1–29.9 s | 3.4–5.2 s |
+ | 7 d @ 1 d (7 buckets) | 5.2–6.1 s | 0.79–0.80 s |
+ | 1 d @ 1 min (1440 buckets) | 0.31–0.40 s | 0.16 s |
+ | 1 d @ 7 min, reverse, sub-µs start | 0.46–0.47 s | 0.12 s |
+
+ (The dev SQL container is slow in absolute terms — the ratio is the point.) ⚠️ The seeks are separate reads of the table within one
+ statement, so a sample **deleted** by HistoryUpdate between the aggregate and its seek leaves that pick NULL while the counts include
+ it: the null-tolerant reader below answers that bucket's Start/End/Delta `BadNoData`, transiently. Min/MaxActualTime
keep their counts + filtered-rank form on both backends: every single-scan variant measured 1.2–2.4× slower, and QuestDB's
`arg_max` cannot express the earliest-time tie-break. So, on both backends, a row whose picks are absent anyway maps to "no such
sample" — `BadNoData` for that interval, never an exception and never a value-less Good. `LiveWriteReadRaceStressTests` (QuestDB
diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
index d57120e5..50bbdf0d 100644
--- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
+++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
@@ -78,36 +78,42 @@ internal static class SqlAggregateQueries
/// StatusCode) and 22..27 the last (same order) — present for every returned (non-empty) bucket.
///
///
- /// One scan (live-gate 8 F1). The counts and all four picked samples come from one reference to s:
- /// a windowed pass ranks each row within its bucket (rfa/rla over every sample, rfu/rlu
- /// within its usable / not-usable partition) and the outer GROUP BY both counts and lifts each picked row's
- /// columns with a conditional MAX (one row per bucket matches each pick). The previous form referenced the
- /// CTE five times; SQL Server expands each reference into its own scan, and under READ COMMITTED nothing makes those
- /// scans see the same rows, so a sample committed between two of them could be counted but not picked (QuestDB hit
- /// exactly that). It is also fewer scans of the key range.
+ ///
+ /// Aggregate, then join back on the key (#683). One GROUP BY over the window computes the counts and
+ /// each bucket's four pick TIMES — MIN/MAX(SourceTimeUtc) over every sample (any quality) and over
+ /// the usable ones — and each pick's columns are then fetched by a seek on the clustered key
+ /// (TagId, SourceTimeUtc) (unique, so one row per pick; a NULL usable time joins nothing ⇒ NULL columns,
+ /// as before). Output is identical to the previous windowed form (one ROW_NUMBER pass + conditional
+ /// MAX, live-gate 8 F1) — EXCEPT both ways is empty on 6M rows, forward and reverse — without that
+ /// form's four window sorts, which spilled: on 6M rows / 30 days, 30 d @ 1 h ~3.4 s vs 18–32 s, 7 d @ 1 d
+ /// ~0.8 s vs 5.2–6.1 s, 1 d @ 1 min ~0.16 s vs 0.31–0.40 s (docs/Historian.md).
+ ///
+ ///
+ /// ⚠️ The key seeks are separate reads of the table within one statement, so under READ COMMITTED a sample
+ /// deleted (HistoryUpdate DeleteRaw/DeleteAtTime) between the aggregate and its seek leaves that pick NULL while
+ /// the counts still include it. The reader maps a NULL pick to "absent" (ReadStoredOrNull, live-gate 8 F1)
+ /// and the evaluator answers that bucket's Start/End/Delta BadNoData — transient, until the next read. A
+ /// sample replaced in between is picked with its new value (same time); a sample inserted in between is in
+ /// neither half (the pick times come from the aggregate).
+ ///
///
/// Schema-qualified Sample table name.
/// The SQL text.
/// True for a reverse read (window (@start, @end]).
public static string StartEnd(string sample, bool reverse = false)
{
- static string Pick(string when, bool any) =>
- (any ? $"MAX(CASE WHEN {when} THEN SourceTimeUtc END), " : "") +
- $"MAX(CASE WHEN {when} THEN ValueKind END), MAX(CASE WHEN {when} THEN NumValue END), " +
- $"MAX(CASE WHEN {when} THEN IntValue END), MAX(CASE WHEN {when} THEN StrValue END), " +
- (any ? $"MAX(CASE WHEN {when} THEN StatusCode END)" : $"MAX(CASE WHEN {when} THEN SourceTimeUtc END)");
+ // A pick's columns from its joined row, in the established order: usable picks (kind, num, int, str, time), any-
+ // quality picks (time, kind, num, int, str, status).
+ static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc";
+ static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode";
+ string Join(string a, string time) => $"LEFT JOIN {sample} {a} ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time} ";
return Classified(sample, reverse) +
- ", w AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, u, g, " +
- "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rfa, " +
- "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rla, " +
- "ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc ASC) AS rfu, " +
- "ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc DESC) AS rlu FROM s) " +
- "SELECT b, " + CommonCounts() + ", " +
- Pick("u = 1 AND rfu = 1", any: false) + ", " +
- Pick("u = 1 AND rlu = 1", any: false) + ", " +
- Pick("rfa = 1", any: true) + ", " +
- Pick("rla = 1", any: true) + " " +
- "FROM w GROUP BY b ORDER BY b;";
+ ", st AS (SELECT b, " + CommonCounts() + ", " +
+ "MIN(CASE WHEN u = 1 THEN SourceTimeUtc END) AS fu, MAX(CASE WHEN u = 1 THEN SourceTimeUtc END) AS lu, " +
+ "MIN(SourceTimeUtc) AS fa, MAX(SourceTimeUtc) AS la FROM s GROUP BY b) " +
+ "SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
+ Usable("x") + ", " + Usable("y") + ", " + Any("p") + ", " + Any("q") + " FROM st " +
+ Join("x", "fu") + Join("y", "lu") + Join("p", "fa") + Join("q", "la") + "ORDER BY st.b;";
}
///
diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs
index 24f40663..626db115 100644
--- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs
+++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs
@@ -76,6 +76,25 @@ public sealed class BucketRowMappingTests
AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
}
+ [Fact]
+ public void A_pick_nulled_by_a_delete_racing_the_key_join_is_no_data_for_that_end_only()
+ {
+ // #683 join-back: the bucket's first sample (usable, so both first picks) was deleted between the GROUP BY and its
+ // key seek — the counts still include it, its pick columns are NULL; the last picks joined normally.
+ var t = StartEndTable();
+ var row = Counts(t, 0, 2);
+ row["k1"] = (byte)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
+ row["at1"] = T.AddSeconds(1); row["ak1"] = (byte)1; row["an1"] = 20d; row["st1"] = 0;
+ t.Rows.Add(row);
+
+ var b = Map(t, AggregateFamily.StartEnd);
+ b.FirstAny.ShouldBeNull();
+ b.First.Kind.ShouldBe(HistorianValueKind.Empty);
+ AggregateEvaluator.Evaluate(HistoryAggregateType.Start, b, T, T).StatusCode.ShouldBe(NoDataCalc);
+ AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).StatusCode.ShouldBe(NoDataCalc);
+ AggregateEvaluator.Evaluate(HistoryAggregateType.End, b, T, T).Value.ShouldBe(20d);
+ }
+
[Fact]
public void A_consistent_row_still_maps_every_pick()
{