diff --git a/docs/Historian.md b/docs/Historian.md index 71eda810..c1b85d3e 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -1337,6 +1337,12 @@ bounding values back to the source integer type, so its `Interpolative` returns `TimeAverage`/`TimeAverage2`/`Total`/`Total2` differ from ours by the truncated bound values (~0.002–0.02 on the gate's data). Ours always computes and returns `Double` (as stated above for every aggregate but Count and Start/End), which is the exact Part 13 value; the parity datasets are all `Double`, so this one is not masked there but recorded here. +Two more, found by live gate 9 (both kept deliberately): **Minimum/MaximumActualTime carry the Calculated bit on every interval** +(`0x…401`, `0x…405` with Partial) where the SDK returns the picked raw sample's own status (`0x…000`, `0x…404`) — the same value and +timestamp, and the same treatment as the Min/Max-at-interval-start case above; masked in `SdkAggregateParityTests` (class remarks). +And an interval holding a **Good String (or DateTime) sample** is `BadAggregateInvalidInputs` for Average/Minimum/Maximum/Range/ +*ActualTime here (the documented #684 refusal rule — a Good non-numeric value is not a number to aggregate), while the SDK skips the +string and serves the numeric samples (e.g. Average 4 Good); an Uncertain non-numeric sample refuses nothing on either side. **Pushdown trade-off.** Unlike the per-sample aggregates, the time-weighted family is **not** pushed down to SQL: the backend streams the window's rows plus the four bounds (one batch of five result sets on SQL Server; a bounds `UNION ALL` + a window select on QuestDB) @@ -1606,28 +1612,40 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t into a windowed pass + conditional `MAX`; **#683 replaced that** — its four `ROW_NUMBER` sorts spilled and it measured *slower* than the five-reference form on long windows. It is now **aggregate, then join back on the key**: one `GROUP BY` computes the counts and each bucket's four pick *times* (`MIN`/`MAX(SourceTimeUtc)`, any quality and usable), and each pick's columns are fetched by a seek on the - clustered key `(TagId, SourceTimeUtc)`. Output is identical (`EXCEPT` both ways empty on 6 M rows, forward and reverse); measured on - 6 M rows / 30 days, one tag (so a 1 d or 7 d window leaves up to 5.8 M rows of the tag's history outside it), average of 3 runs - over 2 rounds, plans compiled by the 30 d read and reused (sniffed) by the rest: + clustered key `(TagId, SourceTimeUtc)` — **forced** with `WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc)))`, so every pick is a + nested-loops seek on both key columns, one row per bucket per pick, whatever the statistics say. The optimizer's own choices were + both pathological, and plans are cached and reused across reads and tags: keyed on the pick time alone, a hash/merge join whose inner + input was the tag's **whole history** (#683 review: 43,657 logical reads vs 361 for a 1 d @ 1 h read of a tag with 3 M rows outside the + window); bounded to the window as well, a nested-loops **range** seek over the window with the pick time as a residual, once per + bucket — O(buckets × window rows) — compiled whenever the statistics understate the table (a fresh or freshly bulk-loaded + historian, ascending-key estimates for recent windows) and then reused for a wide read (**live gate 9 F1**: 1 d @ 1 min 20 s). With + the hint neither shape exists: a forced `HASH`/`MERGE JOIN` is refused (error 8622). ⚠️ The hint names the primary key `PK_Sample` + created by `001_initial.sql`; a hand-built schema must keep that name. - | Window | windowed form (before) | join-back | - |---|---|---| - | 30 d @ 1 h (720 buckets) | 19.0–21.7 s | 3.4–3.5 s | - | 30 d @ 1 h, reverse | 24.7–30.3 s | 3.4 s | - | 7 d @ 1 d (7 buckets) | 5.5–6.0 s | 0.78–0.81 s | - | 7 d @ 1 min (10,080 buckets) | 2.1–2.3 s | 1.08–1.11 s | - | 1 d @ 1 h (24 buckets) | 0.55–0.60 s | 0.12–0.13 s | - | 1 d @ 1 h, reverse | 0.52–0.55 s | 0.11 s | - | 1 d @ 1 min (1440 buckets) | 0.29–0.32 s | 0.16 s | + Output is identical to every earlier form (`EXCEPT` both ways empty, forward and reverse, including a tag of mixed Double/Int64/ + Boolean/String/DateTime/outage samples). Measured on 6 M rows / 30 days for one tag (a 1 d or 7 d window leaves up to 5.8 M rows of + that tag outside it) plus an 86 k-row mixed-kinds tag; elapsed ms / session logical reads, each variant's plan compiled on its first + read and reused by the rest: - ⚠️ **Every pick join is bounded to the window** (`@start`/`@end`, the same predicate as the window CTE), not only keyed on the pick - time. Keyed alone, the optimizer may choose a hash/merge/adaptive join whose inner input is the tag's **whole history**, and - parameter sniffing then reuses that plan for every tag — a review measured 43,657 logical reads vs 361 for a 1 d @ 1 h read of a tag - with 3 M rows outside the window. Forcing that shape here, 1 d @ 1 h: **152,524** session logical reads unbounded vs **6,756** bounded - under `OPTION (HASH JOIN)` (6,655 under `MERGE JOIN`; the unforced plan reads 1,846 either way). `StartEndJoinBackTests` pins it: - a tag with 300 k rows either side of a 1 h window, forced hash and merge joins, < 1,000 reads (183 bounded, 8,055 with the bound - removed). Wider windows trade reads for time — 7 d @ 1 min reads 174 k pages (four key seeks per bucket) against the windowed form's - 46 k, yet runs in half the time. (The dev SQL container is slow in absolute terms — the ratios are the point.) + | Window | before #683 (windowed) | #683 as merged, accurate stats | #683 as merged, understated stats ‡ | **now (FORCESEEK)**, accurate | **now**, understated ‡ | + |---|---|---|---|---|---| + | 1 d @ 1 min (1440 buckets) | 329 / 6.7 k | 121 / 14.9 k | **47,223 / 3.6 M** | 160 / 25.0 k | 253 / 25.6 k | + | 1 d @ 1 min, reverse | 371 / 6.6 k | 129 / 14.9 k | **46,806 / 3.6 M** | 167 / 25.0 k | 246 / 25.6 k | + | 1 d @ 1 h (24 buckets) | 578 / 5.4 k | 79 / 5.5 k | 984 / 63 k | 117 / 1.8 k | 206 / 3.6 k | + | 7 d @ 1 min (10,080 buckets) | 3,426 / 103 k | 1,019 / 92 k | (hours — not run) | 1,472 / 180 k | 1,819 / 179 k | + | 30 d @ 1 h (720 buckets) † | 35,339 / 322 k | 1,487 / 191 k | (hours — not run) | 5,720 / 73 k | 6,921 / 101 k | + | mixed kinds 1 d @ 1 min | 190 / 5.9 k | 92 / 12.0 k | **20,164 / 1.66 M** | 107 / 24.4 k | 137 / 24.0 k | + | mixed kinds 1 d @ 7 min, reverse | 199 / 1.5 k | 71 / 3.9 k | 2,977 / 238 k | 59 / 4.2 k | 93 / 4.7 k | + + ‡ `UPDATE STATISTICS … WITH ROWCOUNT = 34, PAGECOUNT = 1`, then the plan compiled on a 1 h @ 1 min read and reused — gate 9's + recipe. † The 30 d row is dominated by the window `GROUP BY`, not the picks: compiled on the 30 d read itself, the merged query and + the hinted one both take ~6.9 s (serial, a small memory grant, spills); the merged query's 1.5 s above is a plan compiled on the + 1 d read and reused. Alternatives measured and rejected: the same seek written as `OUTER APPLY (SELECT TOP (1) … WITH (FORCESEEK + …))` (identical plans and numbers — the join form is shorter); `OPTION (RECOMPILE)` on the merged query (compiles under the bad + statistics on every call: still 20.4 s / 1.66 M reads on the mixed-kinds tag, and a compile per read). `StartEndJoinBackTests` + pins it — one sample a second over a day with 300 k rows of history either side: under understated statistics 18,645 reads + (the merged query: **1,645,845 reads, 23 s**, so the test failed against it), with accurate statistics 22,616, and forced + `HASH`/`MERGE` joins refused with 8622. (The dev SQL container is slow in absolute terms — the ratios are the point.) ⚠️ The seeks are separate reads of the table within one statement, so a sample **deleted** by HistoryUpdate between the aggregate and its seek leaves that pick NULL while the counts include it: the null-tolerant reader below answers that bucket's Start/End/Delta diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs index c2d953cc..76698531 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs @@ -99,9 +99,17 @@ internal static class SqlAggregateQueries /// (the pick times come from the aggregate). /// /// - /// Every join is also bounded to the window (the same @start/@end predicate as s): keyed - /// on the pick time alone, the optimizer may choose a hash/merge/adaptive join whose inner input is the tag's - /// whole history, and parameter sniffing then reuses that plan for every tag. + /// Every pick is an exact-key seek, by hint (#683 live gate 9 F1). Each pick join carries + /// WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc))), so it can only be a nested-loops seek on BOTH key + /// columns — one row per bucket per pick — whatever the statistics say. Without it the plan was the optimizer's + /// choice, and both alternatives it found were pathological: keyed on the pick time alone, a hash/merge join whose + /// inner input was the tag's whole history (review of 910befc8: 43,657 vs 361 logical reads); bounded to the window + /// as well (4a3b1256), a nested-loops RANGE seek over the window with the pick time as a residual, once per bucket + /// — O(buckets × window rows), chosen under understated statistics and then reused from the plan cache (gate 9: + /// 1 d @ 1 min 20 s; here 47 s, 3.6M reads, vs 0.25 s). With the hint neither shape exists (a forced + /// HASH/MERGE JOIN is refused, error 8622), so the window bound is no longer needed in the join. + /// ⚠️ The hint names the clustered primary key PK_Sample created by 001_initial.sql; a hand-built + /// schema must keep that name. /// /// /// Schema-qualified Sample table name. @@ -113,16 +121,11 @@ internal static class SqlAggregateQueries // quality picks (time, kind, num, int, str, status). static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc"; static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode"; - // Each seek is bounded to the window as well as keyed on the pick time: without the bounds the optimizer may pick a - // hash/merge/adaptive join whose inner input is the tag's WHOLE history (TagId = @id only), and parameter sniffing - // then reuses that plan for every tag (review of 910befc8: 43,657 logical reads vs 361 on a tag with 3M rows of - // history outside a 1 d window). A usable pick also re-checks usability (the exact `u` predicate), so a sample a - // HistoryUpdate Replace turned Bad/Empty in between joins nothing ⇒ absent ⇒ BadNoData, never a Bad row's value. - string Window(string a) => reverse - ? $"{a}.SourceTimeUtc > @start AND {a}.SourceTimeUtc <= @end" - : $"{a}.SourceTimeUtc >= @start AND {a}.SourceTimeUtc < @end"; + // An exact-key seek per pick, forced (see remarks). A usable pick also re-checks usability (the exact `u` + // predicate), so a sample a HistoryUpdate Replace turned Bad/Empty in between joins nothing ⇒ absent ⇒ BadNoData, + // never a Bad row's value. string Join(string a, string time, bool usable) => - $"LEFT JOIN {sample} {a} ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time} AND {Window(a)}" + + $"LEFT JOIN {sample} {a} WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc))) ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time}" + (usable ? $" AND {a}.StatusCode >= 0 AND {a}.ValueKind <> 0" : "") + " "; return Classified(sample, reverse) + ", st AS (SELECT b, " + CommonCounts() + ", " + diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/StartEndJoinBackTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/StartEndJoinBackTests.cs index 04e40336..704ba4b0 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/StartEndJoinBackTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/StartEndJoinBackTests.cs @@ -9,58 +9,53 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests; /// /// #683: aggregates the pick times, then fetches each pick by a key join. -/// The joins must be bounded to the window: keyed on the pick time alone, a hash/merge/adaptive join reads the tag's -/// whole history (review of 910befc8: 43,657 logical reads vs 361 on a tag with 3M rows outside a 1 d window). The -/// plan choice depends on estimates, so the test FORCES the worst shape (OPTION (HASH JOIN)) on a tag with long -/// history either side of the window and asserts the statement reads only window-sized pages — and returns exactly -/// what the unhinted plan and the data source return. +/// Every pick must be an exact-key seek whatever plan the optimizer would like, because both alternatives it finds are +/// pathological and plans are cached and reused across reads/tags: +/// +/// keyed on the pick time alone, a hash/merge/adaptive join reads the tag's whole history (review of 910befc8); +/// bounded to the window, a nested-loops RANGE seek over the window per bucket — O(buckets × window rows) — chosen +/// when the statistics understate the table and then reused for a wide read (live gate 9 F1: 1 d @ 1 min 20 s). +/// +/// The tests use a tag with one sample a second over a day plus long history either side, and bound the session's +/// logical reads (sys.dm_exec_sessions.logical_reads before/after — STATISTICS IO messages do not reach +/// SqlClient's InfoMessage here). /// [Collection(SqlHistorianCollection.Name)] public sealed class StartEndJoinBackTests(SqlHistorianFixture fx) { private static readonly CancellationToken Ct = CancellationToken.None; private static DateTime At(double s) => ReadTestDb.At(s); + private const int WindowSeconds = 86_400; private const int HistoryRowsEachSide = 150_000; /// - /// Tag H: one sample a minute over the hour [+0, +3600) of mixed quality (Good, Uncertain, Bad, an outage - /// marker, a string), plus rows a second before +0 and after +3600. + /// Tag H: one sample a second at +i.25 s over [+0, +1 d) of mixed quality (Good, Bad, Uncertain, an + /// outage marker, a String), plus rows a second before +0 and after +1 d. /// - private static async Task<(ReadTestDb Db, int Id, List<(DateTime T, HistorianValue V, uint S)> Window)> SeedAsync(SqlHistorianFixture fx) + private static async Task<(ReadTestDb Db, int Id)> SeedAsync(SqlHistorianFixture fx) { var db = await ReadTestDb.CreateAsync(fx); - var window = new List<(DateTime T, HistorianValue V, uint S)>(); - for (var i = 0; i < 60; i++) - { - var (v, s) = (i % 7) switch - { - 3 => (HistorianValue.FromDouble(i), ReadTestDb.Bad), - 4 => (HistorianValue.FromDouble(i), ReadTestDb.Uncertain), - 5 => (HistorianValue.Empty, ReadTestDb.BadNoCommunication), - 6 => (HistorianValue.FromString("s" + i), ReadTestDb.Good), - _ => (HistorianValue.FromDouble(i), ReadTestDb.Good), - }; - window.Add((At(i * 60 + 7), v, s)); - } - var id = await db.SeedAsync("H", window.ToArray()); - // The history: inserted set-based (a per-row INSERT of 300k rows would dominate the test). + var id = await db.SeedAsync("H", (At(-1e6), HistorianValue.FromDouble(0), ReadTestDb.Good)); + string T(DateTime t) => $"CAST('{t:yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))"; + var cols = "(TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)"; await db.ScalarAsync( - $"WITH n AS (SELECT TOP ({HistoryRowsEachSide}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS x " + + $"WITH n AS (SELECT TOP ({Math.Max(WindowSeconds, HistoryRowsEachSide)}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) - 1 AS x " + "FROM sys.all_objects a CROSS JOIN sys.all_objects b) " + - $"INSERT {db.Conn.Q("Sample")} (TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc) " + - $"SELECT {id}, DATEADD(second, -x, CAST('{At(0):yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n " + - $"UNION ALL SELECT {id}, DATEADD(second, x, CAST('{At(3600):yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n; " + + $"INSERT {db.Conn.Q("Sample")} {cols} " + + $"SELECT {id}, DATEADD(millisecond, 250, DATEADD(second, x, {T(At(0))})), " + + "CASE x % 7 WHEN 5 THEN 0 WHEN 6 THEN 4 ELSE 3 END, CASE WHEN x % 7 IN (5, 6) THEN NULL ELSE x END, NULL, " + + "CASE WHEN x % 7 = 6 THEN N's' + CAST(x AS nvarchar(10)) END, " + + $"CASE x % 7 WHEN 3 THEN {unchecked((int)ReadTestDb.Bad)} WHEN 4 THEN {unchecked((int)ReadTestDb.Uncertain)} " + + $"WHEN 5 THEN {unchecked((int)ReadTestDb.BadNoCommunication)} ELSE 0 END, SYSUTCDATETIME() FROM n WHERE x < {WindowSeconds} " + + $"UNION ALL SELECT {id}, DATEADD(second, -1 - x, {T(At(0))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide} " + + $"UNION ALL SELECT {id}, DATEADD(second, 1 + x, {T(At(WindowSeconds))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide}; " + "SELECT 0"); - await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")}; SELECT 0"); - return (db, id, window); + await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH FULLSCAN; SELECT 0"); + return (db, id); } - /// - /// Runs the StartEnd SQL (optionally with a query hint) and returns its rows as strings plus the session's logical - /// reads during the statement (sys.dm_exec_sessions.logical_reads before/after — STATISTICS IO messages do - /// not reach SqlClient's InfoMessage here). The two probes add a few reads of their own; the bound is far apart. - /// - private static async Task<(List Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint) + /// Runs the StartEnd SQL (optional query hint) and returns its rows as strings plus the session's logical reads. + private static async Task<(List Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint = null) { var sql = SqlAggregateQueries.StartEnd(db.Conn.Q("Sample"), plan.Reversed); if (hint is not null) sql = sql.TrimEnd().TrimEnd(';') + $" OPTION ({hint});"; @@ -72,6 +67,7 @@ public sealed class StartEndJoinBackTests(SqlHistorianFixture fx) } var before = await SessionReadsAsync(); await using var cmd = db.Conn.CreateCommand(c, sql); + cmd.CommandTimeout = 300; cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id }); SqlHistorianConnection.AddDateTime2(cmd, "@start", plan.StartUtc); SqlHistorianConnection.AddDateTime2(cmd, "@end", plan.EndUtc); @@ -80,44 +76,75 @@ public sealed class StartEndJoinBackTests(SqlHistorianFixture fx) var rows = new List(); await using (var r = await cmd.ExecuteReaderAsync(Ct)) while (await r.ReadAsync(Ct)) - rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture)))); + rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : r.GetValue(i) is DateTime d ? d.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture) : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture)))); return (rows, await SessionReadsAsync() - before); } + private static BucketPlan Day(bool reverse, TimeSpan interval) => + reverse ? BucketPlan.Create(At(WindowSeconds), At(0), interval) : BucketPlan.Create(At(0), At(WindowSeconds), interval); + + /// + /// Forward: bucket b = [b·60, b·60 + 60) holds +b·60.25 … +b·60+59.25. Reverse: (b·60, b·60 + 60] holds the + /// same samples (none sits on an edge). + /// + private static void AssertPicks(List rows) + { + rows.Count.ShouldBe(WindowSeconds / 60); + for (var b = 0; b < rows.Count; b++) + { + var f = rows[b].Split('|'); + f[0].ShouldBe(b.ToString(System.Globalization.CultureInfo.InvariantCulture)); + f[1].ShouldBe("60"); // TotalCount + // any-quality first / last times (ordinals 16, 22) + long.Parse(f[16], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 0.25).Ticks); + long.Parse(f[22], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 59.25).Ticks); + } + } + [Theory] [InlineData(false)] [InlineData(true)] - public async Task The_pick_joins_read_only_the_window_even_under_a_forced_hash_join(bool reverse) + public async Task A_plan_compiled_under_understated_statistics_still_seeks_each_pick_by_its_exact_key(bool reverse) { fx.SkipIfUnavailable(); - var (db, id, window) = await SeedAsync(fx); - var plan = reverse - ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(10)) - : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(10)); + var (db, id) = await SeedAsync(fx); + // Live gate 9 F1's recipe: the optimizer believes the table is tiny, compiles on a small read, and the cached plan is + // then reused for a wide many-bucket read. (The schema is unique to this test, so the plan is compiled here.) + await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH ROWCOUNT = 34, PAGECOUNT = 1; SELECT 0"); + var small = reverse ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(1)) : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(1)); + await RunAsync(db, id, small); - var (plain, _) = await RunAsync(db, id, plan, hint: null); - var (hashed, hashReads) = await RunAsync(db, id, plan, hint: "HASH JOIN"); - var (merged, mergeReads) = await RunAsync(db, id, plan, hint: "MERGE JOIN"); + var (rows, reads) = await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1))); - plain.Count.ShouldBe(6); - hashed.ShouldBe(plain); - merged.ShouldBe(plain); - // Measured: 183 session logical reads bounded vs 8,055 with the window bound removed from the joins (300k history - // rows either side; the threshold sits ~5x above the one and ~8x below the other). Every join - // bounded ⇒ each input is a window-range seek, whatever the join algorithm. - hashReads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)"); - hashReads.ShouldBeLessThan(1000, "a hash join's inner input must be bounded to the window"); - mergeReads.ShouldBeLessThan(1000, "a merge join's inner input must be bounded to the window"); + AssertPicks(rows); + reads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)"); + // Measured: 18,645 reads (one window scan + 4 key seeks per bucket); the previous query (4a3b1256, bounded joins, + // no hint) took the window-range-seek-per-bucket plan here: 1,645,845 reads, 23 s — this test failed against it. + reads.ShouldBeLessThan(200_000, "every pick must be an exact-key seek, never a per-bucket window range seek"); + (await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)), "RECOMPILE")).Rows.ShouldBe(rows); + } - // And the picks are right: Start / End are each bucket's first / last sample of any quality. - var src = db.Source(); - var start = await src.ReadProcessedAsync("H", reverse ? At(3600) : At(0), reverse ? At(0) : At(3600), - TimeSpan.FromMinutes(10), HistoryAggregateType.Start, Ct); - var end = await src.ReadProcessedAsync("H", reverse ? At(3600) : At(0), reverse ? At(0) : At(3600), - TimeSpan.FromMinutes(10), HistoryAggregateType.End, Ct); - var buckets = Enumerable.Range(0, 6).Select(b => window.Skip(b * 10).Take(10).ToList()).ToList(); - if (reverse) buckets.Reverse(); - start.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(buckets.Select(b => (DateTime?)b[0].T)); - end.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(buckets.Select(b => (DateTime?)b[^1].T)); + [Fact] + public async Task With_accurate_statistics_the_long_history_outside_the_window_is_never_read() + { + fx.SkipIfUnavailable(); + var (db, id) = await SeedAsync(fx); + var (rows, reads) = await RunAsync(db, id, Day(false, TimeSpan.FromMinutes(1))); + AssertPicks(rows); + reads.ShouldBeGreaterThan(0L); + reads.ShouldBeLessThan(200_000, "the 300k history rows outside the window must not be read"); + } + + [Theory] + [InlineData("HASH JOIN")] + [InlineData("MERGE JOIN")] + public async Task No_hash_or_merge_pick_join_exists_to_fall_back_to(string hint) + { + // The FORCESEEK hint makes the exact-key nested-loops seek the ONLY legal pick join: forcing a hash or merge join + // (the whole-history shape of 910befc8) is refused outright (8622) rather than silently chosen. + fx.SkipIfUnavailable(); + var (db, id) = await SeedAsync(fx); + var ex = await Should.ThrowAsync(() => RunAsync(db, id, Day(false, TimeSpan.FromHours(1)), hint)); + ex.Number.ShouldBe(8622); } }