using System.Data; using Microsoft.Data.SqlClient; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests; /// /// #683: aggregates the pick times, then fetches each pick by a key join. /// Every pick must be an exact-key seek whatever plan the optimizer would like, because both alternatives it finds are /// pathological and plans are cached and reused across reads/tags: /// /// keyed on the pick time alone, a hash/merge/adaptive join reads the tag's whole history (review of 910befc8); /// bounded to the window, a nested-loops RANGE seek over the window per bucket — O(buckets × window rows) — chosen /// when the statistics understate the table and then reused for a wide read (live gate 9 F1: 1 d @ 1 min 20 s). /// /// The tests use a tag with one sample a second over a day plus long history either side, and bound the session's /// logical reads (sys.dm_exec_sessions.logical_reads before/after — STATISTICS IO messages do not reach /// SqlClient's InfoMessage here). /// [Collection(SqlHistorianCollection.Name)] public sealed class StartEndJoinBackTests(SqlHistorianFixture fx) { private static readonly CancellationToken Ct = CancellationToken.None; private static DateTime At(double s) => ReadTestDb.At(s); private const int WindowSeconds = 86_400; private const int HistoryRowsEachSide = 150_000; /// /// Tag H: one sample a second at +i.25 s over [+0, +1 d) of mixed quality (Good, Bad, Uncertain, an /// outage marker, a String), plus rows a second before +0 and after +1 d. /// private static async Task<(ReadTestDb Db, int Id)> SeedAsync(SqlHistorianFixture fx) { var db = await ReadTestDb.CreateAsync(fx); var id = await db.SeedAsync("H", (At(-1e6), HistorianValue.FromDouble(0), ReadTestDb.Good)); string T(DateTime t) => $"CAST('{t:yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))"; var cols = "(TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)"; await db.ScalarAsync( $"WITH n AS (SELECT TOP ({Math.Max(WindowSeconds, HistoryRowsEachSide)}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) - 1 AS x " + "FROM sys.all_objects a CROSS JOIN sys.all_objects b) " + $"INSERT {db.Conn.Q("Sample")} {cols} " + $"SELECT {id}, DATEADD(millisecond, 250, DATEADD(second, x, {T(At(0))})), " + "CASE x % 7 WHEN 5 THEN 0 WHEN 6 THEN 4 ELSE 3 END, CASE WHEN x % 7 IN (5, 6) THEN NULL ELSE x END, NULL, " + "CASE WHEN x % 7 = 6 THEN N's' + CAST(x AS nvarchar(10)) END, " + $"CASE x % 7 WHEN 3 THEN {unchecked((int)ReadTestDb.Bad)} WHEN 4 THEN {unchecked((int)ReadTestDb.Uncertain)} " + $"WHEN 5 THEN {unchecked((int)ReadTestDb.BadNoCommunication)} ELSE 0 END, SYSUTCDATETIME() FROM n WHERE x < {WindowSeconds} " + $"UNION ALL SELECT {id}, DATEADD(second, -1 - x, {T(At(0))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide} " + $"UNION ALL SELECT {id}, DATEADD(second, 1 + x, {T(At(WindowSeconds))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide}; " + "SELECT 0"); await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH FULLSCAN; SELECT 0"); return (db, id); } /// Runs the StartEnd SQL (optional query hint) and returns its rows as strings plus the session's logical reads. private static async Task<(List Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint = null) { var sql = SqlAggregateQueries.StartEnd(db.Conn.Q("Sample"), plan.Reversed); if (hint is not null) sql = sql.TrimEnd().TrimEnd(';') + $" OPTION ({hint});"; await using var c = await db.Conn.OpenAsync(Ct); async Task SessionReadsAsync() { await using var probe = db.Conn.CreateCommand(c, "SELECT logical_reads FROM sys.dm_exec_sessions WHERE session_id = @@SPID"); return Convert.ToInt64(await probe.ExecuteScalarAsync(Ct)); } var before = await SessionReadsAsync(); await using var cmd = db.Conn.CreateCommand(c, sql); cmd.CommandTimeout = 300; cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id }); SqlHistorianConnection.AddDateTime2(cmd, "@start", plan.StartUtc); SqlHistorianConnection.AddDateTime2(cmd, "@end", plan.EndUtc); cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks }); cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks }); var rows = new List(); await using (var r = await cmd.ExecuteReaderAsync(Ct)) while (await r.ReadAsync(Ct)) rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : r.GetValue(i) is DateTime d ? d.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture) : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture)))); return (rows, await SessionReadsAsync() - before); } private static BucketPlan Day(bool reverse, TimeSpan interval) => reverse ? BucketPlan.Create(At(WindowSeconds), At(0), interval) : BucketPlan.Create(At(0), At(WindowSeconds), interval); /// /// Forward: bucket b = [b·60, b·60 + 60) holds +b·60.25 … +b·60+59.25. Reverse: (b·60, b·60 + 60] holds the /// same samples (none sits on an edge). /// private static void AssertPicks(List rows) { rows.Count.ShouldBe(WindowSeconds / 60); for (var b = 0; b < rows.Count; b++) { var f = rows[b].Split('|'); f[0].ShouldBe(b.ToString(System.Globalization.CultureInfo.InvariantCulture)); f[1].ShouldBe("60"); // TotalCount // any-quality first / last times (ordinals 16, 22) long.Parse(f[16], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 0.25).Ticks); long.Parse(f[22], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 59.25).Ticks); } } [Theory] [InlineData(false)] [InlineData(true)] public async Task A_plan_compiled_under_understated_statistics_still_seeks_each_pick_by_its_exact_key(bool reverse) { fx.SkipIfUnavailable(); var (db, id) = await SeedAsync(fx); // Live gate 9 F1's recipe: the optimizer believes the table is tiny, compiles on a small read, and the cached plan is // then reused for a wide many-bucket read. (The schema is unique to this test, so the plan is compiled here.) await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH ROWCOUNT = 34, PAGECOUNT = 1; SELECT 0"); var small = reverse ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(1)) : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(1)); await RunAsync(db, id, small); var (rows, reads) = await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1))); AssertPicks(rows); reads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)"); // Measured: 18,645 reads (one window scan + 4 key seeks per bucket); the previous query (4a3b1256, bounded joins, // no hint) took the window-range-seek-per-bucket plan here: 1,645,845 reads, 23 s — this test failed against it. reads.ShouldBeLessThan(200_000, "every pick must be an exact-key seek, never a per-bucket window range seek"); (await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)), "RECOMPILE")).Rows.ShouldBe(rows); } [Fact] public async Task With_accurate_statistics_the_long_history_outside_the_window_is_never_read() { fx.SkipIfUnavailable(); var (db, id) = await SeedAsync(fx); var (rows, reads) = await RunAsync(db, id, Day(false, TimeSpan.FromMinutes(1))); AssertPicks(rows); reads.ShouldBeGreaterThan(0L); reads.ShouldBeLessThan(200_000, "the 300k history rows outside the window must not be read"); } [Theory] [InlineData("HASH JOIN")] [InlineData("MERGE JOIN")] public async Task No_hash_or_merge_pick_join_exists_to_fall_back_to(string hint) { // The FORCESEEK hint makes the exact-key nested-loops seek the ONLY legal pick join: forcing a hash or merge join // (the whole-history shape of 910befc8) is refused outright (8622) rather than silently chosen. fx.SkipIfUnavailable(); var (db, id) = await SeedAsync(fx); var ex = await Should.ThrowAsync(() => RunAsync(db, id, Day(false, TimeSpan.FromHours(1)), hint)); ex.Number.ShouldBe(8622); } }