using System.Data; using Microsoft.Data.SqlClient; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests; /// /// #683: aggregates the pick times, then fetches each pick by a key join. /// The joins must be bounded to the window: keyed on the pick time alone, a hash/merge/adaptive join reads the tag's /// whole history (review of 910befc8: 43,657 logical reads vs 361 on a tag with 3M rows outside a 1 d window). The /// plan choice depends on estimates, so the test FORCES the worst shape (OPTION (HASH JOIN)) on a tag with long /// history either side of the window and asserts the statement reads only window-sized pages — and returns exactly /// what the unhinted plan and the data source return. /// [Collection(SqlHistorianCollection.Name)] public sealed class StartEndJoinBackTests(SqlHistorianFixture fx) { private static readonly CancellationToken Ct = CancellationToken.None; private static DateTime At(double s) => ReadTestDb.At(s); private const int HistoryRowsEachSide = 150_000; /// /// Tag H: one sample a minute over the hour [+0, +3600) of mixed quality (Good, Uncertain, Bad, an outage /// marker, a string), plus rows a second before +0 and after +3600. /// private static async Task<(ReadTestDb Db, int Id, List<(DateTime T, HistorianValue V, uint S)> Window)> SeedAsync(SqlHistorianFixture fx) { var db = await ReadTestDb.CreateAsync(fx); var window = new List<(DateTime T, HistorianValue V, uint S)>(); for (var i = 0; i < 60; i++) { var (v, s) = (i % 7) switch { 3 => (HistorianValue.FromDouble(i), ReadTestDb.Bad), 4 => (HistorianValue.FromDouble(i), ReadTestDb.Uncertain), 5 => (HistorianValue.Empty, ReadTestDb.BadNoCommunication), 6 => (HistorianValue.FromString("s" + i), ReadTestDb.Good), _ => (HistorianValue.FromDouble(i), ReadTestDb.Good), }; window.Add((At(i * 60 + 7), v, s)); } var id = await db.SeedAsync("H", window.ToArray()); // The history: inserted set-based (a per-row INSERT of 300k rows would dominate the test). await db.ScalarAsync( $"WITH n AS (SELECT TOP ({HistoryRowsEachSide}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS x " + "FROM sys.all_objects a CROSS JOIN sys.all_objects b) " + $"INSERT {db.Conn.Q("Sample")} (TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc) " + $"SELECT {id}, DATEADD(second, -x, CAST('{At(0):yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n " + $"UNION ALL SELECT {id}, DATEADD(second, x, CAST('{At(3600):yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n; " + "SELECT 0"); await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")}; SELECT 0"); return (db, id, window); } /// /// Runs the StartEnd SQL (optionally with a query hint) and returns its rows as strings plus the session's logical /// reads during the statement (sys.dm_exec_sessions.logical_reads before/after — STATISTICS IO messages do /// not reach SqlClient's InfoMessage here). The two probes add a few reads of their own; the bound is far apart. /// private static async Task<(List Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint) { var sql = SqlAggregateQueries.StartEnd(db.Conn.Q("Sample"), plan.Reversed); if (hint is not null) sql = sql.TrimEnd().TrimEnd(';') + $" OPTION ({hint});"; await using var c = await db.Conn.OpenAsync(Ct); async Task SessionReadsAsync() { await using var probe = db.Conn.CreateCommand(c, "SELECT logical_reads FROM sys.dm_exec_sessions WHERE session_id = @@SPID"); return Convert.ToInt64(await probe.ExecuteScalarAsync(Ct)); } var before = await SessionReadsAsync(); await using var cmd = db.Conn.CreateCommand(c, sql); cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id }); SqlHistorianConnection.AddDateTime2(cmd, "@start", plan.StartUtc); SqlHistorianConnection.AddDateTime2(cmd, "@end", plan.EndUtc); cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks }); cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks }); var rows = new List(); await using (var r = await cmd.ExecuteReaderAsync(Ct)) while (await r.ReadAsync(Ct)) rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture)))); return (rows, await SessionReadsAsync() - before); } [Theory] [InlineData(false)] [InlineData(true)] public async Task The_pick_joins_read_only_the_window_even_under_a_forced_hash_join(bool reverse) { fx.SkipIfUnavailable(); var (db, id, window) = await SeedAsync(fx); var plan = reverse ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(10)) : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(10)); var (plain, _) = await RunAsync(db, id, plan, hint: null); var (hashed, hashReads) = await RunAsync(db, id, plan, hint: "HASH JOIN"); var (merged, mergeReads) = await RunAsync(db, id, plan, hint: "MERGE JOIN"); plain.Count.ShouldBe(6); hashed.ShouldBe(plain); merged.ShouldBe(plain); // Measured: 183 session logical reads bounded vs 8,055 with the window bound removed from the joins (300k history // rows either side; the threshold sits ~5x above the one and ~8x below the other). Every join // bounded ⇒ each input is a window-range seek, whatever the join algorithm. hashReads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)"); hashReads.ShouldBeLessThan(1000, "a hash join's inner input must be bounded to the window"); mergeReads.ShouldBeLessThan(1000, "a merge join's inner input must be bounded to the window"); // And the picks are right: Start / End are each bucket's first / last sample of any quality. var src = db.Source(); var start = await src.ReadProcessedAsync("H", reverse ? At(3600) : At(0), reverse ? At(0) : At(3600), TimeSpan.FromMinutes(10), HistoryAggregateType.Start, Ct); var end = await src.ReadProcessedAsync("H", reverse ? At(3600) : At(0), reverse ? At(0) : At(3600), TimeSpan.FromMinutes(10), HistoryAggregateType.End, Ct); var buckets = Enumerable.Range(0, 6).Select(b => window.Skip(b * 10).Take(10).ToList()).ToList(); if (reverse) buckets.Reverse(); start.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(buckets.Select(b => (DateTime?)b[0].T)); end.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(buckets.Select(b => (DateTime?)b[^1].T)); } }