using System.Data;
using Microsoft.Data.SqlClient;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
///
/// #683: aggregates the pick times, then fetches each pick by a key join.
/// Every pick must be an exact-key seek whatever plan the optimizer would like, because both alternatives it finds are
/// pathological and plans are cached and reused across reads/tags:
///
/// - keyed on the pick time alone, a hash/merge/adaptive join reads the tag's whole history (review of 910befc8);
/// - bounded to the window, a nested-loops RANGE seek over the window per bucket — O(buckets × window rows) — chosen
/// when the statistics understate the table and then reused for a wide read (live gate 9 F1: 1 d @ 1 min 20 s).
///
/// The tests use a tag with one sample a second over a day plus long history either side, and bound the session's
/// logical reads (sys.dm_exec_sessions.logical_reads before/after — STATISTICS IO messages do not reach
/// SqlClient's InfoMessage here).
///
[Collection(SqlHistorianCollection.Name)]
public sealed class StartEndJoinBackTests(SqlHistorianFixture fx)
{
private static readonly CancellationToken Ct = CancellationToken.None;
private static DateTime At(double s) => ReadTestDb.At(s);
private const int WindowSeconds = 86_400;
private const int HistoryRowsEachSide = 150_000;
///
/// Tag H: one sample a second at +i.25 s over [+0, +1 d) of mixed quality (Good, Bad, Uncertain, an
/// outage marker, a String), plus rows a second before +0 and after +1 d.
///
private static async Task<(ReadTestDb Db, int Id)> SeedAsync(SqlHistorianFixture fx)
{
var db = await ReadTestDb.CreateAsync(fx);
var id = await db.SeedAsync("H", (At(-1e6), HistorianValue.FromDouble(0), ReadTestDb.Good));
string T(DateTime t) => $"CAST('{t:yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))";
var cols = "(TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)";
await db.ScalarAsync(
$"WITH n AS (SELECT TOP ({Math.Max(WindowSeconds, HistoryRowsEachSide)}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) - 1 AS x " +
"FROM sys.all_objects a CROSS JOIN sys.all_objects b) " +
$"INSERT {db.Conn.Q("Sample")} {cols} " +
$"SELECT {id}, DATEADD(millisecond, 250, DATEADD(second, x, {T(At(0))})), " +
"CASE x % 7 WHEN 5 THEN 0 WHEN 6 THEN 4 ELSE 3 END, CASE WHEN x % 7 IN (5, 6) THEN NULL ELSE x END, NULL, " +
"CASE WHEN x % 7 = 6 THEN N's' + CAST(x AS nvarchar(10)) END, " +
$"CASE x % 7 WHEN 3 THEN {unchecked((int)ReadTestDb.Bad)} WHEN 4 THEN {unchecked((int)ReadTestDb.Uncertain)} " +
$"WHEN 5 THEN {unchecked((int)ReadTestDb.BadNoCommunication)} ELSE 0 END, SYSUTCDATETIME() FROM n WHERE x < {WindowSeconds} " +
$"UNION ALL SELECT {id}, DATEADD(second, -1 - x, {T(At(0))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide} " +
$"UNION ALL SELECT {id}, DATEADD(second, 1 + x, {T(At(WindowSeconds))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide}; " +
"SELECT 0");
await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH FULLSCAN; SELECT 0");
return (db, id);
}
/// Runs the StartEnd SQL (optional query hint) and returns its rows as strings plus the session's logical reads.
private static async Task<(List Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint = null)
{
var sql = SqlAggregateQueries.StartEnd(db.Conn.Q("Sample"), plan.Reversed);
if (hint is not null) sql = sql.TrimEnd().TrimEnd(';') + $" OPTION ({hint});";
await using var c = await db.Conn.OpenAsync(Ct);
async Task SessionReadsAsync()
{
await using var probe = db.Conn.CreateCommand(c, "SELECT logical_reads FROM sys.dm_exec_sessions WHERE session_id = @@SPID");
return Convert.ToInt64(await probe.ExecuteScalarAsync(Ct));
}
var before = await SessionReadsAsync();
await using var cmd = db.Conn.CreateCommand(c, sql);
cmd.CommandTimeout = 300;
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
SqlHistorianConnection.AddDateTime2(cmd, "@start", plan.StartUtc);
SqlHistorianConnection.AddDateTime2(cmd, "@end", plan.EndUtc);
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
var rows = new List();
await using (var r = await cmd.ExecuteReaderAsync(Ct))
while (await r.ReadAsync(Ct))
rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : r.GetValue(i) is DateTime d ? d.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture) : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture))));
return (rows, await SessionReadsAsync() - before);
}
private static BucketPlan Day(bool reverse, TimeSpan interval) =>
reverse ? BucketPlan.Create(At(WindowSeconds), At(0), interval) : BucketPlan.Create(At(0), At(WindowSeconds), interval);
///
/// Forward: bucket b = [b·60, b·60 + 60) holds +b·60.25 … +b·60+59.25. Reverse: (b·60, b·60 + 60] holds the
/// same samples (none sits on an edge).
///
private static void AssertPicks(List rows)
{
rows.Count.ShouldBe(WindowSeconds / 60);
for (var b = 0; b < rows.Count; b++)
{
var f = rows[b].Split('|');
f[0].ShouldBe(b.ToString(System.Globalization.CultureInfo.InvariantCulture));
f[1].ShouldBe("60"); // TotalCount
// any-quality first / last times (ordinals 16, 22)
long.Parse(f[16], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 0.25).Ticks);
long.Parse(f[22], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 59.25).Ticks);
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task A_plan_compiled_under_understated_statistics_still_seeks_each_pick_by_its_exact_key(bool reverse)
{
fx.SkipIfUnavailable();
var (db, id) = await SeedAsync(fx);
// Live gate 9 F1's recipe: the optimizer believes the table is tiny, compiles on a small read, and the cached plan is
// then reused for a wide many-bucket read. (The schema is unique to this test, so the plan is compiled here.)
await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH ROWCOUNT = 34, PAGECOUNT = 1; SELECT 0");
var small = reverse ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(1)) : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(1));
await RunAsync(db, id, small);
var (rows, reads) = await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)));
AssertPicks(rows);
reads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)");
// Measured: 18,645 reads (one window scan + 4 key seeks per bucket); the previous query (4a3b1256, bounded joins,
// no hint) took the window-range-seek-per-bucket plan here: 1,645,845 reads, 23 s — this test failed against it.
reads.ShouldBeLessThan(200_000, "every pick must be an exact-key seek, never a per-bucket window range seek");
(await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)), "RECOMPILE")).Rows.ShouldBe(rows);
}
[Fact]
public async Task With_accurate_statistics_the_long_history_outside_the_window_is_never_read()
{
fx.SkipIfUnavailable();
var (db, id) = await SeedAsync(fx);
var (rows, reads) = await RunAsync(db, id, Day(false, TimeSpan.FromMinutes(1)));
AssertPicks(rows);
reads.ShouldBeGreaterThan(0L);
reads.ShouldBeLessThan(200_000, "the 300k history rows outside the window must not be read");
}
[Theory]
[InlineData("HASH JOIN")]
[InlineData("MERGE JOIN")]
public async Task No_hash_or_merge_pick_join_exists_to_fall_back_to(string hint)
{
// The FORCESEEK hint makes the exact-key nested-loops seek the ONLY legal pick join: forcing a hash or merge join
// (the whole-history shape of 910befc8) is refused outright (8622) rather than silently chosen.
fx.SkipIfUnavailable();
var (db, id) = await SeedAsync(fx);
var ex = await Should.ThrowAsync(() => RunAsync(db, id, Day(false, TimeSpan.FromHours(1)), hint));
ex.Number.ShouldBe(8622);
}
}