lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/StartEndJoinBackTests.cs

using System.Data;
using Microsoft.Data.SqlClient;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;

namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;

/// <summary>
///     #683: <see cref="SqlAggregateQueries.StartEnd"/> aggregates the pick times, then fetches each pick by a key join.
///     Every pick must be an exact-key seek whatever plan the optimizer would like, because both alternatives it finds are
///     pathological and plans are cached and reused across reads/tags:
///     <list type="bullet">
///       <item>keyed on the pick time alone, a hash/merge/adaptive join reads the tag's whole history (review of 910befc8);</item>
///       <item>bounded to the window, a nested-loops RANGE seek over the window per bucket — O(buckets × window rows) — chosen
///         when the statistics understate the table and then reused for a wide read (live gate 9 F1: 1 d @ 1 min 20 s).</item>
///     </list>
///     The tests use a tag with one sample a second over a day plus long history either side, and bound the session's
///     logical reads (<c>sys.dm_exec_sessions.logical_reads</c> before/after — STATISTICS IO messages do not reach
///     SqlClient's InfoMessage here).
/// </summary>
[Collection(SqlHistorianCollection.Name)]
public sealed class StartEndJoinBackTests(SqlHistorianFixture fx)
{
    private static readonly CancellationToken Ct = CancellationToken.None;
    private static DateTime At(double s) => ReadTestDb.At(s);
    private const int WindowSeconds = 86_400;
    private const int HistoryRowsEachSide = 150_000;

    /// <summary>
    ///     Tag <c>H</c>: one sample a second at <c>+i.25 s</c> over [+0, +1 d) of mixed quality (Good, Bad, Uncertain, an
    ///     outage marker, a String), plus <see cref="HistoryRowsEachSide"/> rows a second before +0 and after +1 d.
    /// </summary>
    private static async Task<(ReadTestDb Db, int Id)> SeedAsync(SqlHistorianFixture fx)
    {
        var db = await ReadTestDb.CreateAsync(fx);
        var id = await db.SeedAsync("H", (At(-1e6), HistorianValue.FromDouble(0), ReadTestDb.Good));
        string T(DateTime t) => $"CAST('{t:yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))";
        var cols = "(TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)";
        await db.ScalarAsync(
            $"WITH n AS (SELECT TOP ({Math.Max(WindowSeconds, HistoryRowsEachSide)}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) - 1 AS x " +
            "FROM sys.all_objects a CROSS JOIN sys.all_objects b) " +
            $"INSERT {db.Conn.Q("Sample")} {cols} " +
            $"SELECT {id}, DATEADD(millisecond, 250, DATEADD(second, x, {T(At(0))})), " +
            "CASE x % 7 WHEN 5 THEN 0 WHEN 6 THEN 4 ELSE 3 END, CASE WHEN x % 7 IN (5, 6) THEN NULL ELSE x END, NULL, " +
            "CASE WHEN x % 7 = 6 THEN N's' + CAST(x AS nvarchar(10)) END, " +
            $"CASE x % 7 WHEN 3 THEN {unchecked((int)ReadTestDb.Bad)} WHEN 4 THEN {unchecked((int)ReadTestDb.Uncertain)} " +
            $"WHEN 5 THEN {unchecked((int)ReadTestDb.BadNoCommunication)} ELSE 0 END, SYSUTCDATETIME() FROM n WHERE x < {WindowSeconds} " +
            $"UNION ALL SELECT {id}, DATEADD(second, -1 - x, {T(At(0))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide} " +
            $"UNION ALL SELECT {id}, DATEADD(second, 1 + x, {T(At(WindowSeconds))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide}; " +
            "SELECT 0");
        await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH FULLSCAN; SELECT 0");
        return (db, id);
    }

    /// <summary>Runs the StartEnd SQL (optional query hint) and returns its rows as strings plus the session's logical reads.</summary>
    private static async Task<(List<string> Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint = null)
    {
        var sql = SqlAggregateQueries.StartEnd(db.Conn.Q("Sample"), plan.Reversed);
        if (hint is not null) sql = sql.TrimEnd().TrimEnd(';') + $" OPTION ({hint});";
        await using var c = await db.Conn.OpenAsync(Ct);
        async Task<long> SessionReadsAsync()
        {
            await using var probe = db.Conn.CreateCommand(c, "SELECT logical_reads FROM sys.dm_exec_sessions WHERE session_id = @@SPID");
            return Convert.ToInt64(await probe.ExecuteScalarAsync(Ct));
        }
        var before = await SessionReadsAsync();
        await using var cmd = db.Conn.CreateCommand(c, sql);
        cmd.CommandTimeout = 300;
        cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
        SqlHistorianConnection.AddDateTime2(cmd, "@start", plan.StartUtc);
        SqlHistorianConnection.AddDateTime2(cmd, "@end", plan.EndUtc);
        cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
        cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
        var rows = new List<string>();
        await using (var r = await cmd.ExecuteReaderAsync(Ct))
            while (await r.ReadAsync(Ct))
                rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : r.GetValue(i) is DateTime d ? d.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture) : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture))));
        return (rows, await SessionReadsAsync() - before);
    }

    private static BucketPlan Day(bool reverse, TimeSpan interval) =>
        reverse ? BucketPlan.Create(At(WindowSeconds), At(0), interval) : BucketPlan.Create(At(0), At(WindowSeconds), interval);

    /// <summary>
    ///     Forward: bucket <c>b</c> = [b·60, b·60 + 60) holds +b·60.25 … +b·60+59.25. Reverse: (b·60, b·60 + 60] holds the
    ///     same samples (none sits on an edge).
    /// </summary>
    private static void AssertPicks(List<string> rows)
    {
        rows.Count.ShouldBe(WindowSeconds / 60);
        for (var b = 0; b < rows.Count; b++)
        {
            var f = rows[b].Split('|');
            f[0].ShouldBe(b.ToString(System.Globalization.CultureInfo.InvariantCulture));
            f[1].ShouldBe("60"); // TotalCount
            // any-quality first / last times (ordinals 16, 22)
            long.Parse(f[16], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 0.25).Ticks);
            long.Parse(f[22], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 59.25).Ticks);
        }
    }

    [Theory]
    [InlineData(false)]
    [InlineData(true)]
    public async Task A_plan_compiled_under_understated_statistics_still_seeks_each_pick_by_its_exact_key(bool reverse)
    {
        fx.SkipIfUnavailable();
        var (db, id) = await SeedAsync(fx);
        // Live gate 9 F1's recipe: the optimizer believes the table is tiny, compiles on a small read, and the cached plan is
        // then reused for a wide many-bucket read. (The schema is unique to this test, so the plan is compiled here.)
        await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH ROWCOUNT = 34, PAGECOUNT = 1; SELECT 0");
        var small = reverse ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(1)) : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(1));
        await RunAsync(db, id, small);

        var (rows, reads) = await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)));

        AssertPicks(rows);
        reads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)");
        // Measured: 18,645 reads (one window scan + 4 key seeks per bucket); the previous query (4a3b1256, bounded joins,
        // no hint) took the window-range-seek-per-bucket plan here: 1,645,845 reads, 23 s — this test failed against it.
        reads.ShouldBeLessThan(200_000, "every pick must be an exact-key seek, never a per-bucket window range seek");
        (await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)), "RECOMPILE")).Rows.ShouldBe(rows);
    }

    [Fact]
    public async Task With_accurate_statistics_the_long_history_outside_the_window_is_never_read()
    {
        fx.SkipIfUnavailable();
        var (db, id) = await SeedAsync(fx);
        var (rows, reads) = await RunAsync(db, id, Day(false, TimeSpan.FromMinutes(1)));
        AssertPicks(rows);
        reads.ShouldBeGreaterThan(0L);
        reads.ShouldBeLessThan(200_000, "the 300k history rows outside the window must not be read");
    }

    [Theory]
    [InlineData("HASH JOIN")]
    [InlineData("MERGE JOIN")]
    public async Task No_hash_or_merge_pick_join_exists_to_fall_back_to(string hint)
    {
        // The FORCESEEK hint makes the exact-key nested-loops seek the ONLY legal pick join: forcing a hash or merge join
        // (the whole-history shape of 910befc8) is refused outright (8622) rather than silently chosen.
        fx.SkipIfUnavailable();
        var (db, id) = await SeedAsync(fx);
        var ex = await Should.ThrowAsync<SqlException>(() => RunAsync(db, id, Day(false, TimeSpan.FromHours(1)), hint));
        ex.Number.ShouldBe(8622);
    }
}