dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(historian): index the QuestDB sample tag column; read the data range before the buckets (#683)

Browse Source 
QuestDB: {p}sample.tag is now SYMBOL INDEX. The processed-read data-range
lookups (EarliestSample / LatestSample) were partition scans of the unindexed
symbol; on 53.6M rows / 120 tags / 61 daily partitions the indexed seeks are
1-1.5 ms warm / 2-4 ms cold whatever the tag's history (unindexed: up to 11 ms
warm, 14 ms cold; cost grew with the table). The bootstrapper adds the index to a
pre-#683 table (ALTER ... ADD INDEX, a WAL txn, idempotent, ~3 s for 68M rows);
AutoCreateSchema=false only logs the statement. Disk +~12%; write apply time
within noise; main per-sample/raw/time-weighted queries unchanged or faster.

Rejected by measurement/analysis: bounded existence probes (still full scans
when a side is absent), LATEST ON (slowest unindexed), per-tag first/last side
table (cannot track TTL drops or deletes INSERT-only).

Both backends now read the data range before the bucket statement: an append
racing the read reaches the buckets but not the range, so it can no longer
produce a Good Count of 0 for an interval that has samples.

Tests: TagIndexTests (new-table index, bound-param seek uses the index,
tombstones excluded / outage marker counted, pre-#683 migration with identical
processed results across 10 aggregates x 4 windows x both directions).
This commit is contained in:
Joseph Doherty
2026-10-12 09:40:28 -04:00
parent 197246c3c1
commit 549d2d68b3
9 changed files with 331 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs
+34
View File
@@ -62,6 +62,40 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
return new QuestDbConformanceBackend(fx, prefix, o, conn, boot);
}
/// <summary>
/// A backend whose sample table has the pre-#683 shape — <c>tag</c> NOT indexed — as a deployment created before
/// the index existed has it. Its bootstrapper runs with <c>AutoCreateSchema=false</c>, so it leaves the table as is.
/// </summary>
/// <param name="fx">The shared fixture.</param>
/// <returns>The backend.</returns>
public static async Task<QuestDbConformanceBackend> CreatePre683Async(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var prefix = fx.UniquePrefix();
var o = fx.Options(prefix, autoCreate: false);
var conn = new QuestDbConnection(o);
await using (var c = await conn.OpenAsync(CancellationToken.None))
{
foreach (var ddl in QuestDbTables.CreateAll(prefix, null))
{
await using var cmd = conn.CreateCommand(c, ddl.Replace("tag SYMBOL INDEX", "tag SYMBOL", StringComparison.Ordinal));
await cmd.ExecuteNonQueryAsync();
}
}
var boot = new QuestDbSchemaBootstrapper(conn, o);
await boot.EnsureAsync(CancellationToken.None);
return new QuestDbConformanceBackend(fx, prefix, o, conn, boot);
}
/// <summary>Whether this instance's sample table has its <c>tag</c> column indexed (#683).</summary>
/// <returns>The <c>table_columns()</c> flag.</returns>
public async Task<bool> TagIndexedAsync()
{
await using var c = await Conn.OpenAsync(CancellationToken.None);
await using var cmd = Conn.CreateCommand(c, QuestDbTables.TagIndexedQuery(Conn.SampleTable));
return Convert.ToBoolean(await cmd.ExecuteScalarAsync());
}
/// <inheritdoc/>
public long ResolutionTicks => 10;
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/TagIndexTests.cs
+160
View File
@@ -0,0 +1,160 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests;
/// <summary>
/// #683: the sample table's <c>tag</c> symbol index — created with new tables, added by the bootstrapper to a table
/// from before it existed — and the data-range lookups it serves (the start/end-of-data Partial bits and Count's
/// in-range empty interval), which must answer exactly as they did from the unindexed scan: tombstones excluded, an
/// outage marker counted as a sample.
/// </summary>
[Collection(QuestDbCollection.Name)]
public sealed class TagIndexTests(QuestDbFixture fx)
{
private static readonly CancellationToken Ct = CancellationToken.None;
private const uint GoodCalc = 0x00000401u;
private const uint NoDataCalc = 0x809B0401u;
private static DateTime At(double s) => ConformanceData.At(s);
/// <summary>
/// Tag <c>R</c>: an outage marker at +0 (its earliest sample), 1 at +100, 2 at +200 (its latest live sample), a row
/// at +300 replaced by a tombstone and a tombstone alone at +400 — neither may extend the data range.
/// </summary>
private static async Task SeedAsync(QuestDbConformanceBackend b)
{
await b.SeedAsync("R",
(At(0), HistorianValue.Empty, ConformanceData.BadNoCommunication),
(At(100), HistorianValue.FromDouble(1), ConformanceData.Good),
(At(200), HistorianValue.FromDouble(2), ConformanceData.Good),
(At(300), HistorianValue.FromDouble(3), ConformanceData.Good));
await b.InsertRowsAsync("R",
[
(At(300), HistorianValue.Empty, HistorianStatusCodes.BadNoData, true),
(At(400), HistorianValue.Empty, HistorianStatusCodes.BadNoData, true),
]);
}
private static readonly HistoryAggregateType[] Aggregates =
[
HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, HistoryAggregateType.Average,
HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
];
private static readonly (double From, double To)[] Windows = [(-100, 100), (50, 450), (150, 250), (250, 500)];
/// <summary>Every per-sample aggregate over every window, forward and reverse, as comparable tuples.</summary>
private static async Task<List<string>> ReadMatrixAsync(QuestDbConformanceBackend b)
{
var src = b.CreateDataSource();
var all = new List<string>();
foreach (var a in Aggregates)
foreach (var (from, to) in Windows)
foreach (var reverse in new[] { false, true })
{
var r = reverse
? await src.ReadProcessedAsync("R", At(to), At(from), TimeSpan.FromSeconds(50), a, Ct)
: await src.ReadProcessedAsync("R", At(from), At(to), TimeSpan.FromSeconds(50), a, Ct);
all.AddRange(r.Samples.Select(s =>
$"{a} [{from},{to}) rev={reverse} {s.SourceTimestampUtc:O} {s.Value} 0x{s.StatusCode:X8}"));
}
return all;
}
private static async Task AwaitIndexedAsync(QuestDbConformanceBackend b)
{
var deadline = DateTime.UtcNow.AddSeconds(15);
while (!await b.TagIndexedAsync())
{
if (DateTime.UtcNow > deadline) (await b.TagIndexedAsync()).ShouldBeTrue("the ADD INDEX was never applied");
await Task.Delay(50); // the ALTER is a WAL transaction: visible once applied
}
}
[Fact]
public async Task A_new_sample_table_is_created_with_the_tag_index()
{
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
(await b.TagIndexedAsync()).ShouldBeTrue();
}
[Fact]
public async Task The_data_range_seeks_the_tag_index()
{
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
foreach (var sql in new[]
{
QuestDbAggregateQueries.EarliestSample(b.Conn.SampleTable),
QuestDbAggregateQueries.LatestSample(b.Conn.SampleTable),
})
{
await using var c = await b.Conn.OpenAsync(Ct);
await using var cmd = b.Conn.CreateCommand(c, "EXPLAIN " + sql);
QuestDbConnection.BindText(cmd, "R");
QuestDbConnection.BindTimestamp(cmd, At(0));
var plan = new List<string>();
await using var r = await cmd.ExecuteReaderAsync(Ct);
while (await r.ReadAsync(Ct)) plan.Add(r.GetString(0));
string.Join("\n", plan).ShouldContain("Index", Case.Sensitive, "a bound-parameter tag seek must use the symbol index");
}
}
[Fact]
public async Task The_data_range_excludes_tombstones_and_counts_outage_markers()
{
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
await SeedAsync(b);
var src = b.CreateDataSource();
// [+50, +450) by 50 s: data range [+0 outage marker, +200]; the +300/+400 tombstones must not extend it.
var count = (await src.ReadProcessedAsync("R", At(50), At(450), TimeSpan.FromSeconds(50), HistoryAggregateType.Count, Ct)).Samples;
count.Select(s => s.Value is null ? (long?)null : Convert.ToInt64(s.Value)).ShouldBe([0L, 1L, 0L, 1L, null, null, null, null]);
count.Select(s => s.StatusCode).ShouldBe(
[
GoodCalc, GoodCalc, GoodCalc,
GoodCalc | HistorianInfoBits.HistorianPartial, // the data ends inside [+200, +250)
NoDataCalc, NoDataCalc, NoDataCalc, NoDataCalc,
]);
// [-100, +100): the outage marker at +0 starts the data range, so [-50, 0] reaches it (edges inclusive) ⇒ 0, Good.
var early = (await src.ReadProcessedAsync("R", At(-100), At(100), TimeSpan.FromSeconds(50), HistoryAggregateType.Count, Ct)).Samples;
early[0].StatusCode.ShouldBe(NoDataCalc);
Convert.ToInt64(early[1].Value).ShouldBe(0L);
early[1].StatusCode.ShouldBe(GoodCalc);
Convert.ToInt64(early[3].Value).ShouldBe(0L);
early[3].StatusCode.ShouldBe(GoodCalc);
}
[Fact]
public async Task A_pre_683_table_gains_the_index_on_boot_and_every_processed_read_answers_the_same()
{
await using var b = await QuestDbConformanceBackend.CreatePre683Async(fx);
(await b.TagIndexedAsync()).ShouldBeFalse();
await SeedAsync(b);
var unindexed = await ReadMatrixAsync(b);
// AutoCreateSchema=false never alters (it only logs the statement to run).
await new QuestDbSchemaBootstrapper(b.Conn, b.Options).EnsureAsync(Ct);
await Task.Delay(300);
(await b.TagIndexedAsync()).ShouldBeFalse();
// AutoCreateSchema=true migrates; a second, concurrent-style re-send on the same column is harmless.
var migrate = fx.Options(b.Options.TablePrefix);
await new QuestDbSchemaBootstrapper(new QuestDbConnection(migrate), migrate).EnsureAsync(Ct);
await new QuestDbSchemaBootstrapper(new QuestDbConnection(migrate), migrate).EnsureAsync(Ct);
await AwaitIndexedAsync(b);
await b.Wal.WaitAsync(b.Conn.SampleTable, Ct);
var indexed = await ReadMatrixAsync(b);
indexed.ShouldBe(unindexed);
// Still writable and readable after the migration (the table was not suspended by the double ALTER).
await b.SeedAsync("R", (At(500), HistorianValue.FromDouble(5), ConformanceData.Good));
(await b.ReadBackAsync("R")).Count.ShouldBe(4);
}
}
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbTablesTests.cs
+11 -2
View File
@@ -8,17 +8,26 @@ public sealed class QuestDbTablesTests
[Fact]
public void Sample_ddl_without_ttl_is_pinned() =>
QuestDbTables.CreateSample("hist_", null).ShouldBe(
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
"TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, tag)");
[Fact]
public void Sample_ddl_with_ttl_puts_ttl_between_partition_and_wal() =>
QuestDbTables.CreateSample("hist_", 7).ShouldBe(
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
"TIMESTAMP(ts) PARTITION BY DAY TTL 7 DAYS WAL DEDUP UPSERT KEYS(ts, tag)");
[Fact]
public void Tag_index_migration_statements_are_pinned()
{
// #683: the migration for a sample table created before the tag index existed, and its probe.
QuestDbTables.AddTagIndex("hist_sample").ShouldBe("ALTER TABLE hist_sample ALTER COLUMN tag ADD INDEX");
QuestDbTables.TagIndexedQuery("hist_sample").ShouldBe(
"SELECT indexed FROM table_columns('hist_sample') WHERE \"column\" = 'tag'");
}
[Fact]
public void Event_ddl_is_pinned()
{