perf(historian): index QuestDB event.source and sample_modified.tag (lmxopcua#746)

Event reads filtered on source and every modified-value read filtered on tag
scanned that symbol column in every partition of the window; only
sample.tag carried an index (#683).

Both columns are now SYMBOL INDEX in the DDL, and the bootstrapper's index
migration is generalised over QuestDbTables.SymbolIndexes (sample.tag,
event.source, sample_modified.tag): a missing index is added with the same
idempotent ALTER ... ADD INDEX, Warning-only when AutoCreateSchema is false
or the ALTER is refused. Integration tests: new tables carry all three; a
pre-index table set gains them on boot and event reads answer the same.

This reverses #683's documented choice to leave the two unindexed;
docs/Historian.md carries the reasoning.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 17:00:54 -04:00
parent d44e58dcd6
commit bc2f5e1889
6 changed files with 134 additions and 32 deletions
@@ -78,7 +78,8 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
{
foreach (var ddl in QuestDbTables.CreateAll(prefix, null))
{
await using var cmd = conn.CreateCommand(c, ddl.Replace("tag SYMBOL INDEX", "tag SYMBOL", StringComparison.Ordinal));
// Tables from before ANY symbol index existed: the sample / trail tag and the event source unindexed.
await using var cmd = conn.CreateCommand(c, ddl.Replace(" SYMBOL INDEX", " SYMBOL", StringComparison.Ordinal));
await cmd.ExecuteNonQueryAsync();
}
}
@@ -87,6 +88,17 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
return new QuestDbConformanceBackend(fx, prefix, o, conn, boot);
}
/// <summary>Whether <paramref name="table"/>'s <paramref name="column"/> is indexed.</summary>
/// <param name="table">The table.</param>
/// <param name="column">The symbol column.</param>
/// <returns>The <c>table_columns()</c> flag.</returns>
public async Task<bool> IndexedAsync(string table, string column)
{
await using var c = await Conn.OpenAsync(CancellationToken.None);
await using var cmd = Conn.CreateCommand(c, QuestDbTables.IndexedQuery(table, column));
return Convert.ToBoolean(await cmd.ExecuteScalarAsync());
}
/// <summary>Whether this instance's sample table has its <c>tag</c> column indexed (#683).</summary>
/// <returns>The <c>table_columns()</c> flag.</returns>
public async Task<bool> TagIndexedAsync()
@@ -83,6 +83,42 @@ public sealed class TagIndexTests(QuestDbFixture fx)
(await b.TagIndexedAsync()).ShouldBeTrue();
}
[Fact]
public async Task New_tables_index_every_filtered_symbol_column()
{
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
foreach (var (table, column) in QuestDbTables.SymbolIndexes(b.Options.TablePrefix))
(await b.IndexedAsync(table, column)).ShouldBeTrue($"{table}.{column}");
}
[Fact]
public async Task Pre_index_event_and_trail_tables_gain_their_indexes_on_boot_and_event_reads_answer_the_same()
{
await using var b = await QuestDbConformanceBackend.CreatePre683Async(fx);
var p = b.Options.TablePrefix;
foreach (var (table, column) in QuestDbTables.SymbolIndexes(p))
(await b.IndexedAsync(table, column)).ShouldBeFalse($"precondition: {table}.{column} unindexed");
await b.SeedEventsAsync((At(10), "e1", "S1"), (At(20), "e2", "S2"), (At(30), "e3", "S1"));
var src = b.CreateDataSource();
async Task<string> ReadAsync() => string.Join(",",
(await src.ReadEventsAsync("S1", At(0), At(100), 0, Ct)).Events.Select(e => $"{e.EventId}@{e.EventTimeUtc:O}"));
var before = await ReadAsync();
before.ShouldNotBeEmpty();
var migrate = fx.Options(p);
await new QuestDbSchemaBootstrapper(new QuestDbConnection(migrate), migrate).EnsureAsync(Ct);
var deadline = DateTime.UtcNow.AddSeconds(15);
foreach (var (table, column) in QuestDbTables.SymbolIndexes(p))
{
while (!await b.IndexedAsync(table, column))
{
DateTime.UtcNow.ShouldBeLessThan(deadline, $"{table}.{column} was never indexed");
await Task.Delay(50);
}
}
(await ReadAsync()).ShouldBe(before);
}
[Fact]
public async Task A_refused_ADD_INDEX_does_not_fail_the_bootstrap()
{
@@ -28,11 +28,26 @@ public sealed class QuestDbTablesTests
"SELECT indexed FROM table_columns('hist_sample') WHERE \"column\" = 'tag'");
}
[Fact]
public void Every_filtered_symbol_column_is_indexed_and_has_a_migration()
{
QuestDbTables.SymbolIndexes("hist_").ShouldBe(
[("hist_sample", "tag"), ("hist_event", "source"), ("hist_sample_modified", "tag")]);
QuestDbTables.AddIndex("hist_event", "source").ShouldBe("ALTER TABLE hist_event ALTER COLUMN source ADD INDEX");
QuestDbTables.IndexedQuery("hist_event", "source").ShouldBe(
"SELECT indexed FROM table_columns('hist_event') WHERE \"column\" = 'source'");
// The CREATE statements carry every index the migration would add.
var ddl = QuestDbTables.CreateAll("hist_", null);
foreach (var (table, column) in QuestDbTables.SymbolIndexes("hist_"))
ddl.Single(d => d.StartsWith($"CREATE TABLE IF NOT EXISTS {table} (", StringComparison.Ordinal))
.ShouldContain($"{column} SYMBOL INDEX");
}
[Fact]
public void Event_ddl_is_pinned()
{
QuestDbTables.CreateEvent("hist_", null).ShouldBe(
"CREATE TABLE IF NOT EXISTS hist_event (ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, " +
"CREATE TABLE IF NOT EXISTS hist_event (ts TIMESTAMP, event_id VARCHAR, source SYMBOL INDEX, alarm_id VARCHAR, " +
"alarm_name VARCHAR, alarm_type_name VARCHAR, severity INT, event_kind VARCHAR, message VARCHAR, " +
"user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP) " +
"TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, event_id)");
@@ -57,7 +72,7 @@ public sealed class QuestDbTablesTests
public void Sample_modified_ddl_is_pinned_append_only_and_carries_the_ttl()
{
QuestDbTables.CreateSampleModified("hist_", null).ShouldBe(
"CREATE TABLE IF NOT EXISTS hist_sample_modified (ts TIMESTAMP, tag SYMBOL, modified_id LONG, " +
"CREATE TABLE IF NOT EXISTS hist_sample_modified (ts TIMESTAMP, tag SYMBOL INDEX, modified_id LONG, " +
"modification_ts TIMESTAMP, update_type BYTE, principal VARCHAR, api_key_id VARCHAR, kind BYTE, num_value DOUBLE, " +
"int_value LONG, str_value VARCHAR, status LONG, server_ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL");
QuestDbTables.CreateSampleModified("hist_", 30).ShouldEndWith("PARTITION BY DAY TTL 30 DAYS WAL");