perf(historian): index QuestDB event.source and sample_modified.tag (lmxopcua#746)
Event reads filtered on source and every modified-value read filtered on tag scanned that symbol column in every partition of the window; only sample.tag carried an index (#683). Both columns are now SYMBOL INDEX in the DDL, and the bootstrapper's index migration is generalised over QuestDbTables.SymbolIndexes (sample.tag, event.source, sample_modified.tag): a missing index is added with the same idempotent ALTER ... ADD INDEX, Warning-only when AutoCreateSchema is false or the ALTER is refused. Integration tests: new tables carry all three; a pre-index table set gains them on boot and event reads answer the same. This reverses #683's documented choice to leave the two unindexed; docs/Historian.md carries the reasoning. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+13
-1
@@ -78,7 +78,8 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
|
||||
{
|
||||
foreach (var ddl in QuestDbTables.CreateAll(prefix, null))
|
||||
{
|
||||
await using var cmd = conn.CreateCommand(c, ddl.Replace("tag SYMBOL INDEX", "tag SYMBOL", StringComparison.Ordinal));
|
||||
// Tables from before ANY symbol index existed: the sample / trail tag and the event source unindexed.
|
||||
await using var cmd = conn.CreateCommand(c, ddl.Replace(" SYMBOL INDEX", " SYMBOL", StringComparison.Ordinal));
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
}
|
||||
@@ -87,6 +88,17 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
|
||||
return new QuestDbConformanceBackend(fx, prefix, o, conn, boot);
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="table"/>'s <paramref name="column"/> is indexed.</summary>
|
||||
/// <param name="table">The table.</param>
|
||||
/// <param name="column">The symbol column.</param>
|
||||
/// <returns>The <c>table_columns()</c> flag.</returns>
|
||||
public async Task<bool> IndexedAsync(string table, string column)
|
||||
{
|
||||
await using var c = await Conn.OpenAsync(CancellationToken.None);
|
||||
await using var cmd = Conn.CreateCommand(c, QuestDbTables.IndexedQuery(table, column));
|
||||
return Convert.ToBoolean(await cmd.ExecuteScalarAsync());
|
||||
}
|
||||
|
||||
/// <summary>Whether this instance's sample table has its <c>tag</c> column indexed (#683).</summary>
|
||||
/// <returns>The <c>table_columns()</c> flag.</returns>
|
||||
public async Task<bool> TagIndexedAsync()
|
||||
|
||||
+36
@@ -83,6 +83,42 @@ public sealed class TagIndexTests(QuestDbFixture fx)
|
||||
(await b.TagIndexedAsync()).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task New_tables_index_every_filtered_symbol_column()
|
||||
{
|
||||
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
|
||||
foreach (var (table, column) in QuestDbTables.SymbolIndexes(b.Options.TablePrefix))
|
||||
(await b.IndexedAsync(table, column)).ShouldBeTrue($"{table}.{column}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pre_index_event_and_trail_tables_gain_their_indexes_on_boot_and_event_reads_answer_the_same()
|
||||
{
|
||||
await using var b = await QuestDbConformanceBackend.CreatePre683Async(fx);
|
||||
var p = b.Options.TablePrefix;
|
||||
foreach (var (table, column) in QuestDbTables.SymbolIndexes(p))
|
||||
(await b.IndexedAsync(table, column)).ShouldBeFalse($"precondition: {table}.{column} unindexed");
|
||||
await b.SeedEventsAsync((At(10), "e1", "S1"), (At(20), "e2", "S2"), (At(30), "e3", "S1"));
|
||||
var src = b.CreateDataSource();
|
||||
async Task<string> ReadAsync() => string.Join(",",
|
||||
(await src.ReadEventsAsync("S1", At(0), At(100), 0, Ct)).Events.Select(e => $"{e.EventId}@{e.EventTimeUtc:O}"));
|
||||
var before = await ReadAsync();
|
||||
before.ShouldNotBeEmpty();
|
||||
|
||||
var migrate = fx.Options(p);
|
||||
await new QuestDbSchemaBootstrapper(new QuestDbConnection(migrate), migrate).EnsureAsync(Ct);
|
||||
var deadline = DateTime.UtcNow.AddSeconds(15);
|
||||
foreach (var (table, column) in QuestDbTables.SymbolIndexes(p))
|
||||
{
|
||||
while (!await b.IndexedAsync(table, column))
|
||||
{
|
||||
DateTime.UtcNow.ShouldBeLessThan(deadline, $"{table}.{column} was never indexed");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
(await ReadAsync()).ShouldBe(before);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_refused_ADD_INDEX_does_not_fail_the_bootstrap()
|
||||
{
|
||||
|
||||
+17
-2
@@ -28,11 +28,26 @@ public sealed class QuestDbTablesTests
|
||||
"SELECT indexed FROM table_columns('hist_sample') WHERE \"column\" = 'tag'");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_filtered_symbol_column_is_indexed_and_has_a_migration()
|
||||
{
|
||||
QuestDbTables.SymbolIndexes("hist_").ShouldBe(
|
||||
[("hist_sample", "tag"), ("hist_event", "source"), ("hist_sample_modified", "tag")]);
|
||||
QuestDbTables.AddIndex("hist_event", "source").ShouldBe("ALTER TABLE hist_event ALTER COLUMN source ADD INDEX");
|
||||
QuestDbTables.IndexedQuery("hist_event", "source").ShouldBe(
|
||||
"SELECT indexed FROM table_columns('hist_event') WHERE \"column\" = 'source'");
|
||||
// The CREATE statements carry every index the migration would add.
|
||||
var ddl = QuestDbTables.CreateAll("hist_", null);
|
||||
foreach (var (table, column) in QuestDbTables.SymbolIndexes("hist_"))
|
||||
ddl.Single(d => d.StartsWith($"CREATE TABLE IF NOT EXISTS {table} (", StringComparison.Ordinal))
|
||||
.ShouldContain($"{column} SYMBOL INDEX");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Event_ddl_is_pinned()
|
||||
{
|
||||
QuestDbTables.CreateEvent("hist_", null).ShouldBe(
|
||||
"CREATE TABLE IF NOT EXISTS hist_event (ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, " +
|
||||
"CREATE TABLE IF NOT EXISTS hist_event (ts TIMESTAMP, event_id VARCHAR, source SYMBOL INDEX, alarm_id VARCHAR, " +
|
||||
"alarm_name VARCHAR, alarm_type_name VARCHAR, severity INT, event_kind VARCHAR, message VARCHAR, " +
|
||||
"user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP) " +
|
||||
"TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, event_id)");
|
||||
@@ -57,7 +72,7 @@ public sealed class QuestDbTablesTests
|
||||
public void Sample_modified_ddl_is_pinned_append_only_and_carries_the_ttl()
|
||||
{
|
||||
QuestDbTables.CreateSampleModified("hist_", null).ShouldBe(
|
||||
"CREATE TABLE IF NOT EXISTS hist_sample_modified (ts TIMESTAMP, tag SYMBOL, modified_id LONG, " +
|
||||
"CREATE TABLE IF NOT EXISTS hist_sample_modified (ts TIMESTAMP, tag SYMBOL INDEX, modified_id LONG, " +
|
||||
"modification_ts TIMESTAMP, update_type BYTE, principal VARCHAR, api_key_id VARCHAR, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, server_ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL");
|
||||
QuestDbTables.CreateSampleModified("hist_", 30).ShouldEndWith("PARTITION BY DAY TTL 30 DAYS WAL");
|
||||
|
||||
Reference in New Issue
Block a user