Joseph Doherty
549d2d68b3
QuestDB: {p}sample.tag is now SYMBOL INDEX. The processed-read data-range
lookups (EarliestSample / LatestSample) were partition scans of the unindexed
symbol; on 53.6M rows / 120 tags / 61 daily partitions the indexed seeks are
1-1.5 ms warm / 2-4 ms cold whatever the tag's history (unindexed: up to 11 ms
warm, 14 ms cold; cost grew with the table). The bootstrapper adds the index to a
pre-#683 table (ALTER ... ADD INDEX, a WAL txn, idempotent, ~3 s for 68M rows);
AutoCreateSchema=false only logs the statement. Disk +~12%; write apply time
within noise; main per-sample/raw/time-weighted queries unchanged or faster.
Rejected by measurement/analysis: bounded existence probes (still full scans
when a side is absent), LATEST ON (slowest unindexed), per-tag first/last side
table (cannot track TTL drops or deletes INSERT-only).
Both backends now read the data range before the bucket statement: an append
racing the read reaches the buckets but not the range, so it can no longer
produce a Good Count of 0 for an interval that has samples.
Tests: TagIndexTests (new-table index, bound-param seek uses the index,
tombstones excluded / outage marker counted, pre-#683 migration with identical
processed results across 10 aggregates x 4 windows x both directions).
125 lines
7.5 KiB
C#
125 lines
7.5 KiB
C#
using System.Globalization;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
|
|
|
/// <summary>
|
|
/// Table names and DDL for the QuestDB backend (T0-confirmed against QuestDB 10.0.1). The TTL clause sits
|
|
/// between <c>PARTITION BY</c> and <c>WAL</c> and is an inlined integer (QuestDB cannot parameterise it).
|
|
/// Prefixes are regex-validated by the caller; these builders also refuse an invalid one.
|
|
/// </summary>
|
|
internal static class QuestDbTables
|
|
{
|
|
/// <summary>Base name of the sample table (prefix + this).</summary>
|
|
public const string SampleBase = "sample";
|
|
/// <summary>Base name of the event table.</summary>
|
|
public const string EventBase = "event";
|
|
/// <summary>Base name of the tag catalog table.</summary>
|
|
public const string TagBase = "tag";
|
|
/// <summary>Base name of the Part 11 modification-trail table (#668).</summary>
|
|
public const string SampleModifiedBase = "sample_modified";
|
|
|
|
/// <summary>The fixed designated timestamp of every <c>tag</c> row, which makes the dedup key an upsert by name.</summary>
|
|
public static readonly DateTime TagEpoch = DateTime.UnixEpoch;
|
|
|
|
/// <summary>The prefixed sample table name.</summary>
|
|
public static string Sample(string prefix) => Name(prefix, SampleBase);
|
|
/// <summary>The prefixed event table name.</summary>
|
|
public static string Event(string prefix) => Name(prefix, EventBase);
|
|
/// <summary>The prefixed tag table name.</summary>
|
|
public static string Tag(string prefix) => Name(prefix, TagBase);
|
|
/// <summary>The prefixed modification-trail table name.</summary>
|
|
public static string SampleModified(string prefix) => Name(prefix, SampleModifiedBase);
|
|
|
|
/// <summary>All four table names in creation order.</summary>
|
|
public static string[] All(string prefix) => [Sample(prefix), Event(prefix), Tag(prefix), SampleModified(prefix)];
|
|
|
|
/// <summary>
|
|
/// The tables that carry a TTL (sample, event and the modification trail, which ages out with the samples it
|
|
/// describes; the tag catalog never expires).
|
|
/// </summary>
|
|
public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix), SampleModified(prefix)];
|
|
|
|
/// <summary>
|
|
/// <c>CREATE TABLE IF NOT EXISTS</c> for the sample table. <c>tag</c> carries a symbol (bitmap) index (#683): every
|
|
/// read filters on one tag, and without the index QuestDB answers a <c>tag = x</c> seek by scanning the tag column of
|
|
/// every partition in the time range — the data-range lookups (<see cref="QuestDbAggregateQueries.EarliestSample"/> /
|
|
/// <c>LatestSample</c>) of a stopped or late-starting tag then scan every partition on one side of the window.
|
|
/// A table created before #683 gains the index through <see cref="AddTagIndex"/>.
|
|
/// </summary>
|
|
public static string CreateSample(string prefix, int? retentionDays) =>
|
|
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
|
|
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
|
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, tag)";
|
|
|
|
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the event table.</summary>
|
|
public static string CreateEvent(string prefix, int? retentionDays) =>
|
|
$"CREATE TABLE IF NOT EXISTS {Event(prefix)} (ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, " +
|
|
"alarm_name VARCHAR, alarm_type_name VARCHAR, severity INT, event_kind VARCHAR, message VARCHAR, " +
|
|
"user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP) " +
|
|
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, event_id)";
|
|
|
|
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the tag catalog (YEAR partitions, no TTL ever).</summary>
|
|
public static string CreateTag(string prefix) =>
|
|
$"CREATE TABLE IF NOT EXISTS {Tag(prefix)} (ts TIMESTAMP, name SYMBOL, data_type BYTE, eng_unit VARCHAR, " +
|
|
"description VARCHAR, updated_utc TIMESTAMP) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name)";
|
|
|
|
/// <summary>
|
|
/// <c>CREATE TABLE IF NOT EXISTS</c> for the Part 11 modification trail (#668): one row per sample a HistoryUpdate
|
|
/// inserted, replaced or deleted, carrying the PRIOR value (NULL kind/value/status for an insert). <c>ts</c> is the
|
|
/// sample's source time (so the TTL ages it out with the samples); <c>modified_id</c> orders the trail within one
|
|
/// source time. Append-only: no DEDUP, never updated or deleted.
|
|
/// </summary>
|
|
public static string CreateSampleModified(string prefix, int? retentionDays) =>
|
|
$"CREATE TABLE IF NOT EXISTS {SampleModified(prefix)} (ts TIMESTAMP, tag SYMBOL, modified_id LONG, " +
|
|
"modification_ts TIMESTAMP, update_type BYTE, principal VARCHAR, api_key_id VARCHAR, kind BYTE, num_value DOUBLE, " +
|
|
"int_value LONG, str_value VARCHAR, status LONG, server_ts TIMESTAMP) " +
|
|
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL";
|
|
|
|
/// <summary>The four CREATE statements in creation order (parallel to <see cref="All"/>).</summary>
|
|
public static string[] CreateAll(string prefix, int? retentionDays) =>
|
|
[CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix),
|
|
CreateSampleModified(prefix, retentionDays)];
|
|
|
|
/// <summary>
|
|
/// <c>ALTER TABLE t ALTER COLUMN tag ADD INDEX</c> — the #683 migration of a sample table created without the tag
|
|
/// index. A WAL transaction (visible once applied); re-issuing it on an indexed column is a harmless no-op (the table
|
|
/// is not suspended), so two nodes booting together may both send it.
|
|
/// </summary>
|
|
public static string AddTagIndex(string table) => $"ALTER TABLE {table} ALTER COLUMN tag ADD INDEX";
|
|
|
|
/// <summary>
|
|
/// Whether <paramref name="table"/>'s <c>tag</c> column is indexed: one row, one BOOLEAN column, no row when the
|
|
/// table or column is missing. The name is inlined (<c>table_columns()</c> takes a constant); it is always a
|
|
/// prefix-validated name built here.
|
|
/// </summary>
|
|
public static string TagIndexedQuery(string table) =>
|
|
$"SELECT indexed FROM table_columns('{table}') WHERE \"column\" = 'tag'";
|
|
|
|
/// <summary><c>ALTER TABLE t SET TTL n DAYS</c>; <c>null</c> removes the TTL (<c>SET TTL 0 DAYS</c>).</summary>
|
|
public static string AlterTtl(string table, int? retentionDays) =>
|
|
$"ALTER TABLE {table} SET TTL {(retentionDays ?? 0).ToString(CultureInfo.InvariantCulture)} DAYS";
|
|
|
|
/// <summary>
|
|
/// Converts a <c>tables()</c> <c>ttlValue</c>/<c>ttlUnit</c> pair to hours. QuestDB normalises the unit (7 days
|
|
/// reads back as <c>1 WEEK</c>), so equality is judged in hours. An unrecognised unit returns -1 (never equal to
|
|
/// a wanted value, so the bootstrapper re-applies the TTL).
|
|
/// </summary>
|
|
public static long TtlHours(long value, string? unit) => (unit ?? "").ToUpperInvariant() switch
|
|
{
|
|
"HOUR" => value,
|
|
"DAY" => value * 24,
|
|
"WEEK" => value * 168,
|
|
_ => value == 0 ? 0 : -1,
|
|
};
|
|
|
|
private static string Ttl(int? days) =>
|
|
days is { } d ? $" TTL {d.ToString(CultureInfo.InvariantCulture)} DAYS" : "";
|
|
|
|
private static string Name(string prefix, string baseName)
|
|
{
|
|
if (!QuestDbHistorianOptions.IsValidPrefix(prefix))
|
|
throw new ArgumentException("Invalid table prefix.", nameof(prefix));
|
|
return prefix + baseName;
|
|
}
|
|
}
|