Joseph Doherty
cfd403710e
Every HistoryUpdate a SQL backend applies now leaves a modification trail
(prior value, principal, Part 11 update type, modification time), and
HistoryReadRaw with IsReadModified=true serves it as HistoryModifiedData.
- Core: HistoryUpdateContext threaded into all three IHistorianDataUpdater
methods; IHistorianDataSource.ReadModifiedAsync (default NotSupported =>
BadHistoryOperationUnsupported on Gateway/Null); HistoryModified* contracts.
- SqlServer: schema v2 (002_sample_modified.sql, RequiredVersion=2, docs copy);
trail written by OUTPUT ... INTO on the same MERGE/DELETE statement, so a
failed update leaves no audit row; retention purges SampleModified per tag on
the source-time cutoff; ReadModifiedAsync keyed on (SourceTimeUtc, ModifiedId).
- QuestDB: append-only {prefix}sample_modified (TTL like samples), trail row
written BEFORE the change (orphan-on-crash, never an untracked change);
existence reads now carry the prior value; ReadModifiedAsync.
- Node manager: modified reads dispatch + page via the #669 continuation store
(HistoryContinuationState gains HistoryCursorKind + LastModifiedId), gated by
the existing HistoryRead decision; session principal / key id passed to the
updater. IsDeleteModified stays unsupported.
- Client.CLI: historyread --modified.
- Tests: node-manager dispatch/paging/cursor binding/principal threading,
over-the-wire HistoryModifiedData, shared ModifiedValueConformance on both
backends, SqlServer atomicity (forced trail failure => no change) + v1->v2
upgrade + retention trail purge.
- Docs: Historian.md (Modified-value history section, schema v2 upgrade note),
security.md, Client.CLI.md, CLAUDE.md.
104 lines
6.0 KiB
C#
104 lines
6.0 KiB
C#
using System.Globalization;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
|
|
|
/// <summary>
|
|
/// Table names and DDL for the QuestDB backend (T0-confirmed against QuestDB 10.0.1). The TTL clause sits
|
|
/// between <c>PARTITION BY</c> and <c>WAL</c> and is an inlined integer (QuestDB cannot parameterise it).
|
|
/// Prefixes are regex-validated by the caller; these builders also refuse an invalid one.
|
|
/// </summary>
|
|
internal static class QuestDbTables
|
|
{
|
|
/// <summary>Base name of the sample table (prefix + this).</summary>
|
|
public const string SampleBase = "sample";
|
|
/// <summary>Base name of the event table.</summary>
|
|
public const string EventBase = "event";
|
|
/// <summary>Base name of the tag catalog table.</summary>
|
|
public const string TagBase = "tag";
|
|
/// <summary>Base name of the Part 11 modification-trail table (#668).</summary>
|
|
public const string SampleModifiedBase = "sample_modified";
|
|
|
|
/// <summary>The fixed designated timestamp of every <c>tag</c> row, which makes the dedup key an upsert by name.</summary>
|
|
public static readonly DateTime TagEpoch = DateTime.UnixEpoch;
|
|
|
|
/// <summary>The prefixed sample table name.</summary>
|
|
public static string Sample(string prefix) => Name(prefix, SampleBase);
|
|
/// <summary>The prefixed event table name.</summary>
|
|
public static string Event(string prefix) => Name(prefix, EventBase);
|
|
/// <summary>The prefixed tag table name.</summary>
|
|
public static string Tag(string prefix) => Name(prefix, TagBase);
|
|
/// <summary>The prefixed modification-trail table name.</summary>
|
|
public static string SampleModified(string prefix) => Name(prefix, SampleModifiedBase);
|
|
|
|
/// <summary>All four table names in creation order.</summary>
|
|
public static string[] All(string prefix) => [Sample(prefix), Event(prefix), Tag(prefix), SampleModified(prefix)];
|
|
|
|
/// <summary>
|
|
/// The tables that carry a TTL (sample, event and the modification trail, which ages out with the samples it
|
|
/// describes; the tag catalog never expires).
|
|
/// </summary>
|
|
public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix), SampleModified(prefix)];
|
|
|
|
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the sample table.</summary>
|
|
public static string CreateSample(string prefix, int? retentionDays) =>
|
|
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
|
|
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
|
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, tag)";
|
|
|
|
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the event table.</summary>
|
|
public static string CreateEvent(string prefix, int? retentionDays) =>
|
|
$"CREATE TABLE IF NOT EXISTS {Event(prefix)} (ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, " +
|
|
"alarm_name VARCHAR, alarm_type_name VARCHAR, severity INT, event_kind VARCHAR, message VARCHAR, " +
|
|
"user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP) " +
|
|
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, event_id)";
|
|
|
|
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the tag catalog (YEAR partitions, no TTL ever).</summary>
|
|
public static string CreateTag(string prefix) =>
|
|
$"CREATE TABLE IF NOT EXISTS {Tag(prefix)} (ts TIMESTAMP, name SYMBOL, data_type BYTE, eng_unit VARCHAR, " +
|
|
"description VARCHAR, updated_utc TIMESTAMP) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name)";
|
|
|
|
/// <summary>
|
|
/// <c>CREATE TABLE IF NOT EXISTS</c> for the Part 11 modification trail (#668): one row per sample a HistoryUpdate
|
|
/// inserted, replaced or deleted, carrying the PRIOR value (NULL kind/value/status for an insert). <c>ts</c> is the
|
|
/// sample's source time (so the TTL ages it out with the samples); <c>modified_id</c> orders the trail within one
|
|
/// source time. Append-only: no DEDUP, never updated or deleted.
|
|
/// </summary>
|
|
public static string CreateSampleModified(string prefix, int? retentionDays) =>
|
|
$"CREATE TABLE IF NOT EXISTS {SampleModified(prefix)} (ts TIMESTAMP, tag SYMBOL, modified_id LONG, " +
|
|
"modification_ts TIMESTAMP, update_type BYTE, principal VARCHAR, api_key_id VARCHAR, kind BYTE, num_value DOUBLE, " +
|
|
"int_value LONG, str_value VARCHAR, status LONG, server_ts TIMESTAMP) " +
|
|
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL";
|
|
|
|
/// <summary>The four CREATE statements in creation order (parallel to <see cref="All"/>).</summary>
|
|
public static string[] CreateAll(string prefix, int? retentionDays) =>
|
|
[CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix),
|
|
CreateSampleModified(prefix, retentionDays)];
|
|
|
|
/// <summary><c>ALTER TABLE t SET TTL n DAYS</c>; <c>null</c> removes the TTL (<c>SET TTL 0 DAYS</c>).</summary>
|
|
public static string AlterTtl(string table, int? retentionDays) =>
|
|
$"ALTER TABLE {table} SET TTL {(retentionDays ?? 0).ToString(CultureInfo.InvariantCulture)} DAYS";
|
|
|
|
/// <summary>
|
|
/// Converts a <c>tables()</c> <c>ttlValue</c>/<c>ttlUnit</c> pair to hours. QuestDB normalises the unit (7 days
|
|
/// reads back as <c>1 WEEK</c>), so equality is judged in hours. An unrecognised unit returns -1 (never equal to
|
|
/// a wanted value, so the bootstrapper re-applies the TTL).
|
|
/// </summary>
|
|
public static long TtlHours(long value, string? unit) => (unit ?? "").ToUpperInvariant() switch
|
|
{
|
|
"HOUR" => value,
|
|
"DAY" => value * 24,
|
|
"WEEK" => value * 168,
|
|
_ => value == 0 ? 0 : -1,
|
|
};
|
|
|
|
private static string Ttl(int? days) =>
|
|
days is { } d ? $" TTL {d.ToString(CultureInfo.InvariantCulture)} DAYS" : "";
|
|
|
|
private static string Name(string prefix, string baseName)
|
|
{
|
|
if (!QuestDbHistorianOptions.IsValidPrefix(prefix))
|
|
throw new ArgumentException("Invalid table prefix.", nameof(prefix));
|
|
return prefix + baseName;
|
|
}
|
|
}
|