diff --git a/ZB.MOM.WW.OtOpcUa.slnx b/ZB.MOM.WW.OtOpcUa.slnx index 16290f6c..4621530b 100644 --- a/ZB.MOM.WW.OtOpcUa.slnx +++ b/ZB.MOM.WW.OtOpcUa.slnx @@ -27,6 +27,7 @@ + @@ -102,6 +103,8 @@ + + diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbConnection.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbConnection.cs new file mode 100644 index 00000000..1b003425 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbConnection.cs @@ -0,0 +1,209 @@ +using System.Net.Sockets; +using Npgsql; +using NpgsqlTypes; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb; + +/// +/// Pooled Npgsql access to QuestDB plus every T0-confirmed typing rule, in one place: how timestamps are rounded and +/// bound (R6), how BYTE columns are bound and read, and how errors are classified. +/// Contract for callers: bind every inbound through (never +/// AddWithValue), call before writing a designated timestamp, and classify +/// failures with / . Every QuestDB server error carries +/// SQLSTATE 00000, so classification is by message text, never by SQLSTATE. +/// +internal sealed class QuestDbConnection : IAsyncDisposable +{ + private static readonly long MinSafeTicks = 10; // DateTime.MinValue itself binds as -infinity + private static readonly long MaxSafeTicks = DateTime.MaxValue.Ticks - DateTime.MaxValue.Ticks % 10; // MaxValue itself binds as +infinity + + private readonly NpgsqlDataSource _dataSource; + + /// Creates the connection helper; options must already be valid. + /// Validated options. + public QuestDbConnection(QuestDbHistorianOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (!QuestDbHistorianOptions.IsValidPrefix(options.TablePrefix)) + throw new ArgumentException("Invalid table prefix.", nameof(options)); + TablePrefix = options.TablePrefix; + CommandTimeoutSeconds = Math.Max(1, (int)Math.Ceiling(options.CommandTimeout.TotalSeconds)); + _dataSource = new NpgsqlDataSourceBuilder(options.ConnectionString).Build(); + } + + /// The validated table prefix. + public string TablePrefix { get; } + + /// Command timeout applied by , in seconds. + public int CommandTimeoutSeconds { get; } + + /// Prefixes a table base name (sample, event, tag) with the configured prefix. + /// Fixed base name from this project (never user input). + /// The full table name. + public string T(string table) => TablePrefix + table; + + /// The prefixed sample table. + public string SampleTable => T(QuestDbTables.SampleBase); + /// The prefixed event table. + public string EventTable => T(QuestDbTables.EventBase); + /// The prefixed tag table. + public string TagTable => T(QuestDbTables.TagBase); + + /// Opens a pooled connection (no bootstrap; callers run the bootstrapper first). + /// Cancellation. + /// An open connection the caller disposes. + public async Task OpenAsync(CancellationToken ct) => + await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false); + + /// Creates a text command with the configured timeout. Parameters are positional ($1, $2…). + /// Open connection. + /// Command text. + /// The command. + public NpgsqlCommand CreateCommand(NpgsqlConnection conn, string sql) + { + var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.CommandTimeout = CommandTimeoutSeconds; + return cmd; + } + + /// + public ValueTask DisposeAsync() => _dataSource.DisposeAsync(); + + // ---- timestamps (R6) ------------------------------------------------------------------------------------------- + + /// + /// Normalises a timestamp for QuestDB (microsecond precision). Rule: Unspecified is treated as UTC + /// (SpecifyKind); Local is converted with ToUniversalTime; then ticks are truncated to a + /// microsecond (ticks - ticks % 10, because Npgsql truncates relative to 2000-01-01 and so rounds UP + /// before 2000); finally the result is nudged into [0001-01-01T00:00:00.000001, 9999-12-31T23:59:59.999999] + /// so a value equal to MinValue/MaxValue never reaches Npgsql, which would bind it as ±infinity + /// (a [Min, Max) range then silently matches nothing). The result always has Kind=Utc. + /// + /// Any timestamp. + /// A UTC, microsecond-aligned, finite timestamp. + public static DateTime Round(DateTime value) + { + var utc = value.Kind switch + { + DateTimeKind.Local => value.ToUniversalTime(), + DateTimeKind.Utc => value, + _ => DateTime.SpecifyKind(value, DateTimeKind.Utc), + }; + var ticks = utc.Ticks - utc.Ticks % 10; + ticks = Math.Clamp(ticks, MinSafeTicks, MaxSafeTicks); + return new DateTime(ticks, DateTimeKind.Utc); + } + + /// + /// True when QuestDB can store the timestamp as a designated timestamp: after it is on or + /// after 1970-01-01 (RC1). Writers drop (live) or refuse (HistoryUpdate) samples for which this is false; reads + /// with earlier bounds are fine. + /// + /// Candidate sample time. + /// Whether an INSERT with this designated timestamp can succeed. + public static bool IsStorable(DateTime value) => Round(value) >= DateTime.UnixEpoch; + + /// + /// Appends the next positional parameter ($n) as timestamp with time zone carrying + /// 's UTC value (T0-confirmed: Timestamp+Utc throws in Npgsql). It never emits + /// MinValue/MaxValue. By default a pre-1970 value is kept as is (a pre-1970 range bound is valid + /// and clamping it would move the bound across the epoch row); pass for a + /// designated-timestamp column so a pre-1970 value can only ever become the epoch itself, never a failed batch. + /// + /// Target command. + /// Any timestamp. + /// Clamp anything before 1970-01-01 to the epoch (writes only, after ). + /// The added parameter. + public static NpgsqlParameter BindTimestamp(NpgsqlCommand cmd, DateTime value, bool clampToEpoch = false) + { + var rounded = Round(value); + if (clampToEpoch && rounded < DateTime.UnixEpoch) rounded = DateTime.UnixEpoch; + var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.TimestampTz, Value = rounded }; + cmd.Parameters.Add(p); + return p; + } + + /// Appends a BYTE parameter (bound as smallint, T0-confirmed). + /// Target command. + /// The byte. + /// The added parameter. + public static NpgsqlParameter BindByte(NpgsqlCommand cmd, byte value) + { + var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Smallint, Value = (short)value }; + cmd.Parameters.Add(p); + return p; + } + + /// Appends a LONG parameter (bigint; also the rule for integer arithmetic in tick math). + /// Target command. + /// The value. + /// The added parameter. + public static NpgsqlParameter BindLong(NpgsqlCommand cmd, long value) + { + var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Bigint, Value = value }; + cmd.Parameters.Add(p); + return p; + } + + /// Appends a SYMBOL/VARCHAR parameter (varchar); null binds as NULL. + /// Target command. + /// The string. + /// The added parameter. + public static NpgsqlParameter BindText(NpgsqlCommand cmd, string? value) + { + var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Varchar, Value = (object?)value ?? DBNull.Value }; + cmd.Parameters.Add(p); + return p; + } + + /// Reads a TIMESTAMP column as UTC (QuestDB returns Kind=Unspecified holding UTC wall clock). + /// Reader positioned on a row. + /// Column ordinal. + /// The timestamp with Kind=Utc. + public static DateTime ReadUtc(NpgsqlDataReader r, int i) => + DateTime.SpecifyKind(r.GetDateTime(i), DateTimeKind.Utc); + + /// Reads a BYTE column (QuestDB returns smallint); NULL (never stored) reads as 0 (Empty). + /// Reader positioned on a row. + /// Column ordinal. + /// The byte value. + public static byte ReadKind(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i); + + // ---- error classification -------------------------------------------------------------------------------------- + + /// + /// True for a transport-level or timeout failure worth retrying. A is a + /// server-side rejection (QuestDB reports every one as SQLSTATE 00000), so it is never transient here; decide + /// those by message text. + /// + /// The failure. + /// Whether the failure is transient. + public static bool IsTransient(Exception ex) + { + for (var e = ex; e is not null; e = e.InnerException) + { + if (e is PostgresException or OperationCanceledException) return false; + if (e is NpgsqlException { IsTransient: true } or SocketException or IOException or TimeoutException) return true; + } + return false; + } + + /// True when a server error says the object already exists (message text; SQLSTATE is always 00000). + /// The failure. + /// Whether it is an "already exists" rejection. + public static bool IsAlreadyExists(Exception ex) => + ex is PostgresException pg && pg.MessageText.Contains("already exists", StringComparison.OrdinalIgnoreCase); + + /// Whether a table exists, per tables(). + /// Open connection. + /// Full table name. + /// Cancellation. + /// True when listed. + public async Task TableExistsAsync(NpgsqlConnection conn, string table, CancellationToken ct) + { + await using var cmd = CreateCommand(conn, "SELECT count() FROM tables() WHERE table_name = $1"); + BindText(cmd, table); + return Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) > 0; + } +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianOptions.cs new file mode 100644 index 00000000..c7d71be2 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianOptions.cs @@ -0,0 +1,57 @@ +using System.Text.RegularExpressions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb; + +/// Options for the QuestDB historian backend (the ServerHistorian:QuestDb config block). +public sealed partial class QuestDbHistorianOptions +{ + /// + /// Required. An Npgsql connection string for QuestDB's PG-wire port, e.g. + /// Host=questdb;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading. + /// + public string ConnectionString { get; set; } = ""; + + /// Prefix of the three tables (QuestDB has no schemas). Spliced into DDL/DML, so it is regex-validated. + public string TablePrefix { get; set; } = "hist_"; + + /// Retention in days applied as a TTL to the sample and event tables; null keeps data forever. + public int? RetentionDays { get; set; } + + /// Per-command timeout. Must be positive. + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// true creates the tables (and applies the TTL) on first use; false only verifies they exist and + /// fails every historian call with HistorianSchemaMissingException otherwise. + /// + public bool AutoCreateSchema { get; set; } = true; + + /// How long a write waits for its rows to become visible to queries (WAL apply). 0 disables the wait. + public TimeSpan VisibilityTimeout { get; set; } = TimeSpan.FromSeconds(5); + + [GeneratedRegex("^[A-Za-z0-9_]+$")] + private static partial Regex PrefixRegex(); + + /// True when is safe to splice into a table name. + /// Candidate prefix. + /// Whether it matches ^[A-Za-z0-9_]+$. + public static bool IsValidPrefix(string? prefix) => prefix is not null && PrefixRegex().IsMatch(prefix); + + /// Validates the options without touching QuestDB. + /// One message per problem (each names its ServerHistorian:QuestDb:<Key>); empty when valid. + public IReadOnlyList Validate() + { + var errors = new List(); + if (string.IsNullOrWhiteSpace(ConnectionString)) + errors.Add("ServerHistorian:QuestDb:ConnectionString is required."); + if (!IsValidPrefix(TablePrefix)) + errors.Add("ServerHistorian:QuestDb:TablePrefix must match ^[A-Za-z0-9_]+$."); + if (RetentionDays is < 1) + errors.Add("ServerHistorian:QuestDb:RetentionDays must be at least 1 when set."); + if (CommandTimeout <= TimeSpan.Zero) + errors.Add("ServerHistorian:QuestDb:CommandTimeout must be greater than zero."); + if (VisibilityTimeout < TimeSpan.Zero) + errors.Add("ServerHistorian:QuestDb:VisibilityTimeout must not be negative (0 disables the wait)."); + return errors; + } +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs new file mode 100644 index 00000000..45d753f0 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbSchemaBootstrapper.cs @@ -0,0 +1,143 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb; + +/// +/// Lazily creates the three tables once per instance (CREATE TABLE IF NOT EXISTS), then reconciles the TTL +/// of the sample and event tables with . +/// +/// Concurrency: two nodes starting together are safe. If a CREATE still fails with a server error +/// whose message says already exists, tables() is re-checked and the error is swallowed only when the +/// table is really there; anything else is rethrown (SQLSTATE is always 00000, so it cannot classify). +/// TTL (RC3): tables() normalises units (7 days reads back as 1 WEEK) so values are compared +/// in hours, and an ALTER becomes visible only after the WAL applies it, so the bootstrapper never re-reads to verify +/// its own ALTER. A boot whose TTL already matches issues no ALTER. +/// Failure: not latched. Within the last failure is +/// re-thrown without touching QuestDB. +/// AutoCreateSchema=false only verifies the tables exist (no TTL changes) and throws +/// otherwise. +/// +/// +internal sealed class QuestDbSchemaBootstrapper +{ + private readonly QuestDbConnection _conn; + private readonly QuestDbHistorianOptions _options; + private readonly ILogger _log; + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly BootstrapFailureCache _failure; // guarded by _gate + private volatile bool _ready; + + /// Creates the bootstrapper. + /// Connection helper. + /// Validated options. + /// Optional logger. + /// Clock for the failure cache; system by default. + public QuestDbSchemaBootstrapper(QuestDbConnection conn, QuestDbHistorianOptions options, ILogger? log = null, + TimeProvider? timeProvider = null) + { + _conn = conn; + _options = options; + _log = log ?? NullLogger.Instance; + _failure = new BootstrapFailureCache(timeProvider ?? TimeProvider.System, BootstrapFailureCache.DefaultDuration); + } + + /// Ensures the schema is ready; no-op after the first success. + /// Cancellation. + /// A task completing when the tables are usable. + /// AutoCreateSchema is false and a table is absent. + public async Task EnsureAsync(CancellationToken ct) + { + if (_ready) return; + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_ready) return; + _failure.ThrowIfCached(); + try + { + await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false); + if (_options.AutoCreateSchema) + { + await CreateTablesAsync(conn, ct).ConfigureAwait(false); + await ReconcileTtlAsync(conn, ct).ConfigureAwait(false); + } + else + { + await VerifyAsync(conn, ct).ConfigureAwait(false); + } + _ready = true; + _failure.Clear(); + } + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + _failure.Record(ex); + _log.LogWarning("QuestDB historian schema bootstrap (prefix '{Prefix}') failed ({Exception}: {Message}); " + + "historian calls fail fast for {Seconds} s before retrying.", + _conn.TablePrefix, ex.GetType().Name, ex.Message, BootstrapFailureCache.DefaultDuration.TotalSeconds); + throw; + } + } + finally + { + _gate.Release(); + } + } + + private async Task VerifyAsync(NpgsqlConnection conn, CancellationToken ct) + { + foreach (var table in QuestDbTables.All(_conn.TablePrefix)) + { + if (!await _conn.TableExistsAsync(conn, table, ct).ConfigureAwait(false)) + throw new HistorianSchemaMissingException( + $"QuestDB historian table '{table}' is missing and AutoCreateSchema is false. " + + "Create the tables (see docs/Historian.md) or enable AutoCreateSchema."); + } + } + + private async Task CreateTablesAsync(NpgsqlConnection conn, CancellationToken ct) + { + var statements = QuestDbTables.CreateAll(_conn.TablePrefix, _options.RetentionDays); + var tables = QuestDbTables.All(_conn.TablePrefix); + for (var i = 0; i < statements.Length; i++) + { + try + { + await using var cmd = _conn.CreateCommand(conn, statements[i]); + await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + catch (PostgresException ex) when (QuestDbConnection.IsAlreadyExists(ex)) + { + // A concurrent creator won the race - swallow ONLY if the table is really there. + if (!await _conn.TableExistsAsync(conn, tables[i], ct).ConfigureAwait(false)) throw; + _log.LogDebug("QuestDB table {Table} was created concurrently.", tables[i]); + } + } + } + + private async Task ReconcileTtlAsync(NpgsqlConnection conn, CancellationToken ct) + { + var wantHours = _options.RetentionDays is { } d ? d * 24L : 0L; + foreach (var table in QuestDbTables.Expiring(_conn.TablePrefix)) + { + long haveHours; + await using (var cmd = _conn.CreateCommand(conn, "SELECT ttlValue, ttlUnit FROM tables() WHERE table_name = $1")) + { + QuestDbConnection.BindText(cmd, table); + await using var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + if (!await r.ReadAsync(ct).ConfigureAwait(false)) continue; // vanished (dropped between create and here) + var value = r.IsDBNull(0) ? 0L : Convert.ToInt64(r.GetValue(0)); + var unit = r.IsDBNull(1) ? null : r.GetString(1); + haveHours = QuestDbTables.TtlHours(value, unit); + } + if (haveHours == wantHours) continue; + + // The ALTER is a WAL transaction: tables() keeps the old value until it applies, so do NOT re-read to verify. + await using (var alter = _conn.CreateCommand(conn, QuestDbTables.AlterTtl(table, _options.RetentionDays))) + await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + _log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", _options.RetentionDays, table); + } + } +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbTables.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbTables.cs new file mode 100644 index 00000000..c7a1ad21 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbTables.cs @@ -0,0 +1,83 @@ +using System.Globalization; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb; + +/// +/// Table names and DDL for the QuestDB backend (T0-confirmed against QuestDB 10.0.1). The TTL clause sits +/// between PARTITION BY and WAL and is an inlined integer (QuestDB cannot parameterise it). +/// Prefixes are regex-validated by the caller; these builders also refuse an invalid one. +/// +internal static class QuestDbTables +{ + /// Base name of the sample table (prefix + this). + public const string SampleBase = "sample"; + /// Base name of the event table. + public const string EventBase = "event"; + /// Base name of the tag catalog table. + public const string TagBase = "tag"; + + /// The fixed designated timestamp of every tag row, which makes the dedup key an upsert by name. + public static readonly DateTime TagEpoch = DateTime.UnixEpoch; + + /// The prefixed sample table name. + public static string Sample(string prefix) => Name(prefix, SampleBase); + /// The prefixed event table name. + public static string Event(string prefix) => Name(prefix, EventBase); + /// The prefixed tag table name. + public static string Tag(string prefix) => Name(prefix, TagBase); + + /// All three table names in creation order. + public static string[] All(string prefix) => [Sample(prefix), Event(prefix), Tag(prefix)]; + + /// The tables that carry a TTL (sample and event; the tag catalog never expires). + public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix)]; + + /// CREATE TABLE IF NOT EXISTS for the sample table. + public static string CreateSample(string prefix, int? retentionDays) => + $"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " + + "int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " + + $"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, tag)"; + + /// CREATE TABLE IF NOT EXISTS for the event table. + public static string CreateEvent(string prefix, int? retentionDays) => + $"CREATE TABLE IF NOT EXISTS {Event(prefix)} (ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, " + + "alarm_name VARCHAR, alarm_type_name VARCHAR, severity INT, event_kind VARCHAR, message VARCHAR, " + + "user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP) " + + $"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, event_id)"; + + /// CREATE TABLE IF NOT EXISTS for the tag catalog (YEAR partitions, no TTL ever). + public static string CreateTag(string prefix) => + $"CREATE TABLE IF NOT EXISTS {Tag(prefix)} (ts TIMESTAMP, name SYMBOL, data_type BYTE, eng_unit VARCHAR, " + + "description VARCHAR, updated_utc TIMESTAMP) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name)"; + + /// The three CREATE statements in creation order. + public static string[] CreateAll(string prefix, int? retentionDays) => + [CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix)]; + + /// ALTER TABLE t SET TTL n DAYS; null removes the TTL (SET TTL 0 DAYS). + public static string AlterTtl(string table, int? retentionDays) => + $"ALTER TABLE {table} SET TTL {(retentionDays ?? 0).ToString(CultureInfo.InvariantCulture)} DAYS"; + + /// + /// Converts a tables() ttlValue/ttlUnit pair to hours. QuestDB normalises the unit (7 days + /// reads back as 1 WEEK), so equality is judged in hours. An unrecognised unit returns -1 (never equal to + /// a wanted value, so the bootstrapper re-applies the TTL). + /// + public static long TtlHours(long value, string? unit) => (unit ?? "").ToUpperInvariant() switch + { + "HOUR" => value, + "DAY" => value * 24, + "WEEK" => value * 168, + _ => value == 0 ? 0 : -1, + }; + + private static string Ttl(int? days) => + days is { } d ? $" TTL {d.ToString(CultureInfo.InvariantCulture)} DAYS" : ""; + + private static string Name(string prefix, string baseName) + { + if (!QuestDbHistorianOptions.IsValidPrefix(prefix)) + throw new ArgumentException("Invalid table prefix.", nameof(prefix)); + return prefix + baseName; + } +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbTagCatalog.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbTagCatalog.cs new file mode 100644 index 00000000..341ba89d --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbTagCatalog.cs @@ -0,0 +1,82 @@ +using System.Collections.Concurrent; +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb; + +/// +/// The only "is this tag known" authority for the QuestDB backend (R7): a row in the tag table (fixed designated +/// timestamp 1970-01-01, deduplicated by name). Caches positive results only. Both methods run the schema +/// bootstrap first. Names longer than are never truncated: +/// answers false and throws. +/// inserts only when the tag is absent, because the table's dedup key replaces a +/// whole row: re-inserting a known tag would erase the engineering unit and description the provisioner wrote. +/// +internal sealed class QuestDbTagCatalog +{ + private readonly QuestDbConnection _conn; + private readonly QuestDbSchemaBootstrapper _bootstrap; + private readonly QuestDbWalVisibility _wal; + private readonly TimeProvider _clock; + private readonly ConcurrentDictionary _known = new(StringComparer.Ordinal); + + /// Creates the catalog. + /// Connection helper. + /// Schema bootstrapper. + /// WAL visibility wait. + /// Clock for updated_utc; system by default. + public QuestDbTagCatalog(QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbWalVisibility wal, + TimeProvider? clock = null) + { + _conn = conn; + _bootstrap = bootstrap; + _wal = wal; + _clock = clock ?? TimeProvider.System; + } + + /// Read-only lookup; never creates a row. + /// Historian tag name. + /// Cancellation. + /// True when the tag is in the catalog; false for an unknown or over-long name. + public async Task ExistsAsync(string name, CancellationToken ct) + { + if (!HistorianTagNames.IsValid(name)) return false; + if (_known.ContainsKey(name)) return true; + await _bootstrap.EnsureAsync(ct).ConfigureAwait(false); + await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false); + await using var cmd = _conn.CreateCommand(conn, $"SELECT count() FROM {_conn.TagTable} WHERE name = $1"); + QuestDbConnection.BindText(cmd, name); + var found = Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) > 0; + if (found) _known[name] = 0; + return found; + } + + /// Registers the tag when absent (no-op when present) and waits for the row to become visible. + /// Historian tag name. + /// Initial data_type (only written on create; provisioning corrects it later). + /// Cancellation. + /// A task completing when the tag is registered and visible. + /// The name is longer than . + public async Task EnsureAsync(string name, byte dataType, CancellationToken ct) + { + if (!HistorianTagNames.IsValid(name)) + throw new ArgumentException($"Historian tag name exceeds {HistorianTagNames.MaxLength} characters.", nameof(name)); + if (await ExistsAsync(name, ct).ConfigureAwait(false)) return; + + await using (var conn = await _conn.OpenAsync(ct).ConfigureAwait(false)) + { + await using var cmd = _conn.CreateCommand(conn, + $"INSERT INTO {_conn.TagTable} (ts, name, data_type, updated_utc) VALUES ($1, $2, $3, $4)"); + QuestDbConnection.BindTimestamp(cmd, QuestDbTables.TagEpoch); + QuestDbConnection.BindText(cmd, name); + QuestDbConnection.BindByte(cmd, dataType); + QuestDbConnection.BindTimestamp(cmd, _clock.GetUtcNow().UtcDateTime); + await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + await _wal.WaitAsync(_conn.TagTable, ct).ConfigureAwait(false); + _known[name] = 0; + } + + /// Marks a tag as known after another component (the provisioner) has written its row and waited. + /// Historian tag name. + internal void MarkKnown(string name) => _known[name] = 0; +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbWalVisibility.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbWalVisibility.cs new file mode 100644 index 00000000..578ff3bd --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbWalVisibility.cs @@ -0,0 +1,92 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb; + +/// +/// The read-your-writes wait (R5, T0 Q4). QuestDB applies WAL commits asynchronously, so a row just inserted may +/// not be visible to the next query. After its INSERT(s) a writer calls , which captures +/// sequencerTxn from wal_tables() (every autocommitted INSERT is one sequencer transaction; a concurrent +/// writer can only raise the target) and polls writerTxn every ~10 ms until it reaches the target, the table +/// is suspended, or elapses. +/// +internal sealed class QuestDbWalVisibility +{ + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(10); + + private readonly QuestDbConnection _conn; + private readonly TimeSpan _timeout; + private readonly ILogger _log; + + /// Creates the waiter. + /// Connection helper. + /// Maximum wait; (or less) disables the wait. + /// Optional logger. + public QuestDbWalVisibility(QuestDbConnection conn, TimeSpan timeout, ILogger? log = null) + { + _conn = conn; + _timeout = timeout; + _log = log ?? NullLogger.Instance; + } + + /// True when the wait is disabled (VisibilityTimeout is 0). + public bool Disabled => _timeout <= TimeSpan.Zero; + + /// True when the applied transaction has caught up with the target, or the table cannot progress. + /// Last applied transaction. + /// The sequencer transaction captured after the INSERT. + /// Whether polling can stop because the rows are visible. + internal static bool Reached(long writerTxn, long targetTxn) => writerTxn >= targetTxn; + + /// Waits until everything this caller inserted into is visible to queries. + /// Full table name written to. + /// Cancellation. + /// + /// True when visible (or the wait is disabled, or the table is not a WAL table); false when the table is suspended + /// or the timeout elapsed (a Warning is logged; the rows are still committed and become visible later). + /// + public async Task WaitAsync(string table, CancellationToken ct) + { + if (Disabled) return true; + var sw = Stopwatch.StartNew(); + await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false); + + long target; + await using (var cmd = _conn.CreateCommand(conn, "SELECT sequencerTxn FROM wal_tables() WHERE name = $1")) + { + QuestDbConnection.BindText(cmd, table); + var o = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false); + if (o is null or DBNull) return true; // not a WAL table: writes are immediately visible + target = Convert.ToInt64(o); + } + + while (true) + { + long writer; + bool suspended; + await using (var cmd = _conn.CreateCommand(conn, "SELECT writerTxn, suspended FROM wal_tables() WHERE name = $1")) + { + QuestDbConnection.BindText(cmd, table); + await using var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + if (!await r.ReadAsync(ct).ConfigureAwait(false)) return true; + writer = r.GetInt64(0); + suspended = r.GetBoolean(1); + } + if (Reached(writer, target)) return true; + if (suspended) + { + _log.LogWarning("QuestDB table {Table} is suspended (writerTxn {Writer} < sequencerTxn {Target}); not waiting.", + table, writer, target); + return false; + } + if (sw.Elapsed >= _timeout) + { + _log.LogWarning("QuestDB table {Table} did not become visible within {Timeout} ms (writerTxn {Writer} < sequencerTxn {Target}).", + table, _timeout.TotalMilliseconds, writer, target); + return false; + } + await Task.Delay(PollInterval, ct).ConfigureAwait(false); + } + } +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.csproj b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.csproj new file mode 100644 index 00000000..0a2487e9 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.csproj @@ -0,0 +1,30 @@ + + + + true + true + $(NoWarn);CS1591 + AnyCPU;x64 + ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.csproj b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.csproj index 6cae72d6..4061cda3 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.csproj +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.csproj @@ -44,6 +44,7 @@ puts it in StatusCodeParityTests' scope. --> + diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/BootstrapTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/BootstrapTests.cs new file mode 100644 index 00000000..cae1231e --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/BootstrapTests.cs @@ -0,0 +1,241 @@ +using Npgsql; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests; + +[Collection(QuestDbCollection.Name)] +public sealed class BootstrapTests(QuestDbFixture fx) +{ + private (QuestDbConnection Conn, QuestDbSchemaBootstrapper Boot, QuestDbHistorianOptions Opts) Make( + string prefix, int? retention = null, bool autoCreate = true) + { + var o = fx.Options(prefix, retention, autoCreate); + var c = new QuestDbConnection(o); + return (c, new QuestDbSchemaBootstrapper(c, o), o); + } + + private static async Task> TablesAsync(QuestDbConnection c, string prefix) + { + await using var conn = await c.OpenAsync(default); + await using var cmd = c.CreateCommand(conn, "SELECT table_name FROM tables() WHERE starts_with(table_name, $1) ORDER BY table_name"); + QuestDbConnection.BindText(cmd, prefix); + var names = new List(); + await using var r = await cmd.ExecuteReaderAsync(); + while (await r.ReadAsync()) names.Add(r.GetString(0)); + return names; + } + + private static async Task<(long Value, string Unit)> TtlAsync(QuestDbConnection c, string table) + { + await using var conn = await c.OpenAsync(default); + await using var cmd = c.CreateCommand(conn, "SELECT ttlValue, ttlUnit FROM tables() WHERE table_name = $1"); + QuestDbConnection.BindText(cmd, table); + await using var r = await cmd.ExecuteReaderAsync(); + (await r.ReadAsync()).ShouldBeTrue(); + return (Convert.ToInt64(r.GetValue(0)), r.GetString(1)); + } + + private static async Task SequencerTxnAsync(QuestDbConnection c, string table) + { + await using var conn = await c.OpenAsync(default); + await using var cmd = c.CreateCommand(conn, "SELECT sequencerTxn FROM wal_tables() WHERE name = $1"); + QuestDbConnection.BindText(cmd, table); + return Convert.ToInt64(await cmd.ExecuteScalarAsync()); + } + + [Fact] + public async Task Tables_are_created() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, _) = Make(p); + await boot.EnsureAsync(default); + (await TablesAsync(c, p)).ShouldBe([p + "event", p + "sample", p + "tag"]); + } + + [Fact] + public async Task A_second_run_is_a_noop_and_issues_no_alter() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, _) = Make(p, retention: 7); + await boot.EnsureAsync(default); + var before = await SequencerTxnAsync(c, p + "sample"); + + var (c2, boot2, _) = Make(p, retention: 7); // a fresh instance on a table whose TTL already matches (stored as 1 WEEK) + await boot2.EnsureAsync(default); + await Task.Delay(300); // an ALTER would have been applied by now + (await SequencerTxnAsync(c2, p + "sample")).ShouldBe(before); + (await SequencerTxnAsync(c2, p + "event")).ShouldBe(await SequencerTxnAsync(c, p + "event")); + (await TablesAsync(c2, p)).Count.ShouldBe(3); + } + + [Fact] + public async Task Two_concurrent_bootstrappers_both_succeed() + { + fx.SkipIfUnavailable(); + for (var round = 0; round < 5; round++) + { + var p = fx.UniquePrefix(); + var a = Make(p, retention: 7); + var b = Make(p, retention: 7); + await Task.WhenAll( + Task.Run(() => a.Boot.EnsureAsync(default)), + Task.Run(() => b.Boot.EnsureAsync(default))); + (await TablesAsync(a.Conn, p)).Count.ShouldBe(3); + } + } + + [Fact] + public async Task Ttl_is_in_metadata_when_retention_is_7_days_and_absent_when_unset() + { + fx.SkipIfUnavailable(); + var withTtl = fx.UniquePrefix(); + var (c1, b1, _) = Make(withTtl, retention: 7); + await b1.EnsureAsync(default); + QuestDbTables.TtlHours((await TtlAsync(c1, withTtl + "sample")).Value, (await TtlAsync(c1, withTtl + "sample")).Unit).ShouldBe(168); + QuestDbTables.TtlHours((await TtlAsync(c1, withTtl + "event")).Value, (await TtlAsync(c1, withTtl + "event")).Unit).ShouldBe(168); + (await TtlAsync(c1, withTtl + "tag")).Value.ShouldBe(0); // the tag catalog never expires + + var noTtl = fx.UniquePrefix(); + var (c2, b2, _) = Make(noTtl); + await b2.EnsureAsync(default); + (await TtlAsync(c2, noTtl + "sample")).Value.ShouldBe(0); + (await TtlAsync(c2, noTtl + "event")).Value.ShouldBe(0); + } + + [Fact] + public async Task A_changed_retention_is_reapplied_and_removal_clears_it() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, b7, _) = Make(p, retention: 7); + await b7.EnsureAsync(default); + + await Make(p, retention: 30).Boot.EnsureAsync(default); + await AwaitHoursAsync(c, p + "sample", 720); + await AwaitHoursAsync(c, p + "event", 720); + + await Make(p).Boot.EnsureAsync(default); // unset => SET TTL 0 + await AwaitHoursAsync(c, p + "sample", 0); + } + + private static async Task AwaitHoursAsync(QuestDbConnection c, string table, long hours) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + long got; + do + { + var (v, u) = await TtlAsync(c, table); + got = QuestDbTables.TtlHours(v, u); + if (got == hours) return; + await Task.Delay(50); // the ALTER is a WAL transaction: visible only once applied + } while (DateTime.UtcNow < deadline); + got.ShouldBe(hours); + } + + [Fact] + public async Task AutoCreateSchema_false_against_missing_tables_throws_HistorianSchemaMissingException() + { + fx.SkipIfUnavailable(); + var (_, boot, _) = Make(fx.UniquePrefix(), autoCreate: false); + var ex = await Should.ThrowAsync(() => boot.EnsureAsync(default)); + ex.Message.ShouldContain("AutoCreateSchema"); + } + + [Fact] + public async Task AutoCreateSchema_false_passes_once_the_tables_exist() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + await Make(p).Boot.EnsureAsync(default); + await Make(p, autoCreate: false).Boot.EnsureAsync(default); + } + + [Fact] + public async Task ExistsAsync_is_false_for_an_unknown_tag_and_true_after_EnsureAsync() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, o) = Make(p); + var catalog = new QuestDbTagCatalog(c, boot, new QuestDbWalVisibility(c, o.VisibilityTimeout)); + + (await catalog.ExistsAsync("Plant.Pump1.Speed", default)).ShouldBeFalse(); + await catalog.EnsureAsync("Plant.Pump1.Speed", 3, default); + (await catalog.ExistsAsync("Plant.Pump1.Speed", default)).ShouldBeTrue(); + // A fresh catalog (no cache) sees it through the table, and a different name is still unknown. + var fresh = new QuestDbTagCatalog(c, boot, new QuestDbWalVisibility(c, o.VisibilityTimeout)); + (await fresh.ExistsAsync("Plant.Pump1.Speed", default)).ShouldBeTrue(); + (await fresh.ExistsAsync("Plant.Pump1.Other", default)).ShouldBeFalse(); + await catalog.EnsureAsync("Plant.Pump1.Speed", 3, default); // idempotent + } + + [Fact] + public async Task An_over_long_tag_name_is_unknown_and_cannot_be_ensured() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, o) = Make(p); + var catalog = new QuestDbTagCatalog(c, boot, new QuestDbWalVisibility(c, o.VisibilityTimeout)); + var name = new string('x', HistorianTagNames.MaxLength + 1); + (await catalog.ExistsAsync(name, default)).ShouldBeFalse(); + await Should.ThrowAsync(() => catalog.EnsureAsync(name, 3, default)); + } + + [Fact] + public async Task Wait_returns_once_writerTxn_reaches_the_captured_sequencerTxn() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, o) = Make(p); + await boot.EnsureAsync(default); + var wal = new QuestDbWalVisibility(c, TimeSpan.FromSeconds(5)); + for (var i = 0; i < 20; i++) + { + var ts = new DateTime(2026, 3, 1, 0, 0, i, DateTimeKind.Utc); + await using (var conn = await c.OpenAsync(default)) + { + await using var cmd = c.CreateCommand(conn, + $"INSERT INTO {c.SampleTable} (ts, tag, kind, num_value, status, deleted) VALUES ($1, $2, $3, $4, $5, false)"); + QuestDbConnection.BindTimestamp(cmd, ts); + QuestDbConnection.BindText(cmd, "wal-tag"); + QuestDbConnection.BindByte(cmd, 3); + cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Double, Value = (double)i }); + QuestDbConnection.BindLong(cmd, 0); + await cmd.ExecuteNonQueryAsync(); + } + (await wal.WaitAsync(c.SampleTable, default)).ShouldBeTrue(); + // After the wait the row is visible on the very next query (the T0 gap was 7/20 without it). + await using var q = await c.OpenAsync(default); + await using var count = c.CreateCommand(q, $"SELECT count() FROM {c.SampleTable} WHERE tag = $1"); + QuestDbConnection.BindText(count, "wal-tag"); + Convert.ToInt64(await count.ExecuteScalarAsync()).ShouldBe(i + 1); + } + } + + [Fact] + public async Task Timeout_zero_skips_the_wait() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, _) = Make(p); + await boot.EnsureAsync(default); + var wal = new QuestDbWalVisibility(c, TimeSpan.Zero); + var sw = System.Diagnostics.Stopwatch.StartNew(); + (await wal.WaitAsync(c.SampleTable, default)).ShouldBeTrue(); + sw.ElapsedMilliseconds.ShouldBeLessThan(50); // no round-trip at all + } + + [Fact] + public async Task Waiting_on_a_table_that_is_not_a_WAL_table_returns_immediately() + { + fx.SkipIfUnavailable(); + var p = fx.UniquePrefix(); + var (c, boot, _) = Make(p); + await boot.EnsureAsync(default); + var wal = new QuestDbWalVisibility(c, TimeSpan.FromSeconds(5)); + (await wal.WaitAsync(p + "no_such_table", default)).ShouldBeTrue(); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbFixture.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbFixture.cs new file mode 100644 index 00000000..dcaba660 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbFixture.cs @@ -0,0 +1,106 @@ +using System.Collections.Concurrent; +using Npgsql; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests; + +/// +/// Env-gated QuestDB for the historian integration tests. HISTORIAN_QUESTDB_TEST_CONNECTION absent ⇒ +/// is set, nothing connects and every test calls Assert.Skip. Present ⇒ tests isolate +/// from each other with ; every prefix handed out is dropped on dispose (and a test may +/// drop its own early with ). Never touches a table outside those prefixes. +/// +public sealed class QuestDbFixture : IAsyncLifetime +{ + /// Env var holding an Npgsql connection string for QuestDB's PG-wire port. + public const string ConnectionEnvVar = "HISTORIAN_QUESTDB_TEST_CONNECTION"; + + private readonly ConcurrentBag _prefixes = new(); + + /// Why tests must skip, or null when QuestDB is available. + public string? SkipReason { get; private set; } + + /// The Npgsql connection string. Valid only when is null. + public string ConnectionString { get; private set; } = ""; + + /// Skips the calling test when the gate is closed. + public void SkipIfUnavailable() + { + if (SkipReason is not null) Assert.Skip(SkipReason); + } + + /// A fresh, valid table prefix ("t" + 12 hex + "_") that nothing uses yet; dropped on dispose. + /// The prefix. + public string UniquePrefix() + { + var p = "t" + Guid.NewGuid().ToString("N")[..12] + "_"; + _prefixes.Add(p); + return p; + } + + /// Options targeting this fixture's QuestDB under . + /// Table prefix (normally from ). + /// Optional TTL. + /// AutoCreateSchema. + /// Valid options. + public QuestDbHistorianOptions Options(string prefix, int? retentionDays = null, bool autoCreate = true) => new() + { + ConnectionString = ConnectionString, + TablePrefix = prefix, + RetentionDays = retentionDays, + AutoCreateSchema = autoCreate, + }; + + /// Drops every table whose name starts with (found through tables()). + /// Table prefix. + /// A task completing when they are gone. + public async Task DropPrefixAsync(string prefix) + { + if (SkipReason is not null) return; + if (!QuestDbHistorianOptions.IsValidPrefix(prefix)) throw new ArgumentException("Invalid prefix.", nameof(prefix)); + await using var ds = NpgsqlDataSource.Create(ConnectionString); + await using var conn = await ds.OpenConnectionAsync().ConfigureAwait(false); + var names = new List(); + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = "SELECT table_name FROM tables() WHERE starts_with(table_name, $1)"; // never LIKE: '_' is a wildcard + cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Varchar, Value = prefix }); + await using var r = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (await r.ReadAsync().ConfigureAwait(false)) names.Add(r.GetString(0)); + } + foreach (var n in names) + { + await using var drop = conn.CreateCommand(); + drop.CommandText = $"DROP TABLE IF EXISTS {n}"; + await drop.ExecuteNonQueryAsync().ConfigureAwait(false); + } + } + + /// + public ValueTask InitializeAsync() + { + var cs = Environment.GetEnvironmentVariable(ConnectionEnvVar); + if (string.IsNullOrWhiteSpace(cs)) + SkipReason = $"{ConnectionEnvVar} is not set; QuestDB historian integration tests skipped."; + else + ConnectionString = cs; + return ValueTask.CompletedTask; + } + + /// + public async ValueTask DisposeAsync() + { + if (SkipReason is not null) return; + foreach (var p in _prefixes.Distinct()) + await DropPrefixAsync(p).ConfigureAwait(false); + NpgsqlConnection.ClearAllPools(); + } +} + +/// Shares one fixture across the QuestDB integration tests. +[CollectionDefinition(Name)] +public sealed class QuestDbCollection : ICollectionFixture +{ + /// Collection name. + public const string Name = "QuestDb"; +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests.csproj b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests.csproj new file mode 100644 index 00000000..c34ce291 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests.csproj @@ -0,0 +1,33 @@ + + + + net10.0 + enable + enable + false + true + ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests + $(NoWarn);xUnit1051 + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbConnectionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbConnectionTests.cs new file mode 100644 index 00000000..0f39bf6e --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbConnectionTests.cs @@ -0,0 +1,140 @@ +using System.Net.Sockets; +using Npgsql; +using NpgsqlTypes; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests; + +public sealed class QuestDbConnectionTests +{ + private static DateTime Bound(DateTime v, bool clamp = false) + { + var cmd = new NpgsqlCommand(); + var p = QuestDbConnection.BindTimestamp(cmd, v, clamp); + p.NpgsqlDbType.ShouldBe(NpgsqlDbType.TimestampTz); + var bound = (DateTime)p.Value!; + bound.Kind.ShouldBe(DateTimeKind.Utc); + return bound; + } + + [Fact] + public void Round_truncates_sub_microsecond_ticks() + { + var t = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc).AddTicks(1234567); + QuestDbConnection.Round(t).ShouldBe(new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc).AddTicks(1234560)); + } + + [Fact] + public void Round_truncates_rather_than_rounds_before_2000() + { + var t = new DateTime(1990, 5, 5, 0, 0, 0, DateTimeKind.Utc).AddTicks(1234565); + QuestDbConnection.Round(t).Ticks.ShouldBe(t.Ticks - 5); + } + + [Fact] + public void Round_treats_Unspecified_as_UTC() + { + var u = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Unspecified); + var r = QuestDbConnection.Round(u); + r.Kind.ShouldBe(DateTimeKind.Utc); + r.Ticks.ShouldBe(u.Ticks); + } + + [Fact] + public void Round_converts_Local_to_UTC() + { + var local = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Local); + var r = QuestDbConnection.Round(local); + r.Kind.ShouldBe(DateTimeKind.Utc); + r.ShouldBe(local.ToUniversalTime()); + } + + [Fact] + public void Round_keeps_a_UTC_value_that_is_already_aligned() + { + var t = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + QuestDbConnection.Round(t).ShouldBe(t); + } + + [Theory] + [InlineData(0)] // MinValue itself + [InlineData(5)] // truncates to MinValue + public void BindTimestamp_never_emits_MinValue(long ticks) + { + var b = Bound(new DateTime(ticks, DateTimeKind.Utc)); + b.ShouldNotBe(DateTime.MinValue); + b.Ticks.ShouldBe(10); + b.ShouldBe(new DateTime(10, DateTimeKind.Utc)); // 0001-01-01T00:00:00.000001 (T0-confirmed) + } + + [Fact] + public void BindTimestamp_never_emits_MaxValue() + { + var b = Bound(DateTime.MaxValue); + b.ShouldNotBe(DateTime.MaxValue); + b.ShouldBe(new DateTime(9999, 12, 31, 23, 59, 59, DateTimeKind.Utc).AddTicks(9999990)); // T0-confirmed bound + Bound(DateTime.MinValue).ShouldNotBe(DateTime.MinValue); + } + + [Fact] + public void BindTimestamp_never_emits_either_extreme_for_any_kind() + { + foreach (var kind in new[] { DateTimeKind.Utc, DateTimeKind.Unspecified, DateTimeKind.Local }) + foreach (var t in new[] { new DateTime(DateTime.MinValue.Ticks, kind), new DateTime(DateTime.MaxValue.Ticks, kind) }) + { + var b = Bound(t); + b.ShouldNotBe(DateTime.MinValue); + b.ShouldNotBe(DateTime.MaxValue); + } + } + + [Fact] + public void BindTimestamp_keeps_a_pre_1970_range_bound_unless_asked_to_clamp() + { + var pre = new DateTime(1960, 1, 1, 0, 0, 0, DateTimeKind.Utc); + Bound(pre).ShouldBe(pre); + Bound(pre, clamp: true).ShouldBe(DateTime.UnixEpoch); + Bound(new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), clamp: true).Year.ShouldBe(2026); + } + + [Fact] + public void IsStorable_is_false_only_before_1970() + { + QuestDbConnection.IsStorable(DateTime.UnixEpoch).ShouldBeTrue(); + QuestDbConnection.IsStorable(DateTime.UnixEpoch.AddTicks(-1)).ShouldBeFalse(); + QuestDbConnection.IsStorable(new DateTime(1601, 1, 1, 0, 0, 0, DateTimeKind.Utc)).ShouldBeFalse(); + QuestDbConnection.IsStorable(DateTime.MinValue).ShouldBeFalse(); + QuestDbConnection.IsStorable(DateTime.MaxValue).ShouldBeTrue(); + QuestDbConnection.IsStorable(new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc)).ShouldBeTrue(); + } + + [Fact] + public void BindByte_binds_smallint() + { + var p = QuestDbConnection.BindByte(new NpgsqlCommand(), 200); + p.NpgsqlDbType.ShouldBe(NpgsqlDbType.Smallint); + p.Value.ShouldBe((short)200); + } + + [Fact] + public void IsTransient_classifies_transport_failures_not_server_rejections() + { + QuestDbConnection.IsTransient(new SocketException()).ShouldBeTrue(); + QuestDbConnection.IsTransient(new IOException("x")).ShouldBeTrue(); + QuestDbConnection.IsTransient(new TimeoutException()).ShouldBeTrue(); + QuestDbConnection.IsTransient(new NpgsqlException("x", new IOException())).ShouldBeTrue(); + QuestDbConnection.IsTransient(new InvalidOperationException("x")).ShouldBeFalse(); + QuestDbConnection.IsTransient(new OperationCanceledException()).ShouldBeFalse(); + } + + [Fact] + public void T_prefixes_the_table_base_name() + { + var c = new QuestDbConnection(new QuestDbHistorianOptions { ConnectionString = "Host=127.0.0.1;Port=1", TablePrefix = "p_" }); + c.T("sample").ShouldBe("p_sample"); + c.SampleTable.ShouldBe("p_sample"); + c.EventTable.ShouldBe("p_event"); + c.TagTable.ShouldBe("p_tag"); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianOptionsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianOptionsTests.cs new file mode 100644 index 00000000..6c37c63d --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianOptionsTests.cs @@ -0,0 +1,79 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests; + +public sealed class QuestDbHistorianOptionsTests +{ + private static QuestDbHistorianOptions Valid() => new() { ConnectionString = "Host=h;Port=8812" }; + + [Fact] + public void Defaults_are_the_documented_ones() + { + var o = new QuestDbHistorianOptions(); + o.ConnectionString.ShouldBe(""); + o.TablePrefix.ShouldBe("hist_"); + o.RetentionDays.ShouldBeNull(); + o.CommandTimeout.ShouldBe(TimeSpan.FromSeconds(30)); + o.AutoCreateSchema.ShouldBeTrue(); + o.VisibilityTimeout.ShouldBe(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void A_valid_configuration_has_no_errors() => Valid().Validate().ShouldBeEmpty(); + + [Fact] + public void Missing_connection_string_names_the_key() + { + var o = Valid(); o.ConnectionString = " "; + o.Validate().ShouldHaveSingleItem().ShouldContain("ServerHistorian:QuestDb:ConnectionString"); + } + + [Theory] + [InlineData("")] + [InlineData("hist-")] + [InlineData("hist.")] + [InlineData("hist_; DROP")] + public void Bad_prefix_names_the_key(string prefix) + { + var o = Valid(); o.TablePrefix = prefix; + o.Validate().ShouldHaveSingleItem().ShouldContain("ServerHistorian:QuestDb:TablePrefix"); + } + + [Theory] + [InlineData("t0123456789ab_")] + [InlineData("A_9")] + public void Good_prefixes_pass(string prefix) + { + var o = Valid(); o.TablePrefix = prefix; + o.Validate().ShouldBeEmpty(); + } + + [Theory] + [InlineData(0)] + [InlineData(-3)] + public void Retention_below_one_names_the_key(int days) + { + var o = Valid(); o.RetentionDays = days; + o.Validate().ShouldHaveSingleItem().ShouldContain("ServerHistorian:QuestDb:RetentionDays"); + } + + [Fact] + public void Retention_of_one_is_valid() { var o = Valid(); o.RetentionDays = 1; o.Validate().ShouldBeEmpty(); } + + [Fact] + public void Non_positive_command_timeout_names_the_key() + { + var o = Valid(); o.CommandTimeout = TimeSpan.Zero; + o.Validate().ShouldHaveSingleItem().ShouldContain("ServerHistorian:QuestDb:CommandTimeout"); + } + + [Fact] + public void Negative_visibility_timeout_names_the_key_and_zero_is_valid() + { + var o = Valid(); o.VisibilityTimeout = TimeSpan.FromMilliseconds(-1); + o.Validate().ShouldHaveSingleItem().ShouldContain("ServerHistorian:QuestDb:VisibilityTimeout"); + o.VisibilityTimeout = TimeSpan.Zero; + o.Validate().ShouldBeEmpty(); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbSchemaBootstrapperFailureTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbSchemaBootstrapperFailureTests.cs new file mode 100644 index 00000000..d1730696 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbSchemaBootstrapperFailureTests.cs @@ -0,0 +1,32 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests; + +public sealed class QuestDbSchemaBootstrapperFailureTests +{ + private sealed class ManualClock : TimeProvider + { + private DateTimeOffset _now = new(2026, 10, 11, 0, 0, 0, TimeSpan.Zero); + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan by) => _now += by; + } + + [Fact] + public async Task A_failure_is_cached_for_five_seconds_then_retried() + { + var opts = new QuestDbHistorianOptions { ConnectionString = "Host=127.0.0.1;Port=1;Timeout=1" }; + var clock = new ManualClock(); + var boot = new QuestDbSchemaBootstrapper(new QuestDbConnection(opts), opts, timeProvider: clock); + + var first = await Should.ThrowAsync(() => boot.EnsureAsync(CancellationToken.None)); + // Within the window the SAME failure is re-thrown without another connection attempt. + var second = await Should.ThrowAsync(() => boot.EnsureAsync(CancellationToken.None)); + second.ShouldBeSameAs(first); + + clock.Advance(BootstrapFailureCache.DefaultDuration + TimeSpan.FromSeconds(1)); + var third = await Should.ThrowAsync(() => boot.EnsureAsync(CancellationToken.None)); + third.ShouldNotBeSameAs(first); // the window closed: QuestDB was tried again + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbTablesTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbTablesTests.cs new file mode 100644 index 00000000..ef42fb50 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbTablesTests.cs @@ -0,0 +1,66 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests; + +public sealed class QuestDbTablesTests +{ + [Fact] + public void Sample_ddl_without_ttl_is_pinned() => + QuestDbTables.CreateSample("hist_", null).ShouldBe( + "CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " + + "int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " + + "TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, tag)"); + + [Fact] + public void Sample_ddl_with_ttl_puts_ttl_between_partition_and_wal() => + QuestDbTables.CreateSample("hist_", 7).ShouldBe( + "CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " + + "int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " + + "TIMESTAMP(ts) PARTITION BY DAY TTL 7 DAYS WAL DEDUP UPSERT KEYS(ts, tag)"); + + [Fact] + public void Event_ddl_is_pinned() + { + QuestDbTables.CreateEvent("hist_", null).ShouldBe( + "CREATE TABLE IF NOT EXISTS hist_event (ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, " + + "alarm_name VARCHAR, alarm_type_name VARCHAR, severity INT, event_kind VARCHAR, message VARCHAR, " + + "user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP) " + + "TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, event_id)"); + QuestDbTables.CreateEvent("hist_", 30).ShouldContain("PARTITION BY DAY TTL 30 DAYS WAL DEDUP"); + } + + [Fact] + public void Tag_ddl_is_pinned_and_never_carries_a_ttl() => + QuestDbTables.CreateTag("hist_").ShouldBe( + "CREATE TABLE IF NOT EXISTS hist_tag (ts TIMESTAMP, name SYMBOL, data_type BYTE, eng_unit VARCHAR, " + + "description VARCHAR, updated_utc TIMESTAMP) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name)"); + + [Fact] + public void The_prefix_is_applied_to_every_table_name() + { + QuestDbTables.All("t1_").ShouldBe(["t1_sample", "t1_event", "t1_tag"]); + QuestDbTables.CreateAll("t1_", 3).ShouldAllBe(s => s.Contains("t1_")); + } + + [Fact] + public void An_invalid_prefix_is_refused() => + Should.Throw(() => QuestDbTables.CreateSample("a b", null)); + + [Fact] + public void Alter_ttl_is_pinned_and_null_removes_it() + { + QuestDbTables.AlterTtl("hist_sample", 30).ShouldBe("ALTER TABLE hist_sample SET TTL 30 DAYS"); + QuestDbTables.AlterTtl("hist_sample", null).ShouldBe("ALTER TABLE hist_sample SET TTL 0 DAYS"); + } + + [Theory] + [InlineData(1, "WEEK", 168)] // 7 days reads back as 1 WEEK + [InlineData(2, "WEEK", 336)] + [InlineData(30, "DAY", 720)] + [InlineData(0, "HOUR", 0)] // no TTL + [InlineData(5, "hour", 5)] + [InlineData(1, "MONTH", -1)] // unknown unit never equals a wanted value + public void Ttl_is_compared_in_hours(long value, string unit, long hours) => + QuestDbTables.TtlHours(value, unit).ShouldBe(hours); +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbWalVisibilityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbWalVisibilityTests.cs new file mode 100644 index 00000000..b4f71714 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbWalVisibilityTests.cs @@ -0,0 +1,33 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests; + +public sealed class QuestDbWalVisibilityTests +{ + [Fact] + public async Task Timeout_zero_skips_the_wait_without_touching_the_database() + { + // Port 1 refuses connections: any attempt to query would throw, so returning true proves nothing was opened. + var conn = new QuestDbConnection(new QuestDbHistorianOptions { ConnectionString = "Host=127.0.0.1;Port=1;Timeout=1" }); + var wal = new QuestDbWalVisibility(conn, TimeSpan.Zero); + wal.Disabled.ShouldBeTrue(); + (await wal.WaitAsync("hist_sample", CancellationToken.None)).ShouldBeTrue(); + } + + [Fact] + public async Task A_positive_timeout_does_touch_the_database_control() + { + var conn = new QuestDbConnection(new QuestDbHistorianOptions { ConnectionString = "Host=127.0.0.1;Port=1;Timeout=1" }); + var wal = new QuestDbWalVisibility(conn, TimeSpan.FromSeconds(1)); + wal.Disabled.ShouldBeFalse(); + await Should.ThrowAsync(() => wal.WaitAsync("hist_sample", CancellationToken.None)); + } + + [Theory] + [InlineData(5, 5, true)] + [InlineData(6, 5, true)] + [InlineData(4, 5, false)] + public void Reached_is_writerTxn_at_or_beyond_the_target(long writer, long target, bool expected) => + QuestDbWalVisibility.Reached(writer, target).ShouldBe(expected); +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests.csproj b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests.csproj new file mode 100644 index 00000000..98c12258 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests.csproj @@ -0,0 +1,27 @@ + + + + net10.0 + enable + enable + false + true + ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + +