Joseph Doherty
8d82d049e9
- I-1: QuestDbConnection caps Npgsql Maximum Pool Size at 24 (DefaultMaxPoolSize) when the connection string does not set it; an explicit value is kept. The effective value is logged once at Information. - M-1: QuestDbTagCatalog.EnsureAsync waits for the tag table's sequenced commits and re-checks before inserting, so it no longer overwrites a provisioner row (unit/description) that is committed but not yet applied. Integration test reproduces the clobber against real QuestDB (falsified). - M-3: the pre-1970 drop Warning is rate-limited per tag (once per 10 min); DroppedPre1970 still counts every sample. - M-5: QuestDbHistorianProvider is IAsyncDisposable/IDisposable and disposes the shared Npgsql pool; Program.cs registers the provider through a factory and every seam resolves it from the container, so DI disposes it at shutdown. - M-8, M-9: docs fixes (HistoryUpdate transaction wording; folded duplicate HistoryUpdate paragraph). Pool-size docs updated for the new default cap. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
248 lines
13 KiB
C#
248 lines
13 KiB
C#
using System.Net.Sockets;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Npgsql;
|
|
using NpgsqlTypes;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
|
|
|
/// <summary>
|
|
/// Pooled Npgsql access to QuestDB plus every T0-confirmed typing rule, in one place: how timestamps are rounded and
|
|
/// bound (R6), how BYTE columns are bound and read, and how errors are classified.
|
|
/// <para>Contract for callers: bind every inbound <see cref="DateTime"/> through <see cref="BindTimestamp"/> (never
|
|
/// <c>AddWithValue</c>), call <see cref="IsStorable"/> before writing a designated timestamp, and classify
|
|
/// failures with <see cref="IsTransient"/> / <see cref="IsAlreadyExists"/>. Every QuestDB server error carries
|
|
/// SQLSTATE <c>00000</c>, so classification is by message text, never by SQLSTATE.</para>
|
|
/// </summary>
|
|
internal sealed class QuestDbConnection : IAsyncDisposable, IDisposable
|
|
{
|
|
/// <summary>
|
|
/// Npgsql pool cap applied when the connection string does not set <c>Maximum Pool Size</c> itself. Npgsql's own
|
|
/// default (100) exceeds QuestDB's default PG-wire connection limit (64), and the server's default read
|
|
/// concurrency alone can reach 64; an explicit value in the connection string is always kept.
|
|
/// </summary>
|
|
internal const int DefaultMaxPoolSize = 24;
|
|
|
|
private static readonly long MinSafeTicks = 10; // DateTime.MinValue itself binds as -infinity
|
|
private static readonly long MaxSafeTicks = DateTime.MaxValue.Ticks - DateTime.MaxValue.Ticks % 10; // MaxValue itself binds as +infinity
|
|
|
|
private readonly NpgsqlDataSource _dataSource;
|
|
|
|
/// <summary>Creates the connection helper; options must already be valid.</summary>
|
|
/// <param name="options">Validated options.</param>
|
|
/// <param name="log">Optional logger; the effective pool size is logged once at Information.</param>
|
|
public QuestDbConnection(QuestDbHistorianOptions options, ILogger? log = null)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
if (!QuestDbHistorianOptions.IsValidPrefix(options.TablePrefix))
|
|
throw new ArgumentException("Invalid table prefix.", nameof(options));
|
|
TablePrefix = options.TablePrefix;
|
|
CommandTimeoutSeconds = Math.Max(1, (int)Math.Ceiling(options.CommandTimeout.TotalSeconds));
|
|
var csb = new NpgsqlConnectionStringBuilder(options.ConnectionString);
|
|
var explicitPool = HasExplicitMaxPoolSize(options.ConnectionString);
|
|
if (!explicitPool) csb.MaxPoolSize = DefaultMaxPoolSize;
|
|
EffectiveMaxPoolSize = csb.MaxPoolSize;
|
|
(log ?? NullLogger.Instance).LogInformation(
|
|
"QuestDB historian connection pool: Maximum Pool Size = {MaxPoolSize} ({Source}).",
|
|
EffectiveMaxPoolSize, explicitPool ? "from the connection string" : "default cap");
|
|
_dataSource = new NpgsqlDataSourceBuilder(csb.ConnectionString).Build();
|
|
}
|
|
|
|
/// <summary>The Npgsql <c>Maximum Pool Size</c> in effect (the connection string's own value, else <see cref="DefaultMaxPoolSize"/>).</summary>
|
|
internal int EffectiveMaxPoolSize { get; }
|
|
|
|
// Npgsql accepts "Maximum Pool Size" and "MaxPoolSize" (case-insensitive); a generic parse keeps the caller's own
|
|
// keywords so "was it set" is answered by the string itself, not by the builder's default.
|
|
private static bool HasExplicitMaxPoolSize(string connectionString)
|
|
{
|
|
var generic = new System.Data.Common.DbConnectionStringBuilder { ConnectionString = connectionString };
|
|
foreach (string key in generic.Keys)
|
|
{
|
|
var k = key.Replace(" ", "", StringComparison.Ordinal);
|
|
if (string.Equals(k, "MaxPoolSize", StringComparison.OrdinalIgnoreCase)
|
|
|| string.Equals(k, "MaximumPoolSize", StringComparison.OrdinalIgnoreCase))
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// <summary>The validated table prefix.</summary>
|
|
public string TablePrefix { get; }
|
|
|
|
/// <summary>Command timeout applied by <see cref="CreateCommand"/>, in seconds.</summary>
|
|
public int CommandTimeoutSeconds { get; }
|
|
|
|
/// <summary>Prefixes a table base name (<c>sample</c>, <c>event</c>, <c>tag</c>) with the configured prefix.</summary>
|
|
/// <param name="table">Fixed base name from this project (never user input).</param>
|
|
/// <returns>The full table name.</returns>
|
|
public string T(string table) => TablePrefix + table;
|
|
|
|
/// <summary>The prefixed sample table.</summary>
|
|
public string SampleTable => T(QuestDbTables.SampleBase);
|
|
/// <summary>The prefixed event table.</summary>
|
|
public string EventTable => T(QuestDbTables.EventBase);
|
|
/// <summary>The prefixed tag table.</summary>
|
|
public string TagTable => T(QuestDbTables.TagBase);
|
|
|
|
/// <summary>Opens a pooled connection (no bootstrap; callers run the bootstrapper first).</summary>
|
|
/// <param name="ct">Cancellation.</param>
|
|
/// <returns>An open connection the caller disposes.</returns>
|
|
public async Task<NpgsqlConnection> OpenAsync(CancellationToken ct) =>
|
|
await _dataSource.OpenConnectionAsync(ct).ConfigureAwait(false);
|
|
|
|
/// <summary>Creates a text command with the configured timeout. Parameters are positional (<c>$1</c>, <c>$2</c>…).</summary>
|
|
/// <param name="conn">Open connection.</param>
|
|
/// <param name="sql">Command text.</param>
|
|
/// <returns>The command.</returns>
|
|
public NpgsqlCommand CreateCommand(NpgsqlConnection conn, string sql)
|
|
{
|
|
var cmd = conn.CreateCommand();
|
|
cmd.CommandText = sql;
|
|
cmd.CommandTimeout = CommandTimeoutSeconds;
|
|
return cmd;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public ValueTask DisposeAsync() => _dataSource.DisposeAsync();
|
|
|
|
/// <inheritdoc/>
|
|
public void Dispose() => _dataSource.Dispose();
|
|
|
|
// ---- timestamps (R6) -------------------------------------------------------------------------------------------
|
|
|
|
/// <summary>
|
|
/// Normalises a timestamp for QuestDB (microsecond precision). Rule: <c>Unspecified</c> is treated as UTC
|
|
/// (<c>SpecifyKind</c>); <c>Local</c> is converted with <c>ToUniversalTime</c>; then ticks are truncated to a
|
|
/// microsecond (<c>ticks - ticks % 10</c>, because Npgsql truncates relative to 2000-01-01 and so rounds UP
|
|
/// before 2000); finally the result is nudged into <c>[0001-01-01T00:00:00.000001, 9999-12-31T23:59:59.999999]</c>
|
|
/// so a value equal to <c>MinValue</c>/<c>MaxValue</c> never reaches Npgsql, which would bind it as ±infinity
|
|
/// (a <c>[Min, Max)</c> range then silently matches nothing). The result always has <c>Kind=Utc</c>.
|
|
/// </summary>
|
|
/// <param name="value">Any timestamp.</param>
|
|
/// <returns>A UTC, microsecond-aligned, finite timestamp.</returns>
|
|
public static DateTime Round(DateTime value)
|
|
{
|
|
var utc = value.Kind switch
|
|
{
|
|
DateTimeKind.Local => value.ToUniversalTime(),
|
|
DateTimeKind.Utc => value,
|
|
_ => DateTime.SpecifyKind(value, DateTimeKind.Utc),
|
|
};
|
|
var ticks = utc.Ticks - utc.Ticks % 10;
|
|
ticks = Math.Clamp(ticks, MinSafeTicks, MaxSafeTicks);
|
|
return new DateTime(ticks, DateTimeKind.Utc);
|
|
}
|
|
|
|
/// <summary>
|
|
/// True when QuestDB can store the timestamp as a designated timestamp: after <see cref="Round"/> it is on or
|
|
/// after 1970-01-01 (RC1). Writers drop (live) or refuse (HistoryUpdate) samples for which this is false; reads
|
|
/// with earlier bounds are fine.
|
|
/// </summary>
|
|
/// <param name="value">Candidate sample time.</param>
|
|
/// <returns>Whether an INSERT with this designated timestamp can succeed.</returns>
|
|
public static bool IsStorable(DateTime value) => Round(value) >= DateTime.UnixEpoch;
|
|
|
|
/// <summary>
|
|
/// Appends the next positional parameter (<c>$n</c>) as <c>timestamp with time zone</c> carrying
|
|
/// <see cref="Round"/>'s UTC value (T0-confirmed: <c>Timestamp</c>+<c>Utc</c> throws in Npgsql). It never emits
|
|
/// <c>MinValue</c>/<c>MaxValue</c>. By default a pre-1970 value is kept as is (a pre-1970 range bound is valid
|
|
/// and clamping it would move the bound across the epoch row); pass <paramref name="clampToEpoch"/> for a
|
|
/// designated-timestamp column so a pre-1970 value can only ever become the epoch itself, never a failed batch.
|
|
/// </summary>
|
|
/// <param name="cmd">Target command.</param>
|
|
/// <param name="value">Any timestamp.</param>
|
|
/// <param name="clampToEpoch">Clamp anything before 1970-01-01 to the epoch (writes only, after <see cref="IsStorable"/>).</param>
|
|
/// <returns>The added parameter.</returns>
|
|
public static NpgsqlParameter BindTimestamp(NpgsqlCommand cmd, DateTime value, bool clampToEpoch = false)
|
|
{
|
|
var rounded = Round(value);
|
|
if (clampToEpoch && rounded < DateTime.UnixEpoch) rounded = DateTime.UnixEpoch;
|
|
var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.TimestampTz, Value = rounded };
|
|
cmd.Parameters.Add(p);
|
|
return p;
|
|
}
|
|
|
|
/// <summary>Appends a <c>BYTE</c> parameter (bound as <c>smallint</c>, T0-confirmed).</summary>
|
|
/// <param name="cmd">Target command.</param>
|
|
/// <param name="value">The byte.</param>
|
|
/// <returns>The added parameter.</returns>
|
|
public static NpgsqlParameter BindByte(NpgsqlCommand cmd, byte value)
|
|
{
|
|
var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Smallint, Value = (short)value };
|
|
cmd.Parameters.Add(p);
|
|
return p;
|
|
}
|
|
|
|
/// <summary>Appends a <c>LONG</c> parameter (<c>bigint</c>; also the rule for integer arithmetic in tick math).</summary>
|
|
/// <param name="cmd">Target command.</param>
|
|
/// <param name="value">The value.</param>
|
|
/// <returns>The added parameter.</returns>
|
|
public static NpgsqlParameter BindLong(NpgsqlCommand cmd, long value)
|
|
{
|
|
var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Bigint, Value = value };
|
|
cmd.Parameters.Add(p);
|
|
return p;
|
|
}
|
|
|
|
/// <summary>Appends a <c>SYMBOL</c>/<c>VARCHAR</c> parameter (<c>varchar</c>); null binds as NULL.</summary>
|
|
/// <param name="cmd">Target command.</param>
|
|
/// <param name="value">The string.</param>
|
|
/// <returns>The added parameter.</returns>
|
|
public static NpgsqlParameter BindText(NpgsqlCommand cmd, string? value)
|
|
{
|
|
var p = new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Varchar, Value = (object?)value ?? DBNull.Value };
|
|
cmd.Parameters.Add(p);
|
|
return p;
|
|
}
|
|
|
|
/// <summary>Reads a <c>TIMESTAMP</c> column as UTC (QuestDB returns <c>Kind=Unspecified</c> holding UTC wall clock).</summary>
|
|
/// <param name="r">Reader positioned on a row.</param>
|
|
/// <param name="i">Column ordinal.</param>
|
|
/// <returns>The timestamp with <c>Kind=Utc</c>.</returns>
|
|
public static DateTime ReadUtc(NpgsqlDataReader r, int i) =>
|
|
DateTime.SpecifyKind(r.GetDateTime(i), DateTimeKind.Utc);
|
|
|
|
/// <summary>Reads a <c>BYTE</c> column (QuestDB returns <c>smallint</c>); NULL (never stored) reads as 0 (Empty).</summary>
|
|
/// <param name="r">Reader positioned on a row.</param>
|
|
/// <param name="i">Column ordinal.</param>
|
|
/// <returns>The byte value.</returns>
|
|
public static byte ReadKind(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
|
|
|
// ---- error classification --------------------------------------------------------------------------------------
|
|
|
|
/// <summary>
|
|
/// True for a transport-level or timeout failure worth retrying. A <see cref="PostgresException"/> is a
|
|
/// server-side rejection (QuestDB reports every one as SQLSTATE 00000), so it is never transient here; decide
|
|
/// those by message text.
|
|
/// </summary>
|
|
/// <param name="ex">The failure.</param>
|
|
/// <returns>Whether the failure is transient.</returns>
|
|
public static bool IsTransient(Exception ex)
|
|
{
|
|
for (var e = ex; e is not null; e = e.InnerException)
|
|
{
|
|
if (e is PostgresException or OperationCanceledException) return false;
|
|
if (e is NpgsqlException { IsTransient: true } or SocketException or IOException or TimeoutException) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// <summary>True when a server error says the object already exists (message text; SQLSTATE is always 00000).</summary>
|
|
/// <param name="ex">The failure.</param>
|
|
/// <returns>Whether it is an "already exists" rejection.</returns>
|
|
public static bool IsAlreadyExists(Exception ex) =>
|
|
ex is PostgresException pg && pg.MessageText.Contains("already exists", StringComparison.OrdinalIgnoreCase);
|
|
|
|
/// <summary>Whether a table exists, per <c>tables()</c>.</summary>
|
|
/// <param name="conn">Open connection.</param>
|
|
/// <param name="table">Full table name.</param>
|
|
/// <param name="ct">Cancellation.</param>
|
|
/// <returns>True when listed.</returns>
|
|
public async Task<bool> TableExistsAsync(NpgsqlConnection conn, string table, CancellationToken ct)
|
|
{
|
|
await using var cmd = CreateCommand(conn, "SELECT count() FROM tables() WHERE table_name = $1");
|
|
BindText(cmd, table);
|
|
return Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) > 0;
|
|
}
|
|
}
|