using System.Globalization;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Npgsql;
using NpgsqlTypes;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
///
/// QuestDB read side of the historian (spec §3 "Reads"): Raw (forward / reverse / point), AtTime (chunked
/// LIMIT 1 neighbour seeks, T0 Q9), Events, and Processed aggregates pushed down to QuestDB SQL
/// (, T0 Q8). Results match the SQL Server data source except that every
/// timestamp is at microsecond resolution.
///
///
///
/// Timestamps (R6). Every inbound time is normalised with and every
/// is bound only through . Read bounds are not
/// clamped to the epoch (a pre-1970 bound is valid); only MinValue/MaxValue are nudged. Results
/// carry the rounded times, so an AtTime result is stamped at the rounded request time.
///
///
/// Reads never create tags (R7). A tag resolves only through ;
/// an unknown tag is a successful empty read (Raw), an all-BadNoData read (AtTime / Processed), never an
/// error. Every sample query filters NOT deleted, so a DeleteRaw/DeleteAtTime tombstone is invisible.
///
///
/// Values. Every row goes through (an Empty row with a non-Bad status
/// is served BadNoData). The value kind, not which column is populated, decides the value. QuestDB stores
/// in a LONG as NULL (its NULL sentinel), so an Int64 row whose
/// int_value is NULL and whose num_value is exactly −2^63 is served as
/// (ruling RC2).
///
///
/// Faults. As in the SQL Server data source: a database / bootstrap exception is recorded in
/// and rethrown, failing only that call. Client-request errors — an unsupported
/// aggregate () and a processed window over
/// intervals () — are thrown before
/// any I/O and are not counted as backend failures.
///
///
public sealed class QuestDbHistorianDataSource : IHistorianDataSource
{
/// Most AtTime request times per statement (two UNION ALL branches each; QuestDB 10.0.1 refuses more than 126 branches).
public const int AtTimeChunkSize = 50;
/// as a double (−2^63, exact): what an Int64 sample of that value leaves in num_value.
internal const double LongNullSentinelAsDouble = -9223372036854775808d;
private const string SampleColumns = "ts, kind, num_value, int_value, str_value, status, server_ts";
private readonly QuestDbConnection _conn;
private readonly QuestDbSchemaBootstrapper _bootstrap;
private readonly QuestDbTagCatalog _catalog;
private readonly ILogger _log;
private readonly TimeProvider _time;
private readonly HistorianHealthCounters _health = new();
/// Creates the data source over the provider's shared connection, bootstrapper and tag catalog.
/// Connection helper.
/// Schema bootstrapper (shared with the other QuestDB seams).
/// Tag catalog (reads use only its read-only ).
/// Optional logger.
/// Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.
internal QuestDbHistorianDataSource(
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
TimeProvider? timeProvider = null)
{
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
_log = log ?? NullLogger.Instance;
_time = timeProvider ?? TimeProvider.System;
}
// ------------------------------------------------------------------------------------------ Raw
///
///
/// Forward (start < end): [start, end) ascending. Reverse (end < start):
/// (end, start] descending. start == end: the sample exactly at that (rounded) time.
/// 0 is "unbounded" (); larger values clamp.
/// At most top samples, ContinuationPoint always null (R8): the node manager pages by re-reading
/// from the last returned timestamp inclusive, which the (ts, tag) dedup key makes exact.
///
public async Task ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
try
{
var samples = new List();
if (await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
{
var start = QuestDbConnection.Round(startUtc);
var end = QuestDbConnection.Round(endUtc);
string where, order;
if (start == end) { where = "ts = $2 AND ts = $3"; order = "ASC"; }
else if (start < end) { where = "ts >= $2 AND ts < $3"; order = "ASC"; }
else { where = "ts > $2 AND ts <= $3"; order = "DESC"; }
var (lo, hi) = start <= end ? (start, end) : (end, start);
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn,
$"SELECT {SampleColumns} FROM {_conn.SampleTable} " +
$"WHERE tag = $1 AND {where} AND NOT deleted ORDER BY ts {order} LIMIT $4");
QuestDbConnection.BindText(cmd, fullReference);
QuestDbConnection.BindTimestamp(cmd, lo);
QuestDbConnection.BindTimestamp(cmd, hi);
cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = top });
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
var now = _time.GetUtcNow().UtcDateTime;
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var s = ReadStored(reader, 0);
samples.Add(new DataValueSnapshot(s.Value.ToObject(), s.Status, s.TimeUtc, ReadServerTime(reader, 6, now)));
}
}
RecordSuccess();
return new HistoryReadResult(samples, ContinuationPoint: null);
}
catch (Exception ex)
{
RecordFailure(ex);
throw;
}
}
// ------------------------------------------------------------------------------------------ AtTime
///
///
/// Each distinct (rounded) requested time picks up its previous-or-equal and next live sample through two
/// LIMIT 1 seeks, UNION ALL-ed, at most times per statement (spec §3,
/// T0 Q9). then decides each value exactly as for SQL Server (exact hit,
/// no previous ⇒ BadNoData, inside an outage, held before an outage, linear, stepped, after now). The
/// result matches the request in length and order (duplicates included), each entry stamped at its rounded
/// time. Unknown tag ⇒ every entry BadNoData.
///
public async Task ReadAtTimeAsync(
string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
ArgumentNullException.ThrowIfNull(timestampsUtc);
try
{
var now = _time.GetUtcNow().UtcDateTime;
var rounded = timestampsUtc.Select(QuestDbConnection.Round).ToArray();
var byTicks = new Dictionary(rounded.Length);
if (rounded.Length > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
{
var distinct = rounded.Distinct().ToList();
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
for (var offset = 0; offset < distinct.Count; offset += AtTimeChunkSize)
{
var chunk = distinct.GetRange(offset, Math.Min(AtTimeChunkSize, distinct.Count - offset));
var prev = new StoredSample?[chunk.Count];
var next = new StoredSample?[chunk.Count];
var prevServer = new DateTime?[chunk.Count];
await using (var cmd = _conn.CreateCommand(conn, AtTimeSql(_conn.SampleTable, chunk.Count)))
{
QuestDbConnection.BindText(cmd, fullReference);
foreach (var t in chunk) QuestDbConnection.BindTimestamp(cmd, t);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var k = Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture);
var dir = Convert.ToInt32(reader.GetValue(1), CultureInfo.InvariantCulture);
var s = ReadStored(reader, 2);
if (dir == 0)
{
prev[k] = s;
prevServer[k] = reader.IsDBNull(8) ? null : QuestDbConnection.ReadUtc(reader, 8);
}
else
{
next[k] = s;
}
}
}
for (var k = 0; k < chunk.Count; k++)
{
var t = chunk[k];
var serverTime = prev[k] is { } ps && ps.TimeUtc == t ? prevServer[k] ?? now : now;
byTicks[t.Ticks] = AtTimeEvaluator.Evaluate(t, prev[k], next[k], serverTime, now);
}
}
}
var result = new DataValueSnapshot[rounded.Length];
for (var i = 0; i < result.Length; i++)
{
var t = rounded[i];
result[i] = byTicks.TryGetValue(t.Ticks, out var v)
? v
: new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, t, now);
}
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
}
catch (Exception ex)
{
RecordFailure(ex);
throw;
}
}
///
/// The T0 Q9 statement for request times: $1 = tag, $(k+2) = time k.
/// Columns: req, dir (0 = previous-or-equal, 1 = next), then ts, kind, num_value, int_value, str_value,
/// status, server_ts. req/dir are generated integers, never user input.
///
internal static string AtTimeSql(string sample, int count)
{
if (count is < 1 or > AtTimeChunkSize) throw new ArgumentOutOfRangeException(nameof(count));
var sb = new StringBuilder();
for (var k = 0; k < count; k++)
{
var p = "$" + (k + 2).ToString(CultureInfo.InvariantCulture);
var req = k.ToString(CultureInfo.InvariantCulture);
if (k > 0) sb.Append(" UNION ALL ");
sb.Append("SELECT * FROM (SELECT ").Append(req).Append(" AS req, 0 AS dir, ").Append(SampleColumns)
.Append(" FROM ").Append(sample).Append(" WHERE tag = $1 AND ts <= ").Append(p)
.Append(" AND NOT deleted ORDER BY ts DESC LIMIT 1)");
sb.Append(" UNION ALL ");
sb.Append("SELECT * FROM (SELECT ").Append(req).Append(" AS req, 1 AS dir, ").Append(SampleColumns)
.Append(" FROM ").Append(sample).Append(" WHERE tag = $1 AND ts > ").Append(p)
.Append(" AND NOT deleted ORDER BY ts ASC LIMIT 1)");
}
return sb.ToString();
}
// ------------------------------------------------------------------------------------------ Events
///
///
/// [start, end) by event time (a reversed window is swapped), ordered by (ts, event_id) ascending.
/// <= 0 or ⇒ unbounded. Otherwise
/// maxEvents + 1 rows are fetched; when the extra row arrives it is dropped and the continuation point is
/// the UTF-8 of for that first not-returned event.
///
public async Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
{
// No stored event can carry a source longer than the limit, so an over-long filter matches nothing.
if (sourceName is not null && sourceName.Length > HistorianTagNames.MaxLength)
return new HistoricalEventsResult([], null);
try
{
await _bootstrap.EnsureAsync(cancellationToken).ConfigureAwait(false);
var start = QuestDbConnection.Round(startUtc);
var end = QuestDbConnection.Round(endUtc);
var (lo, hi) = start <= end ? (start, end) : (end, start);
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
var sql =
$"SELECT event_id, source, ts, received_utc, message, severity FROM {_conn.EventTable} " +
"WHERE ts >= $1 AND ts < $2" +
(sourceName is null ? "" : " AND source = $3") +
" ORDER BY ts ASC, event_id ASC" +
(bounded ? (sourceName is null ? " LIMIT $3" : " LIMIT $4") : "");
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
QuestDbConnection.BindTimestamp(cmd, lo);
QuestDbConnection.BindTimestamp(cmd, hi);
if (sourceName is not null) QuestDbConnection.BindText(cmd, sourceName);
if (bounded) QuestDbConnection.BindLong(cmd, (long)maxEvents + 1);
var events = new List();
byte[]? continuation = null;
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var eventTime = QuestDbConnection.ReadUtc(reader, 2);
var e = new HistoricalEvent(
reader.IsDBNull(0) ? "" : reader.GetString(0),
reader.IsDBNull(1) ? null : reader.GetString(1),
eventTime,
reader.IsDBNull(3) ? eventTime : QuestDbConnection.ReadUtc(reader, 3),
reader.IsDBNull(4) ? null : reader.GetString(4),
HistoricalEventRows.ClampSeverity(reader.IsDBNull(5) ? 0 : reader.GetInt32(5)));
if (bounded && events.Count == maxEvents)
{
continuation = Encoding.UTF8.GetBytes(EventContinuation.Encode(e.EventTimeUtc, e.EventId));
break;
}
events.Add(e);
}
RecordSuccess();
return new HistoricalEventsResult(events, continuation);
}
catch (Exception ex)
{
RecordFailure(ex);
throw;
}
}
// ------------------------------------------------------------------------------------------ Processed
///
///
/// The (rounded) window is planned by : bucket i =
/// [start + i·interval, start + (i+1)·interval), start-inclusive / end-exclusive, the last bucket truncated
/// at end, interval <= 0 ⇒ one bucket, start == end ⇒ none, a reversed window computed forward
/// and returned descending, at most buckets. One statement per node returns
/// the statistics of each non-empty bucket (bucket index
/// (cast(ts AS LONG)*10 − StartUnixTicks) / IntervalTicks, never SAMPLE BY);
/// produces each value and fills the
/// empty buckets with BadNoData.
///
/// The aggregate is not one of the thirteen served.
/// The window holds more than intervals.
public async Task ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
var family = AggregateFamilies.FamilyOf(aggregate) ?? throw new HistoryAggregateNotSupportedException(aggregate);
var plan = BucketPlan.Create(QuestDbConnection.Round(startUtc), QuestDbConnection.Round(endUtc), interval);
try
{
var now = _time.GetUtcNow().UtcDateTime;
var evaluated = new Dictionary();
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
{
var sample = _conn.SampleTable;
var sql = family switch
{
AggregateFamily.Simple => QuestDbAggregateQueries.Simple(sample),
AggregateFamily.StartEnd => QuestDbAggregateQueries.StartEnd(sample),
_ => QuestDbAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
};
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
QuestDbConnection.BindLong(cmd, plan.StartUnixTicks);
QuestDbConnection.BindLong(cmd, plan.IntervalTicks);
QuestDbConnection.BindText(cmd, fullReference);
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var row = ReadBucket(reader, family);
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
}
}
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
}
catch (Exception ex)
{
RecordFailure(ex);
throw;
}
}
private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family)
{
static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
static int N(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
// A bucket without a usable sample has NULL times but kind 0 (BYTE has no NULL): absence is keyed on the time.
static HistorianValue V(NpgsqlDataReader r, int i) =>
r.IsDBNull(i + 4)
? HistorianValue.Empty
: ToValue(QuestDbConnection.ReadKind(r, i), D(r, i + 1), r.IsDBNull(i + 2) ? null : r.GetInt64(i + 2),
r.IsDBNull(i + 3) ? null : r.GetString(i + 3));
var row = new BucketRow
{
Bucket = Convert.ToInt64(reader.GetValue(0), CultureInfo.InvariantCulture),
Total = N(reader, 1),
Usable = N(reader, 2),
Good = N(reader, 3),
Bad = N(reader, 4),
NonNumeric = N(reader, 5),
};
var f = QuestDbAggregateQueries.FirstFamilyColumn;
switch (family)
{
case AggregateFamily.Simple:
row.Avg = D(reader, f);
row.Min = D(reader, f + 1);
row.Max = D(reader, f + 2);
row.Sum = D(reader, f + 3);
break;
case AggregateFamily.StartEnd:
row.First = V(reader, f);
row.Last = V(reader, f + 5);
break;
default:
row.Extreme = D(reader, f);
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1);
break;
}
return row;
}
// ------------------------------------------------------------------------------------------ Shared
///
/// Reads ts, kind, num_value, int_value, str_value, status starting at and applies the
/// shared quality clamp ().
///
private static StoredSample ReadStored(NpgsqlDataReader r, int o)
{
var value = ToValue(
QuestDbConnection.ReadKind(r, o + 1),
r.IsDBNull(o + 2) ? null : r.GetDouble(o + 2),
r.IsDBNull(o + 3) ? null : r.GetInt64(o + 3),
r.IsDBNull(o + 4) ? null : r.GetString(o + 4));
// status is always written; a NULL (only possible for a hand-made row) is served as "no data", never Good.
var status = r.IsDBNull(o + 5) ? HistorianStatusCodes.BadNoData : SampleColumnMapper.FromLongStatus(r.GetInt64(o + 5));
return StoredSample.FromStored(QuestDbConnection.ReadUtc(r, o), value, status);
}
///
/// Rebuilds a value from its stored columns via the shared , after the
/// RC2 repair: QuestDB reads a LONG holding back as NULL, so an Int64 row with
/// a NULL int_value and a num_value of exactly −2^63 is that value.
///
internal static HistorianValue ToValue(byte kind, double? num, long? integer, string? str)
{
if (kind == (byte)HistorianValueKind.Int64 && integer is null && num == LongNullSentinelAsDouble)
integer = long.MinValue;
return SampleColumnMapper.FromColumns(kind, num, integer, str);
}
private static DateTime ReadServerTime(NpgsqlDataReader r, int i, DateTime fallback) =>
r.IsDBNull(i) ? fallback : QuestDbConnection.ReadUtc(r, i);
/// Read-only tag lookup (R7; never creates a row); runs the schema bootstrap on a cache miss.
private Task ResolveAsync(string name, CancellationToken ct) => _catalog.ExistsAsync(name, ct);
// ------------------------------------------------------------------------------------------ Health
///
public HistorianHealthSnapshot GetHealthSnapshot() => _health.Snapshot();
private void RecordSuccess() => _health.RecordSuccess();
private void RecordFailure(Exception ex)
{
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
if (ex is OperationCanceledException) return;
_health.RecordFailure(ex);
_log.LogDebug("QuestDB historian read failed ({ExceptionType})", ex.GetType().Name);
}
///
public void Dispose()
{
// Connections are per call (pooled by the shared NpgsqlDataSource, owned by the provider); nothing to release.
}
}