2ceed561ad
QuestDB Start/End/Delta reads of a tag being written intermittently failed BadHistoryOperationUnsupported (InvalidCastException: Column 'ts_3' is null): the StartEnd query referenced its CTE four times and QuestDB opens a table reader per reference, so a row committed between two of them was counted but not picked. MinimumActualTime/MaximumActualTime had the same race and returned a value-less Good. (a) SQL - QuestDB StartEnd: one scan, one GROUP BY — first()/last() for the any-quality picks, arg_min/arg_max keyed on the usable rows' time for the usable picks (no str_value: arg_min has no VARCHAR form; only Delta reads them). Byte-identical output to the old query on a 1M-row tag, ~15% faster. - SqlServer StartEnd: one windowed pass + conditional MAX (was five CTE references); identical output, 1.4-3x faster. - ActualTime kept on both backends: every single-scan form measured 1.2-2.4x slower; its inconsistency is absorbed by (b). (b) Defence in depth on both backends - ReadBucket takes DbDataReader and reads absent picks as null (ReadStoredOrNull) instead of throwing. - AggregateEvaluator: Delta with a missing first/last usable pick and ActualTime with a missing extreme are BadNoData (were BadAggregateInvalidInputs / a value-less Good); Start/End with no pick already were. - Unit tests (Sql.Common evaluator, QuestDb + SqlServer mappers over a DataTableReader) fail with the guards removed. LiveWriteReadRaceStressTests (QuestDB integration, opt-in via HISTORIAN_QUESTDB_STRESS_SECONDS, Category=SlowUnit): single-row writer vs 4 tight-loop readers. Old code: 6 faults in ~105k reads; new: 0 in 106k. Docs: Historian.md QuestDB processed-read note; F2 (Min/Max on the interval start: SDK raw, ours Calculated) and F3 (integer tags: SDK casts interpolated values to the integer type, ours Double) added to the SDK-parity deviations.
131 lines
6.0 KiB
C#
131 lines
6.0 KiB
C#
using System.Data;
|
|
using Microsoft.Data.SqlClient;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
|
|
|
/// <summary>
|
|
/// Connection + command helpers shared by every SQL historian component.
|
|
/// <para>Usage contract for writers/readers: call <see cref="SqlServerSchemaBootstrapper.EnsureAsync"/>
|
|
/// once (it is cheap after the first success), then <see cref="OpenAsync"/>. Build SQL with <see cref="Q"/>
|
|
/// for every object name and bind every timestamp through <see cref="AddDateTime2"/> or a TVP built by
|
|
/// <see cref="SampleRowMapper"/> — never a <c>DataTable</c> (it infers 3.33 ms <c>datetime</c>).</para>
|
|
/// </summary>
|
|
internal sealed class SqlHistorianConnection
|
|
{
|
|
private static readonly int[] TransientNumbers =
|
|
[1205, -2, 40613, 40197, 40501, 49918, 4060, 233, 10053, 10054, 10060, 11001];
|
|
|
|
private readonly string _connectionString;
|
|
|
|
/// <summary>Creates the helper; options must already be valid.</summary>
|
|
/// <param name="options">Validated options.</param>
|
|
public SqlHistorianConnection(SqlServerHistorianOptions options)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
if (!SqlServerHistorianOptions.IsValidSchemaName(options.Schema))
|
|
throw new ArgumentException("Invalid schema name.", nameof(options));
|
|
_connectionString = options.ConnectionString;
|
|
Schema = options.Schema;
|
|
CommandTimeoutSeconds = Math.Max(1, (int)Math.Ceiling(options.CommandTimeout.TotalSeconds));
|
|
}
|
|
|
|
/// <summary>The validated schema name.</summary>
|
|
public string Schema { get; }
|
|
|
|
/// <summary>Command timeout applied by <see cref="CreateCommand"/>, in seconds.</summary>
|
|
public int CommandTimeoutSeconds { get; }
|
|
|
|
/// <summary>Schema-qualifies an object name: <c>[schema].[obj]</c>.</summary>
|
|
/// <param name="obj">Unquoted object name (a fixed literal from this project, never user input).</param>
|
|
/// <returns>The quoted two-part name.</returns>
|
|
public string Q(string obj) => $"[{Schema}].[{obj}]";
|
|
|
|
/// <summary>Opens a new connection (no bootstrap; see the class remarks).</summary>
|
|
/// <param name="ct">Cancellation.</param>
|
|
/// <returns>An open connection the caller disposes.</returns>
|
|
public async Task<SqlConnection> OpenAsync(CancellationToken ct)
|
|
{
|
|
var conn = new SqlConnection(_connectionString);
|
|
try
|
|
{
|
|
await conn.OpenAsync(ct).ConfigureAwait(false);
|
|
return conn;
|
|
}
|
|
catch
|
|
{
|
|
await conn.DisposeAsync().ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Creates a text command with the configured timeout.</summary>
|
|
/// <param name="conn">Open connection.</param>
|
|
/// <param name="sql">Command text.</param>
|
|
/// <param name="tx">Optional transaction.</param>
|
|
/// <returns>The command.</returns>
|
|
public SqlCommand CreateCommand(SqlConnection conn, string sql, SqlTransaction? tx = null)
|
|
{
|
|
var cmd = conn.CreateCommand();
|
|
cmd.CommandText = sql;
|
|
cmd.CommandTimeout = CommandTimeoutSeconds;
|
|
cmd.Transaction = tx;
|
|
return cmd;
|
|
}
|
|
|
|
/// <summary>Adds a <c>datetime2(7)</c> scalar parameter (full 100 ns tick precision, UTC kind).</summary>
|
|
/// <param name="cmd">Target command.</param>
|
|
/// <param name="name">Parameter name including <c>@</c>.</param>
|
|
/// <param name="value">Timestamp; treated as UTC.</param>
|
|
/// <returns>The added parameter.</returns>
|
|
public static SqlParameter AddDateTime2(SqlCommand cmd, string name, DateTime value)
|
|
{
|
|
var p = new SqlParameter(name, SqlDbType.DateTime2)
|
|
{
|
|
Scale = 7,
|
|
Value = DateTime.SpecifyKind(value, DateTimeKind.Utc),
|
|
};
|
|
cmd.Parameters.Add(p);
|
|
return p;
|
|
}
|
|
|
|
/// <summary>Adds a table-valued parameter of one of this schema's table types.</summary>
|
|
/// <param name="cmd">Target command.</param>
|
|
/// <param name="name">Parameter name including <c>@</c>.</param>
|
|
/// <param name="typeName">Unqualified type name (<c>SampleRow</c> or <c>TimeRow</c>).</param>
|
|
/// <param name="rows">Non-empty <c>SqlDataRecord</c> sequence (SqlClient rejects an empty one — skip the call).</param>
|
|
/// <returns>The added parameter.</returns>
|
|
public SqlParameter AddStructured(SqlCommand cmd, string name, string typeName,
|
|
IEnumerable<Microsoft.Data.SqlClient.Server.SqlDataRecord> rows)
|
|
{
|
|
var p = new SqlParameter(name, SqlDbType.Structured) { TypeName = Q(typeName), Value = rows };
|
|
cmd.Parameters.Add(p);
|
|
return p;
|
|
}
|
|
|
|
/// <summary>Reads a <c>datetime2</c> column as a UTC <see cref="DateTime"/> (the driver returns Unspecified).</summary>
|
|
/// <param name="reader">Reader positioned on a row.</param>
|
|
/// <param name="ordinal">Column ordinal.</param>
|
|
/// <returns>UTC timestamp.</returns>
|
|
public static DateTime GetUtc(System.Data.Common.DbDataReader reader, int ordinal) =>
|
|
DateTime.SpecifyKind(reader.GetDateTime(ordinal), DateTimeKind.Utc);
|
|
|
|
/// <summary>True for connection/timeout/deadlock/throttling errors worth retrying.</summary>
|
|
/// <param name="ex">Exception from a SQL call.</param>
|
|
/// <returns>Whether a retry may succeed.</returns>
|
|
public static bool IsTransient(Exception ex)
|
|
{
|
|
if (ex is TimeoutException) return true;
|
|
if (ex is not SqlException sql) return false;
|
|
if (sql.IsTransient) return true;
|
|
foreach (SqlError e in sql.Errors)
|
|
if (Array.IndexOf(TransientNumbers, e.Number) >= 0) return true;
|
|
return Array.IndexOf(TransientNumbers, sql.Number) >= 0;
|
|
}
|
|
|
|
/// <summary>True when <paramref name="ex"/> is a unique-key violation (2627/2601): "already present".</summary>
|
|
/// <param name="ex">Exception from a SQL call.</param>
|
|
/// <returns>Whether it is a duplicate-key error.</returns>
|
|
public static bool IsDuplicateKey(Exception ex) =>
|
|
ex is SqlException sql && sql.Errors.Cast<SqlError>().Any(e => e.Number is 2627 or 2601);
|
|
}
|