dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(historian): extract Driver.Historian.Sql.Common from the SQL Server backend (no behaviour change) (1a)

Browse Source 
Moves the database-independent logic of the SQL Server historian backend into a new
shared library for the upcoming QuestDB backend (#666): status codes and bits, value/column
mapping, StoredSample + quality clamp, the AtTime and aggregate evaluators, BucketPlan,
LiveWritePreparer and HistoryUpdatePlanner (both with a resolutionTicks parameter, default 1),
tag-name rules, the bootstrap failure cache, health counters, alarm event ids, the event
continuation and severity clamp. The SQL Server backend now calls it; its T-SQL, TVP
builders, connection, bootstrapper and catalog stay.

Tests: StatusCodeOracleTests, ValueWriterPrepareTests, the column half of SampleRowMapperTests
and ProcessedReadTests.Every_enum_member_is_either_served_or_refused move to the new
Sql.Common.Tests under the same names. SqlServer integration tests get mechanical renames
only. Test-name set across the three projects is identical to the baseline (160 tests).

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-11 01:26:49 -04:00
parent cdea173bd1
commit 8c6d6aff6a
49 changed files with 1262 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs
+29 -86
View File
@@ -1,18 +1,9 @@
using Microsoft.Data.SqlClient;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>Status codes owned by the updater (named consts so <c>StatusCodeParityTests</c> sees them).</summary>
internal static class SqlHistorianUpdateStatusCodes
{
/// <summary>
/// A value that cannot be stored: a Double NaN / ±Infinity, an Empty (null) value with a non-Bad status, or a
/// tag name longer than <see cref="SqlServerTagCatalog.MaxNameLength"/>.
/// </summary>
public const uint BadInvalidArgument = 0x80AB0000u;
}
/// <summary>
/// OPC UA HistoryUpdate against the SQL Server sample table (spec §4 "Updates"). One transaction per call;
/// exactly one status per input item, in input order.
@@ -26,7 +17,7 @@ internal static class SqlHistorianUpdateStatusCodes
/// accepted with a Bad status (an operator-authored outage marker); with a Good or Uncertain status it is
/// <c>BadInvalidArgument</c>, because a stored sample is never served better than "no value". Other items
/// still proceed.</para>
/// <para><b>Tag names over <see cref="SqlServerTagCatalog.MaxNameLength"/> characters</b> are refused, never
/// <para><b>Tag names over <see cref="HistorianTagNames.MaxLength"/> characters</b> are refused, never
/// truncated: every item of the call (and DeleteRaw's single result) reports <c>BadInvalidArgument</c> and nothing
/// is written.</para>
/// <para><b>Unknown tag.</b> Insert/Update create it via <see cref="SqlServerTagCatalog"/>; Replace and the
@@ -52,70 +43,26 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
{
ArgumentNullException.ThrowIfNull(tag);
ArgumentNullException.ThrowIfNull(values);
var results = new uint[values.Count];
if (!SqlServerTagCatalog.IsValidName(tag))
{
Array.Fill(results, SqlHistorianUpdateStatusCodes.BadInvalidArgument);
return results;
}
var winner = new Dictionary<long, int>(); // tick -> index of the item actually sent
for (var i = 0; i < values.Count; i++)
{
var v = values[i];
if (v.TimestampUtc is not { } ts || ts == DateTime.MinValue)
{
results[i] = SqlHistorianStatusCodes.BadInvalidTimestamp;
continue;
}
if ((v.Value.Kind == HistorianValueKind.Double && !double.IsFinite(v.Value.Number))
|| (v.Value.Kind == HistorianValueKind.Empty && !StatusBits.IsBad(v.StatusCode)))
{
results[i] = SqlHistorianUpdateStatusCodes.BadInvalidArgument;
continue;
}
if (!winner.TryGetValue(ts.Ticks, out _))
winner[ts.Ticks] = i;
else if (mode == HistoryUpdateMode.Insert)
results[i] = SqlHistorianStatusCodes.BadEntryExists; // first occurrence stays the winner
else
winner[ts.Ticks] = i; // last wins
}
var send = winner.Values.OrderBy(i => i).ToList();
if (send.Count == 0) return results;
var plan = HistoryUpdatePlanner.Plan(tag, mode, values);
if (plan.Winners.Count == 0) return HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());
var tagId = mode == HistoryUpdateMode.Replace
? await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false)
: await _tags.GetOrCreateTagIdAsync(tag, SqlHistorianDataTypes.FromKind(values[send[0]].Value.Kind), ct).ConfigureAwait(false);
: await _tags.GetOrCreateTagIdAsync(tag, HistorianDataTypes.FromKind(plan.Winners[0].Row.Value.Kind), ct).ConfigureAwait(false);
var outcome = tagId is null
? new Dictionary<long, uint>()
: await MergeAsync(tagId.Value, mode, send.Select(i => new SqlSampleRow(
values[i].TimestampUtc!.Value, values[i].Value, values[i].StatusCode)).ToList(), ct).ConfigureAwait(false);
// No OUTPUT row means the MERGE's absent branch: Insert => matched (exists); Replace => not matched.
var absent = mode == HistoryUpdateMode.Replace
? SqlHistorianStatusCodes.BadNoEntryExists
: SqlHistorianStatusCodes.BadEntryExists;
foreach (var i in send)
results[i] = outcome.TryGetValue(values[i].TimestampUtc!.Value.Ticks, out var code) ? code : absent;
// Superseded earlier duplicates (Replace/Update) share the winner's code.
if (mode != HistoryUpdateMode.Insert)
for (var i = 0; i < values.Count; i++)
if (values[i].TimestampUtc is { } ts && results[i] == 0
&& winner.TryGetValue(ts.Ticks, out var w) && w != i)
results[i] = results[w];
return results;
? new Dictionary<long, UpsertOutcome>()
: await MergeAsync(tagId.Value, mode, plan.Winners.Select(w => w.Row).ToList(), ct).ConfigureAwait(false);
return HistoryUpdatePlanner.Complete(plan, mode, outcome);
}
/// <inheritdoc/>
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(tag);
if (!SqlServerTagCatalog.IsValidName(tag)) return SqlHistorianUpdateStatusCodes.BadInvalidArgument;
if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
if (tagId is null) return SqlHistorianStatusCodes.BadNoData;
if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
int rows;
@@ -128,7 +75,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return rows > 0 ? SqlHistorianStatusCodes.Good : SqlHistorianStatusCodes.BadNoData;
return HistoryUpdatePlanner.DeleteRawResult(tagKnown: true, rows);
}
/// <inheritdoc/>
@@ -136,22 +83,19 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
{
ArgumentNullException.ThrowIfNull(tag);
ArgumentNullException.ThrowIfNull(timesUtc);
var results = new uint[timesUtc.Count];
if (!SqlServerTagCatalog.IsValidName(tag))
if (!HistorianTagNames.IsValid(tag))
{
Array.Fill(results, SqlHistorianUpdateStatusCodes.BadInvalidArgument);
return results;
var refused = new uint[timesUtc.Count];
Array.Fill(refused, HistorianStatusCodes.BadInvalidArgument);
return refused;
}
var first = new Dictionary<long, int>();
for (var i = 0; i < timesUtc.Count; i++)
{
results[i] = SqlHistorianStatusCodes.BadNoEntryExists;
if (timesUtc[i] == DateTime.MinValue) results[i] = SqlHistorianStatusCodes.BadInvalidTimestamp;
else first.TryAdd(timesUtc[i].Ticks, i);
}
if (first.Count == 0) return results;
var distinct = new HashSet<long>();
foreach (var t in timesUtc)
if (t != DateTime.MinValue) distinct.Add(t.Ticks);
var deleted = new HashSet<long>();
if (distinct.Count == 0) return HistoryUpdatePlanner.DeleteAtTimeResults(timesUtc, deleted, tagKnown: false);
var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
if (tagId is null) return results;
if (tagId is null) return HistoryUpdatePlanner.DeleteAtTimeResults(timesUtc, deleted, tagKnown: false);
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
@@ -160,18 +104,17 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
{
cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value });
_conn.AddStructured(cmd, "@times", "TimeRow",
SampleRowMapper.ToTimeRecords(first.Keys.Select(t => new DateTime(t, DateTimeKind.Utc))));
SampleRowMapper.ToTimeRecords(distinct.Select(t => new DateTime(t, DateTimeKind.Utc))));
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await rd.ReadAsync(ct).ConfigureAwait(false))
if (first.TryGetValue(SqlHistorianConnection.GetUtc(rd, 0).Ticks, out var idx))
results[idx] = SqlHistorianStatusCodes.Good;
deleted.Add(SqlHistorianConnection.GetUtc(rd, 0).Ticks);
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return results;
return HistoryUpdatePlanner.DeleteAtTimeResults(timesUtc, deleted, tagKnown: true);
}
private async Task<Dictionary<long, uint>> MergeAsync(int tagId, HistoryUpdateMode mode,
List<SqlSampleRow> rows, CancellationToken ct)
private async Task<Dictionary<long, UpsertOutcome>> MergeAsync(int tagId, HistoryUpdateMode mode,
List<HistorianSampleRow> rows, CancellationToken ct)
{
var sql = $"MERGE {_conn.Q("Sample")} WITH (HOLDLOCK) AS t USING @rows AS r ON t.TagId = @tagId AND t.SourceTimeUtc = r.SourceTimeUtc ";
if (mode != HistoryUpdateMode.Insert)
@@ -182,7 +125,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
"VALUES (@tagId, r.SourceTimeUtc, r.ValueKind, r.NumValue, r.IntValue, r.StrValue, r.StatusCode, SYSUTCDATETIME()) ";
sql += "OUTPUT $action, r.SourceTimeUtc;";
var outcome = new Dictionary<long, uint>();
var outcome = new Dictionary<long, UpsertOutcome>();
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
await using (var cmd = _conn.CreateCommand(conn, sql, tx))
@@ -192,8 +135,8 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await rd.ReadAsync(ct).ConfigureAwait(false))
outcome[SqlHistorianConnection.GetUtc(rd, 1).Ticks] = rd.GetString(0) == "INSERT"
? SqlHistorianStatusCodes.GoodEntryInserted
: SqlHistorianStatusCodes.GoodEntryReplaced;
? UpsertOutcome.Inserted
: UpsertOutcome.Replaced;
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return outcome;