dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(historian): extract Driver.Historian.Sql.Common from the SQL Server backend (no behaviour change) (1a)

Browse Source 
Moves the database-independent logic of the SQL Server historian backend into a new
shared library for the upcoming QuestDB backend (#666): status codes and bits, value/column
mapping, StoredSample + quality clamp, the AtTime and aggregate evaluators, BucketPlan,
LiveWritePreparer and HistoryUpdatePlanner (both with a resolutionTicks parameter, default 1),
tag-name rules, the bootstrap failure cache, health counters, alarm event ids, the event
continuation and severity clamp. The SQL Server backend now calls it; its T-SQL, TVP
builders, connection, bootstrapper and catalog stay.

Tests: StatusCodeOracleTests, ValueWriterPrepareTests, the column half of SampleRowMapperTests
and ProcessedReadTests.Every_enum_member_is_either_served_or_refused move to the new
Sql.Common.Tests under the same names. SqlServer integration tests get mechanical renames
only. Test-name set across the three projects is identical to the baseline (160 tests).

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-11 01:26:49 -04:00
parent cdea173bd1
commit 8c6d6aff6a
49 changed files with 1262 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SampleRowMapper.cs
+6 -55
View File
@@ -1,22 +1,12 @@
using System.Data;
using Microsoft.Data.SqlClient.Server;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>One sample bound for a <c>SampleRow</c> TVP.</summary>
/// <param name="TimeUtc">Source timestamp (UTC, full tick precision).</param>
/// <param name="Value">Typed value.</param>
/// <param name="StatusCode">OPC UA status code (stored bit-for-bit as int).</param>
internal readonly record struct SqlSampleRow(DateTime TimeUtc, HistorianValue Value, uint StatusCode);
/// <summary>The (ValueKind, NumValue, IntValue, StrValue) column quad for a value.</summary>
internal readonly record struct SampleColumns(byte ValueKind, double? NumValue, long? IntValue, string? StrValue);
/// <summary>
/// HistorianValue ⇄ column mapping (spec §4) and the <c>SqlDataRecord</c> TVP builders. Mapping:
/// Double → Num; Int64 → Int + Num(double); Boolean → Int 0/1 + Num 0/1; String → Str; DateTime → Int (ticks);
/// Empty → all NULL.
/// The <c>SqlDataRecord</c> TVP builders. The HistorianValue ⇄ column mapping itself is
/// <see cref="SampleColumnMapper"/> (shared with the other SQL backends).
/// </summary>
internal static class SampleRowMapper
{
@@ -32,60 +22,21 @@ internal static class SampleRowMapper
private static readonly SqlMetaData[] TimeMeta = [new("T", SqlDbType.DateTime2, 0, 7)];
/// <summary>Splits a value into its column quad.</summary>
/// <param name="v">Typed value.</param>
/// <returns>The columns to store.</returns>
public static SampleColumns ToColumns(HistorianValue v) => v.Kind switch
{
HistorianValueKind.Double => new((byte)v.Kind, v.Number, null, null),
HistorianValueKind.Int64 => new((byte)v.Kind, v.Number, v.Integer, null),
HistorianValueKind.Boolean => new((byte)v.Kind, v.Integer != 0 ? 1d : 0d, v.Integer != 0 ? 1L : 0L, null),
HistorianValueKind.String => new((byte)v.Kind, null, null, v.Text ?? ""),
HistorianValueKind.DateTime => new((byte)v.Kind, null, v.Integer, null),
_ => new((byte)HistorianValueKind.Empty, null, null, null),
};
/// <summary>Rebuilds a value from stored columns.</summary>
/// <param name="valueKind">ValueKind column.</param>
/// <param name="num">NumValue column (null = NULL).</param>
/// <param name="integer">IntValue column (null = NULL).</param>
/// <param name="str">StrValue column (null = NULL).</param>
/// <returns>The typed value; an unknown kind yields <see cref="HistorianValue.Empty"/>.</returns>
public static HistorianValue FromColumns(byte valueKind, double? num, long? integer, string? str) => (HistorianValueKind)valueKind switch
{
HistorianValueKind.Double => HistorianValue.FromDouble(num ?? 0d),
HistorianValueKind.Int64 => HistorianValue.FromInt64(integer ?? 0L),
HistorianValueKind.Boolean => HistorianValue.FromBoolean((integer ?? 0L) != 0),
HistorianValueKind.String => HistorianValue.FromString(str ?? ""),
HistorianValueKind.DateTime => new HistorianValue(HistorianValueKind.DateTime, 0d, integer ?? 0L, null),
_ => HistorianValue.Empty,
};
/// <summary>Stores a uint status bit-for-bit in the int column.</summary>
/// <param name="status">Status code.</param>
/// <returns>Same bits as int.</returns>
public static int ToDbStatus(uint status) => unchecked((int)status);
/// <summary>Reads the int status column back to a uint code.</summary>
/// <param name="dbStatus">Column value.</param>
/// <returns>Same bits as uint.</returns>
public static uint FromDbStatus(int dbStatus) => unchecked((uint)dbStatus);
/// <summary>Builds <c>SampleRow</c> TVP records (datetime2(7); never a DataTable). Lazy; enumerate once.</summary>
/// <param name="rows">Samples; must be time-unique (the TVP has a PK on time) and non-empty when bound.</param>
/// <returns>Records for <see cref="SqlHistorianConnection.AddStructured"/>.</returns>
public static IEnumerable<SqlDataRecord> ToSampleRecords(IEnumerable<SqlSampleRow> rows)
public static IEnumerable<SqlDataRecord> ToSampleRecords(IEnumerable<HistorianSampleRow> rows)
{
foreach (var r in rows)
{
var c = ToColumns(r.Value);
var c = SampleColumnMapper.ToColumns(r.Value);
var rec = new SqlDataRecord(SampleMeta);
rec.SetDateTime(0, DateTime.SpecifyKind(r.TimeUtc, DateTimeKind.Utc));
rec.SetByte(1, c.ValueKind);
if (c.NumValue is { } n) rec.SetDouble(2, n); else rec.SetDBNull(2);
if (c.IntValue is { } i) rec.SetInt64(3, i); else rec.SetDBNull(3);
if (c.StrValue is { } s) rec.SetString(4, s); else rec.SetDBNull(4);
rec.SetInt32(5, ToDbStatus(r.StatusCode));
rec.SetInt32(5, SampleColumnMapper.ToDbStatus(r.StatusCode));
yield return rec;
}
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
+3 -28
View File
@@ -1,22 +1,10 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>The SQL shape that serves an aggregate. One query per family.</summary>
internal enum SqlAggregateFamily
{
/// <summary>Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad: one GROUP BY.</summary>
Simple,
/// <summary>Start / End / Delta: first + last usable sample per bucket (<c>ROW_NUMBER</c>).</summary>
StartEnd,
/// <summary>MinimumActualTime / MaximumActualTime: extreme usable sample per bucket, ties ⇒ earliest.</summary>
ActualTime,
}
/// <summary>
/// Push-down T-SQL for processed (aggregate) reads (spec §4 "Processed"). Every query takes
/// Push-down T-SQL for processed (aggregate) reads (spec §4 "Processed"), one query per
/// <see cref="AggregateFamily"/> (<see cref="AggregateFamilies.FamilyOf"/> picks it). Every query takes
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the forward window
/// <c>[start, end)</c>) and <c>@intervalTicks</c> (bigint bucket width in 100 ns ticks), and returns one row
/// per <b>non-empty</b> bucket. The caller generates every interval and fills the empty ones.
@@ -52,19 +40,6 @@ internal static class SqlAggregateQueries
/// <summary><c>ValueKind</c> of String (4) and DateTime (5): usable but not numeric.</summary>
private const string NonNumericKinds = "4, 5";
/// <summary>The family that serves <paramref name="aggregate"/>, or null when this backend does not.</summary>
/// <param name="aggregate">Requested aggregate.</param>
/// <returns>The family, or null for an unsupported aggregate.</returns>
public static SqlAggregateFamily? FamilyOf(HistoryAggregateType aggregate) => aggregate switch
{
HistoryAggregateType.Average or HistoryAggregateType.Minimum or HistoryAggregateType.Maximum
or HistoryAggregateType.Total or HistoryAggregateType.Count or HistoryAggregateType.Range
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => SqlAggregateFamily.Simple,
HistoryAggregateType.Start or HistoryAggregateType.End or HistoryAggregateType.Delta => SqlAggregateFamily.StartEnd,
HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime => SqlAggregateFamily.ActualTime,
_ => null,
};
/// <summary>
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> (float, NULL when no usable
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlHistorianDataTypes.cs
-27
View File
@@ -1,27 +0,0 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>
/// The one place that maps value kinds to <c>Tag.DataType</c> (the schema stores a <see cref="DriverDataType"/> byte).
/// Used by the value writer and the updater only when they have to create a tag row on a miss; provisioning
/// later corrects it.
/// </summary>
internal static class SqlHistorianDataTypes
{
/// <summary>Data type used when a tag is created from a batch that holds no typed value.</summary>
public const DriverDataType DefaultForEmpty = DriverDataType.Float64;
/// <summary>Maps a value kind to the stored <c>Tag.DataType</c> byte (Empty ⇒ <see cref="DefaultForEmpty"/>).</summary>
/// <param name="kind">Value kind.</param>
/// <returns>The <see cref="DriverDataType"/> as a byte.</returns>
public static byte FromKind(HistorianValueKind kind) => (byte)(kind switch
{
HistorianValueKind.Double => DriverDataType.Float64,
HistorianValueKind.Int64 => DriverDataType.Int64,
HistorianValueKind.Boolean => DriverDataType.Boolean,
HistorianValueKind.String => DriverDataType.String,
HistorianValueKind.DateTime => DriverDataType.DateTime,
_ => DefaultForEmpty,
});
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlHistorianStatusCodes.cs
-81
View File
@@ -1,81 +0,0 @@
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>
/// Every OPC UA status this backend emits, as named constants (inline literals are invisible to
/// <c>StatusCodeParityTests</c>). Values mirror <c>Opc.Ua.StatusCodes</c>.
/// </summary>
internal static class SqlHistorianStatusCodes
{
public const uint Good = 0x00000000u;
public const uint GoodEntryInserted = 0x00A20000u;
public const uint GoodEntryReplaced = 0x00A30000u;
public const uint BadNoData = 0x809B0000u;
public const uint BadEntryExists = 0x809F0000u;
public const uint BadNoEntryExists = 0x80A00000u;
public const uint BadInvalidTimestamp = 0x80230000u;
public const uint BadAggregateInvalidInputs = 0x80D60000u;
public const uint UncertainDataSubNormal = 0x40A40000u;
/// <summary>Stored for a non-finite Double (NaN / ±Infinity) that arrived with a non-Bad status.</summary>
public const uint BadOutOfRange = 0x803C0000u;
}
/// <summary>
/// Bit masks over a 32-bit OPC UA status code (Part 4 §7.39). These are masks, not emitted codes, so their
/// names deliberately avoid the Good/Bad/Uncertain prefix the status-code parity scan keys on.
/// </summary>
internal static class StatusBits
{
/// <summary>Severity + sub-code (top 16 bits) of a status, i.e. the code without info bits.</summary>
public const uint CodeBitsMask = 0xFFFF0000u;
/// <summary>The two severity bits (Good = 00, Uncertain = 01, Bad = 10).</summary>
public const uint SeverityMask = 0xC0000000u;
/// <summary>The severity bit set on every Bad code.</summary>
public const uint SeverityBadBit = 0x80000000u;
/// <summary>The two historian info bits (Calculated, Interpolated) in the low bits of the info field.</summary>
public const uint HistorianBitsMask = 0x3u;
/// <summary>True when <paramref name="status"/> has Bad severity.</summary>
/// <param name="status">A status code.</param>
/// <returns>Whether the Bad severity bit is set.</returns>
public static bool IsBad(uint status) => (status & SeverityBadBit) != 0;
/// <summary>True when <paramref name="status"/> has Good severity.</summary>
/// <param name="status">A status code.</param>
/// <returns>Whether both severity bits are clear.</returns>
public static bool IsGood(uint status) => (status & SeverityMask) == 0;
}
/// <summary>Part 4 §7.39 historian info bits OR-ed onto a status code.</summary>
internal static class HistorianInfoBits
{
/// <summary>InfoType = DataValue.</summary>
public const uint InfoTypeDataValue = 0x00000400u;
/// <summary>The value was produced by an aggregate calculation.</summary>
public const uint HistorianCalculated = 0x1u;
/// <summary>The value was interpolated.</summary>
public const uint HistorianInterpolated = 0x2u;
/// <summary>
/// Marks <paramref name="status"/> as a Calculated data value. Any historian bits the status already carries
/// (e.g. a stored Interpolated sample) are cleared first, so the result is never both.
/// </summary>
/// <param name="status">Base status code.</param>
/// <returns>Status with InfoType and Calculated bits set.</returns>
public static uint WithCalculated(uint status) =>
(status & ~StatusBits.HistorianBitsMask) | InfoTypeDataValue | HistorianCalculated;
/// <summary>
/// Marks <paramref name="status"/> as an Interpolated data value. Any historian bits the status already
/// carries (e.g. a stored Calculated sample) are cleared first, so the result is never both.
/// </summary>
/// <param name="status">Base status code.</param>
/// <returns>Status with InfoType and Interpolated bits set.</returns>
public static uint WithInterpolated(uint status) =>
(status & ~StatusBits.HistorianBitsMask) | InfoTypeDataValue | HistorianInterpolated;
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerAlarmHistorianWriter.cs
+4 -51
View File
@@ -1,17 +1,16 @@
using System.Data;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>
/// <see cref="IAlarmHistorianWriter"/> over the SQL <c>Event</c> table. Per event: insert-if-absent keyed by a
/// deterministic <see cref="ComputeEventId"/>, so the store-and-forward drain's at-least-once redelivery (and both
/// deterministic <see cref="AlarmEventIds.ComputeEventId"/>, so the store-and-forward drain's at-least-once redelivery (and both
/// pair nodes writing the same event) converge on one row. Outcomes: Ack on insert or already-present;
/// RetryPlease on transient SQL / schema-bootstrap / unclassified SQL errors and on cancellation; PermanentFail only
/// for data errors (truncation, conversion, null, range, constraint); unknown/non-SQL exceptions retry. Never throws.
@@ -76,12 +75,12 @@ internal sealed class SqlServerAlarmHistorianWriter : IAlarmHistorianWriter
{
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.NVarChar, 64) { Value = ComputeEventId(e) });
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.NVarChar, 64) { Value = AlarmEventIds.ComputeEventId(e) });
cmd.Parameters.Add(new SqlParameter("@src", SqlDbType.NVarChar, -1) { Value = e.EquipmentPath ?? "" });
cmd.Parameters.Add(new SqlParameter("@aid", SqlDbType.NVarChar, -1) { Value = e.AlarmId ?? "" });
cmd.Parameters.Add(new SqlParameter("@an", SqlDbType.NVarChar, -1) { Value = e.AlarmName ?? "" });
cmd.Parameters.Add(new SqlParameter("@at", SqlDbType.NVarChar, -1) { Value = e.AlarmTypeName ?? "" });
cmd.Parameters.Add(new SqlParameter("@sev", SqlDbType.Int) { Value = ToNumericSeverity(e.Severity) });
cmd.Parameters.Add(new SqlParameter("@sev", SqlDbType.Int) { Value = AlarmEventIds.ToNumericSeverity(e.Severity) });
cmd.Parameters.Add(new SqlParameter("@ek", SqlDbType.NVarChar, -1) { Value = e.EventKind ?? "" });
cmd.Parameters.Add(new SqlParameter("@msg", SqlDbType.NVarChar, -1) { Value = (object?)e.Message ?? DBNull.Value });
cmd.Parameters.Add(new SqlParameter("@usr", SqlDbType.NVarChar, -1) { Value = (object?)e.User ?? DBNull.Value });
@@ -116,50 +115,4 @@ internal sealed class SqlServerAlarmHistorianWriter : IAlarmHistorianWriter
return HistorianWriteOutcome.PermanentFail;
return HistorianWriteOutcome.RetryPlease;
}
// Same numeric Part 9 priorities the gateway path sends (AlarmEventMapper.ToNumericSeverity).
private static int ToNumericSeverity(AlarmSeverity s) => s switch
{
AlarmSeverity.Low => 200,
AlarmSeverity.Medium => 500,
AlarmSeverity.High => 700,
AlarmSeverity.Critical => 900,
_ => 500,
};
/// <summary>
/// Lowercase-hex SHA-256 over length-prefixed UTF-8 fields (little-endian int32 byte length, -1 for null):
/// EquipmentPath, AlarmId, EventKind, TimestampUtc.Ticks (decimal), User, Comment, Message. Length-prefixing keeps
/// field boundaries unambiguous (<c>"a|b"+"c"</c> ≠ <c>"a"+"b|c"</c>).
/// </summary>
/// <param name="e">The event.</param>
/// <returns>64 hex characters.</returns>
internal static string ComputeEventId(AlarmHistorianEvent e)
{
using var ms = new MemoryStream();
Field(ms, e.EquipmentPath);
Field(ms, e.AlarmId);
Field(ms, e.EventKind);
Field(ms, e.TimestampUtc.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture));
Field(ms, e.User);
Field(ms, e.Comment);
Field(ms, e.Message);
return Convert.ToHexStringLower(SHA256.HashData(ms.ToArray()));
}
private static void Field(MemoryStream ms, string? value)
{
Span<byte> len = stackalloc byte[4];
if (value is null)
{
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(len, -1);
ms.Write(len);
return;
}
var bytes = Encoding.UTF8.GetBytes(value);
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(len, bytes.Length);
ms.Write(len);
ms.Write(bytes);
}
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs
+34 -196
View File
@@ -4,6 +4,7 @@ using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
@@ -39,7 +40,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// on Raw and AtTime alike.
/// </para>
/// <para>
/// <b>Long names.</b> A tag name or event source name longer than <see cref="SqlServerTagCatalog.MaxNameLength"/>
/// <b>Long names.</b> A tag name or event source name longer than <see cref="HistorianTagNames.MaxLength"/>
/// can never be stored, so it is answered as not found (Raw: empty; AtTime / Processed: <c>BadNoData</c>;
/// Events: none) without being truncated onto another series.
/// </para>
@@ -47,7 +48,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
public sealed class SqlServerHistorianDataSource : IHistorianDataSource
{
/// <summary>Spec §9.7: a processed read covering more intervals than this is refused.</summary>
public const int MaxProcessedBuckets = 100_000;
public const int MaxProcessedBuckets = BucketPlan.MaxBuckets;
private const string SampleColumns = "SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc";
@@ -57,14 +58,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
private readonly ILogger _log;
private readonly TimeProvider _time;
private readonly object _healthLock = new();
private DateTime? _lastSuccessUtc;
private DateTime? _lastFailureUtc;
private string? _lastError;
private long _totalQueries;
private long _totalSuccesses;
private long _totalFailures;
private int _consecutiveFailures;
private readonly HistorianHealthCounters _health = new();
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
/// <param name="conn">Connection helper.</param>
@@ -192,7 +186,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
StoredSample? p = reader.IsDBNull(1) ? null : ReadStored(reader, 1);
StoredSample? n = reader.IsDBNull(8) ? null : ReadStored(reader, 8);
var serverTime = p is { } ps && ps.TimeUtc == t ? SqlHistorianConnection.GetUtc(reader, 7) : now;
byTicks[t.Ticks] = AtTimeValue(t, p, n, serverTime, now);
byTicks[t.Ticks] = AtTimeEvaluator.Evaluate(t, p, n, serverTime, now);
}
}
@@ -202,7 +196,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
var t = DateTime.SpecifyKind(timestampsUtc[i], DateTimeKind.Utc);
result[i] = byTicks.TryGetValue(t.Ticks, out var v)
? v
: new DataValueSnapshot(null, SqlHistorianStatusCodes.BadNoData, t, now);
: new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, t, now);
}
RecordSuccess();
@@ -215,44 +209,6 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
}
}
/// <summary>Applies the AtTime rules (see <see cref="ReadAtTimeAsync"/>) to one requested time.</summary>
private static DataValueSnapshot AtTimeValue(DateTime t, StoredSample? prev, StoredSample? next, DateTime serverTime, DateTime now)
{
if (prev is not { } p)
return new DataValueSnapshot(null, SqlHistorianStatusCodes.BadNoData, t, serverTime);
if (p.TimeUtc == t)
return new DataValueSnapshot(p.Value.ToObject(), p.Status, t, serverTime);
// Inside an outage: report the outage's own Bad code, never an upgraded Uncertain.
if (!p.IsUsable)
return new DataValueSnapshot(null, p.Status & StatusBits.CodeBitsMask, t, serverTime);
// Usable previous, Bad next: hold the previous value up to the outage; do not interpolate into it.
if (next is { IsUsable: false })
return new DataValueSnapshot(p.Value.ToObject(),
HistorianInfoBits.WithInterpolated(SqlHistorianStatusCodes.UncertainDataSubNormal), t, serverTime);
if (next is { } n && p.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
&& n.Value.TryGetDouble(out var nv))
{
var fraction = (double)(t.Ticks - p.TimeUtc.Ticks) / (n.TimeUtc.Ticks - p.TimeUtc.Ticks);
var v = p.Value.Number + (nv - p.Value.Number) * fraction;
object value = p.Value.Kind == HistorianValueKind.Int64
? (long)Math.Round(v, MidpointRounding.AwayFromZero)
: v;
var status = p.IsGood && n.IsGood ? SqlHistorianStatusCodes.Good : SqlHistorianStatusCodes.UncertainDataSubNormal;
return new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), t, serverTime);
}
// Beyond the last sample AND beyond now: the value is held into the future, so it is not known to be Good.
if (next is null && t > now)
return new DataValueSnapshot(p.Value.ToObject(),
HistorianInfoBits.WithInterpolated(SqlHistorianStatusCodes.UncertainDataSubNormal), t, serverTime);
// Stepped (non-numeric kind, or after the last sample): hold the previous value and its status.
return new DataValueSnapshot(p.Value.ToObject(), HistorianInfoBits.WithInterpolated(p.Status), t, serverTime);
}
// ------------------------------------------------------------------------------------------ Events
/// <inheritdoc />
@@ -269,7 +225,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
{
// No stored event can carry a source longer than the column (the alarm writer dead-letters it rather than
// truncating), so an over-long filter matches nothing — never a truncated prefix of someone else's source.
if (sourceName is not null && sourceName.Length > SqlServerTagCatalog.MaxNameLength)
if (sourceName is not null && sourceName.Length > HistorianTagNames.MaxLength)
return new HistoricalEventsResult([], null);
try
{
@@ -301,10 +257,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
SqlHistorianConnection.GetUtc(reader, 2),
SqlHistorianConnection.GetUtc(reader, 3),
reader.IsDBNull(4) ? null : reader.GetString(4),
(ushort)Math.Clamp(reader.GetInt32(5), 0, ushort.MaxValue));
HistoricalEventRows.ClampSeverity(reader.GetInt32(5)));
if (bounded && events.Count == maxEvents)
{
continuation = Encoding.UTF8.GetBytes($"{e.EventTimeUtc.Ticks}|{e.EventId}");
continuation = Encoding.UTF8.GetBytes(EventContinuation.Encode(e.EventTimeUtc, e.EventId));
break;
}
events.Add(e);
@@ -352,57 +308,38 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
HistoryAggregateType aggregate, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
var family = SqlAggregateQueries.FamilyOf(aggregate) ?? throw new HistoryAggregateNotSupportedException(aggregate);
var reverse = endUtc < startUtc;
var (start, end) = reverse ? (endUtc, startUtc) : (startUtc, endUtc);
var windowTicks = end.Ticks - start.Ticks;
// Everything stays in 100 ns ticks (≤ ≈3.2e18 across 0001–9999, inside a long) — no ns scaling that could
// wrap. An interval wider than the window is one bucket, so clamp it to the window; the ceiling division
// is written without "+ interval - 1" so a TimeSpan near MaxValue cannot overflow either.
var intervalTicks = interval > TimeSpan.Zero ? Math.Min(interval.Ticks, windowTicks) : windowTicks;
long bucketCount = windowTicks == 0 ? 0 : windowTicks / intervalTicks + (windowTicks % intervalTicks == 0 ? 0 : 1);
if (bucketCount > MaxProcessedBuckets)
throw new ArgumentOutOfRangeException(nameof(interval), bucketCount,
$"A processed read may cover at most {MaxProcessedBuckets} intervals; this window holds {bucketCount}.");
var family = AggregateFamilies.FamilyOf(aggregate) ?? throw new HistoryAggregateNotSupportedException(aggregate);
var plan = BucketPlan.Create(startUtc, endUtc, interval);
try
{
var now = _time.GetUtcNow().UtcDateTime;
var rows = new Dictionary<long, BucketRow>();
var tagId = bucketCount == 0 ? null : await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
var evaluated = new Dictionary<long, DataValueSnapshot>();
var tagId = plan.BucketCount == 0 ? null : await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
if (tagId is { } id)
{
var sample = _conn.Q("Sample");
var sql = family switch
{
SqlAggregateFamily.Simple => SqlAggregateQueries.Simple(sample),
SqlAggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample),
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample),
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample),
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
};
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(start, DateTimeKind.Utc));
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(end, DateTimeKind.Utc));
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = intervalTicks });
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc));
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc));
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var row = ReadBucket(reader, family);
rows[row.Bucket] = row;
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
}
}
var result = new List<DataValueSnapshot>((int)bucketCount);
for (long i = 0; i < bucketCount; i++)
{
var bucketStart = DateTime.SpecifyKind(new DateTime(start.Ticks + i * intervalTicks), DateTimeKind.Utc);
result.Add(rows.TryGetValue(i, out var row)
? Evaluate(aggregate, row, bucketStart, now)
: NoData(bucketStart, now));
}
if (reverse) result.Reverse();
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
@@ -414,56 +351,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
}
}
private static DataValueSnapshot NoData(DateTime t, DateTime now) =>
new(null, HistorianInfoBits.WithCalculated(SqlHistorianStatusCodes.BadNoData), t, now);
private static DataValueSnapshot Evaluate(HistoryAggregateType aggregate, BucketRow r, DateTime bucketStart, DateTime now)
{
DataValueSnapshot Calc(object? value, uint status, DateTime? t = null) =>
new(value, HistorianInfoBits.WithCalculated(status), t ?? bucketStart, now);
if (r.Total == 0) return NoData(bucketStart, now);
switch (aggregate)
{
case HistoryAggregateType.PercentGood: return Calc(100d * r.Good / r.Total, SqlHistorianStatusCodes.Good);
case HistoryAggregateType.PercentBad: return Calc(100d * r.Bad / r.Total, SqlHistorianStatusCodes.Good);
}
if (r.Usable == 0) return NoData(bucketStart, now);
var anyText = aggregate is HistoryAggregateType.Count or HistoryAggregateType.Start or HistoryAggregateType.End;
if (!anyText && r.NonNumeric > 0)
return Calc(null, SqlHistorianStatusCodes.BadAggregateInvalidInputs);
var status = r.Good == r.Total ? SqlHistorianStatusCodes.Good : SqlHistorianStatusCodes.UncertainDataSubNormal;
switch (aggregate)
{
case HistoryAggregateType.Count: return Calc(r.Usable, status);
case HistoryAggregateType.Average: return Calc(r.Avg, status);
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
case HistoryAggregateType.Total: return Calc(r.Sum, status);
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status);
case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status);
case HistoryAggregateType.Delta:
return r.First.TryGetDouble(out var a) && r.Last.TryGetDouble(out var b)
? Calc(b - a, status)
: Calc(null, SqlHistorianStatusCodes.BadAggregateInvalidInputs);
case HistoryAggregateType.MinimumActualTime:
case HistoryAggregateType.MaximumActualTime:
return Calc(r.Extreme, status, r.ExtremeTime);
default:
throw new HistoryAggregateNotSupportedException(aggregate);
}
}
private static BucketRow ReadBucket(SqlDataReader reader, SqlAggregateFamily family)
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
{
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
static long? L(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
static string? S(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
static HistorianValue V(SqlDataReader r, int i) =>
r.IsDBNull(i) ? HistorianValue.Empty : SampleRowMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
r.IsDBNull(i) ? HistorianValue.Empty : SampleColumnMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
var row = new BucketRow
{
@@ -477,13 +371,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
var f = SqlAggregateQueries.FirstFamilyColumn;
switch (family)
{
case SqlAggregateFamily.Simple:
case AggregateFamily.Simple:
row.Avg = D(reader, f);
row.Min = D(reader, f + 1);
row.Max = D(reader, f + 2);
row.Sum = D(reader, f + 3);
break;
case SqlAggregateFamily.StartEnd:
case AggregateFamily.StartEnd:
row.First = V(reader, f);
row.Last = V(reader, f + 5);
break;
@@ -495,44 +389,23 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
return row;
}
/// <summary>One aggregate query row (one non-empty bucket).</summary>
private sealed class BucketRow
{
public long Bucket;
public int Total, Usable, Good, Bad, NonNumeric;
public double? Avg, Min, Max, Sum, Extreme;
public DateTime? ExtremeTime;
public HistorianValue First, Last;
}
// ------------------------------------------------------------------------------------------ Shared
/// <summary>A stored sample read from the <c>{SampleColumns}</c> column run starting at an ordinal.</summary>
private readonly record struct StoredSample(DateTime TimeUtc, HistorianValue Value, uint Status)
{
/// <summary>Severity Good or Uncertain, and a value present (an Empty sample is an outage marker).</summary>
public bool IsUsable => !StatusBits.IsBad(Status) && Value.Kind != HistorianValueKind.Empty;
/// <summary>Usable with severity Good.</summary>
public bool IsGood => IsUsable && StatusBits.IsGood(Status);
}
/// <summary>
/// Reads <c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode</c> starting at <paramref name="o"/>.
/// An Empty (no-value) row whose stored status is not Bad is clamped to <c>BadNoData</c> — defence in depth for
/// rows written before the writer forced every Empty sample Bad.
/// An Empty (no-value) row whose stored status is not Bad is clamped to <c>BadNoData</c>
/// (<see cref="StoredSample.FromStored"/>) — defence in depth for rows written before the writer forced every
/// Empty sample Bad.
/// </summary>
private static StoredSample ReadStored(SqlDataReader r, int o)
{
var value = SampleRowMapper.FromColumns(
var value = SampleColumnMapper.FromColumns(
r.GetByte(o + 1),
r.IsDBNull(o + 2) ? null : r.GetDouble(o + 2),
r.IsDBNull(o + 3) ? null : r.GetInt64(o + 3),
r.IsDBNull(o + 4) ? null : r.GetString(o + 4));
var status = SampleRowMapper.FromDbStatus(r.GetInt32(o + 5));
if (value.Kind == HistorianValueKind.Empty && !StatusBits.IsBad(status))
status = SqlHistorianStatusCodes.BadNoData;
return new StoredSample(SqlHistorianConnection.GetUtc(r, o), value, status);
var status = SampleColumnMapper.FromDbStatus(r.GetInt32(o + 5));
return StoredSample.FromStored(SqlHistorianConnection.GetUtc(r, o), value, status);
}
/// <summary>Read-only tag lookup (never creates a row); runs the schema bootstrap on a cache miss.</summary>
@@ -541,50 +414,15 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
// ------------------------------------------------------------------------------------------ Health
/// <inheritdoc />
public HistorianHealthSnapshot GetHealthSnapshot()
{
lock (_healthLock)
{
var open = _totalSuccesses > 0 && _consecutiveFailures == 0;
return new HistorianHealthSnapshot(
TotalQueries: _totalQueries,
TotalSuccesses: _totalSuccesses,
TotalFailures: _totalFailures,
ConsecutiveFailures: _consecutiveFailures,
LastSuccessTime: _lastSuccessUtc,
LastFailureTime: _lastFailureUtc,
LastError: _lastError,
ProcessConnectionOpen: open,
EventConnectionOpen: open,
ActiveProcessNode: null,
ActiveEventNode: null,
Nodes: []);
}
}
public HistorianHealthSnapshot GetHealthSnapshot() => _health.Snapshot();
private void RecordSuccess()
{
lock (_healthLock)
{
_totalQueries++;
_totalSuccesses++;
_consecutiveFailures = 0;
_lastSuccessUtc = DateTime.UtcNow;
}
}
private void RecordSuccess() => _health.RecordSuccess();
private void RecordFailure(Exception ex)
{
// A cancelled read (deadline / client abort) is not a backend fault.
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
if (ex is OperationCanceledException) return;
lock (_healthLock)
{
_totalQueries++;
_totalFailures++;
_consecutiveFailures++;
_lastFailureUtc = DateTime.UtcNow;
_lastError = $"{ex.GetType().Name}: {ex.Message}";
}
_health.RecordFailure(ex);
_log.LogDebug("SQL historian read failed ({ExceptionType})", ex.GetType().Name);
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs
+29 -86
View File
@@ -1,18 +1,9 @@
using Microsoft.Data.SqlClient;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>Status codes owned by the updater (named consts so <c>StatusCodeParityTests</c> sees them).</summary>
internal static class SqlHistorianUpdateStatusCodes
{
/// <summary>
/// A value that cannot be stored: a Double NaN / ±Infinity, an Empty (null) value with a non-Bad status, or a
/// tag name longer than <see cref="SqlServerTagCatalog.MaxNameLength"/>.
/// </summary>
public const uint BadInvalidArgument = 0x80AB0000u;
}
/// <summary>
/// OPC UA HistoryUpdate against the SQL Server sample table (spec §4 "Updates"). One transaction per call;
/// exactly one status per input item, in input order.
@@ -26,7 +17,7 @@ internal static class SqlHistorianUpdateStatusCodes
/// accepted with a Bad status (an operator-authored outage marker); with a Good or Uncertain status it is
/// <c>BadInvalidArgument</c>, because a stored sample is never served better than "no value". Other items
/// still proceed.</para>
/// <para><b>Tag names over <see cref="SqlServerTagCatalog.MaxNameLength"/> characters</b> are refused, never
/// <para><b>Tag names over <see cref="HistorianTagNames.MaxLength"/> characters</b> are refused, never
/// truncated: every item of the call (and DeleteRaw's single result) reports <c>BadInvalidArgument</c> and nothing
/// is written.</para>
/// <para><b>Unknown tag.</b> Insert/Update create it via <see cref="SqlServerTagCatalog"/>; Replace and the
@@ -52,70 +43,26 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
{
ArgumentNullException.ThrowIfNull(tag);
ArgumentNullException.ThrowIfNull(values);
var results = new uint[values.Count];
if (!SqlServerTagCatalog.IsValidName(tag))
{
Array.Fill(results, SqlHistorianUpdateStatusCodes.BadInvalidArgument);
return results;
}
var winner = new Dictionary<long, int>(); // tick -> index of the item actually sent
for (var i = 0; i < values.Count; i++)
{
var v = values[i];
if (v.TimestampUtc is not { } ts || ts == DateTime.MinValue)
{
results[i] = SqlHistorianStatusCodes.BadInvalidTimestamp;
continue;
}
if ((v.Value.Kind == HistorianValueKind.Double && !double.IsFinite(v.Value.Number))
|| (v.Value.Kind == HistorianValueKind.Empty && !StatusBits.IsBad(v.StatusCode)))
{
results[i] = SqlHistorianUpdateStatusCodes.BadInvalidArgument;
continue;
}
if (!winner.TryGetValue(ts.Ticks, out _))
winner[ts.Ticks] = i;
else if (mode == HistoryUpdateMode.Insert)
results[i] = SqlHistorianStatusCodes.BadEntryExists; // first occurrence stays the winner
else
winner[ts.Ticks] = i; // last wins
}
var send = winner.Values.OrderBy(i => i).ToList();
if (send.Count == 0) return results;
var plan = HistoryUpdatePlanner.Plan(tag, mode, values);
if (plan.Winners.Count == 0) return HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());
var tagId = mode == HistoryUpdateMode.Replace
? await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false)
: await _tags.GetOrCreateTagIdAsync(tag, SqlHistorianDataTypes.FromKind(values[send[0]].Value.Kind), ct).ConfigureAwait(false);
: await _tags.GetOrCreateTagIdAsync(tag, HistorianDataTypes.FromKind(plan.Winners[0].Row.Value.Kind), ct).ConfigureAwait(false);
var outcome = tagId is null
? new Dictionary<long, uint>()
: await MergeAsync(tagId.Value, mode, send.Select(i => new SqlSampleRow(
values[i].TimestampUtc!.Value, values[i].Value, values[i].StatusCode)).ToList(), ct).ConfigureAwait(false);
// No OUTPUT row means the MERGE's absent branch: Insert => matched (exists); Replace => not matched.
var absent = mode == HistoryUpdateMode.Replace
? SqlHistorianStatusCodes.BadNoEntryExists
: SqlHistorianStatusCodes.BadEntryExists;
foreach (var i in send)
results[i] = outcome.TryGetValue(values[i].TimestampUtc!.Value.Ticks, out var code) ? code : absent;
// Superseded earlier duplicates (Replace/Update) share the winner's code.
if (mode != HistoryUpdateMode.Insert)
for (var i = 0; i < values.Count; i++)
if (values[i].TimestampUtc is { } ts && results[i] == 0
&& winner.TryGetValue(ts.Ticks, out var w) && w != i)
results[i] = results[w];
return results;
? new Dictionary<long, UpsertOutcome>()
: await MergeAsync(tagId.Value, mode, plan.Winners.Select(w => w.Row).ToList(), ct).ConfigureAwait(false);
return HistoryUpdatePlanner.Complete(plan, mode, outcome);
}
/// <inheritdoc/>
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(tag);
if (!SqlServerTagCatalog.IsValidName(tag)) return SqlHistorianUpdateStatusCodes.BadInvalidArgument;
if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
if (tagId is null) return SqlHistorianStatusCodes.BadNoData;
if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
int rows;
@@ -128,7 +75,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return rows > 0 ? SqlHistorianStatusCodes.Good : SqlHistorianStatusCodes.BadNoData;
return HistoryUpdatePlanner.DeleteRawResult(tagKnown: true, rows);
}
/// <inheritdoc/>
@@ -136,22 +83,19 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
{
ArgumentNullException.ThrowIfNull(tag);
ArgumentNullException.ThrowIfNull(timesUtc);
var results = new uint[timesUtc.Count];
if (!SqlServerTagCatalog.IsValidName(tag))
if (!HistorianTagNames.IsValid(tag))
{
Array.Fill(results, SqlHistorianUpdateStatusCodes.BadInvalidArgument);
return results;
var refused = new uint[timesUtc.Count];
Array.Fill(refused, HistorianStatusCodes.BadInvalidArgument);
return refused;
}
var first = new Dictionary<long, int>();
for (var i = 0; i < timesUtc.Count; i++)
{
results[i] = SqlHistorianStatusCodes.BadNoEntryExists;
if (timesUtc[i] == DateTime.MinValue) results[i] = SqlHistorianStatusCodes.BadInvalidTimestamp;
else first.TryAdd(timesUtc[i].Ticks, i);
}
if (first.Count == 0) return results;
var distinct = new HashSet<long>();
foreach (var t in timesUtc)
if (t != DateTime.MinValue) distinct.Add(t.Ticks);
var deleted = new HashSet<long>();
if (distinct.Count == 0) return HistoryUpdatePlanner.DeleteAtTimeResults(timesUtc, deleted, tagKnown: false);
var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
if (tagId is null) return results;
if (tagId is null) return HistoryUpdatePlanner.DeleteAtTimeResults(timesUtc, deleted, tagKnown: false);
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
@@ -160,18 +104,17 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
{
cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value });
_conn.AddStructured(cmd, "@times", "TimeRow",
SampleRowMapper.ToTimeRecords(first.Keys.Select(t => new DateTime(t, DateTimeKind.Utc))));
SampleRowMapper.ToTimeRecords(distinct.Select(t => new DateTime(t, DateTimeKind.Utc))));
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await rd.ReadAsync(ct).ConfigureAwait(false))
if (first.TryGetValue(SqlHistorianConnection.GetUtc(rd, 0).Ticks, out var idx))
results[idx] = SqlHistorianStatusCodes.Good;
deleted.Add(SqlHistorianConnection.GetUtc(rd, 0).Ticks);
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return results;
return HistoryUpdatePlanner.DeleteAtTimeResults(timesUtc, deleted, tagKnown: true);
}
private async Task<Dictionary<long, uint>> MergeAsync(int tagId, HistoryUpdateMode mode,
List<SqlSampleRow> rows, CancellationToken ct)
private async Task<Dictionary<long, UpsertOutcome>> MergeAsync(int tagId, HistoryUpdateMode mode,
List<HistorianSampleRow> rows, CancellationToken ct)
{
var sql = $"MERGE {_conn.Q("Sample")} WITH (HOLDLOCK) AS t USING @rows AS r ON t.TagId = @tagId AND t.SourceTimeUtc = r.SourceTimeUtc ";
if (mode != HistoryUpdateMode.Insert)
@@ -182,7 +125,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
"VALUES (@tagId, r.SourceTimeUtc, r.ValueKind, r.NumValue, r.IntValue, r.StrValue, r.StatusCode, SYSUTCDATETIME()) ";
sql += "OUTPUT $action, r.SourceTimeUtc;";
var outcome = new Dictionary<long, uint>();
var outcome = new Dictionary<long, UpsertOutcome>();
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
await using (var cmd = _conn.CreateCommand(conn, sql, tx))
@@ -192,8 +135,8 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await rd.ReadAsync(ct).ConfigureAwait(false))
outcome[SqlHistorianConnection.GetUtc(rd, 1).Ticks] = rd.GetString(0) == "INSERT"
? SqlHistorianStatusCodes.GoodEntryInserted
: SqlHistorianStatusCodes.GoodEntryReplaced;
? UpsertOutcome.Inserted
: UpsertOutcome.Replaced;
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return outcome;
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianOptions.cs
-8
View File
@@ -44,11 +44,3 @@ public sealed partial class SqlServerHistorianOptions
return errors;
}
}
/// <summary>Thrown when <c>AutoCreateSchema=false</c> and the historian schema is absent or behind.</summary>
public sealed class HistorianSchemaMissingException : Exception
{
/// <summary>Creates the exception.</summary>
/// <param name="message">Message naming the missing version.</param>
public HistorianSchemaMissingException(string message) : base(message) { }
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianValueWriter.cs
+5 -37
View File
@@ -2,6 +2,7 @@ using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
@@ -21,6 +22,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <b>Tag names over <see cref="SqlServerTagCatalog.MaxNameLength"/> characters</b> are refused (the write
/// returns false, a per-tag failure for the recorder's #538 quarantine), never truncated onto another tag.
/// <b>Duplicate timestamps within a batch</b> collapse to the first (matching insert-if-absent across batches).
/// The batch normalisation itself is <see cref="LiveWritePreparer.Prepare"/>.
/// </remarks>
internal sealed class SqlServerHistorianValueWriter : IHistorianValueWriter
{
@@ -51,7 +53,7 @@ internal sealed class SqlServerHistorianValueWriter : IHistorianValueWriter
{
try
{
if (!SqlServerTagCatalog.IsValidName(tag))
if (!HistorianTagNames.IsValid(tag))
{
_log.LogWarning("SQL historian live-value write refused: tag name is missing or longer than {Max} characters.",
SqlServerTagCatalog.MaxNameLength);
@@ -60,8 +62,8 @@ internal sealed class SqlServerHistorianValueWriter : IHistorianValueWriter
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
if (values is null || values.Count == 0) return true;
var rows = Prepare(values, _utcNow);
var dataType = SqlHistorianDataTypes.FromKind(rows.Select(r => r.Value.Kind).FirstOrDefault(k => k != HistorianValueKind.Empty));
var rows = LiveWritePreparer.Prepare(values, _utcNow);
var dataType = HistorianDataTypes.FromKind(rows.Select(r => r.Value.Kind).FirstOrDefault(k => k != HistorianValueKind.Empty));
var tagId = await _catalog.GetOrCreateTagIdAsync(tag, dataType, ct).ConfigureAwait(false);
var sql =
@@ -94,38 +96,4 @@ internal sealed class SqlServerHistorianValueWriter : IHistorianValueWriter
return false;
}
}
/// <summary>
/// Normalises a batch for the TVP: null timestamp ⇒ now, non-finite doubles ⇒ Empty, every Empty forced Bad
/// (non-finite ⇒ <c>BadOutOfRange</c>, otherwise <c>BadNoData</c>, an existing Bad code kept), and duplicate
/// timestamps collapsed to the first.
/// </summary>
/// <param name="values">Input samples, in append order.</param>
/// <param name="utcNow">Clock for null timestamps.</param>
/// <returns>Time-unique rows.</returns>
internal static List<SqlSampleRow> Prepare(IReadOnlyList<HistorizationValue> values, Func<DateTime> utcNow)
{
var seen = new HashSet<long>();
var rows = new List<SqlSampleRow>(values.Count);
foreach (var v in values)
{
var t = v.TimestampUtc ?? utcNow();
t = t.Kind == DateTimeKind.Local ? t.ToUniversalTime() : DateTime.SpecifyKind(t, DateTimeKind.Utc);
if (!seen.Add(t.Ticks)) continue;
var value = v.Value;
var status = v.StatusCode;
if (value.Kind == HistorianValueKind.Double && !double.IsFinite(value.Number))
{
value = HistorianValue.Empty;
if (!StatusBits.IsBad(status)) status = SqlHistorianStatusCodes.BadOutOfRange;
}
else if (value.Kind == HistorianValueKind.Empty && !StatusBits.IsBad(status))
{
status = SqlHistorianStatusCodes.BadNoData;
}
rows.Add(new SqlSampleRow(t, value, status));
}
return rows;
}
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerSchemaBootstrapper.cs
+8 -14
View File
@@ -1,17 +1,17 @@
using System.Data;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text.RegularExpressions;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>
/// Lazily applies the embedded, idempotent schema scripts once per instance, under a SQL application lock
/// so concurrent nodes/instances serialise. A failure is NOT latched: an <see cref="EnsureAsync"/> after
/// <see cref="FailureCacheDuration"/> retries. Within that window the last failure is re-thrown without touching
/// <see cref="BootstrapFailureCache.DefaultDuration"/> retries (<see cref="BootstrapFailureCache"/>). Within that window the last failure is re-thrown without touching
/// SQL, so a missing schema (or a SQL outage) does not serialise every write, read and alarm event behind one
/// semaphore and one round-trip each.
/// With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) &gt;= 1</c>.
@@ -24,17 +24,12 @@ internal sealed partial class SqlServerSchemaBootstrapper
private const string LockResource = "OtOpcUaHistorianSchema";
private const int LockTimeoutMs = 30_000;
/// <summary>How long the last bootstrap failure is re-thrown before SQL is tried again.</summary>
public static readonly TimeSpan FailureCacheDuration = TimeSpan.FromSeconds(5);
private readonly SqlHistorianConnection _conn;
private readonly bool _autoCreate;
private readonly ILogger _log;
private readonly TimeProvider _time;
private readonly SemaphoreSlim _gate = new(1, 1);
private readonly BootstrapFailureCache _failure; // guarded by _gate
private volatile bool _ready;
private ExceptionDispatchInfo? _lastFailure; // guarded by _gate
private DateTimeOffset _retryAfter; // guarded by _gate
/// <summary>Creates the bootstrapper.</summary>
/// <param name="conn">Connection helper.</param>
@@ -47,7 +42,7 @@ internal sealed partial class SqlServerSchemaBootstrapper
_conn = conn;
_autoCreate = options.AutoCreateSchema;
_log = log ?? NullLogger.Instance;
_time = timeProvider ?? TimeProvider.System;
_failure = new BootstrapFailureCache(timeProvider ?? TimeProvider.System, BootstrapFailureCache.DefaultDuration);
}
/// <summary>Ensures the schema is ready; no-op after the first success.</summary>
@@ -61,23 +56,22 @@ internal sealed partial class SqlServerSchemaBootstrapper
try
{
if (_ready) return;
if (_lastFailure is { } cached && _time.GetUtcNow() < _retryAfter) cached.Throw();
_failure.ThrowIfCached();
try
{
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
if (_autoCreate) await ApplyAsync(conn, ct).ConfigureAwait(false);
else await VerifyAsync(conn, ct).ConfigureAwait(false);
_ready = true;
_lastFailure = null;
_failure.Clear();
}
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
{
// A caller's own cancellation is not a backend failure and is not cached.
_lastFailure = ExceptionDispatchInfo.Capture(ex);
_retryAfter = _time.GetUtcNow() + FailureCacheDuration;
_failure.Record(ex);
_log.LogWarning("Historian schema bootstrap for [{Schema}] failed ({Exception}: {Message}); " +
"historian calls fail fast for {Seconds} s before retrying.",
_conn.Schema, ex.GetType().Name, ex.Message, FailureCacheDuration.TotalSeconds);
_conn.Schema, ex.GetType().Name, ex.Message, BootstrapFailureCache.DefaultDuration.TotalSeconds);
throw;
}
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerTagCatalog.cs
+5 -9
View File
@@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using Microsoft.Data.SqlClient;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
@@ -13,8 +14,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// </summary>
internal sealed class SqlServerTagCatalog
{
/// <summary>Width of the <c>Tag.Name</c> column; longer names are refused.</summary>
public const int MaxNameLength = 400;
/// <summary>Width of the <c>Tag.Name</c> column (= <see cref="HistorianTagNames.MaxLength"/>); longer names are refused.</summary>
public const int MaxNameLength = HistorianTagNames.MaxLength;
private readonly SqlHistorianConnection _conn;
private readonly SqlServerSchemaBootstrapper _bootstrap;
@@ -29,18 +30,13 @@ internal sealed class SqlServerTagCatalog
_bootstrap = bootstrap;
}
/// <summary>True when <paramref name="name"/> fits the <c>Tag.Name</c> column without truncation.</summary>
/// <param name="name">Historian tag name.</param>
/// <returns>Whether the name is non-null and at most <see cref="MaxNameLength"/> characters.</returns>
public static bool IsValidName(string? name) => name is not null && name.Length <= MaxNameLength;
/// <summary>Read-only lookup; never creates a row. A name over <see cref="MaxNameLength"/> is not found.</summary>
/// <param name="name">Historian tag name.</param>
/// <param name="ct">Cancellation.</param>
/// <returns>The TagId, or null when the tag does not exist.</returns>
public async Task<int?> TryGetTagIdAsync(string name, CancellationToken ct)
{
if (!IsValidName(name)) return null;
if (!HistorianTagNames.IsValid(name)) return null;
if (_cache.TryGetValue(name, out var cached)) return cached;
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
@@ -57,7 +53,7 @@ internal sealed class SqlServerTagCatalog
/// <exception cref="ArgumentException">The name is longer than <see cref="MaxNameLength"/>.</exception>
public async Task<int> GetOrCreateTagIdAsync(string name, byte dataType, CancellationToken ct)
{
if (!IsValidName(name))
if (!HistorianTagNames.IsValid(name))
throw new ArgumentException($"Historian tag name exceeds {MaxNameLength} characters.", nameof(name));
if (_cache.TryGetValue(name, out var cached)) return cached;
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.csproj
+2
View File
@@ -11,6 +11,8 @@
<ItemGroup>
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.Abstractions\ZB.MOM.WW.OtOpcUa.Core.Abstractions.csproj"/>
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj"/>
<!-- The database-independent historian core (status codes, evaluators, planners) shared with the QuestDB backend. -->
<ProjectReference Include="..\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.csproj"/>
<!-- Runtime owns IHistorianBackendProvider / ServerHistorianOptions; same diamond as Driver.Historian.Gateway
(Host -> {Runtime, SqlServer}, SqlServer -> Runtime). This project does NOT reference the Gateway project. -->
<ProjectReference Include="..\..\Server\ZB.MOM.WW.OtOpcUa.Runtime\ZB.MOM.WW.OtOpcUa.Runtime.csproj"/>