Merge feat/668-modified-values: Part 11 modified-value history (IsReadModified) on SqlServer + QuestDB (#668)

2026-10-11 09:32:13 -04:00
parent 699871040a cfd403710e
commit 062086f9a6
51 changed files with 2173 additions and 195 deletions
@@ -0,0 +1,29 @@
+-- OtOpcUa SQL Server historian backend - schema version 2: OPC UA Part 11 modified-value history (Gitea #668).
+-- $(Schema) is substituted by SqlServerSchemaBootstrapper (identifier-validated) BEFORE the script is split on GO lines.
+-- Every statement is guarded so a re-run is a no-op.
+--
+-- One row per sample a HistoryUpdate inserted, replaced or deleted, carrying the PRIOR value (all NULL for an
+-- insert). Rows are written by the SAME statement that changes hist.Sample (OUTPUT ... INTO), so a failed update
+-- leaves no audit row and a committed one always has its row.
+-- WARNING: OUTPUT ... INTO forbids, on its target table: enabled triggers, either side of a FOREIGN KEY, and CHECK
+-- constraints. Do not add any of those to SampleModified, or every HistoryUpdate fails.
+IF OBJECT_ID(N'[$(Schema)].[SampleModified]', N'U') IS NULL
+CREATE TABLE [$(Schema)].[SampleModified] (
+  TagId int NOT NULL,
+  SourceTimeUtc datetime2(7) NOT NULL,
+  ModifiedId bigint IDENTITY(1,1) NOT NULL,
+  ModificationTimeUtc datetime2(7) NOT NULL,
+  UpdateType tinyint NOT NULL,
+  Principal nvarchar(200) NOT NULL,
+  ApiKeyId nvarchar(200) NULL,
+  ValueKind tinyint NULL,
+  NumValue float NULL,
+  IntValue bigint NULL,
+  StrValue nvarchar(max) NULL,
+  StatusCode int NULL,
+  ServerTimeUtc datetime2(7) NULL,
+  CONSTRAINT PK_SampleModified PRIMARY KEY CLUSTERED (TagId, SourceTimeUtc, ModifiedId))
+GO
+IF NOT EXISTS (SELECT 1 FROM [$(Schema)].[SchemaVersion] WHERE Version = 2)
+INSERT [$(Schema)].[SchemaVersion] (Version, AppliedUtc) VALUES (2, SYSUTCDATETIME())
+GO
@@ -135,6 +135,97 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
        }
    }

+    // ------------------------------------------------------------------------------------------ Modified (Part 11, #668)
+
+    /// <inheritdoc />
+    /// <remarks>
+    ///     Reads <c>SampleModified</c> (schema v2), the trail <see cref="SqlServerHistorianDataUpdater"/> writes in the
+    ///     same statement as each change. Windows follow <see cref="ReadRawAsync"/>; rows are ordered by
+    ///     <c>(SourceTimeUtc, ModifiedId)</c> (the clustered key after <c>TagId</c>, so a seek). <c>maxValues + 1</c> rows
+    ///     are fetched to report <see cref="HistoryModifiedReadResult.HasMore"/> exactly. An insert's audit row carries no
+    ///     prior value and is served <c>null</c> / <c>BadNoData</c>; its server timestamp is the modification time. An
+    ///     unknown tag is an empty result.
+    /// </remarks>
+    public async Task<HistoryModifiedReadResult> ReadModifiedAsync(
+        string fullReference, DateTime startUtc, DateTime endUtc, uint maxValues, HistoryModifiedCursor? resumeAfter,
+        CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(fullReference);
+        var bounded = maxValues > 0 && maxValues < int.MaxValue;
+        try
+        {
+            var values = new List<HistoryModifiedValue>();
+            var hasMore = false;
+            var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
+            if (tagId is { } id)
+            {
+                var reverse = startUtc > endUtc;
+                string where;
+                if (startUtc == endUtc) where = "SourceTimeUtc = @lo";
+                else if (!reverse) where = "SourceTimeUtc >= @lo AND SourceTimeUtc < @hi";
+                else where = "SourceTimeUtc > @lo AND SourceTimeUtc <= @hi";
+                if (resumeAfter is not null)
+                    where += reverse
+                        ? " AND (SourceTimeUtc < @ct OR (SourceTimeUtc = @ct AND ModifiedId < @cid))"
+                        : " AND (SourceTimeUtc > @ct OR (SourceTimeUtc = @ct AND ModifiedId > @cid))";
+                var order = reverse ? "DESC" : "ASC";
+                var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc);
+
+                await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
+                await using var cmd = _conn.CreateCommand(conn,
+                    $"SELECT {(bounded ? "TOP (@take) " : "")}SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
+                    "ServerTimeUtc, ModifiedId, ModificationTimeUtc, UpdateType, Principal " +
+                    $"FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND {where} " +
+                    $"ORDER BY SourceTimeUtc {order}, ModifiedId {order}");
+                if (bounded) cmd.Parameters.Add(new SqlParameter("@take", SqlDbType.BigInt) { Value = (long)maxValues + 1 });
+                cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
+                SqlHistorianConnection.AddDateTime2(cmd, "@lo", lo);
+                SqlHistorianConnection.AddDateTime2(cmd, "@hi", hi);
+                if (resumeAfter is not null)
+                {
+                    SqlHistorianConnection.AddDateTime2(cmd, "@ct", resumeAfter.SourceTimeUtc);
+                    cmd.Parameters.Add(new SqlParameter("@cid", SqlDbType.BigInt) { Value = resumeAfter.ModifiedId });
+                }
+
+                await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
+                while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+                {
+                    if (bounded && values.Count == maxValues)
+                    {
+                        hasMore = true;
+                        break;
+                    }
+
+                    var sourceTime = SqlHistorianConnection.GetUtc(reader, 0);
+                    var modifiedAt = SqlHistorianConnection.GetUtc(reader, 8);
+                    DataValueSnapshot prior;
+                    if (reader.IsDBNull(1))
+                    {
+                        // An insert: there was no prior value.
+                        prior = new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, sourceTime, modifiedAt);
+                    }
+                    else
+                    {
+                        var s = ReadStored(reader, 0);
+                        prior = new DataValueSnapshot(s.Value.ToObject(), s.Status, s.TimeUtc,
+                            reader.IsDBNull(6) ? modifiedAt : SqlHistorianConnection.GetUtc(reader, 6));
+                    }
+
+                    values.Add(new HistoryModifiedValue(
+                        prior, modifiedAt, (HistoryModificationType)reader.GetByte(9), reader.GetString(10), reader.GetInt64(7)));
+                }
+            }
+
+            RecordSuccess();
+            return new HistoryModifiedReadResult(values, hasMore);
+        }
+        catch (Exception ex)
+        {
+            RecordFailure(ex);
+            throw;
+        }
+    }
+
    // ------------------------------------------------------------------------------------------ AtTime

    /// <inheritdoc />
@@ -22,6 +22,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
 ///     is written.</para>
 ///     <para><b>Unknown tag.</b> Insert/Update create it via <see cref="SqlServerTagCatalog"/>; Replace and the
 ///     deletes report <c>BadNoEntryExists</c> (DeleteRaw: <c>BadNoData</c>) without creating anything.</para>
+///     <para><b>Modified-value history (Part 11, #668).</b> Every row the MERGE inserts or updates and every row a
+///     delete removes is copied into <c>SampleModified</c> by an <c>OUTPUT … INTO</c> on the SAME statement — the prior
+///     value from <c>deleted.*</c> (NULL for an insert), the caller's <see cref="HistoryUpdateContext"/>, the
+///     <see cref="HistoryModificationType"/> and <c>SYSUTCDATETIME()</c>. One statement means the sample change and its
+///     audit row commit or roll back together: a failed update leaves no audit row.</para>
 /// </summary>
 internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
 {
@@ -39,10 +44,11 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater

    /// <inheritdoc/>
    public async Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
-        IReadOnlyList<HistorizationValue> values, CancellationToken ct)
+        IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct)
    {
        ArgumentNullException.ThrowIfNull(tag);
        ArgumentNullException.ThrowIfNull(values);
+        ArgumentNullException.ThrowIfNull(context);
        var plan = HistoryUpdatePlanner.Plan(tag, mode, values);
        if (plan.Winners.Count == 0) return HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());

@@ -52,14 +58,16 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater

        var outcome = tagId is null
            ? new Dictionary<long, UpsertOutcome>()
-            : await MergeAsync(tagId.Value, mode, plan.Winners.Select(w => w.Row).ToList(), ct).ConfigureAwait(false);
+            : await MergeAsync(tagId.Value, mode, plan.Winners.Select(w => w.Row).ToList(), context, ct).ConfigureAwait(false);
        return HistoryUpdatePlanner.Complete(plan, mode, outcome);
    }

    /// <inheritdoc/>
-    public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct)
+    public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, HistoryUpdateContext context,
+        CancellationToken ct)
    {
        ArgumentNullException.ThrowIfNull(tag);
+        ArgumentNullException.ThrowIfNull(context);
        if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
        var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
        if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
@@ -67,9 +75,11 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
        int rows;
        await using (var cmd = _conn.CreateCommand(conn,
-                         $"DELETE FROM {_conn.Q("Sample")} WHERE TagId = @tagId AND SourceTimeUtc >= @s AND SourceTimeUtc < @e", tx))
+                         $"DELETE FROM {_conn.Q("Sample")} {AuditInto(HistoryModificationType.Delete)} " +
+                         "WHERE TagId = @tagId AND SourceTimeUtc >= @s AND SourceTimeUtc < @e", tx))
        {
            cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value });
+            AddAuditParameters(cmd, context);
            SqlHistorianConnection.AddDateTime2(cmd, "@s", startUtc);
            SqlHistorianConnection.AddDateTime2(cmd, "@e", endUtc);
            rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
@@ -79,10 +89,12 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
    }

    /// <inheritdoc/>
-    public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, CancellationToken ct)
+    public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc,
+        HistoryUpdateContext context, CancellationToken ct)
    {
        ArgumentNullException.ThrowIfNull(tag);
        ArgumentNullException.ThrowIfNull(timesUtc);
+        ArgumentNullException.ThrowIfNull(context);
        if (!HistorianTagNames.IsValid(tag))
        {
            var refused = new uint[timesUtc.Count];
@@ -100,9 +112,11 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
        await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
        await using (var cmd = _conn.CreateCommand(conn,
-                         $"DELETE s OUTPUT deleted.SourceTimeUtc FROM {_conn.Q("Sample")} AS s JOIN @times AS t ON s.SourceTimeUtc = t.T WHERE s.TagId = @tagId", tx))
+                         $"DELETE s {AuditInto(HistoryModificationType.Delete)} OUTPUT deleted.SourceTimeUtc " +
+                         $"FROM {_conn.Q("Sample")} AS s JOIN @times AS t ON s.SourceTimeUtc = t.T WHERE s.TagId = @tagId", tx))
        {
            cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value });
+            AddAuditParameters(cmd, context);
            _conn.AddStructured(cmd, "@times", "TimeRow",
                SampleRowMapper.ToTimeRecords(distinct.Select(t => new DateTime(t, DateTimeKind.Utc))));
            await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
@@ -114,7 +128,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
    }

    private async Task<Dictionary<long, UpsertOutcome>> MergeAsync(int tagId, HistoryUpdateMode mode,
-        List<HistorianSampleRow> rows, CancellationToken ct)
+        List<HistorianSampleRow> rows, HistoryUpdateContext context, CancellationToken ct)
    {
        var sql = $"MERGE {_conn.Q("Sample")} WITH (HOLDLOCK) AS t USING @rows AS r ON t.TagId = @tagId AND t.SourceTimeUtc = r.SourceTimeUtc ";
        if (mode != HistoryUpdateMode.Insert)
@@ -123,7 +137,9 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        if (mode != HistoryUpdateMode.Replace)
            sql += "WHEN NOT MATCHED THEN INSERT (TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc) " +
                   "VALUES (@tagId, r.SourceTimeUtc, r.ValueKind, r.NumValue, r.IntValue, r.StrValue, r.StatusCode, SYSUTCDATETIME()) ";
-        sql += "OUTPUT $action, r.SourceTimeUtc;";
+        // The MERGE's deleted.* is NULL for an inserted row, so one OUTPUT INTO records both an insert (no prior value)
+        // and a replace (the prior value). TagId comes from inserted.* — present on both branches.
+        sql += $"{AuditInto(ToModificationType(mode), "inserted")} OUTPUT $action, r.SourceTimeUtc;";

        var outcome = new Dictionary<long, UpsertOutcome>();
        await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
@@ -131,6 +147,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        await using (var cmd = _conn.CreateCommand(conn, sql, tx))
        {
            cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId });
+            AddAuditParameters(cmd, context);
            _conn.AddStructured(cmd, "@rows", "SampleRow", SampleRowMapper.ToSampleRecords(rows));
            await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
            while (await rd.ReadAsync(ct).ConfigureAwait(false))
@@ -141,4 +158,33 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        await tx.CommitAsync(ct).ConfigureAwait(false);
        return outcome;
    }
+
+    /// <summary>The Part 11 modification type recorded for an update mode.</summary>
+    /// <param name="mode">The update mode.</param>
+    /// <returns>Insert ⇒ Insert, Replace ⇒ Replace, Update ⇒ Update.</returns>
+    internal static HistoryModificationType ToModificationType(HistoryUpdateMode mode) => mode switch
+    {
+        HistoryUpdateMode.Insert => HistoryModificationType.Insert,
+        HistoryUpdateMode.Replace => HistoryModificationType.Replace,
+        _ => HistoryModificationType.Update,
+    };
+
+    /// <summary>
+    ///     The <c>OUTPUT … INTO SampleModified</c> clause copying each affected row's prior value (<c>deleted.*</c>) into
+    ///     the modification trail. <paramref name="keySource"/> names the pseudo-table that carries <c>TagId</c> /
+    ///     <c>SourceTimeUtc</c> for every affected row (<c>deleted</c> for a DELETE, <c>inserted</c> for the MERGE).
+    /// </summary>
+    private string AuditInto(HistoryModificationType type, string keySource = "deleted") =>
+        $"OUTPUT {keySource}.TagId, {keySource}.SourceTimeUtc, SYSUTCDATETIME(), {(byte)type}, @principal, @apiKeyId, " +
+        "deleted.ValueKind, deleted.NumValue, deleted.IntValue, deleted.StrValue, deleted.StatusCode, deleted.ServerTimeUtc " +
+        $"INTO {_conn.Q("SampleModified")} (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal, ApiKeyId, " +
+        "ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)";
+
+    private static void AddAuditParameters(SqlCommand cmd, HistoryUpdateContext context)
+    {
+        cmd.Parameters.Add(new SqlParameter("@principal", System.Data.SqlDbType.NVarChar, HistoryUpdateContext.MaxIdentityLength)
+            { Value = context.StoredPrincipal });
+        cmd.Parameters.Add(new SqlParameter("@apiKeyId", System.Data.SqlDbType.NVarChar, HistoryUpdateContext.MaxIdentityLength)
+            { Value = (object?)context.StoredApiKeyId ?? DBNull.Value });
+    }
 }
@@ -18,7 +18,8 @@ public sealed partial class SqlServerHistorianOptions

    /// <summary>
    ///     <c>true</c> creates the schema and applies missing scripts on first use; <c>false</c> only verifies
-    ///     that schema version 1 exists and fails every historian call with a clear error otherwise.
+    ///     that the required schema version (<c>SqlServerSchemaBootstrapper.RequiredVersion</c>, currently 2) exists and
+    ///     fails every historian call with a clear error otherwise.
    /// </summary>
    public bool AutoCreateSchema { get; set; } = true;

@@ -7,7 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;

 /// <summary>
-///     Deletes <c>hist.Sample</c> / <c>hist.Event</c> rows older than the retention window, in small
+///     Deletes <c>hist.Sample</c> / <c>hist.SampleModified</c> / <c>hist.Event</c> rows older than the retention window, in small
 ///     <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
 ///     delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
 ///     A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
@@ -69,7 +69,7 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
                return [];
            }

-            long samples = 0;
+            long samples = 0, modified = 0;
            var tagIds = new List<int>();
            await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
            await using (var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false))
@@ -80,13 +80,23 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
                samples += await DeleteLoopAsync(conn,
                    $"DELETE TOP (@n) FROM {_conn.Q("Sample")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
                    cutoff, id, ct).ConfigureAwait(false);
+                // The Part 11 modification trail (#668) ages out on the SAME source-time cutoff as the samples it
+                // describes; a purge never writes to it (no trigger, no OUTPUT INTO).
+                modified += await DeleteLoopAsync(conn,
+                    $"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
+                    cutoff, id, ct).ConfigureAwait(false);
            }

            var events = await DeleteLoopAsync(conn,
                $"DELETE TOP (@n) FROM {_conn.Q("Event")} WHERE EventTimeUtc < @cutoff",
                cutoff, null, ct).ConfigureAwait(false);

-            return [new RetentionPurgeResult("Sample", samples), new RetentionPurgeResult("Event", events)];
+            return
+            [
+                new RetentionPurgeResult("Sample", samples),
+                new RetentionPurgeResult("SampleModified", modified),
+                new RetentionPurgeResult("Event", events),
+            ];
        }
        finally
        {
@@ -14,12 +14,16 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
 ///     <see cref="BootstrapFailureCache.DefaultDuration"/> retries (<see cref="BootstrapFailureCache"/>). Within that window the last failure is re-thrown without touching
 ///     SQL, so a missing schema (or a SQL outage) does not serialise every write, read and alarm event behind one
 ///     semaphore and one round-trip each.
-///     With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) &gt;= 1</c>.
+///     With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) &gt;= </c><see cref="RequiredVersion"/>.
 /// </summary>
 internal sealed partial class SqlServerSchemaBootstrapper
 {
-    /// <summary>The schema version this build requires.</summary>
-    public const int RequiredVersion = 1;
+    /// <summary>
+    ///     The schema version this build requires: 2 = <c>002_sample_modified.sql</c> (Part 11 modified-value
+    ///     history, Gitea #668). An existing v1 schema is upgraded in place on the next bootstrap when
+    ///     <c>AutoCreateSchema</c> is on; otherwise the script must be applied by hand.
+    /// </summary>
+    public const int RequiredVersion = 2;

    private const string LockResource = "OtOpcUaHistorianSchema";
    private const int LockTimeoutMs = 30_000;
@@ -48,7 +52,7 @@ internal sealed partial class SqlServerSchemaBootstrapper
    /// <summary>Ensures the schema is ready; no-op after the first success.</summary>
    /// <param name="ct">Cancellation.</param>
    /// <returns>A task completing when the schema is usable.</returns>
-    /// <exception cref="HistorianSchemaMissingException">AutoCreateSchema is false and version 1 is absent.</exception>
+    /// <exception cref="HistorianSchemaMissingException">AutoCreateSchema is false and <see cref="RequiredVersion"/> is absent.</exception>
    public async Task EnsureAsync(CancellationToken ct)
    {
        if (_ready) return;
@@ -94,7 +98,7 @@ internal sealed partial class SqlServerSchemaBootstrapper
        if (v < RequiredVersion)
            throw new HistorianSchemaMissingException(
                $"Historian schema [{_conn.Schema}] is missing version {RequiredVersion} (found {v}) and AutoCreateSchema is false. " +
-                "Apply docs/historian/sqlserver/001_initial.sql or enable AutoCreateSchema.");
+                "Apply the docs/historian/sqlserver/NNN_*.sql scripts above version " + v + " in order, or enable AutoCreateSchema.");
    }

    private async Task ApplyAsync(SqlConnection conn, CancellationToken ct)